Merge branch 'bililive-go:master' into master

This commit is contained in:
Eric authored and GitHub committed 2026-09-22 15:06:03 +08:00
commit ad8a9f2192
43 files changed
+3027 -436

No files matched your search

+1
View File
@@ -130,6 +130,7 @@ jobs:
dist/**/*.zip
dist/**/*.7z
prerelease: ${{ contains(env.GIT_TAG, 'rc') }}
generate_release_notes: true
- name: "[Dry-Run] 跳过 GitHub Release 发布"
if: ${{ env.IS_DRY_RUN == 'true' }}
run: echo "::notice::Dry-run 模式:跳过 GitHub Release 发布(GIT_TAG=$GIT_TAG)"
+30 -7
View File
@@ -31,22 +31,39 @@ video_split_strategies:
on_record_finished:
convert_to_mp4: false
delete_flv_after_convert: false
# 当 custom_commandline 的值 不为空时,convert_to_mp4 的值会被无视,
# 而是在录制结束后直接执行 custom_commandline 中的命令。
# 在 custom_commandline 执行结束后,程序还会继续查看 delete_flv_after_convert 的值,
# 来判断是否需要删除原始 flv 文件。
# 以下是一个在录制结束后将 flv 视频转换为同名 mp4 视频的示例:
# custom_commandline: '{{ .Ffmpeg }} -hide_banner -i "{{ .FileName }}" -c copy "{{ .FileName | trimSuffix (.FileName | ext)}}.mp4"'
custom_commandline: ""
fix_flv_at_first: true
save_cover: false
cloud_upload:
enable: false
storage_name: ""
upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }}
upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }}
delete_after_upload: false
delete_all_after_upload: false
upload_timing: after_process
burn_subtitles: false
burn_subtitles_codec: libx264
burn_subtitles_crf: "18"
burn_subtitles_preset: medium
burn_delete_ass: false
burn_delete_source: false
timeout_in_us: 60000000
danmaku_enable: false
danmaku:
font_size: 36
font_name: Microsoft YaHei
scroll_area: full
scroll_time: 10
resolution: 1920x1080
outline: 1
opacity: 128
record_gift: true
record_douyu_gift: true
record_douyin_gift: true
record_guard: true
record_super_chat: true
guard_position: bottom-left
sc_position: bottom-left
live_rooms:
# quality参数目前仅B站启用,默认为0
# (B站)0代表原画PRO(HEVC)优先, 其他数值为原画(AVC)
@@ -106,6 +123,10 @@ notify:
icon: ""
# 通知级别(可选): active/timeSensitive/passive/critical
level: ""
wxpusher:
enable: false
appToken: ""
uids: []
app_data_path: .appdata
read_only_tool_folder: ""
tool_root_folder: ""
@@ -126,6 +147,8 @@ proxy:
openlist:
port: 5244
data_path: ""
username: ""
password: ""
update:
auto_check: true
check_interval_hours: 6
+117
View File
@@ -339,6 +339,8 @@ func main() {
// 如果启用了云上传功能,初始化 OpenList 管理器
var openlistManager *openlist.Manager
// 串行化 OpenList 生命周期操作,防止快速 enable/disable 导致服务泄漏
var openlistLifecycleMu sync.Mutex
if config.OnRecordFinished.CloudUpload.Enable {
// 获取 OpenList 数据目录
openlistDataPath := config.OpenList.DataPath
@@ -357,6 +359,9 @@ func main() {
bilisentryPkg.Go(func() {
if err := openlistManager.Start(rootCtx); err != nil {
logger.WithError(err).Error("启动 OpenList 失败,云上传功能将不可用")
// 启动失败时清除全局指针(CAS:仅当仍为当前实例时才清除,避免误清新 Manager)
openlist.ClearGlobalManagerIf(openlistManager)
openlistManager = nil
}
})
@@ -366,6 +371,113 @@ func main() {
logger.Info("云上传功能已启用")
}
// 注册配置更新回调:当用户通过 Web UI 首次启用云上传时,自动创建并启动 OpenList Manager
configs.RegisterAfterUpdate(func(oldCfg, newCfg *configs.Config) {
if newCfg == nil {
return
}
wasEnabled := oldCfg != nil && oldCfg.OnRecordFinished.CloudUpload.Enable
isEnabled := newCfg.OnRecordFinished.CloudUpload.Enable
// 关闭云上传:停止 OpenList 进程
if wasEnabled && !isEnabled {
openlistLifecycleMu.Lock()
defer openlistLifecycleMu.Unlock()
if mgr := openlist.GetGlobalManager(); mgr != nil {
logger.Info("检测到云上传已关闭,正在停止 OpenList...")
mgr.Stop()
servers.SetOpenListManager(nil)
openlistManager = nil
}
return
}
// 开启云上传:启动 OpenList 进程
if wasEnabled || !isEnabled {
// 云上传保持启用:检查是否需要重建 Manager(端口或数据目录变更)
if wasEnabled && isEnabled {
if mgr := openlist.GetGlobalManager(); mgr != nil {
// 解析新的配置值
newPort := newCfg.OpenList.Port
if newPort == 0 {
newPort = 5244
}
newDataPath := newCfg.OpenList.DataPath
if newDataPath == "" {
newDataPath = filepath.Join(newCfg.AppDataPath, "openlist")
}
// 与 Manager 当前值比较
if mgr.GetPort() != newPort || mgr.GetDataPath() != newDataPath {
logger.Infof("检测到 OpenList 配置变更(端口: %d→%d, 数据目录: %s→%s),正在重建 Manager...",
mgr.GetPort(), newPort, mgr.GetDataPath(), newDataPath)
openlistLifecycleMu.Lock()
mgr.Stop()
servers.SetOpenListManager(nil)
// 创建新 Manager 并启动
newMgr := openlist.NewManager(newDataPath, newPort)
servers.SetOpenListManager(newMgr)
openlistManager = newMgr
openlistLifecycleMu.Unlock()
bilisentryPkg.Go(func() {
if err := newMgr.Start(rootCtx); err != nil {
logger.WithError(err).Error("重建 OpenList 失败")
openlistLifecycleMu.Lock()
// CAS:仅当全局仍为 newMgr 时才清除
openlist.ClearGlobalManagerIf(newMgr)
openlistManager = nil
openlistLifecycleMu.Unlock()
}
})
}
}
}
return
}
openlistLifecycleMu.Lock()
// 已有 Manager 则跳过
if openlist.GetGlobalManager() != nil {
openlistLifecycleMu.Unlock()
return
}
logger.Info("检测到云上传已启用,正在初始化 OpenList...")
openlistDataPath := newCfg.OpenList.DataPath
if openlistDataPath == "" {
openlistDataPath = filepath.Join(newCfg.AppDataPath, "openlist")
}
openlistPort := newCfg.OpenList.Port
if openlistPort == 0 {
openlistPort = 5244
}
mgr := openlist.NewManager(openlistDataPath, openlistPort)
// 先设置全局 Manager,让 API 端能立即获取到状态
servers.SetOpenListManager(mgr)
openlistLifecycleMu.Unlock()
// 在后台启动 OpenList 进程(需要 rootCtx 生命周期)
bilisentryPkg.Go(func() {
if err := mgr.Start(rootCtx); err != nil {
logger.WithError(err).Error("动态启动 OpenList 失败,云上传功能将不可用")
// CAS:仅当全局仍为当前实例时才清除,避免误清新 Manager
openlistLifecycleMu.Lock()
openlist.ClearGlobalManagerIf(mgr)
openlistLifecycleMu.Unlock()
return
}
// Start 完成后复核配置:如果在 waitForReady 期间配置已关闭,立即清理
openlistLifecycleMu.Lock()
defer openlistLifecycleMu.Unlock()
if cfg := configs.GetCurrentConfig(); cfg == nil || !cfg.OnRecordFinished.CloudUpload.Enable {
logger.Info("OpenList 启动完成但云上传已关闭,立即停止")
mgr.Stop()
openlist.ClearGlobalManagerIf(mgr)
}
})
})
// 初始化 Pipeline 管道管理器
pipelineDbPath := filepath.Join(config.AppDataPath, "db", "pipeline.db")
pipelineStore, err := pipeline.NewSQLiteStore(pipelineDbPath)
@@ -714,6 +826,11 @@ func main() {
if inst.PipelineManager != nil {
inst.PipelineManager.Close(ctx)
}
// 等待异步通知 goroutine(如 Telegram/Email 摘要发送)完成
// 必须在 PipelineManager.Close 之后——pipeline 回调可能触发最后的通知
if rm, ok := inst.RecorderManager.(recorders.Manager); ok {
rm.WaitForNotifications()
}
// 关闭直播间状态管理器
if liveStateManager != nil {
liveStateManager.Close()
+58 -31
View File
@@ -354,6 +354,7 @@ type CloudUpload struct {
UploadPathTmpl string `yaml:"upload_path_tmpl" json:"upload_path_tmpl"` // 上传路径模板
DeleteAfterUpload bool `yaml:"delete_after_upload" json:"delete_after_upload"` // 上传成功后仅删除已上传的文件
DeleteAllAfterUpload bool `yaml:"delete_all_after_upload" json:"delete_all_after_upload"` // 上传成功后删除全部文件(含中间产物)
UploadSubtitles bool `yaml:"upload_subtitles" json:"upload_subtitles"` // 是否上传关联的 .ass 弹幕字幕文件
AdditionalStorages []string `yaml:"additional_storages,omitempty" json:"additional_storages,omitempty"` // 额外存储(支持多目标上传)
}
@@ -618,6 +619,36 @@ var updateMu sync.Mutex
// 当期望版本与实际版本不一致时返回的错误
var ErrConfigVersionConflict = errors.New("config version conflict")
// AfterUpdateCallback 配置更新后的回调函数类型
// old 为更新前的配置(可能为 nil),newCfg 为更新后的配置
type AfterUpdateCallback func(old, newCfg *Config)
// afterUpdateCallbacks 配置更新后的回调列表
var (
afterUpdateCallbacks []AfterUpdateCallback
afterUpdateMu sync.RWMutex
)
// RegisterAfterUpdate 注册配置更新后的回调
// 回调在配置更新并持久化完成后同步执行,此时已释放 updateMu 锁
func RegisterAfterUpdate(cb AfterUpdateCallback) {
afterUpdateMu.Lock()
defer afterUpdateMu.Unlock()
afterUpdateCallbacks = append(afterUpdateCallbacks, cb)
}
// fireAfterUpdate 触发所有配置更新回调
func fireAfterUpdate(old, newCfg *Config) {
afterUpdateMu.RLock()
callbacks := make([]AfterUpdateCallback, len(afterUpdateCallbacks))
copy(callbacks, afterUpdateCallbacks)
afterUpdateMu.RUnlock()
for _, cb := range callbacks {
cb(old, newCfg)
}
}
func SetCurrentConfig(cfg *Config) {
if cfg == nil {
// 存储 nil 以保持行为一致
@@ -657,19 +688,19 @@ func UpdateTransient(mutator func(c *Config) error) (*Config, error) {
}
func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) {
var newCfg *Config
var oldCfg, newCfg *Config
var updateErr error
func() {
updateMu.Lock()
defer updateMu.Unlock()
old := GetCurrentConfig()
oldCfg = GetCurrentConfig()
// 若当前尚未设置配置,则以默认配置为基础
var base *Config
if old == nil {
if oldCfg == nil {
base = NewConfig()
} else {
base = CloneConfigShallow(old)
base = CloneConfigShallow(oldCfg)
}
if err := mutator(base); err != nil {
updateErr = err
@@ -678,10 +709,10 @@ func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) {
// 维护派生字段
base.RefreshLiveRoomIndexCache()
// 版本号自增
if old == nil {
if oldCfg == nil {
base.Version = 1
} else {
base.Version = old.Version + 1
base.Version = oldCfg.Version + 1
}
newCfg = base
@@ -702,6 +733,7 @@ func updateImpl(mutator func(c *Config) error, persist bool) (*Config, error) {
return nil, errors.New("config update failed")
}
fireAfterUpdate(oldCfg, newCfg)
return newCfg, nil
}
@@ -712,17 +744,17 @@ func UpdateCAS(expectedVersion int64, mutator func(c *Config) error) (*Config, e
}
func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist bool) (*Config, error) {
var newCfg *Config
var oldCfg, newCfg *Config
var updateErr error
func() {
updateMu.Lock()
defer updateMu.Unlock()
cur := GetCurrentConfig()
oldCfg = GetCurrentConfig()
// 校验版本
var curVersion int64
if cur != nil {
curVersion = cur.Version
if oldCfg != nil {
curVersion = oldCfg.Version
}
if curVersion != expectedVersion {
updateErr = ErrConfigVersionConflict
@@ -730,10 +762,10 @@ func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist
}
// 克隆并修改
var base *Config
if cur == nil {
if oldCfg == nil {
base = NewConfig()
} else {
base = CloneConfigShallow(cur)
base = CloneConfigShallow(oldCfg)
}
if err := mutator(base); err != nil {
updateErr = err
@@ -757,6 +789,7 @@ func updateCASImpl(expectedVersion int64, mutator func(c *Config) error, persist
return nil, updateErr
}
fireAfterUpdate(oldCfg, newCfg)
return newCfg, nil
}
@@ -980,7 +1013,7 @@ var defaultConfig = Config{
CloudUpload: CloudUpload{
Enable: false,
StorageName: "",
UploadPathTmpl: "/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date \"2006-01-02\" }}.{{ .Ext }}",
UploadPathTmpl: "/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }}",
DeleteAfterUpload: false,
},
UploadTiming: UploadTimingAfterProcess,
@@ -1137,27 +1170,21 @@ func (c *Config) RefreshLiveRoomIndexCache() {
}
}
func (c *Config) RemoveLiveRoomByUrl(url string) error {
c.RefreshLiveRoomIndexCache()
if index, ok := c.liveRoomIndexCache[url]; ok {
if index >= 0 && index < len(c.LiveRooms) && c.LiveRooms[index].Url == url {
c.LiveRooms = append(c.LiveRooms[:index], c.LiveRooms[index+1:]...)
delete(c.liveRoomIndexCache, url)
return nil
}
}
return errors.New("failed removing room: " + url)
}
func (c *Config) GetLiveRoomByUrl(url string) (*LiveRoom, error) {
room, err := c.getLiveRoomByUrlImpl(url)
if err != nil {
c.RefreshLiveRoomIndexCache()
if room, err = c.getLiveRoomByUrlImpl(url); err != nil {
return nil, err
// 配置快照是不可变的:所有写操作都走 Update 的“复制-修改-原子替换”路径,
// 并在替换前调用 RefreshLiveRoomIndexCache 维护索引缓存。因此这里只做只读查找,
// 绝不在共享快照上写 map/切片,否则会与其它 goroutine 的读发生并发读写,
// 触发 Go 运行时的 fatal error(concurrent map read and map write)。
if room, err := c.getLiveRoomByUrlImpl(url); err == nil {
return room, nil
}
// 索引缓存未命中时回退到只读线性扫描,作为兜底(例如未经 Refresh 的手工构造配置)。
for i := range c.LiveRooms {
if c.LiveRooms[i].Url == url {
return &c.LiveRooms[i], nil
}
}
return room, nil
return nil, errors.New("room " + url + " doesn't exist.")
}
func (c Config) getLiveRoomByUrlImpl(url string) (*LiveRoom, error) {
-55
View File
@@ -32,61 +32,6 @@ func DecorateConfigNode(node *yaml.Node) {
# 负数为非法值,程序会输出 log 提醒,并无视所设定的数值`, "")
}
finishNode := findNode(root, "on_record_finished")
if finishNode != nil {
setFieldComment(finishNode, "custom_commandline",
`# 当 custom_commandline 的值 不为空时,convert_to_mp4 的值会被无视,
# 而是在录制结束后直接执行 custom_commandline 中的命令。
# 在 custom_commandline 执行结束后,程序还会继续查看 delete_flv_after_convert 的值,
# 来判断是否需要删除原始 flv 文件。
# 以下是一个在录制结束后将 flv 视频转换为同名 mp4 视频的示例:
# custom_commandline: '{{ .Ffmpeg }} -hide_banner -i "{{ .FileName }}" -c copy "{{ .FileName | trimSuffix (.FileName | ext)}}.mp4"'`, "")
setFieldComment(finishNode, "delete_flv_after_convert",
`# MP4 转换成功后标记原始 FLV 为待删除,全部处理阶段完成后才真正删除
# 默认 false(保留原始 FLV 文件)`, "")
setFieldComment(finishNode, "burn_subtitles",
`# 是否将 ASS 弹幕字幕硬编码(烧录)到视频中
# 开启后会使用 FFmpeg 重编码视频,将字幕叠加到画面上
# 需要同时开启弹幕录制(danmaku_enable)才有 ASS 文件可烧录`, "")
setFieldComment(finishNode, "burn_subtitles_codec",
`# 烧录字幕时使用的视频编码器
# 默认 libx264(H.264),兼容性最好
# 可选 libx265(H.265,压缩率更高但编码更慢)`, "")
setFieldComment(finishNode, "burn_subtitles_crf",
`# 烧录字幕时的 CRF(恒定速率因子)质量值
# 范围 0-51,数值越小画质越好,文件越大
# 默认 18(视觉无损),推荐范围 15-23`, "")
setFieldComment(finishNode, "burn_subtitles_preset",
`# 烧录字幕时的编码预设
# 影响编码速度和压缩率的平衡
# 可选: ultrafast, superfast, veryfast, faster, fast, medium, slow, slower, veryslow
# 默认 medium,越慢画质越好但耗时越长`, "")
setFieldComment(finishNode, "burn_delete_ass",
`# 烧录成功后标记 ASS 文件为待删除,全部处理阶段完成后才真正删除
# 默认 false(保留 ASS 文件)`, "")
setFieldComment(finishNode, "burn_delete_source",
`# 烧录成功后标记源视频文件为待删除,全部处理阶段完成后才真正删除
# 默认 false(保留源文件,同时存在源文件和烧录后的 MKV)
# 开启后仅保留烧录完成的 MKV 文件`, "")
cloudNode := findNode(finishNode, "cloud_upload")
if cloudNode != nil {
setFieldComment(cloudNode, "delete_after_upload",
`# 选「处理完再上传」时,上传成功后仅删除已上传的文件(如最终视频),不影响其他中间文件
# 选「先上传再处理」时此开关无效`, "")
setFieldComment(cloudNode, "delete_all_after_upload",
`# 选「处理完再上传」时,上传成功后删除所有本地文件(含中间产物)
# 选「先上传再处理」时此开关无效`, "")
}
}
setFieldHeadComment(root, "notify", "# 通知服务配置")
notifyNode := findNode(root, "notify")
if notifyNode != nil {
+40 -5
View File
@@ -2,6 +2,7 @@ package configs
import (
"os"
"sync"
"testing"
"github.com/bililive-go/bililive-go/src/pkg/ratelimit"
@@ -97,7 +98,7 @@ func TestGetPlatformMinAccessInterval(t *testing.T) {
assert.Equal(t, 1, interval) // 默认最小间隔为 1 秒,防止无限制高频访问
}
func TestDefaultPlatformRateLimitSurvivesTransientConfigUpdate(t *testing.T) {
func TestPlatformRateLimitRemainsDisabledAfterConfigUpdate(t *testing.T) {
const roomURL = "https://live.douyin.com/123456"
limiter := ratelimit.GetGlobalRateLimiter()
t.Cleanup(func() {
@@ -109,18 +110,52 @@ func TestDefaultPlatformRateLimitSurvivesTransientConfigUpdate(t *testing.T) {
cfg.RefreshLiveRoomIndexCache()
SetCurrentConfig(cfg)
if got := limiter.GetAllPlatformLimits()[PlatformKeyDouyin]; got != 1 {
t.Fatalf("初始默认平台限流 = %d 秒,期望 1 秒", got)
if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 {
t.Fatalf("配置加载后平台限流应保持关闭,实际限制:%v", limits)
}
if _, err := SetLiveRoomId(roomURL, types.LiveID("douyin-test")); err != nil {
t.Fatalf("写入临时 LiveId 失败: %v", err)
}
if got := limiter.GetAllPlatformLimits()[PlatformKeyDouyin]; got != 1 {
t.Fatalf("临时配置更新后的默认平台限流 = %d 秒,期望仍为 1 秒", got)
if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 {
t.Fatalf("临时配置更新后平台限流应保持关闭,实际限制:%v", limits)
}
}
// TestGetLiveRoomByUrlConcurrent 保证在共享的不可变配置快照上并发查询房间时不会
// 触发 Go 运行时的 "concurrent map read and map write" fatal error。
// 回归此前的 bug:GetLiveRoomByUrl 在缓存未命中时会调用 RefreshLiveRoomIndexCache
// 写入共享快照的 map,与其它 goroutine 的读并发时导致崩溃。
// 使用 -race 运行可进一步检测数据竞争。
func TestGetLiveRoomByUrlConcurrent(t *testing.T) {
cfg := NewConfig()
cfg.LiveRooms = []LiveRoom{
{Url: "https://live.bilibili.com/1", IsListening: true},
{Url: "https://live.bilibili.com/2", IsListening: true},
}
cfg.RefreshLiveRoomIndexCache()
var wg sync.WaitGroup
for i := 0; i < 64; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
for j := 0; j < 500; j++ {
// 交替命中缓存与未命中(触发原先的写-回退路径)。
if (i+j)%2 == 0 {
room, err := cfg.GetLiveRoomByUrl("https://live.bilibili.com/1")
assert.NoError(t, err)
assert.Equal(t, "https://live.bilibili.com/1", room.Url)
} else {
_, err := cfg.GetLiveRoomByUrl("https://live.bilibili.com/does-not-exist")
assert.Error(t, err)
}
}
}(i)
}
wg.Wait()
}
func TestBackwardsCompatibility(t *testing.T) {
// Test that old config files still work
oldConfigYaml := `
+126 -2
View File
@@ -16,8 +16,9 @@ import (
// RecordingFileDetail 录制文件详情
type RecordingFileDetail struct {
Name string // 文件名(不含路径)
Size int64 // 文件大小(字节)
Name string // 文件名(不含路径)
Size int64 // 文件大小(字节)
Uploaded bool // 是否已上传到云端(用于摘要消息标注)
}
// SendNotification 发送统一通知函数
@@ -250,6 +251,77 @@ func buildRecordingSummaryMessage(hostName, platform string, files []RecordingFi
return
}
// buildUploadedSummaryBody 构造"已上传到云端"场景的录制摘要消息体
// 与 buildRecordingSummaryMessage 类似,但标注文件已上传、不显示磁盘空间
func buildUploadedSummaryBody(platform string, files []RecordingFileDetail) string {
const maxDisplayFiles = 30
var sb strings.Builder
fmt.Fprintf(&sb, "平台:%s\n", platform)
fmt.Fprintf(&sb, "录制文件:%d 个(已上传到云端)\n", len(files))
var totalSize int64
for i, f := range files {
totalSize += f.Size
if i < maxDisplayFiles {
fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size))
}
}
if len(files) > maxDisplayFiles {
fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(files)-maxDisplayFiles)
}
fmt.Fprintf(&sb, "总大小:%s\n", formatFileSize(totalSize))
fmt.Fprintf(&sb, "本地文件已清理")
return sb.String()
}
// buildMixedSummaryBody 构造"部分上传、部分本地保留"场景的消息体
func buildMixedSummaryBody(platform string, uploaded, kept []RecordingFileDetail, outputPath string) string {
const maxDisplayFiles = 30
var sb strings.Builder
fmt.Fprintf(&sb, "平台:%s\n", platform)
// 已上传部分
fmt.Fprintf(&sb, "已上传到云端:%d 个\n", len(uploaded))
var uploadedSize int64
for i, f := range uploaded {
uploadedSize += f.Size
if i < maxDisplayFiles {
fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size))
}
}
if len(uploaded) > maxDisplayFiles {
fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(uploaded)-maxDisplayFiles)
}
fmt.Fprintf(&sb, "上传总大小:%s\n", formatFileSize(uploadedSize))
// 本地保留部分
fmt.Fprintf(&sb, "本地保留:%d 个\n", len(kept))
var keptSize int64
for i, f := range kept {
keptSize += f.Size
if i < maxDisplayFiles {
fmt.Fprintf(&sb, " %d. %s (%s)\n", i+1, f.Name, formatFileSize(f.Size))
}
}
if len(kept) > maxDisplayFiles {
fmt.Fprintf(&sb, " ... 还有 %d 个文件未显示\n", len(kept)-maxDisplayFiles)
}
fmt.Fprintf(&sb, "本地总大小:%s", formatFileSize(keptSize))
if outputPath != "" {
if free, err := getDiskFreeSpace(outputPath); err == nil {
fmt.Fprintf(&sb, "\n剩余磁盘空间:%s", formatFileSize(int64(free)))
}
}
return sb.String()
}
// buildRecordingSummaryMessageBody 构造纯本地保留的消息体(不含 title)
func buildRecordingSummaryMessageBody(platform string, files []RecordingFileDetail, outputPath string) string {
_, body := buildRecordingSummaryMessage("", platform, files, outputPath)
return body
}
// SendRecordingSummary 录制结束后发送录制文件摘要通知
// outputPath 为录制输出路径,用于获取剩余磁盘空间
func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform string, files []RecordingFileDetail, outputPath string) {
@@ -262,7 +334,59 @@ func SendRecordingSummary(logger *livelogger.LiveLogger, hostName, platform stri
}
title, body := buildRecordingSummaryMessage(hostName, platform, files, outputPath)
sendToAllChannels(cfg, logger, title, body)
}
// SendPipelineRecordingSummary Pipeline 完成后发送录制摘要通知
// originalFiles: 原始录制文件列表(allUploaded 时用于显示)
// finalFiles: Pipeline 处理后的最终文件列表(含 Uploaded 标记)
func SendPipelineRecordingSummary(
logger *livelogger.LiveLogger,
hostName, platform string,
originalFiles, finalFiles []RecordingFileDetail,
outputPath string,
) {
cfg := configs.GetCurrentConfig()
if cfg == nil || !cfg.Notify.SendRecordingSummary {
return
}
// 分离已上传文件和本地保留文件
var uploaded []RecordingFileDetail
var kept []RecordingFileDetail
for _, f := range finalFiles {
if f.Uploaded {
uploaded = append(uploaded, f)
} else {
kept = append(kept, f)
}
}
var title, body string
if len(uploaded) > 0 && len(kept) == 0 {
// 所有文件已上传:显示上传文件列表
title = fmt.Sprintf("%s 录制完成", hostName)
body = buildUploadedSummaryBody(platform, uploaded)
} else if len(uploaded) > 0 && len(kept) > 0 {
// 部分上传、部分保留:显示两段
title = fmt.Sprintf("%s 录制完成", hostName)
body = buildMixedSummaryBody(platform, uploaded, kept, outputPath)
} else if len(kept) > 0 {
// 全部本地保留
title, body = buildRecordingSummaryMessage(hostName, platform, kept, outputPath)
} else if len(originalFiles) > 0 {
// finalFiles 为空,回退到 originalFiles
title = fmt.Sprintf("%s 录制完成", hostName)
body = buildRecordingSummaryMessageBody(platform, originalFiles, outputPath)
} else {
return
}
sendToAllChannels(cfg, logger, title, body)
}
// sendToAllChannels 向所有已启用的通知通道推送摘要消息
func sendToAllChannels(cfg *configs.Config, logger *livelogger.LiveLogger, title, body string) {
// Telegram
if cfg.Notify.Telegram.Enable {
msg := fmt.Sprintf("%s\n%s", title, body)
+3
View File
@@ -32,6 +32,8 @@ const (
OptionCommand = "command"
// OptionFileTypes 处理的文件类型过滤
OptionFileTypes = "file_types"
// OptionUploadSubtitles 是否上传关联的 .ass 弹幕字幕文件
OptionUploadSubtitles = "upload_subtitles"
// OptionCodec 视频编码器
OptionCodec = "codec"
// OptionCrf CRF 质量值
@@ -85,6 +87,7 @@ func ConvertLegacyConfig(legacy *configs.OnRecordFinished) *PipelineConfig {
OptionDeleteAllAfter: legacy.CloudUpload.DeleteAllAfterUpload,
OptionUploadTiming: string(legacy.UploadTiming),
OptionFileTypes: []string{string(FileTypeVideo), string(FileTypeCover)},
OptionUploadSubtitles: legacy.CloudUpload.UploadSubtitles,
},
}
hasCloudUpload := legacy.CloudUpload.Enable && legacy.CloudUpload.StorageName != ""
+172 -16
View File
@@ -9,6 +9,8 @@ import (
"sync"
"time"
"github.com/bililive-go/bililive-go/src/configs"
"github.com/bililive-go/bililive-go/src/pkg/metadata"
bilisentry "github.com/bililive-go/bililive-go/src/pkg/sentry"
"github.com/sirupsen/logrus"
)
@@ -48,11 +50,15 @@ func (e *Executor) getFactory(name string) (StageFactory, bool) {
}
// Execute 执行管道
// startStage: 从第几个启用的阶段开始执行(0=全部执行,用于重试指定阶段)
// onStageComplete: 每个阶段成功完成后的回调,用于持久化进度(崩溃恢复的关键)
func (e *Executor) Execute(
ctx *PipelineContext,
config *PipelineConfig,
initialFiles []FileInfo,
startStage int,
onProgress func(stageIndex int, stageName string, status StageStatus),
onStageComplete func(stageIndex int, result StageResult),
) ([]StageResult, error) {
if config == nil || len(config.Stages) == 0 {
e.logger.Debug("pipeline config is empty, skipping")
@@ -61,6 +67,7 @@ func (e *Executor) Execute(
files := initialFiles
results := make([]StageResult, 0, len(config.Stages))
ctx.UploadedPaths = make(map[string]bool)
stageIndex := 0
for i, stageCfg := range config.Stages {
@@ -76,6 +83,12 @@ func (e *Executor) Execute(
continue
}
// 跳过 startStage 之前的阶段(用于从指定阶段重试)
if stageIndex < startStage {
stageIndex++
continue
}
// 记录开始
if onProgress != nil {
onProgress(stageIndex, stageCfg.Name, StageStatusRunning)
@@ -86,6 +99,9 @@ func (e *Executor) Execute(
var commands []string
var logs string
// 记录阶段开始时间
stageStartTime := getTimeNow()
// 并行阶段处理
if stageCfg.IsParallel() {
e.logger.WithField("stage_index", i).Debug("executing parallel stages")
@@ -99,6 +115,9 @@ func (e *Executor) Execute(
output, commands, logs, err = e.executeStage(ctx, stageCfg, files)
}
// 回填新产出文件的 Size(阶段内构造 FileInfo 时可能未 stat)
backfillFileSizes(output)
// 记录结果
result := StageResult{
StageName: stageCfg.Name,
@@ -107,7 +126,7 @@ func (e *Executor) Execute(
Commands: commands,
Logs: logs,
}
result.StartedAt = getTimeNow()
result.StartedAt = stageStartTime
if err != nil {
result.Status = StageStatusFailed
@@ -134,6 +153,11 @@ func (e *Executor) Execute(
onProgress(stageIndex, stageCfg.Name, StageStatusCompleted)
}
// 通知外部持久化阶段完成状态(用于崩溃恢复)
if onStageComplete != nil {
onStageComplete(stageIndex, result)
}
// 更新文件列表给下一阶段
files = output
stageIndex++
@@ -144,32 +168,143 @@ func (e *Executor) Execute(
}).Debug("stage completed")
}
// 全部成功:执行延迟删除
e.deleteMarkedFiles(files)
// 去重:多个阶段可能添加同名文件(如 cloud_upload 上传 .ass 后,burn 又添加 Deletable 副本)
// 保留第一个出现的(保留最早阶段的 Metadata,如 uploaded 标记)
seen := map[string]int{}
var deduped []FileInfo
for _, f := range files {
if idx, exists := seen[f.Path]; exists {
// 合并 Deletable 标记到已有条目
if f.Deletable && !deduped[idx].Deletable {
deduped[idx].Deletable = true
}
continue
}
seen[f.Path] = len(deduped)
deduped = append(deduped, f)
}
files = deduped
// 重新标记被后续阶段替换的已上传文件
// immediate 模式下 cloud_upload 最先执行,后续阶段(fix_flv)
// 创建新 FileInfo 替换原始文件,丢失 Metadata["uploaded"]。
// 通过 UploadedPaths 按基名+扩展名匹配,为同类型的替换文件恢复上传标记。
// 不匹配不同扩展名的派生文件(如 .flv 上传后生成的 .mp4/.mkv 不应标记为 uploaded)。
if len(ctx.UploadedPaths) > 0 {
for uploadedPath := range ctx.UploadedPaths {
uploadedBase := strings.TrimSuffix(filepath.Base(uploadedPath), filepath.Ext(uploadedPath))
uploadedExt := strings.ToLower(filepath.Ext(uploadedPath))
for i, f := range files {
if f.Metadata != nil {
if u, ok := f.Metadata["uploaded"].(bool); ok && u {
continue // 已有标记,跳过
}
}
ext := strings.ToLower(filepath.Ext(f.Path))
if ext != uploadedExt {
continue // 不同扩展名,不是直接替换
}
fileBase := strings.TrimSuffix(filepath.Base(f.Path), ext)
if fileBase == uploadedBase {
if files[i].Metadata == nil {
files[i].Metadata = map[string]any{}
}
files[i].Metadata["uploaded"] = true
e.logger.Infof("恢复上传标记: %s (匹配 %s)", f.Path, filepath.Base(uploadedPath))
}
}
}
}
// 全部成功:执行延迟删除,并让最终结果反映磁盘上的文件
// 如果上下文已取消,跳过删除以保留所有文件,返回取消错误让任务标记为 Cancelled
if ctx.Ctx.Err() != nil {
e.logger.Warn("pipeline cancelled after completion, skipping file cleanup")
if len(results) > 0 {
results[len(results)-1].OutputFiles = files
}
return results, ctx.Ctx.Err()
}
// 保存最后阶段输出的快照(用于回调提取上传文件详情),再执行清理
// 必须深拷贝 Metadata,因为 deleteMarkedFiles 会清除 Metadata["uploaded"],
// FileInfo.Metadata 是 map(引用类型),浅拷贝会共享同一 map
snapshot := make([]FileInfo, len(files))
for i, f := range files {
cp := f
if f.Metadata != nil {
cp.Metadata = make(map[string]any, len(f.Metadata))
for k, v := range f.Metadata {
cp.Metadata[k] = v
}
}
snapshot[i] = cp
}
ctx.LastStageFiles = snapshot
keptFiles := e.deleteMarkedFiles(files)
if len(results) > 0 {
results[len(results)-1].OutputFiles = keptFiles
}
return results, nil
}
// deleteMarkedFiles 删除标记为 Deletable 的文件,返回保留的文件列表
// deleteMarkedFiles 删除标记为 Deletable 或 Metadata["uploaded"] 的文件,返回保留的文件列表
func (e *Executor) deleteMarkedFiles(files []FileInfo) []FileInfo {
var kept []FileInfo
deleteAll := false // 标记是否为 deleteAll 模式
deleteAfter := false // 标记是否为 deleteAfter 模式(仅删除已上传文件)
// 获取输出路径用于计算相对路径(清理 DB 上传标记)
cfg := configs.GetCurrentConfig()
for _, f := range files {
if f.Deletable {
// 检查删除模式(CloudUploadStage 标记)
if f.Metadata != nil {
if da, ok := f.Metadata["delete_all"].(bool); ok && da {
deleteAll = true
}
if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded {
deleteAfter = true
}
}
// 判断是否应删除:Deletable 标记 或 CloudUploadStage 标记的已上传文件
shouldDelete := f.Deletable
if !shouldDelete && f.Metadata != nil {
if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded {
shouldDelete = true
}
}
if shouldDelete {
if err := os.Remove(f.Path); err != nil {
if !os.IsNotExist(err) {
e.logger.Warnf("pipeline cleanup: 删除文件失败 %s: %v", f.Path, err)
kept = append(kept, f) // 删除失败,文件仍在磁盘,保留记录
}
} else {
e.logger.Infof("pipeline cleanup: 已删除 %s", f.Path)
// 清除 DB 中的上传标记
if cfg != nil {
outPutPath, _ := filepath.Abs(cfg.OutPutPath)
absFilePath, _ := filepath.Abs(f.Path)
if relPath, relErr := filepath.Rel(outPutPath, absFilePath); relErr == nil {
if delErr := metadata.GetStore().Delete(context.Background(), metadata.NamespaceUploaded, filepath.ToSlash(relPath)); delErr != nil {
e.logger.Warnf("pipeline cleanup: 清除上传标记失败 %s: %v", f.Path, delErr)
}
}
}
}
} else {
kept = append(kept, f)
}
}
// 兜底逻辑:清理无关联视频文件的 .ass 文件
// 传入原始 files 列表,确保即使视频被删除也能扫描到对应目录
kept = e.cleanupOrphanedAssFiles(files, kept)
// 清理无关联视频文件的孤立 .ass 文件
// deleteAll 模式下删除所有文件(含 .ass),deleteAfter 模式下删除已上传视频后也应清理残留 .ass
if deleteAll || deleteAfter {
kept = e.cleanupOrphanedAssFiles(files, kept)
}
return kept
}
@@ -183,7 +318,7 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File
dirs := map[string]bool{}
for _, f := range allFiles {
ext := strings.ToLower(filepath.Ext(f.Path))
if ext == ".flv" || ext == ".mp4" || ext == ".mkv" {
if ext == ".flv" || ext == ".mp4" || ext == ".mkv" || ext == ".ts" {
dir := filepath.Dir(f.Path)
base := strings.TrimSuffix(filepath.Base(f.Path), ext)
videoBaseNames[filepath.Join(dir, base)] = true
@@ -199,7 +334,8 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File
}
}
// 扫描目录中的 .ass 文件(包括未进入 Pipeline 的)
// 扫描目录中的 .ass 文件
// 通过视频基名匹配判断 .ass 是否属于本任务,避免误删其他任务或用户放置的字幕
for dir := range dirs {
entries, err := os.ReadDir(dir)
if err != nil {
@@ -216,12 +352,16 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File
if knownAssFiles[assPath] {
continue // 已在 Pipeline 中处理
}
// 检查磁盘上是否有同名视频文件
// 通过基名匹配判断是否属于本任务的视频文件
// (.ass 文件可能未被传入 Pipeline,不能依赖文件列表过滤)
base := strings.TrimSuffix(entry.Name(), ".ass")
key := filepath.Join(dir, base)
// 检查磁盘上实际存在的视频文件
if !videoBaseNames[key] {
continue // 基名不匹配任何本任务视频,跳过
}
// 检查磁盘上是否有同名视频文件
hasVideo := false
for _, ext := range []string{".flv", ".mp4", ".mkv"} {
for _, ext := range []string{".flv", ".mp4", ".mkv", ".ts"} {
if _, err := os.Stat(key + ext); err == nil {
hasVideo = true
break
@@ -230,10 +370,10 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File
if !hasVideo {
if err := os.Remove(assPath); err != nil {
if !os.IsNotExist(err) {
e.logger.Warnf("pipeline cleanup: 删除磁盘孤立 .ass 文件失败 %s: %v", assPath, err)
e.logger.Warnf("pipeline cleanup: 删除孤立 .ass 文件失败 %s: %v", assPath, err)
}
} else {
e.logger.Infof("pipeline cleanup: 已删除磁盘孤立 .ass 文件 %s(无关联视频)", assPath)
e.logger.Infof("pipeline cleanup: 已删除孤立 .ass 文件 %s(无关联视频)", assPath)
}
}
}
@@ -244,6 +384,10 @@ func (e *Executor) cleanupOrphanedAssFiles(allFiles []FileInfo, keptFiles []File
for _, f := range keptFiles {
ext := strings.ToLower(filepath.Ext(f.Path))
if ext == ".ass" {
// 先检查 .ass 是否还在磁盘上(可能已被 deleteMarkedFiles 删除)
if _, err := os.Stat(f.Path); os.IsNotExist(err) {
continue // 已被删除,不加入 kept
}
dir := filepath.Dir(f.Path)
base := strings.TrimSuffix(filepath.Base(f.Path), ".ass")
key := filepath.Join(dir, base)
@@ -399,6 +543,18 @@ func deduplicateFiles(files []FileInfo) []FileInfo {
return result
}
// backfillFileSizes 对 Size 为 0 的文件执行 os.Stat 回填大小
// 阶段内构造 FileInfo 时可能未 stat,导致摘要通知漏掉这些文件
func backfillFileSizes(files []FileInfo) {
for i := range files {
if files[i].Size == 0 {
if fi, err := os.Stat(files[i].Path); err == nil {
files[i].Size = fi.Size()
}
}
}
}
// getTimeNow 获取当前时间(可用于测试时 mock)
var getTimeNow = func() time.Time {
return time.Now()
@@ -415,7 +571,7 @@ func (e *Executor) ExecuteAsync(
bilisentry.GoWithContext(ctx.Ctx, func(goCtx context.Context) {
// 更新上下文
ctx.Ctx = goCtx
results, err := e.Execute(ctx, config, initialFiles, onProgress)
results, err := e.Execute(ctx, config, initialFiles, 0, onProgress, nil)
if onComplete != nil {
onComplete(results, err)
}
+7 -7
View File
@@ -86,7 +86,7 @@ func TestExecute_AllStagesSucceed_AllDeletableFilesRemoved(t *testing.T) {
results, err := executor.Execute(ctx, config, []FileInfo{
{Path: originalFile, Type: FileTypeVideo},
}, nil)
}, 0, nil, nil)
// 验证:无错误
assert.NoError(t, err)
@@ -142,7 +142,7 @@ func TestExecute_StageFailed_AllFilesPreserved(t *testing.T) {
_, err := executor.Execute(ctx, config, []FileInfo{
{Path: originalFile, Type: FileTypeVideo},
}, nil)
}, 0, nil, nil)
// 验证:返回错误
assert.Error(t, err)
@@ -199,7 +199,7 @@ func TestExecute_Cancelled_AllFilesPreserved(t *testing.T) {
_, _ = executor.Execute(ctx, config, []FileInfo{
{Path: originalFile, Type: FileTypeVideo},
}, nil)
}, 0, nil, nil)
// 验证:stage2 返回成功,但下一轮循环检测到 context 已取消
// 注意:stage2 本身成功了,所以 deleteMarkedFiles 会被调用
@@ -248,7 +248,7 @@ func TestExecute_PassthroughPreservesDeletable(t *testing.T) {
results, err := executor.Execute(ctx, config, []FileInfo{
{Path: file1, Type: FileTypeVideo},
{Path: file2, Type: FileTypeOther, Deletable: true},
}, nil)
}, 0, nil, nil)
assert.NoError(t, err)
assert.Len(t, results, 1)
@@ -304,7 +304,7 @@ func TestExecute_UploadCancelledDuringUpload_FilePreserved(t *testing.T) {
_, err := executor.Execute(ctx, config, []FileInfo{
{Path: localFile, Type: FileTypeVideo},
}, nil)
}, 0, nil, nil)
// 验证:返回错误
assert.Error(t, err)
@@ -363,7 +363,7 @@ func TestExecute_UploadFailsMiddleOfFileList_NoneDeleted(t *testing.T) {
{Path: file1, Type: FileTypeVideo},
{Path: file2, Type: FileTypeVideo},
{Path: file3, Type: FileTypeVideo},
}, nil)
}, 0, nil, nil)
assert.NoError(t, err)
@@ -423,7 +423,7 @@ func TestExecute_StaleDeletableFromDB_ClearedOnFailure(t *testing.T) {
_, err := executor.Execute(ctx, config, []FileInfo{
{Path: file1, Type: FileTypeVideo, Deletable: true}, // 旧标记
{Path: file2, Type: FileTypeVideo, Deletable: true}, // 旧标记
}, nil)
}, 0, nil, nil)
assert.NoError(t, err)
+323 -7
View File
@@ -3,14 +3,21 @@ package pipeline
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/bililive-go/bililive-go/src/configs"
"github.com/bililive-go/bililive-go/src/instance"
"github.com/bililive-go/bililive-go/src/live"
"github.com/bililive-go/bililive-go/src/notify"
"github.com/bililive-go/bililive-go/src/pkg/events"
"github.com/bililive-go/bililive-go/src/pkg/livelogger"
bilisentry "github.com/bililive-go/bililive-go/src/pkg/sentry"
"github.com/bililive-go/bililive-go/src/types"
"github.com/sirupsen/logrus"
)
@@ -23,6 +30,7 @@ type Manager struct {
executor *Executor
config *ManagerConfig
runningTasks map[int64]context.CancelFunc
taskCallbacks map[int64]func(*PipelineTask) // 按 task.ID 索引的完成回调(不经过持久化)
mu sync.RWMutex
wg sync.WaitGroup
eventDispatch events.Dispatcher
@@ -64,6 +72,7 @@ func NewManager(ctx context.Context, store Store, config *ManagerConfig, dispatc
executor: NewExecutor(logrus.StandardLogger()),
config: config,
runningTasks: make(map[int64]context.CancelFunc),
taskCallbacks: make(map[int64]func(*PipelineTask)),
eventDispatch: dispatcher,
}
@@ -201,11 +210,21 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) {
WorkDir: "", // 后续可以从配置获取
}
// 执行管道
// 执行管道(从 startStage 开始,支持从指定阶段重试)
startStage := task.CurrentStage
// 保留目标阶段之前的结果(用于合并)
var preservedResults []StageResult
if startStage > 0 && len(task.StageResults) >= startStage {
preservedResults = make([]StageResult, startStage)
copy(preservedResults, task.StageResults[:startStage])
}
results, err := m.executor.Execute(
pipelineCtx,
task.PipelineConfig,
task.CurrentFiles,
startStage,
func(stageIndex int, stageName string, status StageStatus) {
// 更新任务进度
task.CurrentStage = stageIndex
@@ -215,10 +234,26 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) {
}
m.broadcastTaskUpdate(task)
},
// 阶段完成后持久化结果,确保崩溃恢复时阶段记录完整
func(stageIndex int, result StageResult) {
task.CurrentFiles = result.OutputFiles
// 追加/更新 StageResults,让崩溃恢复后 preservedResults 合并条件成立
for len(task.StageResults) <= stageIndex {
task.StageResults = append(task.StageResults, StageResult{})
}
task.StageResults[stageIndex] = result
if err := m.store.UpdateTask(ctx, task); err != nil {
logrus.WithError(err).Warn("failed to persist stage result")
}
},
)
// 保存阶段结果
task.StageResults = results
// 合并:保留之前的结果 + 新执行的结果
if preservedResults != nil {
task.StageResults = append(preservedResults, results...)
} else {
task.StageResults = results
}
if err != nil {
if ctx.Err() == context.Canceled {
@@ -238,6 +273,9 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) {
logrus.WithField("task_id", task.ID).Info("pipeline task completed successfully")
}
// 保存最后阶段输出的快照(清理前),用于回调提取上传文件详情
task.LastStageFiles = pipelineCtx.LastStageFiles
// 更新任务状态
if err := m.store.UpdateTask(m.ctx, task); err != nil {
logrus.WithError(err).Error("failed to update pipeline task status after execution")
@@ -245,6 +283,16 @@ func (m *Manager) executeTask(ctx context.Context, task *PipelineTask) {
// 广播任务状态变化
m.broadcastTaskUpdate(task)
// 原子地取出并删除回调,确保每个任务的回调至多触发一次
// 避免 executeTask 和 CancelTask 并发时双发回调
m.mu.Lock()
callback := m.taskCallbacks[task.ID]
delete(m.taskCallbacks, task.ID)
m.mu.Unlock()
if callback != nil {
callback(task)
}
}
// broadcastTaskUpdate 广播任务更新事件
@@ -254,11 +302,36 @@ func (m *Manager) broadcastTaskUpdate(task *PipelineTask) {
}
}
// FileInfoToDetails 将 FileInfo 切片转换为 RecordingFileDetail 切片
// 使用 FileInfo 中预存的 Size,无需再次 stat 文件
// 仅包含视频和其他类型文件,排除封面(封面不在录制摘要中展示)
func FileInfoToDetails(files []FileInfo) []notify.RecordingFileDetail {
details := make([]notify.RecordingFileDetail, 0, len(files))
for _, f := range files {
if f.Size > 0 && f.Type != FileTypeCover {
details = append(details, notify.RecordingFileDetail{
Name: filepath.Base(f.Path),
Size: f.Size,
})
}
}
return details
}
// EnqueueTask 添加任务到队列
func (m *Manager) EnqueueTask(task *PipelineTask) error {
// onTaskComplete: 任务完成回调(可选),在任务对调度器可见前注册
func (m *Manager) EnqueueTask(task *PipelineTask, onTaskComplete func(*PipelineTask)) error {
// 持锁覆盖 CreateTask + 回调注册,阻塞 pollLoop 的 scheduleNextTasks(需 RLock)
// 确保任务对调度器可见时回调已就绪
m.mu.Lock()
if err := m.store.CreateTask(m.ctx, task); err != nil {
m.mu.Unlock()
return fmt.Errorf("failed to create pipeline task: %w", err)
}
if onTaskComplete != nil {
m.taskCallbacks[task.ID] = onTaskComplete
}
m.mu.Unlock()
logrus.WithFields(logrus.Fields{
"task_id": task.ID,
@@ -277,10 +350,12 @@ func (m *Manager) EnqueueTask(task *PipelineTask) error {
// EnqueueRecordingTask 创建并入队录制完成后的处理任务
// 这是 recorder.go 调用的主要入口
// onTaskComplete: 任务完成回调(可选),用于通知 recorder 任务结束
func (m *Manager) EnqueueRecordingTask(
info *live.Info,
pipelineConfig *PipelineConfig,
outputFiles []string,
onTaskComplete func(task *PipelineTask),
) error {
// 构建文件信息列表
files := make([]FileInfo, len(outputFiles))
@@ -291,7 +366,8 @@ func (m *Manager) EnqueueRecordingTask(
// 创建任务
task := NewPipelineTask(NewRecordInfo(info), pipelineConfig, files)
return m.EnqueueTask(task)
// 入队并注册回调(EnqueueTask 内部在异步调度前完成注册,避免竞态窗口)
return m.EnqueueTask(task, onTaskComplete)
}
// CancelTask 取消任务
@@ -316,6 +392,15 @@ func (m *Manager) CancelTask(taskID int64) error {
return err
}
m.broadcastTaskUpdate(task)
// 原子地取出并删除回调,避免与 executeTask 并发时双发
m.mu.Lock()
callback := m.taskCallbacks[taskID]
delete(m.taskCallbacks, taskID)
m.mu.Unlock()
if callback != nil {
callback(task)
}
}
return nil
@@ -332,8 +417,16 @@ func (m *Manager) RetryTask(taskID int64) error {
return fmt.Errorf("task cannot be retried")
}
if task.Status != PipelineStatusFailed && task.Status != PipelineStatusCancelled {
return fmt.Errorf("only failed or cancelled tasks can be retried")
// 允许 Failed/Cancelled/Completed 状态
if task.Status != PipelineStatusFailed && task.Status != PipelineStatusCancelled && task.Status != PipelineStatusCompleted {
return fmt.Errorf("only failed, cancelled or completed tasks can be retried")
}
// 校验初始文件存在(Completed 任务的文件可能已被 deleteMarkedFiles 清理)
for _, f := range task.InitialFiles {
if _, err := os.Stat(f.Path); os.IsNotExist(err) {
return fmt.Errorf("initial file not found, cannot retry (may have been deleted after upload): %s", f.Path)
}
}
// 重置任务状态
@@ -342,6 +435,7 @@ func (m *Manager) RetryTask(taskID int64) error {
task.CompletedAt = nil
task.ErrorMessage = ""
task.CurrentStage = 0
task.CurrentFiles = task.InitialFiles // 重置为初始文件,从头开始重试
task.StageResults = nil
task.Progress = 0
@@ -357,6 +451,122 @@ func (m *Manager) RetryTask(taskID int64) error {
return nil
}
// RetryTaskFromStage 从指定阶段开始重试任务
// 允许 Failed/Cancelled/Completed 状态的任务
// stageName: 要从哪个阶段开始重试(如 "cloud_upload")
func (m *Manager) RetryTaskFromStage(taskID int64, stageName string) error {
task, err := m.store.GetTask(m.ctx, taskID)
if err != nil {
return err
}
if !task.CanRetry {
return fmt.Errorf("task cannot be retried")
}
// 允许 Failed/Cancelled/Completed 状态
if task.Status != PipelineStatusFailed &&
task.Status != PipelineStatusCancelled &&
task.Status != PipelineStatusCompleted {
return fmt.Errorf("only failed, cancelled or completed tasks can be retried")
}
// 找到目标阶段在启用阶段中的索引
targetStageIndex := -1
stageIndex := 0
for _, stageCfg := range task.PipelineConfig.Stages {
if !stageCfg.IsEnabled() {
continue
}
if stageCfg.Name == stageName {
targetStageIndex = stageIndex
break
}
stageIndex++
}
if targetStageIndex == -1 {
return fmt.Errorf("stage %s not found in pipeline", stageName)
}
// 校验:目标阶段之前的所有阶段必须已完成
if targetStageIndex > 0 {
for i := 0; i < targetStageIndex; i++ {
if i >= len(task.StageResults) {
return fmt.Errorf("stage %d has not been executed, cannot retry from stage %s", i, stageName)
}
if task.StageResults[i].Status != StageStatusCompleted {
return fmt.Errorf("stage %d (%s) is not completed (status: %s), cannot retry from stage %s",
i, task.StageResults[i].StageName, task.StageResults[i].Status, stageName)
}
}
}
// 获取目标阶段的输入文件
var inputFiles []FileInfo
if targetStageIndex < len(task.StageResults) {
// 从已有的 StageResult 中取输入文件
inputFiles = task.StageResults[targetStageIndex].InputFiles
} else if targetStageIndex > 0 && targetStageIndex-1 < len(task.StageResults) {
// 使用上一阶段的输出文件
inputFiles = task.StageResults[targetStageIndex-1].OutputFiles
} else if targetStageIndex == 0 {
// 从头开始,使用初始文件
inputFiles = task.InitialFiles
}
// 过滤已清理的中间文件,保留仍在磁盘上的文件
// deleteMarkedFiles 仅在管道全部成功后执行,失败/取消时文件仍在磁盘但带有标记
// 因此需要结合磁盘实际存在性判断:标记但仍在 → 保留(管道未清理),标记且已删 → 跳过
var activeFiles []FileInfo
for _, f := range inputFiles {
isDeletable := f.Deletable
isUploaded := false
if f.Metadata != nil {
if uploaded, ok := f.Metadata["uploaded"].(bool); ok && uploaded {
isUploaded = true
}
}
if _, err := os.Stat(f.Path); os.IsNotExist(err) {
if isDeletable || isUploaded {
continue // 已被 deleteMarkedFiles 正常清理,跳过
}
return fmt.Errorf("input file not found, cannot retry: %s", f.Path)
}
activeFiles = append(activeFiles, f)
}
inputFiles = activeFiles
// 校验输入文件不为空
if len(inputFiles) == 0 {
return fmt.Errorf("no input files available for stage %s, cannot retry", stageName)
}
// 重置任务状态
task.Status = PipelineStatusPending
task.StartedAt = nil
task.CompletedAt = nil
task.ErrorMessage = ""
task.CurrentStage = targetStageIndex
task.CurrentFiles = inputFiles
// 保留目标阶段之前的结果,清除目标阶段及之后的结果
if targetStageIndex < len(task.StageResults) {
task.StageResults = task.StageResults[:targetStageIndex]
}
task.UpdateProgress()
if err := m.store.UpdateTask(m.ctx, task); err != nil {
return err
}
m.broadcastTaskUpdate(task)
// 立即尝试调度
bilisentry.Go(func() { m.scheduleNextTasks() })
return nil
}
// GetTask 获取任务详情
func (m *Manager) GetTask(taskID int64) (*PipelineTask, error) {
return m.store.GetTask(m.ctx, taskID)
@@ -432,6 +642,112 @@ type ManagerStats struct {
CancelledCount int `json:"cancelled_count"`
}
// EnqueueUploadTask 创建手动上传的 Pipeline 任务并入队
// 行为与自动上传(CloudUploadStage)完全一致:使用相同的路径模板、配置选项
// 每个文件创建一个独立的单阶段 cloud_upload 任务
// absPaths: 已校验的绝对路径列表(调用方应先通过 getSafePath 校验)
func (m *Manager) EnqueueUploadTask(absPaths []string) (enqueued int, skipped []string, taskIDs []int64, err error) {
config := configs.GetCurrentConfig()
if config == nil {
return 0, nil, nil, fmt.Errorf("配置未初始化")
}
cu := config.OnRecordFinished.CloudUpload
if !cu.Enable {
return 0, nil, nil, fmt.Errorf("云上传功能未启用,请先在设置中开启")
}
if cu.StorageName == "" {
return 0, nil, nil, fmt.Errorf("未配置存储名称")
}
// 构建单阶段 cloud_upload PipelineConfig,与 ConvertLegacyConfig 一致
uploadConfig := &PipelineConfig{
Stages: []StageConfig{
{
Name: StageNameCloudUpload,
Options: map[string]any{
OptionStorage: cu.StorageName,
OptionPathTemplate: cu.UploadPathTmpl,
OptionDeleteAfter: cu.DeleteAfterUpload,
OptionDeleteAllAfter: cu.DeleteAllAfterUpload,
OptionUploadTiming: "after_process", // 手动上传无后续处理阶段,始终允许删除标记(不继承全局 upload_timing)
OptionFileTypes: []string{string(FileTypeVideo), string(FileTypeCover)},
OptionUploadSubtitles: cu.UploadSubtitles,
},
},
},
}
outputPath, _ := filepath.Abs(config.OutPutPath)
skipped = []string{}
for _, absPath := range absPaths {
// 检查文件存在
info, statErr := os.Stat(absPath)
if statErr != nil {
skipped = append(skipped, filepath.Base(absPath)+" - 文件不存在或无法访问")
continue
}
if info.IsDir() {
skipped = append(skipped, filepath.Base(absPath)+" - 不支持上传文件夹")
continue
}
// 从相对路径推断 RecordInfo(用于上传路径模板渲染)
relPath, _ := filepath.Rel(outputPath, absPath)
platform, hostName := inferUploadPathInfo(relPath)
recordInfo := RecordInfo{
LiveID: types.LiveID(ManualUploadLiveID),
Platform: platform, // 用于上传路径模板:{{ .Platform }}
HostName: hostName, // 用于上传路径模板:{{ .HostName }}
RoomName: "", // 手动上传无房间名,不影响路径模板
DisplayName: filepath.Base(absPath), // 任务列表展示文件名
StartTime: info.ModTime(),
}
files := []FileInfo{NewVideoFileInfo(absPath)}
task := NewPipelineTask(recordInfo, uploadConfig, files)
if enqueueErr := m.EnqueueTask(task, nil); enqueueErr != nil {
skipped = append(skipped, filepath.Base(absPath)+" - 入队失败: "+enqueueErr.Error())
continue
}
enqueued++
taskIDs = append(taskIDs, task.ID)
}
return enqueued, skipped, taskIDs, nil
}
// bracketPattern 匹配文件名中的 [xxx] 模式
var bracketPattern = regexp.MustCompile(`\[([^\]]*)\]`)
// inferUploadPathInfo 从文件相对路径推断平台和主播名
// 两级策略:优先从文件名解析([date][HostName][RoomName].flv),回退到目录结构推断
func inferUploadPathInfo(relPath string) (platform, hostName string) {
slashPath := filepath.ToSlash(relPath)
// 策略 1:从文件名解析 HostName(文件名格式通常是 [date][HostName][RoomName].flv)
fileName := filepath.Base(slashPath)
brackets := bracketPattern.FindAllStringSubmatch(fileName, -1)
if len(brackets) >= 2 {
hostName = brackets[1][1] // 第二个方括号通常是主播名
}
// 策略 2:从目录结构推断 Platform,以及作为 HostName 的兜底
parts := strings.Split(slashPath, "/")
if len(parts) >= 2 {
platform = parts[0]
if hostName == "" {
hostName = parts[1]
}
}
return platform, hostName
}
// GetManager 从实例获取管道管理器
func GetManager(inst *instance.Instance) *Manager {
if inst == nil || inst.PipelineManager == nil {
+7
View File
@@ -83,6 +83,13 @@ func (s *BurnSubtitlesStage) Execute(ctx *pipeline.PipelineContext, input []pipe
continue
}
// 跳过已标记待删除的文件(如 ConvertMp4 阶段标记的原始 FLV),
// 但仍将其透传到 output,保证 Executor 的延迟删除机制正常工作。
if file.Deletable {
output = append(output, file)
continue
}
// 检查文件是否存在
if _, err := os.Stat(file.Path); os.IsNotExist(err) {
s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path)
+10
View File
@@ -120,6 +120,12 @@ func (s *ConvertMp4Stage) Execute(ctx *pipeline.PipelineContext, input []pipelin
return nil, fmt.Errorf("failed to create stdout pipe: %w", err)
}
stderr, err := cmd.StderrPipe()
if err != nil {
s.logs += fmt.Sprintf("创建错误管道失败: %s\n", err.Error())
return nil, fmt.Errorf("failed to create stderr pipe: %w", err)
}
if err := cmd.Start(); err != nil {
s.logs += fmt.Sprintf("启动 ffmpeg 失败: %s\n", err.Error())
return nil, fmt.Errorf("failed to start ffmpeg: %w", err)
@@ -130,10 +136,14 @@ func (s *ConvertMp4Stage) Execute(ctx *pipeline.PipelineContext, input []pipelin
s.parseProgress(goCtx, stdout, duration)
})
// 读取 stderr 防止缓冲区满导致 ffmpeg 挂起
stderrBytes, _ := io.ReadAll(stderr)
// 等待命令完成
if err := cmd.Wait(); err != nil {
os.Remove(tempFile)
s.logs += fmt.Sprintf("ffmpeg 转换失败: %s - %s\n", file.Path, err.Error())
s.logs += fmt.Sprintf("ffmpeg stderr: %s\n", string(stderrBytes))
return nil, fmt.Errorf("ffmpeg conversion failed for %s: %w", file.Path, err)
}
+18
View File
@@ -51,6 +51,14 @@ func (s *CustomCommandStage) Execute(ctx *pipeline.PipelineContext, input []pipe
var output []pipeline.FileInfo
for _, file := range input {
// 跳过中间产物(如 ConvertMp4 标记的原始 FLV),避免重复执行命令。
// 但 delete_all_after_upload 模式会将所有文件(含最终成品)标记为 Deletable,
// 此类文件仍需执行自定义命令,不能跳过。
if file.Deletable && !isDeleteAllMarked(file) {
output = append(output, file)
continue
}
// 渲染命令模板
cmdStr, err := s.renderCommand(ctx, file)
if err != nil {
@@ -77,6 +85,16 @@ func (s *CustomCommandStage) Execute(ctx *pipeline.PipelineContext, input []pipe
return output, nil
}
// isDeleteAllMarked 检查文件是否被 delete_all_after_upload 模式标记
// 此类文件虽为 Deletable,但属于最终成品,仍需执行自定义命令
func isDeleteAllMarked(file pipeline.FileInfo) bool {
if file.Metadata == nil {
return false
}
da, ok := file.Metadata["delete_all"].(bool)
return ok && da
}
// renderCommand 渲染命令模板
func (s *CustomCommandStage) renderCommand(ctx *pipeline.PipelineContext, file pipeline.FileInfo) (string, error) {
// 构建模板数据
+247 -125
View File
@@ -1,17 +1,16 @@
package stages
import (
"bytes"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"text/template"
"time"
"github.com/bililive-go/bililive-go/src/configs"
"github.com/bililive-go/bililive-go/src/pipeline"
"github.com/bililive-go/bililive-go/src/pkg/metadata"
"github.com/bililive-go/bililive-go/src/pkg/openlist"
"github.com/bililive-go/bililive-go/src/tools"
)
@@ -52,6 +51,12 @@ func (s *ExtractCoverStage) Execute(ctx *pipeline.PipelineContext, input []pipel
continue
}
// 跳过已标记待删除的文件(如 ConvertMp4 阶段标记的原始 FLV),
// 避免重复提取封面。
if file.Deletable {
continue
}
// 检查文件是否存在
if _, err := os.Stat(file.Path); os.IsNotExist(err) {
s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path)
@@ -92,27 +97,29 @@ func (s *ExtractCoverStage) GetLogs() string {
// CloudUploadStage 云上传阶段
type CloudUploadStage struct {
config pipeline.StageConfig
storageName string
pathTemplate string
deleteAfter bool // 仅删除已上传的文件
deleteAllAfter bool // 删除全部文件(含中间产物)
uploadTiming string // immediate 或 after_process
fileTypes []string // 过滤的文件类型,空表示所有
commands []string
logs string
config pipeline.StageConfig
storageName string
pathTemplate string
deleteAfter bool // 仅删除已上传的文件
deleteAllAfter bool // 删除全部文件(含中间产物)
uploadTiming string // immediate 或 after_process
fileTypes []string // 过滤的文件类型,空表示所有
uploadSubtitles bool // 是否上传关联的 .ass 弹幕字幕文件
commands []string
logs string
}
// NewCloudUploadStage 创建云上传阶段工厂
func NewCloudUploadStage(config pipeline.StageConfig) (pipeline.Stage, error) {
return &CloudUploadStage{
config: config,
storageName: config.GetStringOption(pipeline.OptionStorage, ""),
pathTemplate: config.GetStringOption(pipeline.OptionPathTemplate, ""),
deleteAfter: config.GetBoolOption(pipeline.OptionDeleteAfter, false),
deleteAllAfter: config.GetBoolOption(pipeline.OptionDeleteAllAfter, false),
uploadTiming: config.GetStringOption(pipeline.OptionUploadTiming, ""),
fileTypes: config.GetStringSliceOption(pipeline.OptionFileTypes),
config: config,
storageName: config.GetStringOption(pipeline.OptionStorage, ""),
pathTemplate: config.GetStringOption(pipeline.OptionPathTemplate, ""),
deleteAfter: config.GetBoolOption(pipeline.OptionDeleteAfter, false),
deleteAllAfter: config.GetBoolOption(pipeline.OptionDeleteAllAfter, false),
uploadTiming: config.GetStringOption(pipeline.OptionUploadTiming, ""),
fileTypes: config.GetStringSliceOption(pipeline.OptionFileTypes),
uploadSubtitles: config.GetBoolOption(pipeline.OptionUploadSubtitles, false),
}, nil
}
@@ -134,12 +141,16 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli
// 获取 OpenList 管理器
mgr := openlist.GetGlobalManager()
if mgr == nil {
s.logs = "OpenList 管理器未初始化,跳过上传\n"
return input, nil
s.logs = "OpenList 管理器未初始化\n"
return input, fmt.Errorf("OpenList 管理器未初始化")
}
// 获取配置并创建客户端
config := configs.GetCurrentConfig()
if config == nil {
s.logs = "配置未初始化\n"
return input, fmt.Errorf("配置未初始化")
}
client, err := mgr.GetClient(ctx.Ctx, config.OpenList.Token, config.OpenList.Username, config.OpenList.Password)
if err != nil {
@@ -149,8 +160,11 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli
// 如果 token 变化了(通过用户名密码登录获取了新 token),自动写回配置
if newToken := client.GetCurrentToken(); newToken != "" && newToken != config.OpenList.Token {
config.OpenList.Token = newToken
if err := config.Marshal(); err != nil {
_, err := configs.UpdateWithRetry(func(c *configs.Config) error {
c.OpenList.Token = newToken
return nil
}, 3, 10*time.Millisecond)
if err != nil {
ctx.Logger.Warnf("保存 OpenList token 到配置文件失败: %v", err)
} else {
ctx.Logger.Info("OpenList token 已自动更新到配置文件")
@@ -159,54 +173,111 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli
var output []pipeline.FileInfo
var uploadFailed bool
usedPaths := map[string]bool{} // 检测远程路径冲突
var failureDetails []string // 收集每条失败详情,用于返回给前端
// 清除所有输入文件的 Deletable 标记(防止数据库中残留的旧标记影响本次判断)
for i := range input {
input[i].Deletable = false
}
// 只上传第一个视频文件和封面文件,跳过中间产物
// Pipeline 阶段输出顺序保证:最终视频在最前面,中间文件在后面
videoUploaded := false
// 上传所有非 Deletable 的视频文件和封面文件
// Deletable 检查已能正确区分中间产物(如 ConvertMp4 标记的原始 FLV)
// 与最终视频(如录播姬生成的多分段 _PART 文件),无需额外限制
uploadedIndices := map[int]bool{} // 记录实际上传的文件在 output 中的索引
// 上传弹幕字幕:扫描 input 中的视频文件,从磁盘发现关联的 .ass 文件并加入 input
if s.uploadSubtitles {
assAdded := map[string]bool{}
var discovered []pipeline.FileInfo
for _, f := range input {
ext := strings.ToLower(filepath.Ext(f.Path))
if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" {
continue
}
assPath := strings.TrimSuffix(f.Path, ext) + ".ass"
if assAdded[assPath] {
continue
}
if info, err := os.Stat(assPath); err == nil && info.Size() > 0 {
// 检查 input 和 discovered 中是否已存在
alreadyExists := false
for _, of := range input {
if of.Path == assPath {
alreadyExists = true
break
}
}
if !alreadyExists {
for _, of := range discovered {
if of.Path == assPath {
alreadyExists = true
break
}
}
}
if !alreadyExists {
discovered = append(discovered, pipeline.FileInfo{
Path: assPath,
Size: info.Size(),
Type: pipeline.FileTypeOther,
})
assAdded[assPath] = true
ctx.Logger.Infof("发现关联弹幕字幕文件: %s", assPath)
}
}
}
if len(discovered) > 0 {
input = append(input, discovered...)
}
}
for _, file := range input {
// 跳过标记为待删除的中间文件(由其他阶段产出,不应上传)
// 例外:uploadSubtitles 时放行 Deletable 的 .ass(如 burn_delete_ass 标记的字幕)
if file.Deletable {
output = append(output, file)
continue
}
// 文件类型过滤
if len(s.fileTypes) > 0 && !s.matchFileType(file.Type) {
output = append(output, file)
continue
}
// 视频文件:只上传第一个(最终视频),跳过后续中间视频
if file.Type == pipeline.FileTypeVideo {
if videoUploaded {
if !s.uploadSubtitles || strings.ToLower(filepath.Ext(file.Path)) != ".ass" {
output = append(output, file)
continue
}
}
// 文件类型过滤(uploadSubtitles 时放行 .ass 字幕文件)
if len(s.fileTypes) > 0 && !s.matchFileType(file.Type) {
if !s.uploadSubtitles || strings.ToLower(filepath.Ext(file.Path)) != ".ass" {
output = append(output, file)
continue
}
videoUploaded = true
}
// 检查文件是否存在
if _, err := os.Stat(file.Path); os.IsNotExist(err) {
s.logs += fmt.Sprintf("文件不存在: %s\n", file.Path)
errMsg := fmt.Sprintf("文件不存在: %s", file.Path)
s.logs += errMsg + "\n"
failureDetails = append(failureDetails, errMsg)
output = append(output, file)
uploadFailed = true
continue
}
// 渲染目标路径
targetPath := s.renderTargetPath(ctx, file)
if targetPath == "" {
s.logs += fmt.Sprintf("无法生成目标路径: %s\n", file.Path)
errMsg := fmt.Sprintf("无法生成目标路径: %s", file.Path)
s.logs += errMsg + "\n"
failureDetails = append(failureDetails, errMsg)
output = append(output, file)
uploadFailed = true
continue
}
// 检测远程路径冲突(多分段视频或视频+封面使用相同模板时可能产生同路径)
// 冲突时自动追加原始文件名以区分
if usedPaths[targetPath] {
// 使用 path.* 而非 filepath.*,因为 targetPath 是远程路径,始终用正斜杠
dir := path.Dir(targetPath)
ext := filepath.Ext(targetPath)
base := strings.TrimSuffix(path.Base(targetPath), ext)
targetPath = path.Join(dir, base+"_"+filepath.Base(file.Path))
ctx.Logger.Infof("检测到远程路径冲突,自动追加文件名: %s", targetPath)
}
usedPaths[targetPath] = true
ctx.Logger.Infof("上传文件: %s -> %s", file.Path, targetPath)
s.commands = append(s.commands, fmt.Sprintf("upload %s to %s/%s", file.Path, s.storageName, targetPath))
@@ -218,6 +289,7 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli
}
// 确保存储名和路径之间没有双斜杠
fullRemotePath := "/" + s.storageName + remotePath
fullRemotePath = strings.ReplaceAll(fullRemotePath, "//", "/")
// 确保远程目录存在(递归创建)
// 使用 path.Dir 而非 filepath.Dir,因为远程路径始终用正斜杠
@@ -236,7 +308,9 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli
})
if err != nil {
s.logs += fmt.Sprintf("上传失败: %s -> %s/%s - %s\n", fileName, s.storageName, targetPath, err.Error())
errMsg := fmt.Sprintf("上传失败: %s -> %s/%s - %s", fileName, s.storageName, targetPath, err.Error())
s.logs += errMsg + "\n"
failureDetails = append(failureDetails, errMsg)
ctx.Logger.Errorf("上传失败: %s - %v", file.Path, err)
// 上传失败,保留文件
output = append(output, file)
@@ -246,37 +320,153 @@ func (s *CloudUploadStage) Execute(ctx *pipeline.PipelineContext, input []pipeli
s.logs += fmt.Sprintf("上传成功: %s -> %s/%s\n", fileName, s.storageName, targetPath)
ctx.Logger.Infof("文件上传成功: %s -> %s/%s", fileName, s.storageName, targetPath)
// 标记已上传(立即设置,不依赖后续删除逻辑块,确保 immediate 模式下也有标记)
if file.Metadata == nil {
file.Metadata = map[string]any{}
}
file.Metadata["uploaded"] = true
// 记录上传路径,用于后续阶段替换文件后重新标记
if ctx.UploadedPaths != nil {
ctx.UploadedPaths[file.Path] = true
}
uploadedIndices[len(output)] = true // 记录上传成功的文件索引
output = append(output, file)
// 持久化上传标记(统一用正斜杠存储,与 URL 路径一致)
if cfg := configs.GetCurrentConfig(); cfg != nil {
outPutPath, _ := filepath.Abs(cfg.OutPutPath)
absFilePath, _ := filepath.Abs(file.Path)
if relPath, relErr := filepath.Rel(outPutPath, absFilePath); relErr == nil {
if markErr := metadata.GetStore().Set(ctx.Ctx, metadata.NamespaceUploaded, filepath.ToSlash(relPath), "1"); markErr != nil {
ctx.Logger.Warnf("保存上传标记失败: %v", markErr)
}
}
}
}
// 为已上传的 Deletable 文件标记 Metadata["uploaded"](如 burn_delete_ass 标记的 .ass)
// 这些文件在上传循环中被放行(uploadSubtitles),但 Deletable 跳过逻辑未标记 uploaded
for i, f := range output {
if uploadedIndices[i] && f.Deletable {
if f.Metadata == nil {
f.Metadata = map[string]any{}
}
f.Metadata["uploaded"] = true
output[i] = f
}
}
// 全部上传成功后,根据配置标记文件为可删除(由 Executor 在管道全部成功后统一删除)
// 任意一个上传失败,则不标记任何文件,确保本地文件全部保留
// 注意:不使用 Deletable 标记,避免后续阶段(如 CustomCommand)误跳过最终成品
// 改用 Metadata["uploaded"] 标记,由 Executor 在清理时读取
if !uploadFailed && s.uploadTiming != "immediate" {
if s.deleteAllAfter {
// 删除全部文件(含中间产物)
// 删除全部模式:查找视频文件关联的 .ass 字幕文件并加入 output
// (BurnSubtitlesStage 仅在 burn_delete_ass=true 时才将 .ass 加入 output,
// 但 delete_all 意图是删除所有文件,应包含字幕)
assAdded := map[string]bool{}
for _, f := range output {
ext := strings.ToLower(filepath.Ext(f.Path))
if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" {
continue
}
assPath := strings.TrimSuffix(f.Path, ext) + ".ass"
if assAdded[assPath] {
continue
}
if _, err := os.Stat(assPath); err == nil {
// 检查 output 中是否已存在
alreadyInOutput := false
for _, of := range output {
if of.Path == assPath {
alreadyInOutput = true
break
}
}
if !alreadyInOutput {
output = append(output, pipeline.FileInfo{
Path: assPath,
Type: pipeline.FileTypeOther,
Deletable: true, // 标记为可删除,由 executor 统一清理
})
assAdded[assPath] = true
ctx.Logger.Infof("delete_all 模式:关联字幕文件 %s", assPath)
}
}
}
// 删除全部文件(含中间产物):用 Deletable 标记
// 跳过已是 Deletable 的中间产物(如 BurnSubtitles 标记的源视频),
// 避免 delete_all 标记覆盖其"中间产物"身份,导致 CustomCommandStage 误执行
for i := range output {
if output[i].Deletable {
// 中间产物:保留原始 Deletable 标记,不加 delete_all
continue
}
output[i].Deletable = true
if output[i].Metadata == nil {
output[i].Metadata = map[string]any{}
}
output[i].Metadata["delete_all"] = true
// 为实际上传的文件标记 uploaded(供回调提取上传文件详情)
if uploadedIndices[i] {
output[i].Metadata["uploaded"] = true
}
}
s.logs += fmt.Sprintf("全部上传成功,已标记 %d 个文件待删除(含中间产物)\n", len(output))
ctx.Logger.Infof("全部上传成功,已标记 %d 个文件待删除(含中间产物)", len(output))
} else if s.deleteAfter {
// 仅删除已上传的文件
// 仅删除已上传的文件:用 Metadata["uploaded"] 标记,不设 Deletable
count := 0
for i := range output {
if uploadedIndices[i] {
output[i].Deletable = true
if output[i].Metadata == nil {
output[i].Metadata = map[string]any{}
}
output[i].Metadata["uploaded"] = true
count++
}
}
// 已上传的视频文件会被删除,其关联的 .ass 字幕文件也应标记为可删除
// 避免摘要将已删除的字幕文件误显为本地保留文件
// 注意:这些 .ass 未被上传到云存储,不标记 uploaded,仅标记 Deletable
for i, f := range output {
if !uploadedIndices[i] {
continue // 仅处理已上传成功的视频文件
}
ext := strings.ToLower(filepath.Ext(f.Path))
if ext != ".flv" && ext != ".mp4" && ext != ".mkv" && ext != ".ts" {
continue
}
assPath := strings.TrimSuffix(f.Path, ext) + ".ass"
if _, err := os.Stat(assPath); err == nil {
alreadyInOutput := false
for _, of := range output {
if of.Path == assPath {
alreadyInOutput = true
break
}
}
if !alreadyInOutput {
output = append(output, pipeline.FileInfo{
Path: assPath,
Type: pipeline.FileTypeOther,
Deletable: true, // 标记为可删除,由 executor 统一清理
})
ctx.Logger.Infof("delete_uploaded 模式:关联字幕文件 %s", assPath)
}
}
}
s.logs += fmt.Sprintf("全部上传成功,已标记 %d 个已上传文件待删除\n", count)
ctx.Logger.Infof("全部上传成功,已标记 %d 个已上传文件待删除", count)
}
}
// 任意上传失败,返回错误(让 Executor 将任务标记为 failed)
// 将每条失败详情拼入 error message,前端可直接展示具体原因
if uploadFailed {
return output, fmt.Errorf("部分文件上传失败,详见日志")
return output, fmt.Errorf("部分文件上传失败:\n%s", strings.Join(failureDetails, "\n"))
}
return output, nil
@@ -294,29 +484,12 @@ func (s *CloudUploadStage) matchFileType(fileType pipeline.FileType) bool {
// renderTargetPath 渲染目标路径
func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file pipeline.FileInfo) string {
if s.pathTemplate == "" {
// 默认路径:/录播归档/{平台}/{主播名}/{文件名}
return fmt.Sprintf("/录播归档/%s/%s/%s",
ctx.RecordInfo.Platform,
ctx.RecordInfo.HostName,
filepath.Base(file.Path),
)
}
// 获取扩展名
ext := filepath.Ext(file.Path)
if len(ext) > 0 && ext[0] == '.' {
ext = ext[1:]
}
// 模板数据
data := struct {
Platform string
HostName string
RoomName string
FileName string
Ext string
}{
data := pipeline.UploadPathData{
Platform: ctx.RecordInfo.Platform,
HostName: ctx.RecordInfo.HostName,
RoomName: ctx.RecordInfo.RoomName,
@@ -324,64 +497,13 @@ func (s *CloudUploadStage) renderTargetPath(ctx *pipeline.PipelineContext, file
Ext: ext,
}
// 使用 Go 模板引擎
funcMap := template.FuncMap{
"date": func(format string, t time.Time) string {
return t.Format(format)
},
"now": func() time.Time {
return time.Now()
},
"trimSuffix": func(suffix, s string) string {
return strings.TrimSuffix(s, suffix)
},
"ext": func(path string) string {
return filepath.Ext(path)
},
"filenameFilter": func(s string) string {
// 过滤文件名中的非法字符
replacer := strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_",
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
)
return replacer.Replace(s)
},
}
tmpl, err := template.New("path").Funcs(funcMap).Parse(s.pathTemplate)
path, err := pipeline.RenderUploadPath(s.pathTemplate, data, func() time.Time {
return ctx.RecordInfo.StartTime
})
if err != nil {
// 模板解析失败,回退到简单替换
return s.fallbackRender(data)
ctx.Logger.Errorf("%v", err)
return ""
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, data); err != nil {
// 模板执行失败,回退到简单替换
return s.fallbackRender(data)
}
return buf.String()
}
// fallbackRender 简单字符串替换(模板引擎失败时的回退方案)
func (s *CloudUploadStage) fallbackRender(data struct {
Platform string
HostName string
RoomName string
FileName string
Ext string
}) string {
path := s.pathTemplate
path = strings.ReplaceAll(path, "{{ .Platform }}", data.Platform)
path = strings.ReplaceAll(path, "{{.Platform}}", data.Platform)
path = strings.ReplaceAll(path, "{{ .HostName }}", data.HostName)
path = strings.ReplaceAll(path, "{{.HostName}}", data.HostName)
path = strings.ReplaceAll(path, "{{ .RoomName }}", data.RoomName)
path = strings.ReplaceAll(path, "{{.RoomName}}", data.RoomName)
path = strings.ReplaceAll(path, "{{ .FileName }}", data.FileName)
path = strings.ReplaceAll(path, "{{.FileName}}", data.FileName)
path = strings.ReplaceAll(path, "{{ .Ext }}", data.Ext)
path = strings.ReplaceAll(path, "{{.Ext}}", data.Ext)
return path
}
+27 -7
View File
@@ -4,6 +4,7 @@ package pipeline
import (
"context"
"os"
"time"
"github.com/bililive-go/bililive-go/src/live"
@@ -15,6 +16,9 @@ import (
// PipelineTaskUpdateEvent 管道任务更新事件
const PipelineTaskUpdateEvent events.EventType = "PipelineTaskUpdate"
// ManualUploadLiveID 手动上传任务的 LiveID 标识
const ManualUploadLiveID = "manual-upload"
// FileType 文件类型
type FileType string
@@ -30,18 +34,23 @@ const (
// FileInfo 表示管道中流转的文件信息
type FileInfo struct {
Path string `json:"path"` // 文件绝对路径
Size int64 `json:"size,omitempty"` // 文件大小(字节),在任务创建时记录
Type FileType `json:"type"` // 文件类型
SourcePath string `json:"source_path,omitempty"` // 来源文件路径(用于追踪转换链)
Metadata map[string]any `json:"metadata,omitempty"` // 额外元数据
Deletable bool `json:"deletable,omitempty"` // 标记此文件在管道全部成功后可删除
}
// NewVideoFileInfo 创建视频文件信息
// NewVideoFileInfo 创建视频文件信息,自动记录文件大小
func NewVideoFileInfo(path string) FileInfo {
return FileInfo{
fi := FileInfo{
Path: path,
Type: FileTypeVideo,
}
if stat, err := os.Stat(path); err == nil {
fi.Size = stat.Size()
}
return fi
}
// NewCoverFileInfo 创建封面文件信息
@@ -55,11 +64,12 @@ func NewCoverFileInfo(path, sourcePath string) FileInfo {
// RecordInfo 录制信息
type RecordInfo struct {
LiveID types.LiveID `json:"live_id"`
Platform string `json:"platform"`
HostName string `json:"host_name"`
RoomName string `json:"room_name"`
StartTime time.Time `json:"start_time"`
LiveID types.LiveID `json:"live_id"`
Platform string `json:"platform"`
HostName string `json:"host_name"`
RoomName string `json:"room_name"`
DisplayName string `json:"display_name,omitempty"` // 任务列表展示用,不影响路径渲染
StartTime time.Time `json:"start_time"`
}
// NewRecordInfo 从 live.Info 创建录制信息
@@ -83,6 +93,15 @@ type PipelineContext struct {
// FFmpegPath 是 ffmpeg 可执行文件的路径
FFmpegPath string
// LastStageFiles 最后阶段输出的文件快照(清理前保存)
// 用于回调提取上传文件详情,因为 deleteMarkedFiles 会删除文件
LastStageFiles []FileInfo
// UploadedPaths 记录 cloud_upload 阶段成功上传的文件路径
// 用于在所有阶段执行完毕后,为被后续阶段替换的文件重新标记 Metadata["uploaded"]
// (如 fix_flv 替换 .flv 后,新 FileInfo 丢失上传标记)
UploadedPaths map[string]bool
}
// Stage 管道阶段接口
@@ -242,6 +261,7 @@ type PipelineTask struct {
CompletedAt *time.Time `json:"completed_at,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
CanRetry bool `json:"can_retry"` // 是否可以重试
LastStageFiles []FileInfo `json:"-"` // 最后阶段输出文件(清理前快照,不持久化)
}
// NewPipelineTask 创建新的管道任务
+67
View File
@@ -0,0 +1,67 @@
package pipeline
import (
"bytes"
"fmt"
"path/filepath"
"strings"
"text/template"
"time"
)
// UploadPathData 上传路径模板数据
type UploadPathData struct {
Platform string
HostName string
RoomName string
FileName string
Ext string
}
// DefaultUploadPathFuncs 返回上传路径模板的默认函数集
// nowFn 返回当前时间(可从 RecordInfo.StartTime 或 time.Now() 传入)
func DefaultUploadPathFuncs(nowFn func() time.Time) template.FuncMap {
return template.FuncMap{
"date": func(format string, t time.Time) string {
return t.Format(format)
},
"now": nowFn,
"trimSuffix": func(suffix, s string) string {
return strings.TrimSuffix(s, suffix)
},
"ext": func(path string) string {
return filepath.Ext(path)
},
"filenameFilter": func(s string) string {
replacer := strings.NewReplacer(
"/", "_", "\\", "_", ":", "_", "*", "_",
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
)
return replacer.Replace(s)
},
}
}
// RenderUploadPath 渲染上传目标路径
// pathTemplate 为空时使用默认路径 /录播归档/{Platform}/{HostName}/{FileName}
// 返回空字符串表示模板解析失败
func RenderUploadPath(pathTemplate string, data UploadPathData, nowFn func() time.Time) (string, error) {
if pathTemplate == "" {
path := fmt.Sprintf("/录播归档/%s/%s/%s", data.Platform, data.HostName, data.FileName)
return path, nil
}
funcMap := DefaultUploadPathFuncs(nowFn)
tmpl, err := template.New("path").Funcs(funcMap).Parse(pathTemplate)
if err != nil {
return "", fmt.Errorf("上传路径模板解析失败: %w", err)
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, data); err != nil {
return "", fmt.Errorf("上传路径模板执行失败: %w", err)
}
return buf.String(), nil
}
+18 -1
View File
@@ -92,7 +92,7 @@ func Close() error {
storeMu.Lock()
defer storeMu.Unlock()
if globalStore == nil {
if globalStore == nil || globalStore.db == nil {
return nil
}
@@ -103,6 +103,9 @@ func Close() error {
// Get 从指定命名空间获取值
func (s *Store) Get(ctx context.Context, namespace, key string) (string, error) {
if s == nil {
return "", nil
}
s.mu.RLock()
defer s.mu.RUnlock()
@@ -123,6 +126,9 @@ func (s *Store) Get(ctx context.Context, namespace, key string) (string, error)
// Set 在指定命名空间设置值
func (s *Store) Set(ctx context.Context, namespace, key, value string) error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
@@ -142,6 +148,9 @@ func (s *Store) Set(ctx context.Context, namespace, key, value string) error {
// Delete 从指定命名空间删除键
func (s *Store) Delete(ctx context.Context, namespace, key string) error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
@@ -157,6 +166,9 @@ func (s *Store) Delete(ctx context.Context, namespace, key string) error {
// GetAll 获取指定命名空间的所有键值对
func (s *Store) GetAll(ctx context.Context, namespace string) (map[string]string, error) {
if s == nil {
return nil, nil
}
s.mu.RLock()
defer s.mu.RUnlock()
@@ -183,6 +195,9 @@ func (s *Store) GetAll(ctx context.Context, namespace string) (map[string]string
// DeleteNamespace 删除整个命名空间
func (s *Store) DeleteNamespace(ctx context.Context, namespace string) error {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
@@ -206,6 +221,8 @@ const (
NamespaceUpdate = "update"
// NamespaceMigration 数据库迁移状态
NamespaceMigration = "migration"
// NamespaceUploaded 已上传文件标记(key=相对路径, value="1")
NamespaceUploaded = "uploaded"
)
// 预定义的键常量
+54 -18
View File
@@ -4,12 +4,14 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"sync"
)
// encodePath 对路径的每个段分别编码,保留 / 分隔符
@@ -24,6 +26,7 @@ func encodePath(path string) string {
// Client OpenList API 客户端
type Client struct {
baseURL string
mu sync.RWMutex // 保护 token/username/password 的并发访问
token string
username string // 用于 token 刷新
password string // 用于 token 刷新
@@ -43,40 +46,75 @@ func NewClient(baseURL, token string) *Client {
// SetToken 设置 API Token
func (c *Client) SetToken(token string) {
c.mu.Lock()
defer c.mu.Unlock()
c.token = token
}
// GetToken 获取当前 API Token
// GetCurrentToken 获取当前 API Token
func (c *Client) GetCurrentToken() string {
c.mu.RLock()
defer c.mu.RUnlock()
return c.token
}
// SetCredentials 设置用户名密码(用于 token 自动刷新)
func (c *Client) SetCredentials(username, password string) {
c.mu.Lock()
defer c.mu.Unlock()
c.username = username
c.password = password
}
// getTokenForRequest 获取用于请求的 token 副本(内部使用)
func (c *Client) getTokenForRequest() string {
c.mu.RLock()
defer c.mu.RUnlock()
return c.token
}
// refreshToken 刷新 token
func (c *Client) refreshToken(ctx context.Context) error {
if c.username == "" || c.password == "" {
c.mu.RLock()
username := c.username
password := c.password
c.mu.RUnlock()
if username == "" || password == "" {
return fmt.Errorf("无法刷新 token: 未配置用户名密码")
}
newToken, err := c.GetToken(ctx, c.username, c.password)
newToken, err := c.GetToken(ctx, username, password)
if err != nil {
return fmt.Errorf("刷新 token 失败: %w", err)
}
c.token = newToken
c.SetToken(newToken)
return nil
}
// isAuthError 判断是否为认证错误(HTTP 401/403)
// APIError 表示 OpenList API 返回的结构化错误
// 用于在 isAuthError 中识别认证失败(code 401/403)
type APIError struct {
Code int
Message string
}
func (e *APIError) Error() string {
return fmt.Sprintf("API 错误 (code %d): %s", e.Code, e.Message)
}
// isAuthError 判断是否为认证错误
// 匹配 HTTP 状态码 401/403 或 OpenList JSON code 401/403
func isAuthError(err error) bool {
if err == nil {
return false
}
// 检查结构化 API 错误
var apiErr *APIError
if errors.As(err, &apiErr) {
return apiErr.Code == 401 || apiErr.Code == 403
}
// 兜底:检查错误文本中的 HTTP 状态码
s := err.Error()
// 匹配 "HTTP 401" 或 "HTTP 403" 格式,避免误匹配其他包含 401/403 的字符串
return strings.Contains(s, "HTTP 401") || strings.Contains(s, "HTTP 403")
}
@@ -123,7 +161,7 @@ func (c *Client) doUpload(ctx context.Context, localPath, remotePath string, onP
return fmt.Errorf("创建请求失败: %w", err)
}
req.Header.Set("Authorization", c.token)
req.Header.Set("Authorization", c.getTokenForRequest())
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", fmt.Sprintf("%d", totalSize))
// 对路径的每个段分别编码,保留 / 分隔符
@@ -148,12 +186,10 @@ func (c *Client) doUpload(ctx context.Context, localPath, remotePath string, onP
Message string `json:"message"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
// 响应不是 JSON 格式,可能是空响应(某些 API 成功时返回空 body)
// 由于 HTTP 状态码是 200,视为成功
return nil
return fmt.Errorf("解析上传响应失败: %w", err)
}
if result.Code != 200 {
return fmt.Errorf("上传失败: %s", result.Message)
return fmt.Errorf("上传失败: %w", &APIError{Code: result.Code, Message: result.Message})
}
return nil
@@ -176,7 +212,7 @@ func (c *Client) ListStorages(ctx context.Context) ([]StorageInfo, error) {
if err != nil {
return err
}
req.Header.Set("Authorization", c.token)
req.Header.Set("Authorization", c.getTokenForRequest())
resp, err := c.httpClient.Do(req)
if err != nil {
@@ -197,7 +233,7 @@ func (c *Client) ListStorages(ctx context.Context) ([]StorageInfo, error) {
}
if result.Code != 200 {
return fmt.Errorf("API 错误: %s", result.Message)
return fmt.Errorf("API 错误: %w", &APIError{Code: result.Code, Message: result.Message})
}
storages = result.Data.Content
@@ -218,7 +254,7 @@ func (c *Client) CheckStorageHealth(ctx context.Context, storageName string) err
if err != nil {
return err
}
req.Header.Set("Authorization", c.token)
req.Header.Set("Authorization", c.getTokenForRequest())
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
@@ -237,7 +273,7 @@ func (c *Client) CheckStorageHealth(ctx context.Context, storageName string) err
}
if result.Code != 200 {
return fmt.Errorf("存储不可用: %s", result.Message)
return fmt.Errorf("存储不可用: %w", &APIError{Code: result.Code, Message: result.Message})
}
return nil
@@ -276,7 +312,7 @@ func (c *Client) GetToken(ctx context.Context, username, password string) (strin
}
if result.Code != 200 {
return "", fmt.Errorf("登录失败: %s", result.Message)
return "", fmt.Errorf("登录失败: %w", &APIError{Code: result.Code, Message: result.Message})
}
return result.Data.Token, nil
@@ -293,7 +329,7 @@ func (c *Client) Mkdir(ctx context.Context, remotePath string) error {
if err != nil {
return err
}
req.Header.Set("Authorization", c.token)
req.Header.Set("Authorization", c.getTokenForRequest())
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
@@ -313,7 +349,7 @@ func (c *Client) Mkdir(ctx context.Context, remotePath string) error {
// 目录已存在也算成功
if result.Code != 200 && result.Message != "file exists" {
return fmt.Errorf("创建目录失败: %s", result.Message)
return fmt.Errorf("创建目录失败: %w", &APIError{Code: result.Code, Message: result.Message})
}
return nil
+44 -22
View File
@@ -34,6 +34,19 @@ func SetGlobalManager(m *Manager) {
globalManager = m
}
// ClearGlobalManagerIf 仅当全局管理器等于 expected 时才清除(CAS 语义)
// 用于异步清理路径,避免误清已被新实例替换的管理器
// 返回是否成功清除
func ClearGlobalManagerIf(expected *Manager) bool {
globalMu.Lock()
defer globalMu.Unlock()
if globalManager == expected {
globalManager = nil
return true
}
return false
}
// GetGlobalManager 获取全局 OpenList 管理器
func GetGlobalManager() *Manager {
globalMu.RLock()
@@ -197,9 +210,14 @@ func (m *Manager) saveInitialCredentials(logFile string) {
return
}
// 如果已经配置了用户名密码,跳过
// 如果已经配置了用户名密码,验证有效性
if config.OpenList.Username != "" && config.OpenList.Password != "" {
return
// 用现有凭据尝试登录,验证是否有效
testClient := NewClient(m.apiEndpoint, "")
if _, err := testClient.GetToken(context.Background(), config.OpenList.Username, config.OpenList.Password); err == nil {
return // 凭据有效,无需更新
}
// 凭据无效(如占位值 root/root),继续从日志读取真实密码
}
// 读取日志文件查找初始密码
@@ -226,10 +244,20 @@ func (m *Manager) saveInitialCredentials(logFile string) {
return
}
// 验证密码有效性:日志是追加模式,旧密码可能已被用户修改
// 只有实际能登录时才保存
client := NewClient(m.apiEndpoint, "")
if _, err := client.GetToken(context.Background(), "admin", initialPassword); err != nil {
logrus.WithError(err).Debug("初始密码验证失败,跳过保存(可能已被修改)")
return
}
// 保存到配置
config.OpenList.Username = "admin"
config.OpenList.Password = initialPassword
if err := config.Marshal(); err != nil {
if _, err := configs.UpdateWithRetry(func(c *configs.Config) error {
c.OpenList.Username = "admin"
c.OpenList.Password = initialPassword
return nil
}, 3, 10*time.Millisecond); err != nil {
logrus.WithError(err).Warn("保存 OpenList 初始密码到配置文件失败")
} else {
logrus.Info("OpenList 首次启动,初始密码已保存到配置文件")
@@ -251,10 +279,16 @@ func (m *Manager) watchProcess() {
select {
case <-m.stopCh:
// 正常停止
// 正常停止(stopInternal 已负责关闭 m.logFile)
default:
// 异常退出
// 异常退出:关闭日志文件句柄,避免 fd 泄漏
if wasRunning {
m.mu.Lock()
if m.logFile != nil {
m.logFile.Close()
m.logFile = nil
}
m.mu.Unlock()
logrus.WithError(err).Warn("OpenList 进程异常退出")
}
}
@@ -322,6 +356,7 @@ func (m *Manager) GetDataPath() string {
// GetClient 获取已认证的 API 客户端
// 如果提供了 token,直接使用;否则使用用户名密码登录获取 token
// token 有效性由 withRetry 在实际 API 调用时自动处理(401/403 时自动刷新)
func (m *Manager) GetClient(ctx context.Context, token, username, password string) (*Client, error) {
client := NewClient(m.apiEndpoint, "")
@@ -330,23 +365,10 @@ func (m *Manager) GetClient(ctx context.Context, token, username, password strin
client.SetCredentials(username, password)
}
// 有 token 时,验证其有效性
// 有 token 时直接使用,不预验证(避免需要管理员权限的 API 调用)
// withRetry 会在实际调用遇到 401/403 时自动用密码刷新 token
if token != "" {
client.SetToken(token)
// 尝试用 token 调用一个轻量 API 验证有效性
if _, err := client.ListStorages(ctx); err != nil {
// token 无效,尝试用密码重新登录
if username != "" && password != "" {
logrus.Warn("配置的 OpenList token 无效,尝试用用户名密码重新登录")
newToken, loginErr := client.GetToken(ctx, username, password)
if loginErr != nil {
return nil, fmt.Errorf("OpenList token 无效且登录失败: %w", loginErr)
}
client.SetToken(newToken)
return client, nil
}
return nil, fmt.Errorf("OpenList token 无效且未配置用户名密码")
}
return client, nil
}
+54 -3
View File
@@ -11,6 +11,7 @@ import (
type PlatformRateLimiter struct {
limiters map[string]*PlatformLimiter // 平台名称 -> 限制器
mu sync.RWMutex // 读写锁保护map
enabled bool // 是否实际应用平台级访问限制
}
// PlatformLimiter 单个平台的频率限制器
@@ -21,8 +22,17 @@ type PlatformLimiter struct {
inFlight chan struct{} // 限制同一平台最多一个请求在途
}
var globalRateLimiter = &PlatformRateLimiter{
limiters: make(map[string]*PlatformLimiter),
// 平台级访问限制暂时关闭,以避免同平台请求串行排队导致新直播间长时间无法开始录制。
// 保留限制器实现和配置结构,后续改善调度策略后可重新启用。
const globalRateLimitEnabled = false
var globalRateLimiter = newPlatformRateLimiter(globalRateLimitEnabled)
func newPlatformRateLimiter(enabled bool) *PlatformRateLimiter {
return &PlatformRateLimiter{
limiters: make(map[string]*PlatformLimiter),
enabled: enabled,
}
}
// GetGlobalRateLimiter 获取全局速率限制器实例
@@ -30,8 +40,17 @@ func GetGlobalRateLimiter() *PlatformRateLimiter {
return globalRateLimiter
}
// Enabled 返回当前限制器是否实际应用平台级访问限制。
func (prl *PlatformRateLimiter) Enabled() bool {
return prl.enabled
}
// SetPlatformLimit 设置或更新指定平台的访问频率限制
func (prl *PlatformRateLimiter) SetPlatformLimit(platform string, intervalSec int) {
if !prl.enabled {
return
}
if intervalSec <= 0 {
// 如果间隔为0或负数,移除该平台的限制
prl.mu.Lock()
@@ -63,7 +82,7 @@ func (prl *PlatformRateLimiter) SetPlatformLimit(platform string, intervalSec in
// EnsurePlatformLimit 在平台尚无限制时设置默认值,不覆盖已经显式配置的间隔。
// 用于房间对象早于配置持久化创建的路径,保证首次请求也不会绕过平台保护。
func (prl *PlatformRateLimiter) EnsurePlatformLimit(platform string, intervalSec int) {
if platform == "" || intervalSec <= 0 {
if !prl.enabled || platform == "" || intervalSec <= 0 {
return
}
@@ -90,6 +109,14 @@ func (prl *PlatformRateLimiter) WaitForPlatform(platform string) {
// 如果平台没有设置限制,立即返回 true
// 返回 true 表示成功获取访问权限,false 表示被 context 取消
func (prl *PlatformRateLimiter) WaitForPlatformWithContext(ctx context.Context, platform string) bool {
if ctx.Err() != nil {
return false
}
if !prl.enabled {
return true
}
prl.mu.RLock()
limiter, exists := prl.limiters[platform]
prl.mu.RUnlock()
@@ -108,6 +135,14 @@ func (prl *PlatformRateLimiter) WaitForPlatformWithContext(ctx context.Context,
// 仅限制请求开始间隔不足以保护启动阶段:当前一个请求耗时超过最小间隔时,后续请求仍会
// 重叠执行。这里把并发槽位与开始间隔合并为一次许可,使大量直播间并发初始化时仍按平台串行。
func (prl *PlatformRateLimiter) AcquirePlatformWithContext(ctx context.Context, platform string) (release func(), ok bool) {
if ctx.Err() != nil {
return nil, false
}
if !prl.enabled {
return func() {}, true
}
prl.mu.RLock()
limiter, exists := prl.limiters[platform]
prl.mu.RUnlock()
@@ -172,6 +207,10 @@ func waitForLimiterWithContext(ctx context.Context, limiter *PlatformLimiter) bo
// GetPlatformNextAllowedTime 获取平台下次允许访问的时间
func (prl *PlatformRateLimiter) GetPlatformNextAllowedTime(platform string) time.Time {
if !prl.enabled {
return time.Now()
}
prl.mu.RLock()
limiter, exists := prl.limiters[platform]
prl.mu.RUnlock()
@@ -197,6 +236,10 @@ func (prl *PlatformRateLimiter) RemovePlatformLimit(platform string) {
// GetAllPlatformLimits 获取所有平台的当前限制设置
func (prl *PlatformRateLimiter) GetAllPlatformLimits() map[string]int {
if !prl.enabled {
return map[string]int{}
}
prl.mu.RLock()
defer prl.mu.RUnlock()
@@ -219,6 +262,10 @@ type WaitInfo struct {
// GetPlatformWaitInfo 获取指定平台的等待状态信息
func (prl *PlatformRateLimiter) GetPlatformWaitInfo(platform string) WaitInfo {
if !prl.enabled {
return WaitInfo{}
}
prl.mu.RLock()
limiter, exists := prl.limiters[platform]
prl.mu.RUnlock()
@@ -254,6 +301,10 @@ func (prl *PlatformRateLimiter) GetPlatformWaitInfo(platform string) WaitInfo {
// ForceAccess 强制访问平台,忽略频率限制
// 返回距离上次访问的时间间隔
func (prl *PlatformRateLimiter) ForceAccess(platform string) time.Duration {
if !prl.enabled {
return 0
}
prl.mu.RLock()
limiter, exists := prl.limiters[platform]
prl.mu.RUnlock()
+36 -3
View File
@@ -7,7 +7,7 @@ import (
)
func TestPlatformRateLimiter(t *testing.T) {
limiter := GetGlobalRateLimiter()
limiter := newPlatformRateLimiter(true)
// 设置测试平台限制:2秒间隔
limiter.SetPlatformLimit("test_platform", 2)
@@ -44,7 +44,7 @@ func TestPlatformRateLimiter(t *testing.T) {
}
func TestPlatformRateLimiterUpdate(t *testing.T) {
limiter := GetGlobalRateLimiter()
limiter := newPlatformRateLimiter(true)
// 设置初始限制
limiter.SetPlatformLimit("update_test", 3)
@@ -80,6 +80,7 @@ func TestAcquirePlatformSerializesInFlightRequests(t *testing.T) {
inFlight: make(chan struct{}, 1),
},
},
enabled: true,
}
releaseFirst, ok := limiter.AcquirePlatformWithContext(context.Background(), "test")
@@ -112,7 +113,7 @@ func TestAcquirePlatformSerializesInFlightRequests(t *testing.T) {
}
func TestEnsurePlatformLimitDoesNotOverrideExplicitLimit(t *testing.T) {
limiter := &PlatformRateLimiter{limiters: make(map[string]*PlatformLimiter)}
limiter := newPlatformRateLimiter(true)
limiter.SetPlatformLimit("test", 5)
limiter.EnsurePlatformLimit("test", 1)
@@ -120,3 +121,35 @@ func TestEnsurePlatformLimitDoesNotOverrideExplicitLimit(t *testing.T) {
t.Fatalf("兜底限制覆盖了显式配置:得到 %d 秒,期望 5 秒", got)
}
}
func TestGlobalPlatformRateLimiterDisabled(t *testing.T) {
limiter := GetGlobalRateLimiter()
limiter.SetPlatformLimit("test", 60)
if limiter.Enabled() {
t.Fatal("全局平台限制器应保持关闭")
}
if limits := limiter.GetAllPlatformLimits(); len(limits) != 0 {
t.Fatalf("全局平台限制器应保持关闭,实际限制:%v", limits)
}
release, ok := limiter.AcquirePlatformWithContext(context.Background(), "test")
if !ok {
t.Fatal("关闭平台限制后请求应立即获准")
}
release()
waitInfo := limiter.GetPlatformWaitInfo("test")
if waitInfo.MinIntervalSec != 0 || waitInfo.NextRequestInSec != 0 {
t.Fatalf("关闭平台限制后不应报告等待状态:%+v", waitInfo)
}
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
if limiter.WaitForPlatformWithContext(canceledCtx, "test") {
t.Fatal("关闭平台限制后仍应尊重 context 取消")
}
if release, acquired := limiter.AcquirePlatformWithContext(canceledCtx, "test"); acquired || release != nil {
t.Fatal("context 已取消时不应获取平台许可")
}
}
+4 -3
View File
@@ -3,7 +3,8 @@ package recorders
import "github.com/bililive-go/bililive-go/src/pkg/events"
const (
RecorderStart events.EventType = "RecorderStart"
RecorderStop events.EventType = "RecorderStop"
RecorderRestart events.EventType = "RecorderRestart"
RecorderStart events.EventType = "RecorderStart"
RecorderStop events.EventType = "RecorderStop"
RecorderRestart events.EventType = "RecorderRestart"
RecorderStopRequested events.EventType = "RecorderStopRequested"
)
+34 -4
View File
@@ -73,6 +73,8 @@ type Manager interface {
GetRecorderStatus(ctx context.Context, liveId types.LiveID) (map[string]interface{}, error)
// GetActiveRecordingsCount 获取当前活跃的录制数量
GetActiveRecordingsCount() int
// WaitForNotifications 等待所有异步通知 goroutine 完成(关闭流程中使用)
WaitForNotifications()
}
// for test
@@ -86,6 +88,7 @@ type manager struct {
statusTicker *time.Ticker
statusStopCh chan struct{}
statusWg sync.WaitGroup // 用于等待广播 goroutine 退出
notifyWg sync.WaitGroup // 跟踪异步通知 goroutine(sendAccumulatedSummary),关闭时等待
// restartingCount 追踪正在执行 CloseForRestart 的旧 recorder 数量。
// RestartRecorder 在释放锁后才执行 oldRecorder.CloseForRestart(),
// 此期间 map 中只有新 recorder,但旧 recorder 仍在收尾运行。
@@ -133,6 +136,9 @@ func (m *manager) registryListener(ctx context.Context, ed events.Dispatcher) {
})
ed.AddEventListener(listeners.LiveEnd, removeEvtListener)
ed.AddEventListener(listeners.ListenStop, removeEvtListener)
// RecorderStopRequested 仅表示 recorder 自身遇到不可重试错误,需要被回收;
// 它不具有 LiveEnd 的全局直播状态语义,避免错误结束仍在进行的直播会话。
ed.AddEventListener(RecorderStopRequested, removeEvtListener)
}
func (m *manager) Start(ctx context.Context) error {
@@ -169,6 +175,12 @@ func (m *manager) Close(ctx context.Context) {
inst.WaitGroup.Done()
}
// WaitForNotifications 等待所有异步通知 goroutine 完成
// 应在 PipelineManager.Close 之后调用,确保所有 pipeline 回调触发的通知已完成
func (m *manager) WaitForNotifications() {
m.notifyWg.Wait()
}
func (m *manager) AddRecorder(ctx context.Context, live live.Live) error {
m.lock.Lock()
defer m.lock.Unlock()
@@ -180,11 +192,15 @@ func (m *manager) addRecorderLocked(ctx context.Context, live live.Live) error {
if _, ok := m.savers[live.GetLiveId()]; ok {
return ErrRecorderExist
}
recorder, err := newRecorder(ctx, live)
rec, err := newRecorder(ctx, live)
if err != nil {
return err
}
m.savers[live.GetLiveId()] = recorder
// 设置通知 WaitGroup,使异步摘要发送 goroutine 可被关闭流程等待
if recTyped, ok := rec.(*recorder); ok {
recTyped.notifyWg = &m.notifyWg
}
m.savers[live.GetLiveId()] = rec
cfg := configs.GetCurrentConfig()
if cfg != nil {
@@ -192,12 +208,12 @@ func (m *manager) addRecorderLocked(ctx context.Context, live live.Live) error {
bilisentry.GoWithContext(ctx, func(ctx context.Context) { m.cronRestart(ctx, live) })
}
}
if err := recorder.Start(ctx); err != nil {
if err := rec.Start(ctx); err != nil {
// Start 失败时从 map 删除并异步 Close 新 recorder,防止泄漏/僵尸实例
// 使用异步 Close 避免在持锁时执行耗时操作(如等待 ffmpeg 进程退出),
// 防止长时间阻塞其他 manager 操作
delete(m.savers, live.GetLiveId())
bilisentry.Go(recorder.Close)
bilisentry.Go(rec.Close)
return err
}
return nil
@@ -241,6 +257,11 @@ func (m *manager) RestartRecorder(ctx context.Context, live live.Live) error {
return err
}
newRec := m.savers[live.GetLiveId()]
// 设置分段重启摘要屏障:阻止新 recorder 在 Pipeline 状态转移前发送摘要
if newRecTyped, ok := newRec.(*recorder); ok {
newRecTyped.transferWg = &sync.WaitGroup{}
newRecTyped.transferWg.Add(1)
}
// restartingCount 必须在释放锁之前递增,否则 Unlock 到 Add(1) 之间
// LiveEnd 可能移除新 recorder 并看到 restartingCount==0,
// 导致 GetActiveRecordingsCount() 误判为"无活跃录制"触发优雅更新
@@ -274,6 +295,15 @@ func (m *manager) RestartRecorder(ctx context.Context, live live.Live) error {
}
}
// 4. 转移 Pipeline 状态(确保旧分段的 Pipeline 结果不丢失)
if newRecTyped, ok := newRec.(*recorder); ok {
if oldRecTyped, ok := oldRecorder.(*recorder); ok {
newRecTyped.TransferPipelineState(oldRecTyped)
}
// 释放摘要屏障:转移已完成(或无需转移),允许新 recorder 发送摘要
newRecTyped.transferWg.Done()
}
return nil
}
+22
View File
@@ -12,6 +12,7 @@ import (
"github.com/bililive-go/bililive-go/src/instance"
"github.com/bililive-go/bililive-go/src/live"
livemock "github.com/bililive-go/bililive-go/src/live/mock"
"github.com/bililive-go/bililive-go/src/pkg/events"
"github.com/bililive-go/bililive-go/src/pkg/livelogger"
"github.com/bililive-go/bililive-go/src/types"
)
@@ -56,6 +57,27 @@ func TestManagerAddAndRemoveRecorder(t *testing.T) {
assert.False(t, m.HasRecorder(context.Background(), "test"))
}
// TestRecorderStopRequestedRemovesOnlyRecorder 验证 recorder 的内部停止请求会被
// manager 消费并回收 recorder,无需伪造具有全局直播状态语义的 LiveEnd。
func TestRecorderStopRequestedRemovesOnlyRecorder(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
ctx := context.Background()
ed := events.NewDispatcher(ctx)
l := livemock.NewMockLive(ctrl)
l.EXPECT().GetLiveId().Return(types.LiveID("test")).AnyTimes()
r := NewMockRecorder(ctrl)
r.EXPECT().Close()
m := &manager{savers: map[types.LiveID]Recorder{"test": r}}
m.registryListener(ctx, ed)
ed.DispatchEventSync(events.NewEvent(RecorderStopRequested, l))
assert.False(t, m.HasRecorder(ctx, "test"))
}
// TestRestartRecorderRaceWithLiveEnd 验证 RestartRecorder 和 LiveEnd(RemoveRecorder)
// 并发执行时不会产生僵尸录制器。
//
+392 -27
View File
@@ -265,6 +265,41 @@ var (
_ danmakuRecorder = (*danmaku.DouyuDanmakuRecorder)(nil)
)
// pipelineSharedState 封装可在分段重启时共享的 Pipeline 状态
// 分段重启时新旧 recorder 通过 redirect 链共享同一份状态,
// 确保旧任务的回调能正确递减新 recorder 的计数器
type pipelineSharedState struct {
mu sync.Mutex
pendingCount int // 尚未完成的 Pipeline 任务数
enqueued bool // 是否有 Pipeline 任务已入队
summarySent bool // 摘要是否已发送(幂等保护)
runExited bool // run() 是否已退出(defer 中置位,回调中检查)
details []notify.RecordingFileDetail // 收集的文件详情
sourceNames map[string]bool // 被 Pipeline 消费的原始文件名(用于合并时排除已删除的源文件)
onAllTasksDone func() // 所有任务完成时的回调(分段重启时设置)
suppressSummary bool // 为 true 时,run() 退出不推送摘要(分段重启场景),用 mu 保护
redirectedTo atomic.Pointer[pipelineSharedState] // 分段重启时重定向到新 recorder 的共享状态
}
// resolveState 沿 redirect 链找到当前活跃的 pipelineSharedState。
// 使用无锁遍历 + 锁内验证的方式,避免与 TransferPipelineState 产生死锁。
// 返回时调用者持有 result.mu 锁。
func resolveState(state *pipelineSharedState) *pipelineSharedState {
for {
// 无锁遍历找到链尾候选(通过 atomic.Load 避免与写入端的数据竞争)
final := state
for next := final.redirectedTo.Load(); next != nil; next = final.redirectedTo.Load() {
final = next
}
// 锁住候选状态,验证没有被并发重定向
final.mu.Lock()
if final.redirectedTo.Load() == nil {
return final // 调用者持有 final.mu 锁
}
final.mu.Unlock()
}
}
type recorder struct {
Live live.Live
ed events.Dispatcher
@@ -281,6 +316,25 @@ type recorder struct {
currentFileLock sync.RWMutex
currentFilePath string
// closeForRestart 标记当前 recorder 正在分段重启,run() 退出时不发送摘要。
// 在 CloseForRestart 中设置(早于 Close),run() 的 defer 中检查。
closeForRestart atomic.Bool
// pipelineState 封装 Pipeline 相关的共享状态
// 使用指针,分段重启时新旧 recorder 共享同一份状态
pipelineState *pipelineSharedState
// transferWg 用于分段重启时的摘要屏障。
// RestartRecorder 在创建新 recorder 后 Add(1),
// TransferPipelineState 完成后 Done()。
// sendAccumulatedSummary 在发送前 Wait(),确保旧分段状态已转入。
// 非分段重启场景为 nil,Wait 是 no-op。
transferWg *sync.WaitGroup
// notifyWg 跟踪异步通知 goroutine(go sendAccumulatedSummary)。
// 由 RecorderManager 持有,关闭流程中等待所有通知发送完成。
notifyWg *sync.WaitGroup
// 当前录制的流信息(来自平台 API)
currentStreamInfo *live.AvailableStreamInfo
@@ -300,8 +354,7 @@ type recorder struct {
// done 在 run() 退出时关闭,用于 CloseForRestart 等待 goroutine 完成
done chan struct{}
// suppressSummary 为 true 时,run() 退出不推送摘要(分段重启场景)
suppressSummary bool
// suppressSummary 已移入 pipelineSharedState,用 pipelineState.mu 保护
retryLogMu sync.Mutex
lastRetryLogKey string
@@ -313,14 +366,17 @@ func NewRecorder(ctx context.Context, live live.Live) (Recorder, error) {
inst := instance.GetInstance(ctx)
return &recorder{
Live: live,
cache: inst.Cache,
startTime: time.Now(),
ed: inst.EventDispatcher.(events.Dispatcher),
state: begin,
stop: make(chan struct{}),
done: make(chan struct{}),
parserLock: new(sync.RWMutex),
Live: live,
cache: inst.Cache,
startTime: time.Now(),
ed: inst.EventDispatcher.(events.Dispatcher),
state: begin,
stop: make(chan struct{}),
done: make(chan struct{}),
parserLock: new(sync.RWMutex),
pipelineState: &pipelineSharedState{
sourceNames: make(map[string]bool),
},
}, nil
}
@@ -381,7 +437,12 @@ func (r *recorder) tryRecord(ctx context.Context) {
buf := new(bytes.Buffer)
if err = tmpl.Execute(buf, info); err != nil {
r.getLogger().WithError(err).Error("failed to render filename, recording aborted")
// 文件名模板渲染失败通常是配置错误(如引用了不存在的字段),每次重试都会
// 重复失败。请求 manager 仅回收该 recorder,避免无意义的 5s 重试风暴与
// 日志刷屏,也不再被误计入活跃录制数;不要派发具有全局直播状态语义的
// LiveEnd,否则会错误地提前结束仍在进行的 live session。
r.getLogger().WithError(err).Error("failed to render filename, stopping recorder (check out_put_tmpl)")
r.ed.DispatchEvent(events.NewEvent(RecorderStopRequested, r.Live))
return
}
// 使用层级配置的 OutPutPath
@@ -783,16 +844,26 @@ func (r *recorder) tryRecord(ctx context.Context) {
return
}
// 先递增 pending 计数(在入队前,避免任务被异步调度完成后回调递减到 -1)
r.pipelineState.mu.Lock()
r.pipelineState.pendingCount++
r.pipelineState.mu.Unlock()
// 入队 Pipeline 任务
if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles); err != nil {
if err := pipelineManager.EnqueueRecordingTask(info, pipelineConfig, outputFiles, r.onPipelineTaskComplete); err != nil {
r.getLogger().WithError(err).Error("failed to enqueue pipeline task")
// 入队失败,回滚计数
r.pipelineState.mu.Lock()
r.pipelineState.pendingCount--
r.pipelineState.mu.Unlock()
} else {
r.pipelineState.enqueued = true
// 记录 Pipeline 阶段顺序
var stageNames []string
for _, stage := range pipelineConfig.Stages {
stageNames = append(stageNames, stage.Name)
}
r.getLogger().Infof("pipeline task enqueued: %d files, stages: %s", len(outputFiles), stageNames)
r.getLogger().Infof("pipeline task enqueued: %d files, stages: %v", len(outputFiles), stageNames)
}
}
}
@@ -909,7 +980,18 @@ func (r *recorder) selectPreferredStream(streamInfos []*live.StreamUrlInfo) (ret
func (r *recorder) run(ctx context.Context) {
defer close(r.done)
defer r.sendAccumulatedSummary()
defer func() {
// 标记 run() 已退出,使回调在 remaining==0 时直接发送摘要,
// 而不是等待 r.done 通道关闭(避免 defer 执行顺序导致的漏发窗口)
r.pipelineState.mu.Lock()
r.pipelineState.runExited = true
r.pipelineState.mu.Unlock()
// 分段重启时跳过摘要:CloseForRestart 已设置此标记,
// TransferPipelineState 完成后由回调发送含完整 Pipeline 详情的摘要
if !r.closeForRestart.Load() {
r.sendAccumulatedSummary()
}
}()
const minRetryInterval = 5 * time.Second
@@ -958,22 +1040,41 @@ func (r *recorder) accumulateRecordedFiles(files ...string) {
// sendAccumulatedSummary 录制结束后统一推送录制文件摘要通知
// 在 run() 退出时通过 defer 调用,确保所有分段文件汇总为一条通知
func (r *recorder) sendAccumulatedSummary() {
// 分段重启屏障:等待 TransferPipelineState 完成,
// 确保旧分段的 pipelineDetails/pendingCount 已转入再发送摘要
if r.transferWg != nil {
r.transferWg.Wait()
}
r.pipelineState.mu.Lock()
pendingCount := r.pipelineState.pendingCount
// 幂等保护:防止 defer 和回调并发双发
if r.pipelineState.summarySent {
r.pipelineState.mu.Unlock()
return
}
// Pipeline 任务尚未全部完成,等待回调触发最终摘要
if r.pipelineState.enqueued && pendingCount > 0 {
r.pipelineState.mu.Unlock()
r.getLogger().Infof("Pipeline 任务尚未全部完成(剩余 %d 个),延迟摘要推送", pendingCount)
return
}
// 标记摘要已发送(在实际发送前,防止并发双发)
r.pipelineState.summarySent = true
suppressSummary := r.pipelineState.suppressSummary
pipelineDetails := r.pipelineState.details
pipelineSourceNames := r.pipelineState.sourceNames
r.pipelineState.mu.Unlock()
r.recordedFilesMu.Lock()
defer r.recordedFilesMu.Unlock()
if r.suppressSummary {
if suppressSummary {
r.getLogger().Infof("录制摘要推送已抑制(分段重启),累积 %d 个文件将传递给新 recorder", len(r.recordedFiles))
return
}
if len(r.recordedFiles) == 0 {
if len(r.recordedFiles) == 0 && len(pipelineDetails) == 0 {
r.getLogger().Info("无录制文件,跳过摘要推送")
return
}
obj, err := r.cache.Get(r.Live)
if err != nil {
r.getLogger().WithError(err).Error("获取直播信息失败,无法推送录制摘要")
return
}
info := obj.(*live.Info)
// 获取录制输出路径,用于查询剩余磁盘空间
cfg := configs.GetCurrentConfig()
@@ -984,8 +1085,208 @@ func (r *recorder) sendAccumulatedSummary() {
outputPath = resolved.OutPutPath
}
r.getLogger().Infof("推送录制摘要:%d 个文件", len(r.recordedFiles))
notify.SendRecordingSummary(r.getLogger(), info.HostName, r.Live.GetPlatformCNName(), r.recordedFiles, outputPath)
// 使用 Pipeline 收集的最终文件详情(如有),否则用录制累积的原始文件
if len(pipelineDetails) > 0 {
// 合并未被 Pipeline 覆盖的录制文件(如某分段因 stages==0 未入队)
merged := append([]notify.RecordingFileDetail(nil), pipelineDetails...)
if len(r.recordedFiles) > 0 {
inPipeline := map[string]bool{}
for _, d := range pipelineDetails {
inPipeline[d.Name] = true
}
// 排除已被 Pipeline 转换并删除的源文件(如 convert_mp4 delete_source 场景)
for name := range pipelineSourceNames {
inPipeline[name] = true
}
for _, f := range r.recordedFiles {
if !inPipeline[f.Name] {
merged = append(merged, f)
}
}
// 清理无关联视频的 .ass 文件(DAA/executor 可能已删除视频但 .ass 从 recordedFiles 加入)
merged = filterOrphanedAssFromDetails(merged)
}
r.getLogger().Infof("推送 Pipeline 录制摘要:%d 个文件", len(merged))
hostName := ""
platform := r.Live.GetPlatformCNName()
if obj, cacheErr := r.cache.Get(r.Live); cacheErr == nil {
if info, ok := obj.(*live.Info); ok {
hostName = info.HostName
}
}
notify.SendPipelineRecordingSummary(
r.getLogger(),
hostName,
platform,
r.recordedFiles, // originalFiles(allUploaded 时显示用)
merged, // finalFiles(含未入队分段的文件)
outputPath,
)
} else {
// Pipeline 未产出有效文件详情,回退到录制累积
if len(r.recordedFiles) == 0 {
return
}
obj, err := r.cache.Get(r.Live)
if err != nil {
r.getLogger().WithError(err).Error("获取直播信息失败,无法推送录制摘要")
return
}
info := obj.(*live.Info)
r.getLogger().Infof("推送录制摘要:%d 个文件", len(r.recordedFiles))
notify.SendRecordingSummary(r.getLogger(), info.HostName, r.Live.GetPlatformCNName(), r.recordedFiles, outputPath)
}
}
// onPipelineTaskComplete Pipeline 任务完成回调
// 由 Pipeline Manager 在 executeTask 结束时调用
func (r *recorder) onPipelineTaskComplete(task *pipeline.PipelineTask) {
r.getLogger().Infof("Pipeline 回调触发:状态=%s, CurrentFiles=%d, LastStageFiles=%d, InitialFiles=%d",
task.Status, len(task.CurrentFiles), len(task.LastStageFiles), len(task.InitialFiles))
// 沿 redirect 链找到当前活跃的共享状态
// 支持连续重启 A→B→C 场景:A 的回调通过 resolveState 找到 C 的状态
ps := resolveState(r.pipelineState)
// ps.mu 已被 resolveState 锁住
ps.pendingCount--
// 收集任务的最终文件详情
switch task.Status {
case pipeline.PipelineStatusCompleted:
// 始终从 LastStageFiles 提取已上传的文件(含 Metadata["uploaded"] 标记)
uploadedDetails := extractUploadedDetails(task.LastStageFiles)
if len(uploadedDetails) > 0 {
ps.details = append(ps.details, uploadedDetails...)
}
// 合并本地保留的文件(CurrentFiles 中未被上传标记覆盖的文件)
uploadedNames := map[string]bool{}
for _, d := range uploadedDetails {
uploadedNames[d.Name] = true
}
for _, d := range pipeline.FileInfoToDetails(task.CurrentFiles) {
if !uploadedNames[d.Name] {
ps.details = append(ps.details, d)
}
}
// 回退:无上传标记且无保留文件时用 InitialFiles
if len(uploadedDetails) == 0 && len(task.CurrentFiles) == 0 {
for _, d := range pipeline.FileInfoToDetails(task.InitialFiles) {
d.Uploaded = true
ps.details = append(ps.details, d)
}
}
case pipeline.PipelineStatusFailed, pipeline.PipelineStatusCancelled:
// 失败/取消时优先用 CurrentFiles(最后成功阶段的输出),回退到 InitialFiles
if len(task.CurrentFiles) > 0 {
ps.details = append(ps.details, pipeline.FileInfoToDetails(task.CurrentFiles)...)
} else {
ps.details = append(ps.details, pipeline.FileInfoToDetails(task.InitialFiles)...)
}
}
// 收集被 Pipeline 转换的原始文件名(SourcePath),用于摘要合并时排除已删除的源文件
collectSourceNames(ps.sourceNames, task.CurrentFiles, task.LastStageFiles)
remaining := ps.pendingCount
runExited := ps.runExited
onAllDone := ps.onAllTasksDone
ps.mu.Unlock()
r.getLogger().Infof("Pipeline 任务完成(状态: %s),剩余 %d 个任务", task.Status, remaining)
// 所有任务完成且录制已结束,发送统一摘要
// 使用 goroutine 避免通知发送(Telegram/Email 等)阻塞 Pipeline 任务槽位
if remaining == 0 {
if onAllDone != nil {
// 分段重启场景:通过共享回调触发摘要(回调内部检查 runExited)
onAllDone()
} else if runExited {
// 非分段重启场景:run() 已退出,异步发送摘要
if r.notifyWg != nil {
r.notifyWg.Add(1)
}
go func() {
defer func() {
if r.notifyWg != nil {
r.notifyWg.Done()
}
}()
r.sendAccumulatedSummary()
}()
}
// run() 仍在运行且无回调:等待 run() 退出时由 defer 中的 sendAccumulatedSummary 发送
}
}
// extractUploadedDetails 从最后阶段输出文件中提取有上传标记的文件详情
// CloudUploadStage 会为成功上传的文件设置 Metadata["uploaded"]=true
// 排除封面文件(封面不在录制摘要中展示,与 FileInfoToDetails 保持一致)
func extractUploadedDetails(files []pipeline.FileInfo) []notify.RecordingFileDetail {
var details []notify.RecordingFileDetail
for _, f := range files {
if f.Type == pipeline.FileTypeCover {
continue
}
uploaded := false
if f.Metadata != nil {
if u, ok := f.Metadata["uploaded"].(bool); ok && u {
uploaded = true
}
}
if uploaded && f.Size > 0 {
details = append(details, notify.RecordingFileDetail{
Name: filepath.Base(f.Path),
Size: f.Size,
Uploaded: true,
})
}
}
logrus.WithFields(logrus.Fields{
"input_count": len(files),
"uploaded_count": len(details),
}).Debug("extractUploadedDetails")
return details
}
// collectSourceNames 从 Pipeline 输出文件中收集被转换的原始文件名
// 当 convert_mp4/burn_subtitles 等阶段转换文件时,输出文件的 SourcePath 指向原始文件
// 这些原始文件可能已被 Pipeline 删除,摘要合并时需要排除,避免将已删除的源文件重新纳入
func collectSourceNames(sourceNames map[string]bool, fileSets ...[]pipeline.FileInfo) {
for _, files := range fileSets {
for _, f := range files {
if f.SourcePath != "" {
sourceNames[filepath.Base(f.SourcePath)] = true
}
}
}
}
// filterOrphanedAssFromDetails 从文件详情列表中移除无关联视频的 .ass 文件
// 当 DAA/executor 删除了视频但 .ass 从 recordedFiles 加入 merged 时,
// 需要清理这些孤立的 .ass,避免通知误显为本地保留
func filterOrphanedAssFromDetails(details []notify.RecordingFileDetail) []notify.RecordingFileDetail {
videoExts := map[string]bool{".flv": true, ".mp4": true, ".mkv": true, ".ts": true}
videoBases := map[string]bool{}
for _, d := range details {
ext := strings.ToLower(filepath.Ext(d.Name))
if videoExts[ext] {
base := strings.TrimSuffix(d.Name, ext)
videoBases[base] = true
}
}
var result []notify.RecordingFileDetail
for _, d := range details {
ext := strings.ToLower(filepath.Ext(d.Name))
if ext == ".ass" {
base := strings.TrimSuffix(d.Name, ".ass")
if !videoBases[base] {
continue // 无关联视频,跳过
}
}
result = append(result, d)
}
return result
}
func (r *recorder) logStreamURLRetry(err error) {
@@ -1120,9 +1421,12 @@ func (r *recorder) Close() {
}
func (r *recorder) CloseForRestart() []notify.RecordingFileDetail {
r.recordedFilesMu.Lock()
r.suppressSummary = true
r.recordedFilesMu.Unlock()
// 先设置重启标记,使 run() 的 defer 跳过 sendAccumulatedSummary,
// 避免在 TransferPipelineState 之前发送不含 Pipeline 详情的摘要
r.closeForRestart.Store(true)
r.pipelineState.mu.Lock()
r.pipelineState.suppressSummary = true
r.pipelineState.mu.Unlock()
r.Close()
<-r.done // 等待 run() 完全退出,确保最后一个文件已累积
r.recordedFilesMu.Lock()
@@ -1143,6 +1447,67 @@ func (r *recorder) SetInitialRecordedFiles(files []notify.RecordingFileDetail) {
r.getLogger().Infof("继承上一个 recorder 的 %d 个录制文件", len(files))
}
// TransferPipelineState 从旧 recorder 转移 Pipeline 状态
// 分段重启时由 RestartRecorder 调用,确保旧分段的 Pipeline 结果不丢失
// 使用 redirect 链替代指针覆盖,支持连续重启 A→B→C 场景
func (r *recorder) TransferPipelineState(old *recorder) {
r.pipelineState.mu.Lock()
defer r.pipelineState.mu.Unlock()
// 沿 old 的 redirect 链找到最终活跃状态(可能经过多次重启 A→B→C→...)
oldState := resolveState(old.pipelineState)
// 此时 oldState.mu 已被 resolveState 锁住
// 合并 Pipeline 状态
r.pipelineState.enqueued = r.pipelineState.enqueued || oldState.enqueued
r.pipelineState.pendingCount += oldState.pendingCount
// 注意:suppressSummary 不合并。该标志表示"我是被重启的 recorder,不发摘要",
// 新 recorder 从未调用 CloseForRestart,应保持自己的 suppressSummary=false。
// 回调通过 onAllTasksDone(含 runExited 检查)触发,不依赖 suppressSummary 传播。
// 合并文件详情(旧 recorder 的详情追加到新 recorder)
r.pipelineState.details = append(r.pipelineState.details, oldState.details...)
// 合并已被 Pipeline 消费的原始文件名
for name := range oldState.sourceNames {
r.pipelineState.sourceNames[name] = true
}
// 设置回调:旧任务全部完成时,触发摘要发送
// 回调与 suppressSummary 解耦:suppressSummary 是 recorder 级别的抑制标志,
// 回调是 pipeline 级别的完成通知。多跳 A→B→C 场景下,A 的回调通过 redirect
// 链解析到 C 的状态,如果只在 suppressSummary=true 时设置回调,B→C 转移时
// B.suppressSummary=false 导致 C 没有回调,A 的摘要永远发不出。
// runExited 检查确保只在新 recorder 已退出时发送,避免录制中提前发出摘要。
r.pipelineState.onAllTasksDone = func() {
r.pipelineState.mu.Lock()
shouldSend := r.pipelineState.runExited
r.pipelineState.mu.Unlock()
if shouldSend {
if r.notifyWg != nil {
r.notifyWg.Add(1)
}
go func() {
defer func() {
if r.notifyWg != nil {
r.notifyWg.Done()
}
}()
r.sendAccumulatedSummary()
}()
}
// run() 未退出时不发送:新 recorder 的 defer 会在退出时检查并发送
}
// 关键:设置 redirect 链,使旧 recorder 及更早的 recorder 的回调
// 能通过 resolveState 找到当前活跃的共享状态
oldState.redirectedTo.Store(r.pipelineState)
oldState.mu.Unlock()
r.getLogger().Infof("继承 Pipeline 状态:pending=%d, details=%d 个文件",
r.pipelineState.pendingCount, len(r.pipelineState.details))
}
func (r *recorder) getLogger() *livelogger.LiveLogger {
return r.Live.GetLogger()
}
+340 -2
View File
@@ -11,9 +11,15 @@ import (
gomock "go.uber.org/mock/gomock"
"github.com/bililive-go/bililive-go/src/configs"
"github.com/bililive-go/bililive-go/src/instance"
"github.com/bililive-go/bililive-go/src/live"
livemock "github.com/bililive-go/bililive-go/src/live/mock"
"github.com/bililive-go/bililive-go/src/notify"
"github.com/bililive-go/bililive-go/src/pipeline"
"github.com/bililive-go/bililive-go/src/pkg/events"
eventsmock "github.com/bililive-go/bililive-go/src/pkg/events/mock"
"github.com/bililive-go/bililive-go/src/pkg/livelogger"
"github.com/bililive-go/bililive-go/src/types"
)
func TestTryRecordStopsWithoutPanicWhenFilenameRenderFails(t *testing.T) {
@@ -34,19 +40,351 @@ func TestTryRecordStopsWithoutPanicWhenFilenameRenderFails(t *testing.T) {
l.EXPECT().GetStreamInfos().Return([]*live.StreamUrlInfo{{Url: streamURL}}, nil)
l.EXPECT().GetLogger().Return(logger)
// 渲染失败时 recorder 应请求 manager 仅回收自身,而不是错误派发 LiveEnd。
ed := eventsmock.NewMockDispatcher(ctrl)
ed.EXPECT().DispatchEvent(events.NewEvent(RecorderStopRequested, l))
cache := gcache.New(1).LRU().Build()
if err := cache.Set(l, &live.Info{Live: l}); err != nil {
t.Fatalf("写入直播信息缓存失败: %v", err)
}
r := &recorder{Live: l, cache: cache}
r := &recorder{Live: l, cache: cache, ed: ed}
r.tryRecord(context.Background())
logs := logger.GetLogs()
if !strings.Contains(logs, "failed to render filename, recording aborted") {
if !strings.Contains(logs, "failed to render filename, stopping recorder") {
t.Fatalf("未记录文件名渲染失败日志: %s", logs)
}
if !strings.Contains(logs, "render failed") {
t.Fatalf("日志未保留原始模板错误: %s", logs)
}
}
// newTestRecorder 创建用于测试的 recorder 实例(不启动 run goroutine)
func newTestRecorder(t *testing.T, liveId types.LiveID) *recorder {
t.Helper()
ctrl := gomock.NewController(t)
l := livemock.NewMockLive(ctrl)
l.EXPECT().GetLiveId().Return(liveId).AnyTimes()
l.EXPECT().GetLogger().Return(livelogger.New(0, logrus.Fields{"test": t.Name()})).AnyTimes()
// 创建带 instance 的 context(NewRecorder 需要 instance.Cache 和 EventDispatcher)
inst := &instance.Instance{
Cache: gcache.New(1).LRU().Build(),
}
ctx := context.WithValue(context.Background(), instance.Key, inst)
// NewDispatcher 会自动设置 inst.EventDispatcher
events.NewDispatcher(ctx)
r, err := NewRecorder(ctx, l)
if err != nil {
t.Fatalf("创建 recorder 失败: %v", err)
}
return r.(*recorder)
}
func TestResolveState_NoRedirect(t *testing.T) {
state := &pipelineSharedState{
sourceNames: make(map[string]bool),
pendingCount: 5,
}
resolved := resolveState(state)
if resolved != state {
t.Fatal("无 redirect 时 resolveState 应返回原始状态")
}
if resolved.pendingCount != 5 {
t.Fatalf("pendingCount 期望 5,实际 %d", resolved.pendingCount)
}
resolved.mu.Unlock()
}
func TestResolveState_SingleRedirect(t *testing.T) {
stateA := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 3}
stateB := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 0}
// 模拟 A→B 重定向
stateA.redirectedTo.Store(stateB)
resolved := resolveState(stateA)
if resolved != stateB {
t.Fatal("单跳 redirect 应解析到 stateB")
}
resolved.mu.Unlock()
}
func TestResolveState_MultiHopChain(t *testing.T) {
stateA := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 3}
stateB := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 2}
stateC := &pipelineSharedState{sourceNames: make(map[string]bool), pendingCount: 1}
// 模拟 A→B→C 链
stateA.redirectedTo.Store(stateB)
stateB.redirectedTo.Store(stateC)
resolved := resolveState(stateA)
if resolved != stateC {
t.Fatal("多跳 redirect 链应解析到 stateC")
}
resolved.mu.Unlock()
}
func TestTransferPipelineState_ChainRedirect(t *testing.T) {
// 模拟 A→B→C 连续分段重启
recA := newTestRecorder(t, "test-live")
recB := newTestRecorder(t, "test-live")
recC := newTestRecorder(t, "test-live")
// A 有 3 个待处理任务和一些文件详情
recA.pipelineState.pendingCount = 3
recA.pipelineState.enqueued = true
recA.pipelineState.details = []notify.RecordingFileDetail{
{Name: "a.flv", Size: 100},
}
recA.pipelineState.sourceNames["a_src.flv"] = true
// A 模拟已抑制(CloseForRestart 设置)
recA.pipelineState.suppressSummary = true
// A → B 重启
recB.TransferPipelineState(recA)
// 验证 B 继承了 A 的状态
if recB.pipelineState.pendingCount != 3 {
t.Fatalf("B pendingCount 期望 3,实际 %d", recB.pipelineState.pendingCount)
}
if len(recB.pipelineState.details) != 1 {
t.Fatalf("B details 期望 1,实际 %d", len(recB.pipelineState.details))
}
// B 有额外的 2 个任务
recB.pipelineState.pendingCount += 2 // 模拟 B 自己又入队了 2 个任务
// B → C 重启
recC.TransferPipelineState(recB)
// 验证 C 继承了所有状态
if recC.pipelineState.pendingCount != 5 {
t.Fatalf("C pendingCount 期望 5,实际 %d", recC.pipelineState.pendingCount)
}
// 关键验证:A 的 pipelineState 通过 redirect 链应解析到 C 的状态
resolvedA := resolveState(recA.pipelineState)
if resolvedA != recC.pipelineState {
t.Fatal("A 的状态应通过 redirect 链解析到 C 的状态")
}
resolvedA.mu.Unlock()
// B 的 pipelineState 应直接指向 C
resolvedB := resolveState(recB.pipelineState)
if resolvedB != recC.pipelineState {
t.Fatal("B 的状态应解析到 C 的状态")
}
resolvedB.mu.Unlock()
}
func TestOnPipelineTaskComplete_AfterChainRedirect(t *testing.T) {
// 模拟 A→B→C 后,A 的回调操作 C 的状态
recA := newTestRecorder(t, "test-live")
recB := newTestRecorder(t, "test-live")
recC := newTestRecorder(t, "test-live")
// A 有 1 个待处理任务
recA.pipelineState.pendingCount = 1
recA.pipelineState.enqueued = true
recA.pipelineState.suppressSummary = true
recA.pipelineState.runExited = true
// A → B → C
recB.TransferPipelineState(recA)
recC.TransferPipelineState(recB)
// 模拟 C 的 run() 已退出(C 是当前活跃 recorder,runExited 是 per-recorder 的)
recC.pipelineState.mu.Lock()
recC.pipelineState.runExited = true
recC.pipelineState.mu.Unlock()
// C 的 onAllTasksDone 回调应存在(TransferPipelineState 始终设置)
onAllDoneCalled := false
recC.pipelineState.onAllTasksDone = func() {
recC.pipelineState.mu.Lock()
shouldSend := recC.pipelineState.runExited
recC.pipelineState.mu.Unlock()
if shouldSend {
onAllDoneCalled = true
}
}
// 模拟 A 的 Pipeline 任务完成
task := &pipeline.PipelineTask{
Status: pipeline.PipelineStatusCompleted,
CurrentFiles: []pipeline.FileInfo{},
InitialFiles: []pipeline.FileInfo{
{Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo},
},
LastStageFiles: []pipeline.FileInfo{
{Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo, Metadata: map[string]any{"uploaded": true}},
},
}
recA.onPipelineTaskComplete(task)
// 验证:C 的 pendingCount 应为 0(A 的回调正确递减了 C 的计数)
if recC.pipelineState.pendingCount != 0 {
t.Fatalf("C pendingCount 期望 0,实际 %d(A 的回调未正确操作 C 的状态)", recC.pipelineState.pendingCount)
}
// 验证:A 的回调收集的 details 应在 C 的状态中
if len(recC.pipelineState.details) == 0 {
t.Fatal("C 的 details 不应为空(A 的回调应将文件详情写入 C 的状态)")
}
// 验证:onAllTasksDone 应被触发(C 已退出)
if !onAllDoneCalled {
t.Fatal("onAllTasksDone 回调应被触发(C.runExited=true)")
}
}
func TestExtractUploadedDetails_ExcludesCover(t *testing.T) {
files := []pipeline.FileInfo{
{Path: "/data/video.flv", Size: 1024, Type: pipeline.FileTypeVideo, Metadata: map[string]any{"uploaded": true}},
{Path: "/data/cover.jpg", Size: 512, Type: pipeline.FileTypeCover, Metadata: map[string]any{"uploaded": true}},
{Path: "/data/other.txt", Size: 256, Type: pipeline.FileTypeOther, Metadata: map[string]any{"uploaded": true}},
}
details := extractUploadedDetails(files)
if len(details) != 2 {
t.Fatalf("期望 2 个文件(排除封面),实际 %d", len(details))
}
names := map[string]bool{}
for _, d := range details {
names[d.Name] = true
}
if names["cover.jpg"] {
t.Fatal("封面文件不应出现在上传详情中")
}
if !names["video.flv"] || !names["other.txt"] {
t.Fatal("视频和其他类型文件应出现在上传详情中")
}
}
func TestSuppressSummary_InSharedState(t *testing.T) {
// 验证 suppressSummary 在 pipelineSharedState 中,通过 mu 保护
rec := newTestRecorder(t, "test-live")
// 初始值应为 false
rec.pipelineState.mu.Lock()
if rec.pipelineState.suppressSummary {
t.Fatal("suppressSummary 初始值应为 false")
}
rec.pipelineState.mu.Unlock()
// CloseForRestart 设置 suppressSummary
rec.pipelineState.mu.Lock()
rec.pipelineState.suppressSummary = true
rec.pipelineState.mu.Unlock()
// 验证设置成功
rec.pipelineState.mu.Lock()
if !rec.pipelineState.suppressSummary {
t.Fatal("suppressSummary 应为 true")
}
rec.pipelineState.mu.Unlock()
}
func TestTransferPipelineState_SetRedirect(t *testing.T) {
// 验证 TransferPipelineState 设置了 redirect 链
recA := newTestRecorder(t, "test-live")
recB := newTestRecorder(t, "test-live")
recA.pipelineState.pendingCount = 1
recA.pipelineState.suppressSummary = true
recB.TransferPipelineState(recA)
// A 的状态应被重定向到 B
if recA.pipelineState.redirectedTo.Load() != recB.pipelineState {
t.Fatal("A 的 redirect 应指向 B 的状态")
}
// resolveState 应从 A 找到 B
resolved := resolveState(recA.pipelineState)
if resolved != recB.pipelineState {
t.Fatal("resolveState(A) 应返回 B 的状态")
}
resolved.mu.Unlock()
}
func TestOnAllTasksDone_NotSentWhileRecording(t *testing.T) {
// 验证 Bot 指出的场景:旧任务完成时新 recorder 仍在录制,摘要不应提前发送
recA := newTestRecorder(t, "test-live")
recB := newTestRecorder(t, "test-live")
recA.pipelineState.pendingCount = 1
recA.pipelineState.enqueued = true
recA.pipelineState.suppressSummary = true
recA.pipelineState.runExited = true
// A → B
recB.TransferPipelineState(recA)
// B 仍在录制(runExited=false)
// 记录 B 的 onAllTasksDone 是否被调用
callbackTriggered := false
originalCallback := recB.pipelineState.onAllTasksDone
recB.pipelineState.onAllTasksDone = func() {
callbackTriggered = true
originalCallback()
}
// 模拟 A 的任务完成
task := &pipeline.PipelineTask{
Status: pipeline.PipelineStatusCompleted,
CurrentFiles: []pipeline.FileInfo{},
InitialFiles: []pipeline.FileInfo{
{Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo},
},
LastStageFiles: []pipeline.FileInfo{
{Path: "/data/a.flv", Size: 1024, Type: pipeline.FileTypeVideo},
},
}
recA.onPipelineTaskComplete(task)
// 回调应被触发(remaining=0, suppressSummary=true)
if !callbackTriggered {
t.Fatal("onAllTasksDone 应被触发")
}
// 但摘要不应被发送(B 仍在录制,runExited=false)
// 验证方式:summarySent 应仍为 false
recB.pipelineState.mu.Lock()
summarySent := recB.pipelineState.summarySent
recB.pipelineState.mu.Unlock()
if summarySent {
t.Fatal("摘要不应在新 recorder 仍在录制时发送(summarySent 应为 false)")
}
}
func TestFinalRecorder_SummaryNotSuppressed(t *testing.T) {
// 验证最终 recorder 的摘要不被 suppressSummary 抑制
// (suppressSummary 不应从旧 recorder 传播到新 recorder)
recA := newTestRecorder(t, "test-live")
recB := newTestRecorder(t, "test-live")
recA.pipelineState.pendingCount = 1
recA.pipelineState.enqueued = true
recA.pipelineState.suppressSummary = true
// A → B
recB.TransferPipelineState(recA)
// B 的 suppressSummary 应为 false(不从 A 继承)
recB.pipelineState.mu.Lock()
bSuppress := recB.pipelineState.suppressSummary
recB.pipelineState.mu.Unlock()
if bSuppress {
t.Fatal("最终 recorder 的 suppressSummary 应为 false(不应从旧 recorder 继承)")
}
}
+123 -12
View File
@@ -35,6 +35,7 @@ import (
applog "github.com/bililive-go/bililive-go/src/log"
"github.com/bililive-go/bililive-go/src/pkg/livelogger"
"github.com/bililive-go/bililive-go/src/pkg/memstats"
"github.com/bililive-go/bililive-go/src/pkg/metadata"
"github.com/bililive-go/bililive-go/src/pkg/ratelimit"
"github.com/bililive-go/bililive-go/src/pkg/utils"
"github.com/bililive-go/bililive-go/src/recorders"
@@ -184,7 +185,13 @@ func getLive(writer http.ResponseWriter, r *http.Request) {
}
// 获取平台等待状态信息
waitInfo := ratelimit.GetGlobalRateLimiter().GetPlatformWaitInfo(platformKey)
platformRateLimiter := ratelimit.GetGlobalRateLimiter()
platformRateLimitEnabled := platformRateLimiter.Enabled()
waitInfo := platformRateLimiter.GetPlatformWaitInfo(platformKey)
platformRateLimit := 0
if platformRateLimitEnabled {
platformRateLimit = cfg.GetPlatformMinAccessInterval(platformKey)
}
// 获取调度器状态信息
var schedulerStatus *live.SchedulerStatus
@@ -249,10 +256,12 @@ func getLive(writer http.ResponseWriter, r *http.Request) {
"audio_only": room.AudioOnly,
// 平台访问限制
"platform_rate_limit": cfg.GetPlatformMinAccessInterval(platformKey),
"platform_rate_limit": platformRateLimit,
"platform_rate_limit_enabled": platformRateLimitEnabled,
// 平台等待状态
"rate_limit_info": map[string]interface{}{
"enabled": platformRateLimitEnabled,
"waited_seconds": waitInfo.WaitedSeconds,
"next_request_in_sec": waitInfo.NextRequestInSec,
"min_interval_sec": waitInfo.MinIntervalSec,
@@ -468,7 +477,8 @@ func parseLiveAction(writer http.ResponseWriter, r *http.Request) {
case "forceRefresh":
// 强制刷新:忽略平台访问频率限制,立即获取最新信息
platformKey := configs.GetPlatformKeyFromUrl(live.GetRawUrl())
ratelimit.GetGlobalRateLimiter().ForceAccess(platformKey)
platformRateLimiter := ratelimit.GetGlobalRateLimiter()
platformRateLimiter.ForceAccess(platformKey)
// 手动调用 GetInfo 获取最新信息
info, err := live.GetInfo()
@@ -480,8 +490,9 @@ func parseLiveAction(writer http.ResponseWriter, r *http.Request) {
}
// 广播频率限制更新事件,通知前端更新倒计时
waitInfo := ratelimit.GetGlobalRateLimiter().GetPlatformWaitInfo(platformKey)
waitInfo := platformRateLimiter.GetPlatformWaitInfo(platformKey)
GetSSEHub().BroadcastRateLimitUpdate(live.GetLiveId(), map[string]interface{}{
"enabled": platformRateLimiter.Enabled(),
"waited_seconds": waitInfo.WaitedSeconds,
"next_request_in_sec": waitInfo.NextRequestInSec,
"min_interval_sec": waitInfo.MinIntervalSec,
@@ -941,8 +952,9 @@ func getConfig(writer http.ResponseWriter, r *http.Request) {
}
func putConfig(writer http.ResponseWriter, r *http.Request) {
// 直接序列化当前快照即可;索引缓存不参与序列化,且共享快照不可变,
// 不能在此调用 RefreshLiveRoomIndexCache 写它的 map。
config := configs.GetCurrentConfig()
config.RefreshLiveRoomIndexCache()
if err := config.Marshal(); err != nil {
writeJsonWithStatusCode(writer, http.StatusBadRequest, commonResp{
ErrNo: http.StatusBadRequest,
@@ -991,7 +1003,8 @@ func putRawConfig(writer http.ResponseWriter, r *http.Request) {
return
}
oldConfig := configs.GetCurrentConfig()
oldConfig.RefreshLiveRoomIndexCache()
// 共享快照不可变:其索引缓存在生成时已由 Update 维护好,
// 这里只做只读查找,不能再调用 RefreshLiveRoomIndexCache 写它的 map。
// 继承原配置的文件路径
newConfig.File = oldConfig.File
// 预先将旧配置中的 LiveId 迁移到新配置(相同 URL)
@@ -1186,6 +1199,7 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) {
})
return
}
platformRateLimitEnabled := ratelimit.GetGlobalRateLimiter().Enabled()
// 统计每个平台的直播间(只统计正在监控的)
platformRooms := make(map[string][]map[string]interface{})
@@ -1268,7 +1282,7 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) {
// 检查是否低于最小访问间隔
warningMessage := ""
if listeningCount > 0 && platformConfig.MinAccessIntervalSec > 0 && actualAccessInterval < float64(platformConfig.MinAccessIntervalSec) {
if platformRateLimitEnabled && listeningCount > 0 && platformConfig.MinAccessIntervalSec > 0 && actualAccessInterval < float64(platformConfig.MinAccessIntervalSec) {
effectiveInterval := float64(platformConfig.MinAccessIntervalSec) * float64(listeningCount)
warningMessage = fmt.Sprintf("当前设置下实际每个直播间的检测间隔约为 %.1f 秒(受最小访问间隔限制)", effectiveInterval)
}
@@ -1348,9 +1362,10 @@ func getPlatformStats(writer http.ResponseWriter, r *http.Request) {
}
response := map[string]interface{}{
"platforms": stats,
"available_platforms": availablePlatforms,
"global_interval": cfg.Interval,
"platforms": stats,
"available_platforms": availablePlatforms,
"global_interval": cfg.Interval,
"platform_rate_limit_enabled": platformRateLimitEnabled,
}
writeJSON(writer, response)
@@ -1726,6 +1741,9 @@ func applyConfigUpdates(c *configs.Config, updates map[string]interface{}) error
if deleteAllAfter, ok := cloudUpload["delete_all_after_upload"].(bool); ok {
c.OnRecordFinished.CloudUpload.DeleteAllAfterUpload = deleteAllAfter
}
if uploadSubtitles, ok := cloudUpload["upload_subtitles"].(bool); ok {
c.OnRecordFinished.CloudUpload.UploadSubtitles = uploadSubtitles
}
}
// 处理上传时机
if uploadTiming, ok := orf["upload_timing"].(string); ok {
@@ -2419,6 +2437,7 @@ func getFileInfo(writer http.ResponseWriter, r *http.Request) {
LastModified int64 `json:"last_modified"`
Size int64 `json:"size"`
SubtitleFile string `json:"subtitle_file,omitempty"`
Uploaded bool `json:"uploaded,omitempty"`
}
// First pass: separate ASS files and build base-name -> ASS file map
@@ -2461,6 +2480,14 @@ func getFileInfo(writer http.ResponseWriter, r *http.Request) {
if assName, ok := assFiles[baseName]; ok {
jf.SubtitleFile = assName
}
// Check if this file has been uploaded to cloud
relPath := fe.dir.Name()
if path != "" {
relPath = path + "/" + fe.dir.Name() // key 统一用正斜杠
}
if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, relPath); err == nil && val != "" {
jf.Uploaded = true
}
}
jsonFiles = append(jsonFiles, jf)
}
@@ -2580,6 +2607,30 @@ func renameFile(writer http.ResponseWriter, r *http.Request) {
}
}
// 迁移上传标记(key 统一用正斜杠)
oldRel := path
newRelPath, _ := filepath.Rel(base, newAbsPath)
newRel := filepath.ToSlash(newRelPath)
if info.IsDir() {
// 目录重命名:迁移该目录下所有文件的上传标记
allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded)
oldPrefix := oldRel + "/"
for key, val := range allMarks {
if strings.HasPrefix(key, oldPrefix) {
suffix := key[len(oldPrefix):]
newKey := newRel + "/" + suffix
metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newKey, val)
metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key)
}
}
} else {
// 文件重命名:迁移单个上传标记
if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, oldRel); err == nil {
metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newRel, val)
metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, oldRel)
}
}
writeJSON(writer, commonResp{Data: "OK"})
}
@@ -2599,8 +2650,14 @@ func deleteFile(writer http.ResponseWriter, r *http.Request) {
return
}
// 记录是否为目录(删除前检查)
isDir := false
if info, err := os.Stat(absPath); err == nil {
isDir = info.IsDir()
}
// 删除关联的 ASS 弹幕文件
if info, err := os.Stat(absPath); err == nil && !info.IsDir() {
if !isDir {
assPath := strings.TrimSuffix(absPath, filepath.Ext(absPath)) + ".ass"
if _, err := os.Stat(assPath); err == nil {
os.Remove(assPath)
@@ -2612,6 +2669,20 @@ func deleteFile(writer http.ResponseWriter, r *http.Request) {
return
}
// 清除上传标记
if isDir {
// 目录:清除该目录下所有文件的上传标记
allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded)
prefix := path + "/" // key 统一用正斜杠
for key := range allMarks {
if strings.HasPrefix(key, prefix) {
metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key)
}
}
} else {
metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, path)
}
writeJSON(writer, commonResp{Data: "OK"})
}
@@ -2698,6 +2769,28 @@ func batchRenameFiles(writer http.ResponseWriter, r *http.Request) {
os.Rename(oldAss, newAss)
}
}
// 迁移上传标记(key 统一用正斜杠)
oldRel := path
newRelPath, _ := filepath.Rel(base, newAbsPath)
newRel := filepath.ToSlash(newRelPath)
if info.IsDir() {
allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded)
oldPrefix := oldRel + "/"
for key, val := range allMarks {
if strings.HasPrefix(key, oldPrefix) {
suffix := key[len(oldPrefix):]
newKey := newRel + "/" + suffix
metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newKey, val)
metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key)
}
}
} else {
if val, err := metadata.GetStore().Get(r.Context(), metadata.NamespaceUploaded, oldRel); err == nil {
metadata.GetStore().Set(r.Context(), metadata.NamespaceUploaded, newRel, val)
metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, oldRel)
}
}
}
}
@@ -2734,8 +2827,14 @@ func batchDeleteFiles(writer http.ResponseWriter, r *http.Request) {
continue
}
// 记录是否为目录(删除前检查)
isDir := false
if info, err := os.Stat(absPath); err == nil {
isDir = info.IsDir()
}
// 删除关联的 ASS 弹幕文件
if info, err := os.Stat(absPath); err == nil && !info.IsDir() {
if !isDir {
assPath := strings.TrimSuffix(absPath, filepath.Ext(absPath)) + ".ass"
if _, err := os.Stat(assPath); err == nil {
os.Remove(assPath)
@@ -2745,6 +2844,18 @@ func batchDeleteFiles(writer http.ResponseWriter, r *http.Request) {
if err := os.RemoveAll(absPath); err != nil {
results = append(results, Result{Path: path, Success: false, Message: translateOSError(err)})
} else {
// 清除上传标记
if isDir {
allMarks, _ := metadata.GetStore().GetAll(r.Context(), metadata.NamespaceUploaded)
prefix := path + "/" // key 统一用正斜杠
for key := range allMarks {
if strings.HasPrefix(key, prefix) {
metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, key)
}
}
} else {
metadata.GetStore().Delete(r.Context(), metadata.NamespaceUploaded, path)
}
results = append(results, Result{Path: path, Success: true, Message: "成功"})
}
}
+38
View File
@@ -1,6 +1,7 @@
package servers
import (
"context"
"encoding/json"
"net/http/httptest"
"testing"
@@ -8,6 +9,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/bililive-go/bililive-go/src/configs"
"github.com/bililive-go/bililive-go/src/instance"
)
func TestGetSoopLiveAuthConfigDoesNotExposeSavedPassword(t *testing.T) {
@@ -32,3 +34,39 @@ func TestGetSoopLiveAuthConfigDoesNotExposeSavedPassword(t *testing.T) {
_, exists := data["password"]
assert.False(t, exists)
}
func TestGetPlatformStatsReportsDisabledRateLimit(t *testing.T) {
cfg := configs.NewConfig()
cfg.Interval = 1
cfg.PlatformConfigs[configs.PlatformKeyDouyin] = configs.PlatformConfig{
Name: "抖音",
MinAccessIntervalSec: 60,
}
cfg.LiveRooms = []configs.LiveRoom{{
Url: "https://live.douyin.com/123456",
IsListening: true,
}}
cfg.RefreshLiveRoomIndexCache()
configs.SetCurrentConfig(cfg)
t.Cleanup(func() { configs.SetCurrentConfig(configs.NewConfig()) })
recorder := httptest.NewRecorder()
request := httptest.NewRequest("GET", "/api/config/platforms", nil)
request = request.WithContext(context.WithValue(request.Context(), instance.Key, &instance.Instance{}))
getPlatformStats(recorder, request)
assert.Equal(t, 200, recorder.Code)
var response struct {
PlatformRateLimitEnabled bool `json:"platform_rate_limit_enabled"`
Platforms []struct {
PlatformKey string `json:"platform_key"`
WarningMessage string `json:"warning_message"`
} `json:"platforms"`
}
assert.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &response))
assert.False(t, response.PlatformRateLimitEnabled)
if assert.Len(t, response.Platforms, 1) {
assert.Equal(t, configs.PlatformKeyDouyin, response.Platforms[0].PlatformKey)
assert.Empty(t, response.Platforms[0].WarningMessage)
}
}
+27 -4
View File
@@ -2,6 +2,7 @@ package servers
import (
"encoding/json"
"io"
"net/http"
"os"
"path/filepath"
@@ -165,6 +166,8 @@ func makePipelineCancelTaskHandler(pm *pipeline.Manager) http.HandlerFunc {
}
// makePipelineRetryTaskHandler 重试任务
// 支持可选的 JSON body: {"stage_name": "cloud_upload"}
// 无 body 或 stage_name 为空时全量重试
func makePipelineRetryTaskHandler(pm *pipeline.Manager) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
@@ -174,9 +177,29 @@ func makePipelineRetryTaskHandler(pm *pipeline.Manager) http.HandlerFunc {
return
}
if err := pm.RetryTask(id); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
// 解析可选的 JSON body
var reqBody struct {
StageName string `json:"stage_name"`
}
if r.Body != nil {
if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil && err != io.EOF {
http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest)
return
}
}
if reqBody.StageName != "" {
// 从指定阶段重试
if err := pm.RetryTaskFromStage(id, reqBody.StageName); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
} else {
// 全量重试(向后兼容)
if err := pm.RetryTask(id); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
w.WriteHeader(http.StatusOK)
@@ -324,7 +347,7 @@ func makeBatchBurnHandler(pm *pipeline.Manager) http.HandlerFunc {
}
task := pipeline.NewPipelineTask(recordInfo, burnConfig, files)
if err := pm.EnqueueTask(task); err != nil {
if err := pm.EnqueueTask(task, nil); err != nil {
result.Skipped = append(result.Skipped, filepath.Base(p)+" - 入队失败: "+err.Error())
continue
}
+2 -1
View File
@@ -153,10 +153,11 @@ func initMux(ctx context.Context) *mux.Router {
apiRoute.HandleFunc("/openlist/status", getOpenListStatus).Methods("GET")
apiRoute.HandleFunc("/openlist/check-storage", checkOpenListStorageHealth).Methods("GET")
// Pipeline 任务路由
// Pipeline 任务路由 + 手动上传路由
inst := instance.GetInstance(ctx)
if pm := pipeline.GetManager(inst); pm != nil {
RegisterPipelineHandlers(apiRoute, pm)
RegisterUploadHandlers(apiRoute, pm)
}
// OSRP 开放直播录制协议路由
+125
View File
@@ -0,0 +1,125 @@
package servers
import (
"encoding/json"
"net/http"
"path/filepath"
"github.com/gorilla/mux"
"github.com/bililive-go/bililive-go/src/configs"
"github.com/bililive-go/bililive-go/src/pipeline"
)
// uploadResult 手动上传的响应结构
type uploadResult struct {
Enqueued int `json:"enqueued"`
Skipped []string `json:"skipped"`
TaskIDs []int64 `json:"task_ids"`
}
// RegisterUploadHandlers 注册手动上传相关的 HTTP 处理器
// 通过 Pipeline 系统执行上传,与自动上传行为完全一致
func RegisterUploadHandlers(r *mux.Router, pm *pipeline.Manager) {
if pm == nil {
return
}
r.HandleFunc("/file/upload", makeUploadFileHandler(pm)).Methods("POST")
r.HandleFunc("/batch/file/upload", makeBatchUploadFilesHandler(pm)).Methods("POST")
}
// makeUploadFileHandler 单文件上传到云盘(通过 Pipeline)
func makeUploadFileHandler(pm *pipeline.Manager) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req struct {
Path string `json:"path"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeJSON(w, commonResp{ErrMsg: "请求参数错误"})
return
}
if req.Path == "" {
writeJSON(w, commonResp{ErrMsg: "文件路径不能为空"})
return
}
// 路径校验(复用 handler.go 中的 getSafePath)
cfg := configs.GetCurrentConfig()
if cfg == nil {
writeJSON(w, commonResp{ErrMsg: "配置未初始化"})
return
}
absPath, err := getSafePath(cfg.OutPutPath, req.Path)
if err != nil {
writeJSON(w, commonResp{ErrMsg: "无效路径: " + err.Error()})
return
}
enqueued, skipped, taskIDs, err := pm.EnqueueUploadTask([]string{absPath})
if err != nil {
writeJSON(w, commonResp{ErrMsg: err.Error()})
return
}
writeJSON(w, commonResp{Data: uploadResult{
Enqueued: enqueued,
Skipped: skipped,
TaskIDs: taskIDs,
}})
}
}
// makeBatchUploadFilesHandler 批量上传文件到云盘(通过 Pipeline)
func makeBatchUploadFilesHandler(pm *pipeline.Manager) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
var req struct {
Paths []string `json:"paths"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeJSON(w, commonResp{ErrMsg: "请求参数错误"})
return
}
// 批量路径校验
cfg := configs.GetCurrentConfig()
if cfg == nil {
writeJSON(w, commonResp{ErrMsg: "配置未初始化"})
return
}
absPaths := make([]string, 0, len(req.Paths))
skipped := []string{}
for _, p := range req.Paths {
absPath, err := getSafePath(cfg.OutPutPath, p)
if err != nil {
skipped = append(skipped, filepath.Base(p)+" - 路径越权")
continue
}
absPaths = append(absPaths, absPath)
}
// 如果所有路径都校验失败,直接返回
if len(absPaths) == 0 {
writeJSON(w, commonResp{Data: uploadResult{
Enqueued: 0,
Skipped: skipped,
TaskIDs: []int64{},
}})
return
}
enqueued, moreSkipped, taskIDs, err := pm.EnqueueUploadTask(absPaths)
if err != nil {
writeJSON(w, commonResp{ErrMsg: err.Error()})
return
}
skipped = append(skipped, moreSkipped...)
writeJSON(w, commonResp{Data: uploadResult{
Enqueued: enqueued,
Skipped: skipped,
TaskIDs: taskIDs,
}})
}
}
@@ -44,8 +44,8 @@ const UPLOAD_PRESETS: PresetTemplate[] = [
},
{
name: '简洁归档',
description: '按平台/主播归档,文件名带日期',
template: '/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }}',
description: '按平台/主播归档,文件名带日期时间',
template: '/录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02 15-04-05" }}.{{ .Ext }}',
},
{
name: '按房间归档',
@@ -95,7 +95,7 @@ const MOCK_DATA: MockStream[] = [
const renderUploadTemplate = (template: string, stream: MockStream, room: MockStream['rooms'][0]): string => {
const fileName = `[${room.date} ${room.time}][${stream.hostName}][${room.roomName}].flv`;
let path = template;
path = path.replace(/\{\{ ?\.Platform ?\}\}/g, stream.platform);
path = path.replace(/\{\{ ?\.Platform ?\}\}/g, stream.platformCN);
path = path.replace(/\{\{ ?\.HostName ?\}\}/g, stream.hostName);
path = path.replace(/\{\{ ?\.RoomName ?\}\}/g, room.roomName);
path = path.replace(/\{\{ ?\.FileName ?\}\}/g, fileName);
@@ -123,28 +123,40 @@ const analyzeConfig = (config: any) => {
const upload = cu.enable ?? false;
const delAfter = cu.delete_after_upload ?? false;
const delAllAfter = cu.delete_all_after_upload ?? false;
const uploadAss = cu.upload_subtitles ?? false;
// 文件状态:uploaded(上传), deleted(删除), kept(保留)
// 文件状态:uploaded(上传保留), uploaded_deleted(上传后删除), intermediate(中间产物,不上传),
// deleted(删除), kept(本地保留)
const files: Record<string, { ext: string; desc: string; status: string }> = {};
// 初始文件
files['video.flv'] = { ext: '.flv', desc: '原始录制视频', status: 'kept' };
files['video.ass'] = { ext: '.ass', desc: '弹幕字幕', status: 'kept' };
// immediate 模式:先上传原始文件
// immediate 模式:先上传原始文件(.ass 在录制结束时已存在,可一并上传)
if (isImmediate && upload) {
files['video.flv'].status = 'uploaded';
if (uploadAss) {
files['video.ass'].status = 'uploaded';
}
}
// 转码
if (convert) {
files['video.mp4'] = { ext: '.mp4', desc: '转码后视频', status: 'kept' };
if (deleteFlv) {
// 如果已经被标记为 uploaded,改为 uploaded_deleted
if (files['video.flv'].status === 'uploaded') {
if (isImmediate && upload) {
// immediate 模式:.flv 已在 pipeline 开头被上传,转换后被标记为 Deletable 并删除
files['video.flv'].status = 'uploaded_deleted';
} else if (files['video.flv'].status === 'uploaded') {
files['video.flv'].status = 'uploaded_deleted';
} else {
files['video.flv'].status = 'deleted';
// after_process 模式:源文件是中间产物,不参与上传
files['video.flv'].status = 'intermediate';
}
} else {
if (files['video.flv'].status === 'kept') {
files['video.flv'].status = 'uploaded';
}
}
}
@@ -153,19 +165,37 @@ const analyzeConfig = (config: any) => {
if (burn) {
files['video.mkv'] = { ext: '.mkv', desc: '烧录后视频', status: 'kept' };
if (burnDelSource) {
// 删除转码后的 mp4 或原始 flv
const target = convert ? 'video.mp4' : 'video.flv';
if (files[target].status === 'uploaded') {
if (isImmediate && upload && !convert) {
// immediate 模式(无转码):.flv 已在 pipeline 开头被上传,烧录后被标记为 Deletable 并删除
files[target].status = 'uploaded_deleted';
} else if (files[target].status === 'uploaded') {
files[target].status = 'uploaded_deleted';
} else {
files[target].status = 'deleted';
// after_process 模式或 immediate+convert(.mp4 未被上传):中间产物
files[target].status = 'intermediate';
}
} else {
const src = convert ? 'video.mp4' : 'video.flv';
// after_process 模式:源视频保留在 BurnSubtitlesStage output 中,会被 cloud_upload 上传
// immediate 模式:源视频在 cloud_upload 之后才创建/保留,不会被上传
if (!isImmediate && files[src] && files[src].status === 'kept') {
files[src].status = 'uploaded';
}
}
if (burnDelAss) {
files['video.ass'].status = 'deleted';
if (isImmediate && upload) {
// immediate 模式:.ass 已在 pipeline 开头被上传,烧录后被标记为 Deletable 并删除
files['video.ass'].status = 'uploaded_deleted';
} else {
files['video.ass'].status = 'deleted';
}
}
}
// immediate 模式下,convert 和 burn 产出的文件(.mp4, .mkv)不会被上传
// 它们在 cloud_upload 之后才创建,应保持 'kept' 状态
// 封面
if (cover) {
files['cover.jpg'] = { ext: '.jpg', desc: '视频封面', status: 'kept' };
@@ -173,9 +203,10 @@ const analyzeConfig = (config: any) => {
// after_process 模式:上传最终文件
if (isAfterProcess && upload) {
// 上传第一个视频和封面
// 上传最终产物
if (burn) {
files['video.mkv'].status = 'uploaded';
// 源视频已在烧录阶段标记为 uploaded(burnDelSource=false 时)
} else if (convert) {
files['video.mp4'].status = 'uploaded';
} else {
@@ -184,22 +215,28 @@ const analyzeConfig = (config: any) => {
if (cover) {
files['cover.jpg'].status = 'uploaded';
}
// 上传弹幕字幕(需 after_process 模式,immediate 模式下 .ass 尚未生成)
if (uploadAss) {
files['video.ass'].status = 'uploaded';
}
// 删除逻辑
if (delAllAfter) {
// 删除全部
// 删除全部:先保存已上传文件的状态,再全部标记删除,最后恢复已上传的标记
const uploadedStatus: Record<string, string> = {};
Object.entries(files).forEach(([name, f]) => {
if (f.status === 'uploaded') uploadedStatus[name] = f.status;
});
Object.values(files).forEach(f => { f.status = 'deleted'; });
// 但上传的文件标记为 uploaded_deleted
if (burn) files['video.mkv'].status = 'uploaded_deleted';
else if (convert) files['video.mp4'].status = 'uploaded_deleted';
else files['video.flv'].status = 'uploaded_deleted';
if (cover) files['cover.jpg'].status = 'uploaded_deleted';
// 恢复已上传文件为 uploaded_deleted
Object.entries(uploadedStatus).forEach(([name]) => {
files[name].status = 'uploaded_deleted';
});
} else if (delAfter) {
// 只删除已上传的文件
if (burn) files['video.mkv'].status = 'uploaded_deleted';
else if (convert) files['video.mp4'].status = 'uploaded_deleted';
else files['video.flv'].status = 'uploaded_deleted';
if (cover) files['cover.jpg'].status = 'uploaded_deleted';
Object.entries(files).forEach(([, f]) => {
if (f.status === 'uploaded') f.status = 'uploaded_deleted';
});
}
}
@@ -208,7 +245,10 @@ const analyzeConfig = (config: any) => {
const hasVideo = Object.entries(files).some(([name, f]) => {
if (name === 'video.ass') return false;
const ext = f.ext.toLowerCase();
return (ext === '.flv' || ext === '.mp4' || ext === '.mkv') && f.status !== 'deleted' && f.status !== 'uploaded_deleted';
if (ext !== '.flv' && ext !== '.mp4' && ext !== '.mkv') return false;
// 只有 kept 或 uploaded(本地保留)的视频才算"存在"
// intermediate(中间产物)和 deleted/uploaded_deleted(已删除)不算
return f.status === 'kept' || f.status === 'uploaded';
});
if (!hasVideo) {
files['video.ass'].status = 'deleted';
@@ -228,8 +268,8 @@ const analyzeConfig = (config: any) => {
} else if (f.status === 'uploaded_deleted') {
uploaded.push(name);
deleted.push(name);
} else if (f.status === 'deleted') {
deleted.push(name);
} else if (f.status === 'deleted' || f.status === 'intermediate') {
deleted.push(name); // intermediate 是中间产物,显示为删除
} else {
kept.push(name);
}
@@ -273,6 +313,11 @@ const FileProcessingPreview: React.FC<{ config: any }> = ({ config }) => {
}
</Descriptions.Item>
</Descriptions>
<div style={{ marginTop: 12, fontSize: 12, color: '#888', lineHeight: 1.8 }}>
<div>💡 <b>说明:</b>开启「转换后删除 FLV」或「烧录后删除源视频」后,只会上传最终成品(MP4 或 MKV)。</div>
<div style={{ paddingLeft: 16 }}>如果不开启,原始视频也会一并上传到云端(相当于云端保存了两份视频)。</div>
<div style={{ paddingLeft: 16 }}>上方预览会根据你当前的设置实时显示哪些文件会被上传、删除或保留。</div>
</div>
</Card>
);
};
@@ -291,6 +336,27 @@ interface CloudUploadSettingsProps {
const CloudUploadSettings: React.FC<CloudUploadSettingsProps> = ({ config, form }) => {
const isEnabled = config.on_record_finished?.cloud_upload?.enable;
// 订阅表单中影响文件处理预览的字段,使预览实时反映用户编辑
const watchedOrf = Form.useWatch('on_record_finished', form);
const watchedCloudUpload = Form.useWatch(['on_record_finished', 'cloud_upload'], form);
// 将表单实时值合并到 config 副本中,供 FileProcessingPreview 使用
// Form.useWatch 返回 undefined 时表示字段未被编辑,此时保留 config 中的原始值
const liveConfig = useMemo(() => {
if (!watchedOrf && !watchedCloudUpload) return config;
return {
...config,
on_record_finished: {
...config.on_record_finished,
...watchedOrf,
cloud_upload: {
...config.on_record_finished?.cloud_upload,
...watchedCloudUpload,
},
},
};
}, [config, watchedOrf, watchedCloudUpload]);
// 互斥逻辑:开启一个时关闭另一个
const handleDeleteAfterChange = (checked: boolean) => {
if (checked && form) {
@@ -325,7 +391,24 @@ const CloudUploadSettings: React.FC<CloudUploadSettingsProps> = ({ config, form
>
OpenList 管理页面
</a>{' '}
添加网盘。
添加网盘。{' '}
{config.openlist?.username && config.openlist?.password && (
<>
<span style={{ color: '#999', fontSize: 12 }}>
(登录凭据: {config.openlist.username} / {config.openlist.password})
</span>{' '}
</>
)}
<span style={{ color: '#999', fontSize: 12 }}>
(如无法访问,尝试{' '}
<a
href="/remotetools/tool/openlist/"
target="_blank"
rel="noopener noreferrer"
>
通过代理访问
</a>)
</span>
</>
}
type="info"
@@ -333,8 +416,8 @@ const CloudUploadSettings: React.FC<CloudUploadSettingsProps> = ({ config, form
style={{ marginBottom: 16 }}
/>
{/* 文件处理预览 */}
<FileProcessingPreview config={config} />
{/* 文件处理预览 - 使用表单实时值,无需保存即可反映用户编辑 */}
<FileProcessingPreview config={liveConfig} />
<ConfigField label="启用云上传" description="录制结束后自动把视频上传到网盘">
<Form.Item name={['on_record_finished', 'cloud_upload', 'enable']} valuePropName="checked" noStyle>
@@ -369,7 +452,7 @@ const CloudUploadSettings: React.FC<CloudUploadSettingsProps> = ({ config, form
presets={UPLOAD_PRESETS}
mockData={MOCK_DATA}
renderTemplate={renderUploadTemplate}
placeholder="/录播归档/{{ .Platform }}/{{ .HostName }}/{{ now | date '2006-01-02' }}/{{ .FileName }}"
placeholder={'/录播归档/{{ .Platform }}/{{ .HostName }}/{{ now | date "2006-01-02" }}/{{ .FileName }}'}
showTreePreview={true}
width={500}
/>
@@ -385,6 +468,11 @@ const CloudUploadSettings: React.FC<CloudUploadSettingsProps> = ({ config, form
<Switch onChange={handleDeleteAllAfterChange} />
</Form.Item>
</ConfigField>
<ConfigField label="上传弹幕字幕" description="同时上传与视频同名的 .ass 弹幕字幕文件到云存储。需开启弹幕录制">
<Form.Item name={['on_record_finished', 'cloud_upload', 'upload_subtitles']} valuePropName="checked" noStyle>
<Switch />
</Form.Item>
</ConfigField>
<Card type="inner" title="OpenList 认证配置" size="small" style={{ marginTop: 16, marginBottom: 8 }}>
<Alert
+34 -7
View File
@@ -85,6 +85,7 @@ interface EffectiveConfig {
upload_path_tmpl: string;
delete_after_upload: boolean;
delete_all_after_upload: boolean;
upload_subtitles: boolean;
};
upload_timing: string;
burn_subtitles: boolean;
@@ -182,6 +183,7 @@ interface PlatformStatsResponse {
platforms: PlatformStat[];
available_platforms: string[];
global_interval: number;
platform_rate_limit_enabled?: boolean;
}
// 实际生效值显示组件
@@ -1081,6 +1083,7 @@ const PlatformSettings: React.FC<{
}
const { platforms, available_platforms, global_interval } = platformStats;
const platformRateLimitEnabled = platformStats.platform_rate_limit_enabled ?? true;
// 分组平台:有直播间的 vs 只有配置没有直播间的
const platformsWithRooms = platforms.filter(p => p.has_rooms);
@@ -1185,6 +1188,7 @@ const PlatformSettings: React.FC<{
platform={platform}
globalConfig={globalConfig}
globalInterval={global_interval}
platformRateLimitEnabled={platformRateLimitEnabled}
onSave={(values) => handleSave(platform.platform_key, values)}
onDelete={() => handleDelete(platform.platform_key)}
loading={loading}
@@ -1207,6 +1211,15 @@ const PlatformSettings: React.FC<{
showIcon
style={{ marginBottom: 16 }}
/>
{!platformRateLimitEnabled && (
<Alert
message="平台级访问限流已暂时停用"
description="min_access_interval_sec 配置会被保留,但当前不会限制请求间隔或同平台并发。后续调度策略改善后再恢复生效。"
type="warning"
showIcon
style={{ marginBottom: 16 }}
/>
)}
{/* 正在监控的平台 */}
{platformsWithRooms.length > 0 && (
@@ -1256,11 +1269,12 @@ const PlatformConfigForm: React.FC<{
platform: PlatformStat;
globalConfig: EffectiveConfig;
globalInterval: number;
platformRateLimitEnabled: boolean;
onSave: (values: any) => void;
onDelete: () => void;
loading: boolean;
onNavigateToRoom: (liveId: string) => void;
}> = ({ platform, globalConfig, globalInterval, onSave, onDelete, loading, onNavigateToRoom }) => {
}> = ({ platform, globalConfig, globalInterval, platformRateLimitEnabled, onSave, onDelete, loading, onNavigateToRoom }) => {
const [form] = Form.useForm();
useEffect(() => {
@@ -1340,18 +1354,31 @@ const PlatformConfigForm: React.FC<{
</ConfigField>
<ConfigField
label="最小访问间隔 (秒)"
description="该平台 API 的最小访问间隔,用于防风控。若监控数量过多导致频率过快,系统会自动增加检测间隔。"
label={`最小访问间隔 (秒${platformRateLimitEnabled ? '' : ',暂未启用'})`}
description={platformRateLimitEnabled
? '该平台 API 的最小访问间隔,用于防风控。若监控数量过多导致频率过快,系统会自动增加检测间隔。'
: '配置值会继续保留,但平台级访问限流关闭期间不会生效。'}
id={`platforms-${platformKey}-min_access_interval_sec`}
inheritance={{
inheritance={platformRateLimitEnabled ? {
source: 'default',
isOverridden: (platform.min_access_interval_sec || 0) > 0,
inheritedValue: '不限制 (0)',
}}
valueDisplay={(platform.min_access_interval_sec || 0) > 0 ? platform.min_access_interval_sec : '不限制 (0)'}
} : undefined}
effectiveValue={platformRateLimitEnabled ? undefined : '当前不生效'}
valueDisplay={!platformRateLimitEnabled
? ((platform.min_access_interval_sec || 0) > 0
? `已配置 ${platform.min_access_interval_sec} 秒(暂不生效)`
: '暂未启用')
: ((platform.min_access_interval_sec || 0) > 0 ? platform.min_access_interval_sec : '不限制 (0)')}
>
<Form.Item name="min_access_interval_sec" rules={[{ type: 'number', min: 0, message: '不能为负数' }]}>
<InputNumber min={0} max={3600} style={{ width: 200 }} placeholder="0 (不限制)" />
<InputNumber
min={0}
max={3600}
style={{ width: 200 }}
placeholder="0 (不限制)"
disabled={!platformRateLimitEnabled}
/>
</Form.Item>
</ConfigField>
+95 -1
View File
@@ -11,7 +11,9 @@ import {
// @ts-ignore
EditOutlined,
// @ts-ignore
DeleteOutlined
DeleteOutlined,
// @ts-ignore
CloudUploadOutlined
} from "@ant-design/icons";
import { Link, useNavigate, useParams } from "react-router-dom";
import Utils from "../../utils/common";
@@ -479,6 +481,7 @@ type CurrentFolderFile = {
last_modified: number;
size: number;
subtitle_file?: string;
uploaded?: boolean;
}
const FileList: React.FC = () => {
@@ -780,6 +783,71 @@ const FileList: React.FC = () => {
.catch(err => message.error("删除失败: " + err));
};
const handleUpload = (record: CurrentFolderFile, e?: React.MouseEvent) => {
if (e) e.stopPropagation();
let fullPath = record.name;
if (pathParam) {
fullPath = pathParam + "/" + record.name;
}
message.loading({ content: `正在上传 ${record.name}...`, key: "upload" });
api.uploadFile(fullPath)
.then((rsp: any) => {
if (rsp.data) {
const data = rsp.data;
if (data.enqueued > 0) {
message.success({ content: `已入队上传任务: ${record.name}`, key: "upload" });
} else if (data.skipped && data.skipped.length > 0) {
message.error({ content: data.skipped[0] || "上传失败", key: "upload" });
} else {
message.error({ content: rsp.err_msg || "上传失败", key: "upload" });
}
} else {
message.error({ content: rsp.err_msg || "上传失败", key: "upload" });
}
})
.catch((err: any) => message.error({ content: "上传失败: " + (err.message || err), key: "upload" }));
};
const handleBatchUpload = () => {
if (selectedRowKeys.length === 0) return;
const paths = selectedRowKeys.map(key => {
const fileName = key.toString();
return pathParam ? `${pathParam}/${fileName}` : fileName;
});
message.loading({ content: `正在上传 ${paths.length} 个文件...`, key: "batchUpload" });
api.batchUploadFiles(paths)
.then((rsp: any) => {
if (rsp.data) {
const data = rsp.data;
const enqueued = data.enqueued || 0;
const skipped = data.skipped || [];
const failCount = skipped.length;
if (failCount === 0) {
message.success({ content: `全部 ${enqueued} 个文件已入队上传`, key: "batchUpload" });
} else if (enqueued > 0) {
message.warning({ content: `已入队 ${enqueued} 个,跳过 ${failCount} 个`, key: "batchUpload" });
Modal.info({
title: '部分文件已跳过',
content: (
<div style={{ maxHeight: 300, overflow: 'auto' }}>
{skipped.map((s: string, i: number) => (
<div key={i} style={{ fontSize: '12px', color: '#8c8c8c' }}>{s}</div>
))}
</div>
),
});
} else {
message.error({ content: skipped[0] || "批量上传失败", key: "batchUpload" });
}
} else {
message.error({ content: rsp.err_msg || "批量上传失败", key: "batchUpload" });
}
})
.catch((err: any) => message.error({ content: "批量上传失败: " + (err.message || err), key: "batchUpload" }));
};
const handleBatchDelete = () => {
if (selectedRowKeys.length === 0) return;
const paths = selectedRowKeys.map(key => {
@@ -1153,6 +1221,11 @@ const FileList: React.FC = () => {
无字幕
</span>
)}
{record.uploaded && (
<span style={{ marginLeft: 6, fontSize: 11, color: '#52c41a', background: '#f6ffed', padding: '1px 6px', borderRadius: 4, cursor: 'default' }}>
已上传
</span>
)}
</div>
);
}
@@ -1194,6 +1267,18 @@ const FileList: React.FC = () => {
>
重命名
</Button>
{!record.is_folder && (
<Button
type="link"
size="small"
// @ts-ignore
icon={<CloudUploadOutlined />}
onClick={(e) => handleUpload(record, e)}
className="action-btn"
>
上传
</Button>
)}
<Popconfirm
title={`确定要删除${record.is_folder ? '文件夹' : '文件'} "${record.name}" 吗?`}
onConfirm={() => handleDelete(record)}
@@ -1401,6 +1486,15 @@ const FileList: React.FC = () => {
</Tooltip>
);
})()}
<Button
size="small"
type="default"
// @ts-ignore
icon={<CloudUploadOutlined />}
onClick={handleBatchUpload}
>
批量上传
</Button>
<Popconfirm
title={`确定要删除选中的 ${selectedRowKeys.length} 个项目吗?`}
onConfirm={handleBatchDelete}
+77 -16
View File
@@ -1,5 +1,5 @@
import React from "react";
import { Button, Divider, Table, Tag, Tabs, Row, Col, Tooltip, message, List, Typography, Switch, Space, Popconfirm, Select, Spin } from 'antd';
import { Alert, Button, Divider, Table, Tag, Tabs, Row, Col, Tooltip, message, List, Typography, Switch, Space, Popconfirm, Select, Spin } from 'antd';
import { EditOutlined, SyncOutlined, CloudSyncOutlined, ReloadOutlined, SwapOutlined, CheckCircleOutlined, ExclamationCircleOutlined, CommentOutlined } from '@ant-design/icons';
import PopDialog from '../pop-dialog/index';
import BatchAddRoomDialog from '../batch-add-room-dialog/index';
@@ -824,11 +824,41 @@ class LiveList extends React.Component<Props, IState> {
// 旧的频率限制信息处理逻辑(兼容性保留)
const rateLimitInfo = updateData;
const rateLimitEnabled = rateLimitInfo?.enabled
?? currentDetail?.platform_rate_limit_enabled
?? true;
const updatedDetail = {
...currentDetail,
rate_limit_info: rateLimitInfo
platform_rate_limit_enabled: rateLimitEnabled,
platform_rate_limit: rateLimitEnabled ? currentDetail?.platform_rate_limit : 0,
rate_limit_info: {
...rateLimitInfo,
enabled: rateLimitEnabled
}
};
if (!rateLimitEnabled) {
return {
...prevState,
expandedDetails: {
...prevState.expandedDetails,
[roomId]: updatedDetail
},
countdownTimers: {
...prevState.countdownTimers,
[roomId]: 0
},
lastUpdateTimes: {
...prevState.lastUpdateTimes,
[roomId]: Date.now()
},
refreshStatus: {
...prevState.refreshStatus,
[roomId]: 'idle'
}
};
}
const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0);
const minIntervalSec = rateLimitInfo?.min_interval_sec || currentDetail?.platform_rate_limit || 20;
const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0);
@@ -1235,6 +1265,9 @@ class LiveList extends React.Component<Props, IState> {
// 优先使用 scheduler_status 来确定刷新状态
const schedulerStatus = detail.scheduler_status;
const rateLimitInfo = detail.rate_limit_info;
const rateLimitEnabled = detail.platform_rate_limit_enabled
?? rateLimitInfo?.enabled
?? true;
let initialCountdown = 0;
let initialStatus: RefreshStatus = 'idle';
@@ -1249,14 +1282,14 @@ class LiveList extends React.Component<Props, IState> {
// 有下次请求计划
initialCountdown = Math.ceil(schedulerStatus.seconds_until_next_request);
// 检查是否在等待平台限制
if (rateLimitInfo?.next_request_in_sec > 0) {
if (rateLimitEnabled && rateLimitInfo?.next_request_in_sec > 0) {
initialStatus = 'waiting_rate_limit';
} else {
initialStatus = 'waiting_interval';
}
} else if (schedulerStatus.seconds_until_next_request === 0) {
// 即将发送请求或正在等待平台限制
if (rateLimitInfo?.next_request_in_sec > 0) {
if (rateLimitEnabled && rateLimitInfo?.next_request_in_sec > 0) {
initialCountdown = Math.ceil(rateLimitInfo.next_request_in_sec);
initialStatus = 'waiting_rate_limit';
} else {
@@ -1270,19 +1303,24 @@ class LiveList extends React.Component<Props, IState> {
}
} else {
// 回退到旧逻辑(兼容性)
const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0);
const minIntervalSec = rateLimitInfo?.min_interval_sec || detail.platform_rate_limit || 20;
const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0);
if (nextRequestInSec > 0) {
initialCountdown = nextRequestInSec;
initialStatus = 'waiting_rate_limit';
} else if (waitedSec < minIntervalSec) {
initialCountdown = minIntervalSec - waitedSec;
initialStatus = 'waiting_interval';
} else {
if (!rateLimitEnabled) {
initialCountdown = 0;
initialStatus = 'idle';
} else {
const nextRequestInSec = Math.ceil(rateLimitInfo?.next_request_in_sec || 0);
const minIntervalSec = rateLimitInfo?.min_interval_sec || detail.platform_rate_limit || 20;
const waitedSec = Math.round(rateLimitInfo?.waited_seconds || 0);
if (nextRequestInSec > 0) {
initialCountdown = nextRequestInSec;
initialStatus = 'waiting_rate_limit';
} else if (waitedSec < minIntervalSec) {
initialCountdown = minIntervalSec - waitedSec;
initialStatus = 'waiting_interval';
} else {
initialCountdown = 0;
initialStatus = 'idle';
}
}
}
@@ -1428,6 +1466,9 @@ class LiveList extends React.Component<Props, IState> {
const countdown = countdownTimers[record.roomId] ?? 0;
const status = refreshStatus[record.roomId] ?? 'idle';
const liveId = record.roomId;
const platformRateLimitEnabled = detail?.platform_rate_limit_enabled
?? detail?.rate_limit_info?.enabled
?? true;
// 保存 this 引用供嵌套函数使用
const component = this;
@@ -1782,7 +1823,27 @@ class LiveList extends React.Component<Props, IState> {
<Divider style={{ margin: '8px 0' }}>平台访问频率控制</Divider>
<div style={{ padding: '0 12px 8px' }}>
{detail.rate_limit_info ? (
{!platformRateLimitEnabled ? (
<div>
<Alert
message="平台级访问限流已暂时停用"
description="当前不会限制同平台请求的最小间隔或并发数;已保存的 min_access_interval_sec 配置暂不生效。"
type="warning"
showIcon
/>
<div style={{ marginTop: 12, borderBottom: 'none' }}>
<Button
type="primary"
size="small"
onClick={handleForceRefresh}
loading={status === 'refreshing'}
icon={<ReloadOutlined />}
>
立即刷新
</Button>
</div>
</div>
) : detail.rate_limit_info ? (
<div>
<div style={configRowStyle}>
<Tooltip
@@ -36,6 +36,7 @@ interface RecordInfo {
platform: string;
host_name: string;
room_name: string;
display_name?: string;
start_time: string;
}
@@ -229,11 +230,19 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
}
};
handleRetry = async (taskId: number) => {
handleRetry = async (taskId: number, stageName?: string) => {
try {
const res = await fetch(`/api/pipeline/tasks/${taskId}/retry`, { method: 'POST' });
const body: any = {};
if (stageName) {
body.stage_name = stageName;
}
const res = await fetch(`/api/pipeline/tasks/${taskId}/retry`, {
method: 'POST',
headers: stageName ? { 'Content-Type': 'application/json' } : undefined,
body: stageName ? JSON.stringify(body) : undefined,
});
if (res.ok) {
message.success('任务已重新排队');
message.success(stageName ? `从 ${this.getStageLabel(stageName)} 阶段重试` : '任务已重新排队');
this.loadData();
} else {
message.error('重试失败');
@@ -291,6 +300,7 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
'extract_cover': '提取封面',
'cloud_upload': '云盘上传',
'custom_command': '自定义命令',
'burn_subtitles': '烧录字幕',
};
return labels[stageName] || stageName;
};
@@ -450,6 +460,9 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
const results = task.stage_results || [];
if (results.length === 0) return null;
// 是否允许阶段级重试(Failed/Cancelled/Completed 状态)
const canRetryStage = task.can_retry && (task.status === 'failed' || task.status === 'cancelled' || task.status === 'completed');
return (
<Collapse size="small" style={{ marginTop: 12 }}>
{results.map((result, index) => (
@@ -461,6 +474,19 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
{result.status === 'failed' && (
<Text type="danger">失败</Text>
)}
{canRetryStage && (result.status === 'completed' || result.status === 'failed') && (
<Button
type="link"
size="small"
icon={<PlayCircleOutlined />}
onClick={(e) => {
e.stopPropagation();
this.handleRetry(task.id, result.stage_name);
}}
>
从此阶段重试
</Button>
)}
</Space>
}
key={index}
@@ -470,7 +496,11 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
{result.error_message && (
<Alert
message="错误信息"
description={result.error_message}
description={
<div style={{ whiteSpace: 'pre-wrap', maxHeight: 200, overflow: 'auto' }}>
{result.error_message}
</div>
}
type="error"
style={{ marginBottom: 8 }}
/>
@@ -534,7 +564,7 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
<div>
<Text strong>直播间:</Text>
<Text>
{task.record_info.room_name} ({task.record_info.host_name} - {task.record_info.platform})
{task.record_info.display_name || task.record_info.room_name} ({task.record_info.host_name} - {task.record_info.platform})
</Text>
</div>
)}
@@ -549,7 +579,11 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
{task.error_message && (
<Alert
message="任务失败"
description={task.error_message}
description={
<div style={{ whiteSpace: 'pre-wrap', maxHeight: 300, overflow: 'auto' }}>
{task.error_message}
</div>
}
type="error"
/>
)}
@@ -568,7 +602,7 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
取消任务
</Button>
)}
{(task.status === 'failed' || task.status === 'cancelled') && task.can_retry && (
{(task.status === 'failed' || task.status === 'cancelled' || task.status === 'completed') && task.can_retry && (
<Button icon={<PlayCircleOutlined />} onClick={() => this.handleRetry(task.id)}>
重新执行
</Button>
@@ -609,7 +643,7 @@ class PipelineTaskList extends Component<object, PipelineTaskListState> {
width: 200,
render: (_, record: PipelineTask) => (
<span>
{record.record_info?.room_name || '-'}
{record.record_info?.display_name || record.record_info?.room_name || '-'}
{record.record_info?.platform && (
<Tag style={{ marginLeft: 4 }}>{record.record_info.platform}</Tag>
)}
+16
View File
@@ -170,6 +170,22 @@ class API {
return utils.requestPost(`${BASE_URL}/pipeline/batch-burn`, { paths });
}
/**
* 上传单个文件到云盘
* @param path 文件路径
*/
uploadFile(path: string) {
return utils.requestPost(`${BASE_URL}/file/upload`, { path });
}
/**
* 批量上传文件到云盘
* @param paths 文件路径列表
*/
batchUploadFiles(paths: string[]) {
return utils.requestPost(`${BASE_URL}/batch/file/upload`, { paths });
}
/**
* 获取Cookie列表
*/
@@ -16,7 +16,7 @@ log:
rotate_days: 7
feature:
remove_symbol_other_character: false
out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{.StartTime}}'
out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}.flv'
video_split_strategies:
on_room_name_changed: false
max_duration: 0s
+2 -2
View File
@@ -18,7 +18,7 @@ feature:
# '{{ .Live.GetPlatformCNName }}/{{ .HostName | filenameFilter }}/[{{ now | date "2006-01-02 15-04-05"}}][{{ .HostName | filenameFilter }}][{{ .RoomName | filenameFilter }}].flv'
# ./平台名称/主播名字/[时间戳][主播名字][房间名字].flv
# https://github.com/bililive-go/bililive-go/wiki/More-Tips
out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{.StartTime}}'
out_put_tmpl: '{{.RoomName}}-{{.HostName}}/{{ now | date "2006-01-02 15-04-05" }}.flv'
video_split_strategies:
on_room_name_changed: false
max_duration: 0s
@@ -43,7 +43,7 @@ on_record_finished:
cloud_upload:
enable: false
storage_name: ""
upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .RoomName }}-{{ now | date "2006-01-02" }}.{{ .Ext }}
upload_path_tmpl: /录播归档/{{ .Platform }}/{{ .HostName }}/{{ .FileName }}
delete_after_upload: false
upload_timing: after_process
timeout_in_us: 60000000
+62 -3
View File
@@ -1,8 +1,8 @@
import { test, expect } from '@playwright/test';
import { test, expect, request as apiRequest } from '@playwright/test';
/**
* 直播录制功能测试
*
*
* 使用 osrp-stream-tester 提供的 dev 直播间进行测试
* 测试添加直播间、开始/停止录制等功能
*/
@@ -10,6 +10,66 @@ import { test, expect } from '@playwright/test';
// dev 测试流服务器地址
const DEV_STREAM_SERVER = 'http://127.0.0.1:8888';
const DEV_STREAM_URL = `${DEV_STREAM_SERVER}/live/test.flv`;
// bililive-go 后端地址
const BGO_SERVER = 'http://127.0.0.1:8080';
interface LiveInfo {
id: string;
live_url?: string;
}
const preExistingLiveIds = new Set<string>();
let baselineCaptured = false;
// 本地运行时 Playwright 可能复用开发者已经启动的 bililive-go。先记录既有房间,
// 避免测试清理误删开发者的持久化配置。
test.beforeAll(async () => {
const ctx = await apiRequest.newContext();
try {
const res = await ctx.get(`${BGO_SERVER}/api/lives`);
if (!res.ok()) {
return;
}
const lives: LiveInfo[] = await res.json();
if (!Array.isArray(lives)) {
return;
}
for (const live of lives) {
if (live.id) {
preExistingLiveIds.add(live.id);
}
}
baselineCaptured = true;
} finally {
await ctx.dispose();
}
});
// 本文件会通过 UI 添加 dev 直播间;由于所有 spec 共用同一个后端进程(workers=1),
// 若不清理,残留的监控房间会一直被判定为"在播",导致后续 spec(如 update-*)的
// active_recordings 断言失败。这里只删除本文件新建的 dev 直播间,既保证测试隔离,
// 也不会在复用本地服务时误删开发者已有的监控房间。
test.afterAll(async () => {
if (!baselineCaptured) {
return;
}
const ctx = await apiRequest.newContext();
try {
const res = await ctx.get(`${BGO_SERVER}/api/lives`);
if (res.ok()) {
const lives: LiveInfo[] = await res.json();
if (Array.isArray(lives)) {
for (const live of lives) {
if (live.id && live.live_url === DEV_STREAM_URL && !preExistingLiveIds.has(live.id)) {
await ctx.delete(`${BGO_SERVER}/api/lives/${live.id}`);
}
}
}
}
} finally {
await ctx.dispose();
}
});
test.describe('直播间管理测试', () => {
test.beforeEach(async ({ page }) => {
@@ -224,4 +284,3 @@ test.describe('紧急情况:停止所有录制', () => {
expect(count).toBeGreaterThanOrEqual(0);
});
});
+7 -3
View File
@@ -208,9 +208,13 @@ test.describe('优雅更新逻辑测试', () => {
const response = await request.get('/api/update/launcher');
expect(response.ok()).toBeTruthy();
const data = await response.json();
// 测试环境应该没有活跃录制
expect(data.active_recordings).toBe(0);
// 测试环境应该没有活跃录制。前序 spec 清理房间后录制器回收存在短暂延迟,
// 使用轮询等待其归零,避免时序相关的偶发失败。
await expect.poll(async () => {
const res = await request.get('/api/update/launcher');
const d = await res.json();
return d.active_recordings;
}, { timeout: 10000 }).toBe(0);
});
test('更新状态不影响其他 API', async ({ request }) => {
+7 -3
View File
@@ -254,9 +254,13 @@ test.describe('优雅更新逻辑测试', () => {
const response = await request.get('/api/update/launcher');
expect(response.ok()).toBeTruthy();
const data = await response.json();
// 测试环境应该没有活跃录制
expect(data.active_recordings).toBe(0);
// 测试环境应该没有活跃录制。前序 spec 清理房间后录制器回收存在短暂延迟,
// 使用轮询等待其归零,避免时序相关的偶发失败。
await expect.poll(async () => {
const res = await request.get('/api/update/launcher');
const d = await res.json();
return d.active_recordings;
}, { timeout: 10000 }).toBe(0);
});
test('更新状态不影响其他核心 API', async ({ request }) => {