Compare commits

...

3 Commits

Author SHA1 Message Date
Kevin Wong
e33dfc3031 更新 2026-02-10 13:31:29 +08:00
Kevin Wong
3129d45b25 更新 2026-02-09 14:47:19 +08:00
Kevin Wong
e226224119 更新 2026-02-08 19:54:11 +08:00
70 changed files with 5487 additions and 1605 deletions

17
.gitignore vendored
View File

@@ -20,11 +20,14 @@ node_modules/
out/
.turbo/
# ============ IDE ============
# ============ IDE / AI 工具 ============
.vscode/
.idea/
*.swp
*.swo
.agents/
.opencode/
.claude/
# ============ 系统文件 ============
.DS_Store
@@ -35,11 +38,21 @@ desktop.ini
backend/outputs/
backend/uploads/
backend/cookies/
backend/user_data/
backend/debug_screenshots/
*_cookies.json
# ============ MuseTalk ============
# ============ 模型权重 ============
models/*/checkpoints/
models/MuseTalk/models/
models/MuseTalk/results/
models/LatentSync/temp/
# ============ Remotion 构建 ============
remotion/dist/
# ============ 临时文件 ============
Temp/
# ============ 日志 ============
*.log

View File

@@ -29,15 +29,16 @@ backend/
├── app/
│ ├── core/ # config、deps、security、response
│ ├── modules/ # 业务模块(路由 + 逻辑)
│ │ ├── videos/ # 视频生成任务
│ │ ├── materials/ # 素材管理
│ │ ├── videos/ # 视频生成任务router/schemas/service/workflow
│ │ ├── materials/ # 素材管理router/schemas/service
│ │ ├── publish/ # 多平台发布
│ │ ├── auth/ # 认证与会话
│ │ ├── ai/ # AI 功能(标题标签生成
│ │ ├── ai/ # AI 功能(标题标签生成、多语言翻译
│ │ ├── assets/ # 静态资源(字体/样式/BGM
│ │ ├── ref_audios/ # 声音克隆参考音频
│ │ ├── ref_audios/ # 声音克隆参考音频router/schemas/service
│ │ ├── generated_audios/ # 预生成配音管理router/schemas/service
│ │ ├── login_helper/ # 扫码登录辅助
│ │ ├── tools/ # 工具接口
│ │ ├── tools/ # 工具接口router/schemas/service
│ │ └── admin/ # 管理员功能
│ ├── repositories/ # Supabase 数据访问
│ ├── services/ # 外部服务集成
@@ -124,10 +125,13 @@ backend/user_data/{user_uuid}/cookies/
## 8. 开发流程建议
- **新增功能**:先建模块,再写 router/service/workflow
- **修复 Bug**:顺手把涉及的逻辑抽到对应 service/workflow。
- **新增功能**:先建模块,**必须**包含 `router.py + schemas.py + service.py`,不允许 router-only
- **修复 Bug**:顺手把涉及的逻辑抽到对应 service/workflow(渐进式改造)
- **改旧模块**:改动哪部分就拆哪部分,不要求一次重构整个文件。
- **核心流程变更**:必跑冒烟(登录/生成/发布)。
> **渐进原则**:新代码高标准,旧代码逐步改。不做大规模一次性重构,避免引入回归风险。
---
## 9. 常用环境变量

View File

@@ -15,16 +15,17 @@ backend/
├── app/
│ ├── core/ # 核心配置 (config.py, security.py, response.py)
│ ├── modules/ # 业务模块 (router/service/workflow/schemas)
│ │ ├── videos/ # 视频生成任务
│ │ ├── materials/ # 素材管理
│ │ ├── videos/ # 视频生成任务router/schemas/service/workflow
│ │ ├── materials/ # 素材管理router/schemas/service
│ │ ├── publish/ # 多平台发布
│ │ ├── auth/ # 认证与会话
│ │ ├── ai/ # AI 功能(标题标签生成)
│ │ ├── assets/ # 静态资源(字体/样式/BGM
│ │ ├── ref_audios/ # 声音克隆参考音频
│ │ ├── login_helper/ # 扫码登录辅助
│ │ ├── tools/ # 工具接口(文案提取等)
│ │ ── admin/ # 管理员功能
│ │ ├── ai/ # AI 功能(标题标签生成、多语言翻译
│ │ ├── assets/ # 静态资源(字体/样式/BGM
│ │ ├── ref_audios/ # 声音克隆参考音频router/schemas/service
│ │ ├── generated_audios/ # 预生成配音管理router/schemas/service
│ │ ├── login_helper/ # 扫码登录辅助
│ │ ── tools/ # 工具接口router/schemas/service
│ │ └── admin/ # 管理员功能
│ ├── repositories/ # Supabase 数据访问
│ ├── services/ # 外部服务集成 (TTS/Remotion/Storage/Uploader 等)
│ └── tests/ # 单元测试与集成测试
@@ -83,11 +84,19 @@ backend/
7. **AI 功能 (AI)**
* `POST /api/ai/generate-meta`: AI 生成标题和标签
* `POST /api/ai/translate`: AI 多语言翻译(支持 9 种目标语言)
8. **工具 (Tools)**
8. **预生成配音 (Generated Audios)**
* `POST /api/generated-audios/generate`: 异步生成配音(返回 task_id
* `GET /api/generated-audios/tasks/{task_id}`: 轮询生成进度
* `GET /api/generated-audios`: 列出用户所有配音
* `DELETE /api/generated-audios/{audio_id}`: 删除配音
* `PUT /api/generated-audios/{audio_id}`: 重命名配音
9. **工具 (Tools)**
* `POST /api/tools/extract-script`: 从视频链接提取文案
9. **健康检查**
10. **健康检查**
* `GET /api/lipsync/health`: LatentSync 服务健康状态
* `GET /api/voiceclone/health`: Qwen3-TTS 服务健康状态
@@ -108,9 +117,14 @@ backend/
`POST /api/videos/generate` 支持以下可选字段:
- `material_path`: 视频素材路径(单素材模式)
- `material_paths`: 多素材路径数组多机位模式≥2 个素材时按句子自动切换)
- `tts_mode`: TTS 模式 (`edgetts` / `voiceclone`)
- `voice`: EdgeTTS 音色 IDedgetts 模式)
- `ref_audio_id` / `ref_text`: 参考音频 ID 与文本voiceclone 模式)
- `generated_audio_id`: 预生成配音 ID存在时跳过内联 TTS使用已生成的配音文件
- `custom_assignments`: 自定义素材分配数组(每项含 `material_path` / `start` / `end` / `source_start`),存在时跳过 Whisper 均分
- `language`: TTS 语言(默认自动检测,声音克隆时透传给 Qwen3-TTS
- `title`: 片头标题文字
- `subtitle_style_id`: 字幕样式 ID
- `title_style_id`: 标题样式 ID

View File

@@ -165,6 +165,8 @@ playwright install chromium
CREATE POLICY "Allow public read" ON storage.objects FOR SELECT TO anon USING (bucket_id = 'materials' OR bucket_id = 'outputs');
EOF
```
> **注意**:后端启动时会自动创建额外的存储桶(`ref-audios`、`generated-audios`),无需手动创建。
---
@@ -570,6 +572,7 @@ pm2 logs vigent2-qwen-tts
| `next` | React 框架 |
| `swr` | 数据请求与缓存 |
| `tailwindcss` | CSS 样式 |
| `wavesurfer.js` | 音频波形(时间轴编辑器) |
### LatentSync 关键依赖

View File

@@ -342,6 +342,6 @@ models/Qwen3-TTS/
## 🔗 相关文档
- [task_complete.md](../task_complete.md) - 任务总览
- [TASK_COMPLETE.md](../TASK_COMPLETE.md) - 任务总览
- [Day11.md](./Day11.md) - 上传架构重构
- [QWEN3_TTS_DEPLOY.md](../QWEN3_TTS_DEPLOY.md) - Qwen3-TTS 部署指南

View File

@@ -273,7 +273,7 @@ pm2 logs vigent2-qwen-tts --lines 50
## 🔗 相关文档
- [task_complete.md](../task_complete.md) - 任务总览
- [TASK_COMPLETE.md](../TASK_COMPLETE.md) - 任务总览
- [Day12.md](./Day12.md) - iOS 兼容与 Qwen3-TTS 部署
- [QWEN3_TTS_DEPLOY.md](../QWEN3_TTS_DEPLOY.md) - Qwen3-TTS 部署指南
- [SUBTITLE_DEPLOY.md](../SUBTITLE_DEPLOY.md) - 字幕功能部署指南

View File

@@ -397,6 +397,6 @@ if ((status === 401 || status === 403) && !isRedirecting && !isPublicPath) {
## 🔗 相关文档
- [task_complete.md](../task_complete.md) - 任务总览
- [TASK_COMPLETE.md](../TASK_COMPLETE.md) - 任务总览
- [Day13.md](./Day13.md) - 声音克隆功能集成 + 字幕功能
- [QWEN3_TTS_DEPLOY.md](../QWEN3_TTS_DEPLOY.md) - Qwen3-TTS 1.7B 部署指南

View File

@@ -342,7 +342,7 @@ pm2 restart vigent2-backend vigent2-frontend
## 🔗 相关文档
- [task_complete.md](../task_complete.md) - 任务总览
- [TASK_COMPLETE.md](../TASK_COMPLETE.md) - 任务总览
- [Day14.md](./Day14.md) - 模型升级 + AI 标题标签
- [AUTH_DEPLOY.md](../AUTH_DEPLOY.md) - 认证系统部署指南

View File

@@ -136,4 +136,4 @@ if service["failures"] >= service['threshold']:
- [x] `Docs/QWEN3_TTS_DEPLOY.md`: 添加 Flash Attention 安装指南
- [x] `Docs/DEPLOY_MANUAL.md`: 添加 Watchdog 部署说明
- [x] `Docs/task_complete.md`: 更新进度至 100% (Day 16)
- [x] `Docs/TASK_COMPLETE.md`: 更新进度至 100% (Day 16)

View File

@@ -246,3 +246,204 @@ PLATFORM_CONFIGS = {
pm2 restart vigent2-backend # 发布服务 + QR登录
npm run build && pm2 restart vigent2-frontend # 刷脸验证UI
```
---
## 🏗️ 架构优化:前端结构微调 + 后端模块分层 (Day 21)
### 概述
根据架构审计结果,完成前端目录规范化和后端核心模块的分层补全。
### 一、前端结构微调
#### 1. ScriptExtractionModal 迁移
- `components/ScriptExtractionModal.tsx``features/home/ui/ScriptExtractionModal.tsx`
- 连带 `components/script-extraction/` 目录一并迁移到 `features/home/ui/script-extraction/`
- 更新 `HomePage.tsx` 的 import 路径
#### 2. contexts/ 目录归并
- `src/contexts/AuthContext.tsx``src/shared/contexts/AuthContext.tsx`
- `src/contexts/TaskContext.tsx``src/shared/contexts/TaskContext.tsx`
- 更新 6 处 importlayout.tsx, useHomeController.ts, usePublishController.ts, AccountSettingsDropdown.tsx, GlobalTaskIndicator.tsx
- 删除空的 `src/contexts/` 目录
#### 3. 清理重构遗留空目录
- 删除 `src/lib/``src/components/home/``src/hooks/`
### 二、后端模块分层补全
将 3 个 400+ 行的 router-only 模块拆分为 `router.py + schemas.py + service.py`
| 模块 | 改造前 | 改造后 router |
|------|--------|--------------|
| `materials/` | 416 行 | 63 行 |
| `tools/` | 417 行 | 33 行 |
| `ref_audios/` | 421 行 | 71 行 |
业务逻辑全部提取到 `service.py`,数据模型定义在 `schemas.py`router 只做参数校验 + 调用 service + 返回响应。
### 三、开发规范更新
`BACKEND_DEV.md` 第 8 节新增渐进原则:
- 新模块**必须**包含 `router.py + schemas.py + service.py`
- 改旧模块时顺手拆涉及的部分
- 新代码高标准,旧代码逐步改
### 涉及文件汇总
| 文件 | 变更 |
|------|------|
| `frontend/src/features/home/ui/ScriptExtractionModal.tsx` | 从 components/ 迁入 |
| `frontend/src/features/home/ui/script-extraction/` | 从 components/ 迁入 |
| `frontend/src/shared/contexts/AuthContext.tsx` | 从 contexts/ 迁入 |
| `frontend/src/shared/contexts/TaskContext.tsx` | 从 contexts/ 迁入 |
| `backend/app/modules/materials/schemas.py` | **新建** |
| `backend/app/modules/materials/service.py` | **新建** |
| `backend/app/modules/materials/router.py` | 精简为薄路由 |
| `backend/app/modules/tools/schemas.py` | **新建** |
| `backend/app/modules/tools/service.py` | **新建** |
| `backend/app/modules/tools/router.py` | 精简为薄路由 |
| `backend/app/modules/ref_audios/schemas.py` | **新建** |
| `backend/app/modules/ref_audios/service.py` | **新建** |
| `backend/app/modules/ref_audios/router.py` | 精简为薄路由 |
| `Docs/BACKEND_DEV.md` | 目录结构标注分层、新增渐进原则 |
| `Docs/BACKEND_README.md` | 目录结构标注分层 |
| `Docs/FRONTEND_DEV.md` | 更新目录结构contexts 迁移、ScriptExtractionModal 迁移) |
### 重启要求
```bash
pm2 restart vigent2-backend
npm run build && pm2 restart vigent2-frontend
```
---
## 🎬 多素材视频生成(多机位效果)
### 概述
支持用户上传多个不同角度的自拍视频,生成视频时按句子自动切换素材,最终效果类似多机位拍摄。单素材时走原有流程,无额外开销。
### 核心架构
#### 流水线变更
```
【单素材(不变)】
text → TTS → audio → LatentSync(1个素材+完整audio) → Whisper字幕 → Remotion → 成片
【多素材(新增)】
text → TTS → audio → Whisper字幕(提前) → 按素材数量均分时长(对齐字边界)
→ 对每段: 切分audio + LatentSync(素材[i]+音频片段[i])
→ FFmpeg拼接所有片段 → Remotion(完整字幕时间戳) → 成片
```
#### 素材切换逻辑(均分方案)
1. Whisper 对完整音频转录,得到字级别时间戳
2. 按素材数量**均分音频总时长**`total_duration / N`
3. 每个分割点对齐到最近的 Whisper 字边界,避免在字中间切分
4. 首段 start 扩展为 0.0,末段 end 扩展为音频结尾,确保完整覆盖
> **设计决策**:最初方案基于原始文案标点分句,但用户文案往往不含句号(只有逗号),导致只产生 1 段。改为均分方案后不依赖文案标点,对任何输入都能正确切分。
---
### 一、后端改动
#### 1. `backend/app/modules/videos/schemas.py`
- 新增 `material_paths: Optional[List[str]]` 字段
- 保留 `material_path: str` 向后兼容
#### 2. `backend/app/modules/videos/workflow.py`(核心改动)
**新增函数**
- `_split_equal(segments, material_paths)`: 按素材数量均分音频时长,对齐到最近的 Whisper 字边界
**修改 `process_video_generation()`**
- `is_multi = len(material_paths) > 1` 判断走多素材/单素材分支
- 多素材分支Whisper 提前 → 均分切分 → 音频切分 → 逐段 LatentSync → FFmpeg 拼接
#### 3. `backend/app/services/video_service.py`
- 新增 `concat_videos()`: FFmpeg concat demuxer (`-c copy`) 拼接视频片段
- 新增 `split_audio()`: FFmpeg 按时间范围切分音频 (`-ss` + `-t` + `-c copy`)
#### 4. `backend/scripts/watchdog.py`
- 健康检查阈值从 3 次提高到 5 次(容忍期 2.5 分钟)
- 新增重启后 120 秒冷却期,避免模型加载期间被误判为故障
- 启动时给所有服务 60 秒初始冷却期
---
### 二、前端改动
#### 1. 新增依赖
```bash
npm install @dnd-kit/core @dnd-kit/sortable @dnd-kit/utilities
```
#### 2. `frontend/src/features/home/model/useMaterials.ts`
- `selectedMaterial: string``selectedMaterials: string[]`(多选)
- 新增 `toggleMaterial(id)`: 切换选中/取消至少保留1个
- 新增 `reorderMaterials(activeId, overId)`: 拖拽排序
- 上传格式扩展:新增 `.mkv/.webm/.flv/.wmv/.m4v/.ts/.mts`
#### 3. `frontend/src/features/home/ui/MaterialSelector.tsx`(重写)
- 素材列表每行增加复选框 + 序号徽标(①②③)
- 选中 ≥2 个时显示拖拽排序区(@dnd-kit `SortableContext`
- 每个排序项:拖拽把手 + 序号 + 素材名 + 移除按钮
- HTML input accept 改为 `video/*`
#### 4. `frontend/src/features/home/model/useHomeController.ts`
- 多素材 payload`material_paths` 数组 + `material_path` 向后兼容
- `enable_subtitles` 硬编码为 `true`(移除开关)
- 验证:至少选中 1 个素材
#### 5. `frontend/src/features/home/model/useHomePersistence.ts`
- 素材持久化改为 JSON 数组,向后兼容旧格式(单字符串)
- 移除 `enableSubtitles` 持久化
#### 6. `frontend/src/features/home/ui/TitleSubtitlePanel.tsx`
- 移除"逐字高亮字幕"开关,字幕样式区始终显示
#### 7. `frontend/src/features/home/ui/HomePage.tsx`
- 更新 props 传递(`selectedMaterials`, `toggleMaterial`, `reorderMaterials`
---
### 三、Bug 修复记录
#### BUG-1: 多素材只使用第一个视频(基于标点的分句方案失败)
- **现象**: 选了 2 个素材但生成的视频只使用第 1 个,日志显示 `Multi-material: 1 segments, 2 materials`
- **根因 v1**: 最初通过正则 `[。!?!?]` 在 Whisper 输出中分句,但 Whisper 不输出标点。
- **修复 v1**: 改为用原始文案标点分句——但用户文案往往只含逗号(,),无句末标点(。!?),仍退化为 1 段。
- **最终修复**: 彻底放弃基于标点的分句方案,改为 `_split_equal()` **按素材数量均分音频时长**,对齐到最近的 Whisper 字边界。不依赖任何标点符号,对所有文案均有效。
#### BUG-2: 口型对不上(音频时间偏移)
- **根因**: `split_audio` 用 Whisper 的 start/end 时间(如 0.11~7.21)切分音频,但 `compose()` 用完整原始音频0.0~结尾)合成,导致时间偏移。
- **修复**: 强制首段 start=0.0,末段 end=音频实际时长,确保切分音频完整覆盖。
#### BUG-3: min_segment_sec 过度合并导致退化(已随方案切换移除)
- **根因**: 旧方案中 2 个句子第 2 句不足 3 秒时,最短时长检查合并为 1 段,多素材退化为单素材。
- **状态**: 均分方案不存在此问题,相关代码已移除。
---
### 涉及文件汇总
| 文件 | 变更类型 | 说明 |
|------|----------|------|
| `backend/app/modules/videos/schemas.py` | 修改 | 新增 material_paths 字段 |
| `backend/app/modules/videos/workflow.py` | 修改 | 多素材流水线核心逻辑 + 3个 Bug 修复 |
| `backend/app/services/video_service.py` | 修改 | 新增 concat_videos / split_audio |
| `backend/scripts/watchdog.py` | 修改 | 阈值优化 + 冷却期机制 |
| `frontend/package.json` | 修改 | 新增 @dnd-kit 依赖 |
| `frontend/src/features/home/model/useMaterials.ts` | 修改 | 多选 + 排序状态管理 |
| `frontend/src/features/home/ui/MaterialSelector.tsx` | 重写 | 多选复选框 + 拖拽排序 UI |
| `frontend/src/features/home/model/useHomeController.ts` | 修改 | 多素材 payload + 移除字幕开关 |
| `frontend/src/features/home/model/useHomePersistence.ts` | 修改 | JSON 数组持久化 |
| `frontend/src/features/home/ui/TitleSubtitlePanel.tsx` | 修改 | 移除字幕开关 |
| `frontend/src/features/home/ui/HomePage.tsx` | 修改 | 更新 props 传递 |
### 重启要求
```bash
pm2 restart vigent2-backend
npm run build && pm2 restart vigent2-frontend
```

221
Docs/DevLogs/Day22.md Normal file
View File

@@ -0,0 +1,221 @@
## 🔧 多素材生成优化与健壮性加固 (Day 22)
### 概述
对 Day 21 实现的多素材视频生成(多机位)功能进行全面审查,修复 6 个高优先级 Bug、完成 8 项体验优化,并将多素材流水线从"逐段 LatentSync"重构为"先拼接再推理"架构,推理次数从 N 次降为 1 次。
---
### 一、后端高优 Bug 修复
#### 1. `_split_equal()` 素材数 > 字符数边界溢出
- **问题**: 5 个素材但只有 2 个 Whisper 字符时,边界索引重复,部分素材被跳过
- **修复**: 加入 `n = min(n, len(all_chars))` 上限保护
- **文件**: `backend/app/modules/videos/workflow.py`
#### 2. 多素材 LatentSync 单段失败无 fallback
- **问题**: 单素材模式下 LatentSync 失败会 fallback 到原始素材,但多素材模式直接抛异常,整个任务失败
- **修复**: 多素材循环中加 try-except失败时 fallback 到原始素材片段
- **文件**: `backend/app/modules/videos/workflow.py`
#### 3. `num_segments == 0` 时 ZeroDivisionError
- **问题**: 所有 assignments 被跳过后 `i / num_segments` 触发除零
- **修复**: 循环前加 `if num_segments == 0` 检查并抛出明确错误
- **文件**: `backend/app/modules/videos/workflow.py`
#### 4. `split_audio` 未校验 duration > 0
- **问题**: `end <= start` 时 FFmpeg 行为异常
- **修复**: 加入 `if duration <= 0: raise ValueError(...)`
- **文件**: `backend/app/services/video_service.py`
#### 5. Whisper 失败时按时长均分兜底
- **问题**: Whisper 失败后直接退化为单素材,其他素材被浪费
- **修复**: 按 `audio_duration / len(material_paths)` 均分,不依赖字符对齐
- **文件**: `backend/app/modules/videos/workflow.py`
#### 6. `concat_videos` 空列表未检查
- **问题**: 传入空 `video_paths` 时 FFmpeg 报错
- **修复**: 加入 `if not video_paths: raise ValueError(...)`
- **文件**: `backend/app/services/video_service.py`
---
### 二、前端优化
#### 1. payload 构建非空断言修复
- `m!.path``m?.path` + `.filter(Boolean)`,防止素材被删后 crash
- **文件**: `frontend/src/features/home/model/useHomeController.ts`
#### 2. 生成按钮展示后端进度消息
- 新增 `message` prop生成中显示如"(正在处理片段 2/3...)"
- **文件**: `frontend/src/features/home/ui/GenerateActionBar.tsx`, `HomePage.tsx`
#### 3. 新上传素材自动选中
- 上传成功后对比前后素材列表,新增的 ID 自动追加到 `selectedMaterials`
- **文件**: `frontend/src/features/home/model/useMaterials.ts`
#### 4. Material 接口统一
- 三处 `interface Material` 重复定义提取到 `shared/types/material.ts`
- **文件**: `frontend/src/shared/types/material.ts` (新建), `useMaterials.ts`, `useHomeController.ts`, `MaterialSelector.tsx`
#### 5. 拖拽排序修复
- 移除 `DragOverlay``backdrop-blur` 创建新 containing block 导致定位错乱)
- 改为 `useSortable` 原生拖拽 + `CSS.Translate`,拖拽中元素高亮加阴影
- **文件**: `frontend/src/features/home/ui/MaterialSelector.tsx`
#### 6. 素材选择上限 4 个
- `toggleMaterial` 新增 `MAX_MATERIALS = 4` 限制
- UI 选满后未选中项变半透明禁用,提示文字改为"可多选最多4个"
- **文件**: `useMaterials.ts`, `MaterialSelector.tsx`
#### 7. 移动端排序区域响应式
- 素材列表 `max-h-64``max-h-48 sm:max-h-64`
- **文件**: `MaterialSelector.tsx`
#### 8. 多素材耗时提示
- 选中 ≥2 素材时生成按钮下方显示"多素材模式 (N 个机位),生成耗时较长"
- **文件**: `GenerateActionBar.tsx`, `HomePage.tsx`
---
### 三、核心架构重构:先拼接再推理
#### V1 (Day 21): 逐段 LatentSync
```
素材A → LatentSync(素材A, 音频片段1) → lipsync_A
素材B → LatentSync(素材B, 音频片段2) → lipsync_B
FFmpeg concat(lipsync_A, lipsync_B) → 最终视频
```
- 缺点N 个素材 = N 次 LatentSync 推理(每次 ~30s
#### V2 (Day 22): 先拼接再推理
```
素材A → prepare_segment(裁剪到3.67s) → prepared_A
素材B → prepare_segment(裁剪到4.00s) → prepared_B
FFmpeg concat(prepared_A, prepared_B) → concat_video (7.67s)
LatentSync(concat_video, 完整音频) → 最终视频
```
- 优点:只需 **1 次** LatentSync 推理,时间从 N×30s 降为 1×30s
#### 新增 `prepare_segment()` 方法
```python
def prepare_segment(self, video_path, target_duration, output_path, target_resolution=None):
# 素材时长 > 目标: 裁剪 (-t)
# 素材时长 < 目标: 循环 (-stream_loop) + 裁剪
# 分辨率一致: -c copy 无损 (不重编码)
# 分辨率不一致: scale + pad 统一到第一个素材分辨率
```
#### 分辨率处理策略
- 新增 `get_resolution()` 方法检测各素材分辨率
- 所有素材分辨率相同时:`-c copy` 无损裁剪(保持原画质)
- 分辨率不一致时:统一到第一个素材的分辨率,`force_original_aspect_ratio=decrease` + `pad` 居中
- LatentSync 只处理嘴部 512×512 区域,输出保持原分辨率
#### 时间对齐验证
| 环节 | 时间基准 | 对齐关系 |
|------|---------|---------|
| TTS 音频 | 原始时长 (7.67s) | 基准 |
| Whisper 字幕 | 基于 TTS 音频 | 时间戳对齐音频 |
| 均分切分 | assignments 总时长 = 音频时长 | 首段 start=0, 末段 end=audio_duration |
| prepare 各段 | `-t seg_dur` 精确截断 | 总和 ≈ 音频时长 |
| LatentSync | concat_video + 完整音频 | 内部 0.5s 容差 |
| compose | lipsync_video + 音频/BGM | `-shortest` 保证同步 |
| Remotion | 基于 captions_path 渲染字幕 | 时间戳对齐音频 |
---
### 涉及文件汇总
| 文件 | 变更类型 | 说明 |
|------|----------|------|
| `backend/app/modules/videos/workflow.py` | 修改 | 6 个 Bug 修复 + 流水线重构(先拼接再推理)|
| `backend/app/services/video_service.py` | 修改 | 新增 `prepare_segment()``get_resolution()``split_audio` 校验,`concat_videos` 空列表检查 |
| `frontend/src/shared/types/material.ts` | 新建 | 统一 Material 接口 |
| `frontend/src/features/home/model/useMaterials.ts` | 修改 | 上传自动选中、素材上限 4 个 |
| `frontend/src/features/home/model/useHomeController.ts` | 修改 | payload 非空断言修复、Material 接口引用 |
| `frontend/src/features/home/ui/MaterialSelector.tsx` | 修改 | 拖拽修复、上限 4 个 UI、移动端响应式 |
| `frontend/src/features/home/ui/GenerateActionBar.tsx` | 修改 | 进度消息展示、多素材耗时提示 |
| `frontend/src/features/home/ui/HomePage.tsx` | 修改 | 传递 message、materialCount prop |
---
### 四、AI 多语言翻译
#### 功能
在文案编辑区新增「AI多语言」按钮支持将中文口播文案一键翻译为 9 种语言,并可随时还原原文。
#### 支持语言
英语 English、日语 日本語、韩语 한국어、法语 Français、德语 Deutsch、西班牙语 Español、俄语 Русский、意大利语 Italiano、葡萄牙语 Português
#### 实现
##### 后端
- **`backend/app/services/glm_service.py`** — 新增 `translate_text()` 方法,调用智谱 GLM APItemperature=0.3prompt 要求只返回译文、保持语气风格
- **`backend/app/modules/ai/router.py`** — 新增 `POST /api/ai/translate` 接口,接收 `{text, target_lang}`,返回 `{translated_text}`
##### 前端
- **`frontend/src/features/home/ui/ScriptEditor.tsx`** — 新增 `LANGUAGES` 列表9 种语言)、语言下拉菜单(点击外部自动关闭)、翻译中 loading 状态、「还原原文」按钮(翻译过后出现在菜单顶部)
- **`frontend/src/features/home/model/useHomeController.ts`** — 新增 `handleTranslate`(调用翻译 API、首次翻译保存原文`originalText` 状态、`handleRestoreOriginal`(恢复原文)
#### 涉及文件
| 文件 | 变更 | 说明 |
|------|------|------|
| `backend/app/services/glm_service.py` | 修改 | 新增 `translate_text()` 方法 |
| `backend/app/modules/ai/router.py` | 修改 | 新增 `/api/ai/translate` 接口 |
| `frontend/src/features/home/ui/ScriptEditor.tsx` | 修改 | 语言菜单 UI、翻译 loading、还原原文按钮 |
| `frontend/src/features/home/model/useHomeController.ts` | 修改 | `handleTranslate``originalText``handleRestoreOriginal` |
---
### 五、TTS 多语言支持
#### 背景
翻译功能实现后,用户可将中文文案翻译为其他语言。但翻译后生成视频时 TTS 仍只支持中文:
- **EdgeTTS**:声音列表只有 5 个 `zh-CN-*` 中文声音
- **声音克隆 (Qwen3-TTS)**`language` 参数硬编码为 `"Chinese"`
#### 实现方案
##### 1. 前端:语言感知的声音列表
- `VOICES` 从扁平数组扩展为 `Record<string, VoiceOption[]>`,覆盖 10 种语言zh-CN / en-US / ja-JP / ko-KR / fr-FR / de-DE / es-ES / ru-RU / it-IT / pt-BR每种语言 2 个声音(男/女)
- 新增 `LANG_TO_LOCALE` 映射:翻译目标语言名 → EdgeTTS locale`"English" → "en-US"`
- 新增 `textLang` 状态,跟踪当前文案语言,默认 `"zh-CN"`
##### 2. 翻译时自动切换声音
- `handleTranslate` 成功后:根据目标语言设置 `textLang`EdgeTTS 模式下自动切换 `voice` 为目标语言的默认声音
- `handleRestoreOriginal` 还原时:重置 `textLang``"zh-CN"`,恢复中文默认声音
- `VoiceSelector` 根据 `textLang` 动态显示对应语言的声音列表
##### 3. 声音克隆语言透传
- 前端:新增 `LOCALE_TO_QWEN_LANG` 映射(`zh-CN→"Chinese"`, `en-US→"English"`, 其他→`"Auto"`
- 生成请求 payload 加入 `language` 字段(仅声音克隆模式)
- 后端 `GenerateRequest` schema 新增 `language: str = "Chinese"` 字段
- `workflow.py``language="Chinese"` 硬编码改为 `language=req.language`
##### 4. Bug 修复textLang 持久化
- **问题**: `voice` 已持久化但 `textLang` 未持久化,刷新页面后 `voice` 恢复为英文声音但 `textLang` 默认回中文,导致 VoiceSelector 显示中文声音列表却选中英文声音,无高亮按钮
- **修复**: 在 `useHomePersistence` 中加入 `textLang` 的 localStorage 读写
#### 数据流
```
用户翻译 "English"
→ ScriptEditor.onTranslate("English")
→ LANG_TO_LOCALE["English"] = "en-US"
→ setTextLang("en-US"), setVoice("en-US-GuyNeural")
→ VoiceSelector 显示 VOICES["en-US"] = [Guy, Jenny]
→ 生成时:
EdgeTTS: payload.voice = "en-US-GuyNeural"
声音克隆: payload.language = "English" (via getQwenLanguage)
```
#### 涉及文件
| 文件 | 变更 | 说明 |
|------|------|------|
| `frontend/src/features/home/model/useHomeController.ts` | 修改 | VOICES 多语言 Record、textLang 状态、LANG_TO_LOCALE / LOCALE_TO_QWEN_LANG 映射、翻译自动切换 voice |
| `frontend/src/features/home/model/useHomePersistence.ts` | 修改 | textLang 持久化读写 |
| `backend/app/modules/videos/schemas.py` | 修改 | GenerateRequest 加 `language` 字段 |
| `backend/app/modules/videos/workflow.py` | 修改 | 声音克隆调用处用 `req.language` 替代硬编码 |

546
Docs/DevLogs/Day23.md Normal file
View File

@@ -0,0 +1,546 @@
## 🎙️ 配音前置重构 — 第一阶段 (Day 23)
### 概述
将配音从视频生成流程中独立出来,实现"先生成配音 → 选中配音 → 再选素材 → 生成视频"的新工作流。用户可以独立管理配音(生成/试听/改名/删除/选择),并在选中配音后看到时长信息,为第二阶段的素材时间轴编排奠定数据基础。
**旧流程**: 文案 + 选素材 → 一键生成(内联 TTS → Whisper → 均分 → LipSync → 合成)
**新流程**: 文案 → 配音方式 → **生成配音** → 选中配音 → 选素材 → 背景音乐 → 生成视频
---
### 一、后端:新增 `generated_audios` 模块
#### 模块结构
```
backend/app/modules/generated_audios/
├── __init__.py
├── router.py # 5 个 API 端点
├── schemas.py # 请求/响应模型
└── service.py # 生成/列表/删除/改名
```
#### API 端点
| 方法 | 路径 | 说明 |
|------|------|------|
| POST | `/api/generated-audios/generate` | 异步生成配音(返回 task_id |
| GET | `/api/generated-audios/tasks/{task_id}` | 轮询生成进度 |
| GET | `/api/generated-audios` | 列出用户所有配音 |
| DELETE | `/api/generated-audios/{audio_id}` | 删除配音 |
| PUT | `/api/generated-audios/{audio_id}` | 改名 |
#### 存储方案
- Supabase 存储桶:`generated-audios`(启动时自动创建)
- 音频文件:`{user_id}/{timestamp}_audio.wav`
- 元数据文件:`{user_id}/{timestamp}_audio.json`(含 display_name、text、tts_mode、duration_sec 等)
#### 生成流程
复用现有 `TTSService` / `voice_clone_service` / `task_store`
```
POST /generate → 创建 task → BackgroundTask:
1. edgetts → TTSService.generate_audio()
voiceclone → 下载 ref_audio → voice_clone_service.generate_audio()
2. ffprobe 获取时长
3. 上传 .wav + .json 到 generated-audios 桶
4. 更新 task(status=completed, output={audio_id, duration_sec, ...})
```
---
### 二、后端:修改视频生成 workflow
#### `GenerateRequest` 新增字段
```python
generated_audio_id: Optional[str] = None # 预生成配音 ID存在时跳过内联 TTS
```
#### `workflow.py` TTS 阶段新增分支
```python
if req.generated_audio_id:
# 下载预生成配音 + 从元数据读取 language
elif req.tts_mode == "voiceclone":
# 原有声音克隆逻辑
else:
# 原有 EdgeTTS 逻辑
```
向后兼容:不传 `generated_audio_id` 时,原有内联 TTS 流程不受影响。
---
### 三、前端:新增配音列表 hook + 面板
#### `useGeneratedAudios.ts`
- 状态:`generatedAudios[]``selectedAudio``isGeneratingAudio``audioTask`
- 方法:`fetchGeneratedAudios()``generateAudio()``deleteAudio()``renameAudio()``selectAudio()`
- 轮询:生成后 1s 轮询 task 状态,完成后自动刷新列表并选中最新配音
- 独立于视频生成的 TaskContext不互相干扰
#### `GeneratedAudiosPanel.tsx`
- 每条配音:播放/暂停、名称、时长、重命名、删除
- 选中态:`border-purple-500 bg-purple-500/20`
- 内嵌进度条(生成中显示)
- 底部显示选中配音的原始文案(截断)
- 播放逻辑自包含于面板内(`new Audio()` + play/pause toggle
---
### 四、前端UI 面板重排序
**旧顺序**: MaterialSelector → ScriptEditor → TitleSubtitle → VoiceSelector → BgmPanel → GenerateActionBar
**新顺序**:
1. ScriptEditor文案编辑
2. TitleSubtitlePanel标题与字幕样式
3. VoiceSelector配音方式
4. **GeneratedAudiosPanel**(配音列表)← 新增
5. MaterialSelector视频素材← 后移,需选中配音才解锁
6. BgmPanel背景音乐
7. GenerateActionBar生成视频
#### 素材区门控
未选中配音时,素材区显示半透明遮罩 + "请先生成并选中配音"提示。素材上传/预览/改名/删除始终可用,仅选择勾选被遮罩。
#### 时长信息
选中配音后MaterialSelector 顶部显示:
```
当前配音: 45.2 秒 | 已选 3 个素材(自动均分每段 ~15.1 秒)
```
#### 生成按钮条件更新
```typescript
// 旧条件
disabled={isGenerating || selectedMaterials.length === 0 || (ttsMode === "voiceclone" && !selectedRefAudio)}
// 新条件
disabled={isGenerating || selectedMaterials.length === 0 || !selectedAudio}
```
---
### 五、持久化
`useHomePersistence` 新增 `selectedAudioId` 的 localStorage 读写,刷新页面后恢复选中的配音。
---
### 涉及文件汇总
#### 后端新增
| 文件 | 说明 |
|------|------|
| `backend/app/modules/generated_audios/__init__.py` | 模块标记 |
| `backend/app/modules/generated_audios/router.py` | 5 个 API 端点 |
| `backend/app/modules/generated_audios/service.py` | 生成/列表/删除/改名 |
| `backend/app/modules/generated_audios/schemas.py` | 请求/响应模型 |
#### 后端修改
| 文件 | 变更 |
|------|------|
| `backend/app/main.py` | 注册 generated_audios 路由 |
| `backend/app/services/storage.py` | 新增 `BUCKET_GENERATED_AUDIOS`,启动时自动创建桶 |
| `backend/app/modules/videos/schemas.py` | `GenerateRequest` 新增 `generated_audio_id` 字段 |
| `backend/app/modules/videos/workflow.py` | TTS 阶段新增预生成音频分支 |
#### 前端新增
| 文件 | 说明 |
|------|------|
| `frontend/src/features/home/model/useGeneratedAudios.ts` | 配音列表 hook |
| `frontend/src/features/home/ui/GeneratedAudiosPanel.tsx` | 配音列表面板 |
#### 前端修改
| 文件 | 变更 |
|------|------|
| `frontend/src/features/home/ui/HomePage.tsx` | 面板重排序 + 素材区门控 + 插入 GeneratedAudiosPanel |
| `frontend/src/features/home/ui/MaterialSelector.tsx` | 新增 `selectedAudioDuration` prop + 时长信息显示 |
| `frontend/src/features/home/ui/GenerateActionBar.tsx` | 禁用条件改为 `!selectedAudio` |
| `frontend/src/features/home/model/useHomeController.ts` | 集成 useGeneratedAudios、新增 handleGenerateAudio、修改 handleGenerate 使用 generated_audio_id |
| `frontend/src/features/home/model/useHomePersistence.ts` | 新增 selectedAudioId 持久化 |
---
## 🎞️ 素材时间轴编排 — 第二阶段 (Day 23)
### 概述
在第一阶段"配音前置"基础上,新增**时间轴编辑器**,用户可以:
1. 在音频波形上查看各素材块的时长分配
2. 拖拽分割线调整每段素材的时长(无缝铺满,调整一段自动压缩/扩展相邻段)
3. 为每段素材设置**源视频截取起点**(从视频任意位置开始,而非始终从头)
**旧行为**: 多素材时自动均分(`_split_equal`),无法控制每段时长和源视频起始点
**新行为**: 时间轴编辑器可视化分配 + 拖拽调整 + ClipTrimmer 截取设置
---
### 一、后端改动
#### 1.1 新增 `CustomAssignment` 模型
```python
# backend/app/modules/videos/schemas.py
class CustomAssignment(BaseModel):
material_path: str
start: float # 音频时间轴起点
end: float # 音频时间轴终点
source_start: float = 0.0 # 源视频截取起点
```
`GenerateRequest` 新增 `custom_assignments: Optional[List[CustomAssignment]] = None`。存在时跳过 Whisper 均分,直接使用用户定义的分配。
#### 1.2 `prepare_segment` 支持 `source_start`
```python
def prepare_segment(self, video_path, target_duration, output_path,
target_resolution=None, source_start: float = 0.0):
```
关键逻辑:
- `source_start > 0` 时使用 `-ss` 快速 seek并强制重编码避免 stream copy 关键帧不精确)
- 当需要循环且有 `source_start` 时,先裁剪出 `source_start` 到视频结尾的片段,再循环裁剪后的文件(避免 `stream_loop` 从视频 0s 开始循环)
- 裁剪临时文件在 `finally` 中自动清理
#### 1.3 `workflow.py` 支持 `custom_assignments`
- **多素材模式**: `custom_assignments` 存在时,直接使用用户分配(仍运行 Whisper 生成字幕),每个 `prepare_segment` 调用传入 `source_start`
- **单素材模式**: `custom_assignments` 有 1 条且 `source_start > 0` 时,先截取片段再传入 LatentSync
- **向后兼容**: `custom_assignments``None` 时完全走旧路径
---
### 二、前端新增组件
#### 2.1 `useTimelineEditor.ts` — 时间轴段管理 hook
```typescript
interface TimelineSegment {
id: string; // React key
materialId: string; // 素材 ID
materialName: string; // 显示名
start: number; // 音频时间轴开始秒数
end: number; // 音频时间轴结束秒数
sourceStart: number; // 源视频截取起点(默认 0
sourceEnd: number; // 源视频截取终点0 = 到结尾)
color: string; // 色块颜色
}
```
核心方法:
- `initSegments()`: selectedMaterials 变化时按数量均分 audioDuration
- `resizeSegment(id, newEnd)`: 拖拽右边界,约束每段最小 1s
- `setSourceRange(id, sourceStart, sourceEnd)`: 设置截取范围
- `toCustomAssignments()`: 转为后端 `CustomAssignment[]` 格式
#### 2.2 `TimelineEditor.tsx` — 波形 + 色块时间轴
- **wavesurfer.js** 渲染音频波形(仅展示,不播放)
- 色块层按比例排列,显示素材名 + 时长 + 截取标记
- 色块间分割线可拖拽(`onPointerDown/Move/Up` 实现连续像素拖拽)
- 点击色块打开 ClipTrimmer
#### 2.3 `ClipTrimmer.tsx` — 素材截取模态框
- HTML5 `<video>` 实时预览,拖拽滑块时 `video.currentTime` 跟随
- 双端 Range Slider起点/终点),互锁约束 ≥ 0.5s
- 显示截取时长 vs 分配时长对比(循环补足/截断提示)
- `loadedmetadata` 获取源视频时长
---
### 三、前端整合改动
#### 3.1 `useHomeController.ts`
- 集成 `useTimelineEditor` hook
- 新增 `clipTrimmerOpen` / `clipTrimmerSegmentId` 状态
- `handleGenerate` 多素材时始终发送 `custom_assignments`;单素材 + `sourceStart > 0` 时也发送
- 移除不再使用的 `reorderMaterials` 导出
#### 3.2 `HomePage.tsx`
- 在 MaterialSelector 和 BgmPanel 之间插入 TimelineEditor仅当有配音且已选素材时显示
- 底部新增 ClipTrimmer 模态框
- 移除 `reorderMaterials``selectedAudioDuration` prop 传递
#### 3.3 `MaterialSelector.tsx`
- 移除配音时长信息栏(功能迁至 TimelineEditor
- 移除拖拽排序区SortableChip + @dnd-kit 相关代码)
- 移除 `onReorderMaterials` / `selectedAudioDuration` prop
---
### 四、审查修复的 Bug
| # | 严重程度 | 问题 | 修复 |
|---|---------|------|------|
| 1 | **中** | `prepare_segment` 使用 `source_start > 0` + stream copy 时 seek 不精确 | 添加 `source_start > 0` 到重编码条件 |
| 2 | **高** | `stream_loop + source_start` 循环时从视频 0s 开始而非从 source_start 循环 | 改为两步:先裁剪片段再循环裁剪后的文件 |
| 3 | **低** | `useHomeController` 导出已废弃的 `reorderMaterials` | 移除 |
---
### 涉及文件汇总
#### 后端修改
| 文件 | 变更 |
|------|------|
| `backend/app/modules/videos/schemas.py` | 新增 `CustomAssignment` model`GenerateRequest` 新增 `custom_assignments` 字段 |
| `backend/app/services/video_service.py` | `prepare_segment` 新增 `source_start` 参数,循环+截取两步处理 |
| `backend/app/modules/videos/workflow.py` | 多素材/单素材流水线支持 `custom_assignments`,传递 `source_start` |
#### 前端新增
| 文件 | 说明 |
|------|------|
| `frontend/src/features/home/model/useTimelineEditor.ts` | 时间轴段管理 hook |
| `frontend/src/features/home/ui/TimelineEditor.tsx` | 波形 + 色块时间轴组件 |
| `frontend/src/features/home/ui/ClipTrimmer.tsx` | 素材截取模态框 |
#### 前端修改
| 文件 | 变更 |
|------|------|
| `frontend/src/features/home/ui/HomePage.tsx` | 插入 TimelineEditor + ClipTrimmer |
| `frontend/src/features/home/ui/MaterialSelector.tsx` | 移除时长信息 + 拖拽排序区 + 相关 prop |
| `frontend/src/features/home/model/useHomeController.ts` | 集成 useTimelineEditorhandleGenerate 发送 custom_assignments |
| `frontend/package.json` | 新增 `wavesurfer.js` 依赖 |
---
## 🎨 UI 体验优化 + TTS 稳定性修复 — 第三阶段 (Day 23)
### 概述
根据用户反馈,修复 6 项 UI 体验问题,同时修复 Qwen3-TTS 声音克隆服务的 SoX 路径问题和显存缓存管理。
---
### 一、Qwen3-TTS 稳定性修复
#### 1.1 SoX PATH 修复
**问题**: PM2 启动 qwen-tts 时,`sox` 工具安装在 conda env 的 bin 目录中,系统 PATH 找不到,导致音频编解码走 fallback 路径CPU 密集型),日志中出现 `SoX could not be found!` 警告。
**修复**: `run_qwen_tts.sh` 中 export conda env bin 到 PATH
```bash
export PATH="/home/rongye/ProgramFiles/miniconda3/envs/qwen-tts/bin:$PATH"
```
#### 1.2 CUDA 缓存清理
**修复**: `qwen_tts_server.py` 每次生成完成后(无论成功或失败)调用 `torch.cuda.empty_cache()`,防止显存碎片累积。使用 `asyncio.to_thread()` 在线程池中运行推理,避免阻塞事件循环导致健康检查超时。
---
### 二、配音列表按钮布局统一 (反馈 #1 + #6)
**问题**: `GeneratedAudiosPanel` 的试听按钮位于左侧(独立于 Edit/Delete`RefAudioPanel` 的布局不一致。底部文案摘要区域不需要展示。
**修复**:
- Play/Edit/Delete 按钮统一放在右侧同组hover 显示,顺序为 试听→重命名→删除
- 移除选中配音的文案摘要区域
- 布局与 RefAudioPanel 一致:左侧名称+时长,右侧操作按钮组
---
### 三、视频素材区域移除配音依赖遮罩 (反馈 #2)
**问题**: MaterialSelector 被 `!selectedAudio` 遮罩覆盖,必须先选配音才能操作素材。
**修复**: 移除 `HomePage.tsx` 中 MaterialSelector 外层的 disabled overlay `<div>`。素材随时可上传/预览/管理,仅 TimelineEditor 需要选中配音才显示(已有独立条件 `selectedAudio && selectedMaterials.length > 0`)。
---
### 四、时间轴拖拽排序 (反馈 #3)
**问题**: TimelineEditor 不支持调换素材顺序。
**修复**:
- `useTimelineEditor` 已有 `reorderSegments()` 方法(交换两个段的素材信息但保留时间范围)
- 通过 `useHomeController` 暴露 `reorderSegments`,传入 `TimelineEditor`
- 色块支持 HTML5 Drag & Drop`draggable` + `onDragStart/Over/Drop/End`
- 拖拽时:源色块半透明(`opacity-50`),目标色块高亮 ring`ring-2 ring-purple-400 scale-[1.02]`
- 光标样式:`cursor-grab` / `active:cursor-grabbing`
---
### 五、截取设置双手柄 Range Slider (反馈 #4)
**问题**: ClipTrimmer 使用两个独立的 `<input type="range">` 滑块,起点和终点分开操作,体验不直观。
**修复**: 改为自定义双手柄 range slider
- 单条轨道,紫色圆形手柄(起点)+ 粉色圆形手柄(终点)
- 轨道底色 `bg-white/10`,选中范围用素材对应颜色高亮
- Pointer Events 实现拖拽:`onPointerDown` 捕获手柄 → `onPointerMove` 更新位置 → `onPointerUp` 释放
- 手柄互锁约束:起点不超过终点 - 0.5s,终点不低于起点 + 0.5s
- 底部显示起点(紫色)和终点(粉色)时间标签
---
### 六、截取设置视频预览 (反馈 #5)
**问题**: ClipTrimmer 的视频只能静态查看,无法播放预览截取范围。
**修复**:
- 视频区域点击可播放/暂停Play/Pause 图标覆盖层)
- 播放范围:从 sourceStart 播放到 sourceEnd 自动停止
- 播放结束后回到起点
- 拖拽手柄时 `video.currentTime` 实时跟随seek 到当前位置查看画面)
- 播放进度条(白色竖线)叠加在 range slider 轨道上
- `preload="auto"` 预加载视频,确保拖拽时快速 seek
---
### 涉及文件汇总
#### 后端修改
| 文件 | 变更 |
|------|------|
| `run_qwen_tts.sh` | export conda env bin 到 PATH修复 SoX 找不到问题 |
| `models/Qwen3-TTS/qwen_tts_server.py` | 每次生成后 `torch.cuda.empty_cache()`asyncio.to_thread 避免阻塞 |
#### 前端修改
| 文件 | 变更 |
|------|------|
| `frontend/src/features/home/ui/GeneratedAudiosPanel.tsx` | 按钮布局统一Play/Edit/Delete 右侧同组),移除文案摘要 |
| `frontend/src/features/home/ui/HomePage.tsx` | 移除 MaterialSelector 配音遮罩,传入 onReorderSegment |
| `frontend/src/features/home/ui/TimelineEditor.tsx` | 新增 HTML5 Drag & Drop 排序,新增 onReorderSegment prop |
| `frontend/src/features/home/ui/ClipTrimmer.tsx` | 双手柄 range slider + 视频播放预览 + 播放进度指示 |
| `frontend/src/features/home/model/useHomeController.ts` | 暴露 reorderSegments 方法 |
---
## 📝 历史文案保存 + 时间轴拖拽修复 — 第四阶段 (Day 23)
### 概述
新增文案手动保存与加载功能,修复时间轴拖拽排序后素材时长不跟随的 Bug统一按钮视觉规范。
---
### 一、历史文案保存与加载
#### 功能
用户可手动保存当前文案到历史列表,随时从历史中加载恢复。只有手动保存的文案才出现在历史列表中,与自动保存(`useHomePersistence`)完全独立。
#### UI 布局
```
按钮栏: [历史文案▼] [文案提取助手] [AI多语言▼] [AI生成标题标签]
底部栏: 128 字 [保存文案]
```
- **历史文案下拉**: 展示已保存列表(名称 + 日期 + 删除按钮),点击条目加载文案,空列表显示"暂无保存的文案"
- **保存文案按钮**: 文案为空时 disabled点击后 `toast.success("文案已保存")`
- **预计时长已移除**: 底部栏只保留字数 + 保存按钮
#### 实现
##### `useSavedScripts.ts`(新建)
```typescript
interface SavedScript { id: string; name: string; content: string; savedAt: number }
```
- localStorage key: `vigent_{storageKey}_savedScripts`
- `saveScript(content)`: 取前 15 字符自动命名,新条目插入列表头部,**直接写入 localStorage**
- `deleteScript(id)`: 删除指定条目,直接写入 localStorage
- `useEffect([lsKey])`: lsKey 变化时guest → userId重新从 localStorage 读取
- **不使用自动持久化 effect**,避免 storageKey 切换时空数组覆盖已有数据
##### 数据流
```
ScriptEditor (UI)
↑ savedScripts / onSaveScript / onLoadScript / onDeleteScript (纯 props + callbacks)
useHomeController
├── useSavedScripts(storageKey) → { savedScripts, saveScript, deleteScript }
└── handleSaveScript() → saveScript(text) + toast
HomePage
└── 传递 props 到 ScriptEditor
```
---
### 二、时间轴拖拽排序 Bug 修复
#### 问题
拖拽调换素材顺序后各素材的时长没有跟随素材移动而是留在原槽位。例如素材1(3s) + 素材2(8s+4s循环)拖拽后变成素材2(3s) + 素材1(8s+4s循环),时长分配没变。
#### 根因
`reorderSegments` 使用**属性交换**方式:逐个拷贝 `materialId``sourceStart``sourceEnd` 等属性在两个槽位间交换,然后调用 `recalcPositions` 重算位置。
#### 修复
改为**数组移动**splice将整个 segment 对象从旧位置取出插入到新位置。segment 对象携带全部属性materialId、sourceStart、sourceEnd、color 等)作为一个整体移动,再由 `recalcPositions` 重算位置。
```typescript
// 修复前:属性交换
const fromMat = { materialId: next[fromIdx].materialId, ... };
const toMat = { materialId: next[toIdx].materialId, ... };
next[fromIdx] = { ...next[fromIdx], ...toMat };
next[toIdx] = { ...next[toIdx], ...fromMat };
// 修复后:数组移动
const [moved] = next.splice(fromIdx, 1);
next.splice(toIdx, 0, moved);
```
附带优势3+ 素材拖拽行为从"交换"变为"插入",更符合用户直觉。
---
### 三、按钮视觉统一
#### 问题
历史文案、文案提取助手、AI多语言、AI生成标题标签 4 个按钮高度不一致AI 按钮的文本被 `<span>` 嵌套包裹导致内部布局差异。
#### 修复
- 4 个按钮统一为 `h-7 px-2.5 text-xs rounded inline-flex items-center gap-1`(固定高度 28px
- 移除 AI多语言 / AI生成标题标签 按钮内多余的 `<span>` 嵌套,改为 `<>...</>` fragment
---
### 涉及文件汇总
#### 前端新增
| 文件 | 说明 |
|------|------|
| `frontend/src/features/home/model/useSavedScripts.ts` | 历史文案 hooklocalStorage 持久化) |
#### 前端修改
| 文件 | 变更 |
|------|------|
| `frontend/src/features/home/ui/ScriptEditor.tsx` | 历史文案下拉 + 保存按钮 + 移除预计时长 + 按钮高度统一 |
| `frontend/src/features/home/model/useHomeController.ts` | 集成 useSavedScripts新增 handleSaveScript |
| `frontend/src/features/home/ui/HomePage.tsx` | 传递 savedScripts / handleSaveScript / deleteSavedScript 到 ScriptEditor |
| `frontend/src/features/home/model/useTimelineEditor.ts` | reorderSegments 从属性交换改为数组移动splice |

View File

@@ -389,7 +389,7 @@ if not qr_element:
## 📋 文档规则优化 (16:42 - 17:10)
**问题**Doc_Rules需要优化,避免误删历史内容、规范工具使用、防止任务清单遗漏
**问题**DOC_RULES需要优化,避免误删历史内容、规范工具使用、防止任务清单遗漏
**优化内容(最终版)**
@@ -411,7 +411,7 @@ if not qr_element:
- 移除无关项目组件
**修改文件**
- `Docs/Doc_Rules.md` - 包含检查清单的最终完善版
- `Docs/DOC_RULES.md` - 包含检查清单的最终完善版
---

View File

@@ -8,8 +8,8 @@
| 规则 | 说明 |
|------|------|
| **默认更新** | 更新 `DayN.md` |
| **按需更新** | `task_complete.md` 仅在用户**明确要求**时更新 |
| **默认更新** | 更新 `DayN.md``TASK_COMPLETE.md` |
| **按需更新** | 其他文档仅在内容变化涉及时更新 |
| **智能修改** | 错误→替换,改进→追加(见下方详细规则) |
| **先读后写** | 更新前先查看文件当前内容 |
| **日内合并** | 同一天的多次小修改合并为最终版本 |
@@ -23,7 +23,7 @@
| 优先级 | 文件路径 | 检查重点 |
| :---: | :--- | :--- |
| 🔥 **High** | `Docs/DevLogs/DayN.md` | **(最新日志)** 详细记录变更、修复、代码片段 |
| 🔥 **High** | `Docs/task_complete.md` | **(任务总览)** 更新 `[x]`、进度条、时间线 |
| 🔥 **High** | `Docs/TASK_COMPLETE.md` | **(任务总览)** 更新 `[x]`、进度条、时间线 |
| ⚡ **Med** | `README.md` | **(项目主页)** 功能特性、技术栈、最新截图 |
| ⚡ **Med** | `Docs/DEPLOY_MANUAL.md` | **(部署手册)** 环境变量、依赖包、启动命令变更 |
| ⚡ **Med** | `Docs/BACKEND_DEV.md` | **(后端规范)** 接口契约、模块划分、环境变量 |
@@ -186,15 +186,15 @@ new_string: "**状态**:✅ 已修复"
```
ViGent2/Docs/
├── task_complete.md # 任务总览(仅按需更新)
├── Doc_Rules.md # 本文件
├── TASK_COMPLETE.md # 任务总览(仅按需更新)
├── DOC_RULES.md # 本文件
├── BACKEND_DEV.md # 后端开发规范
├── BACKEND_README.md # 后端功能文档
├── FRONTEND_DEV.md # 前端开发规范
├── FRONTEND_README.md # 前端功能文档
├── DEPLOY_MANUAL.md # 部署手册
├── SUPABASE_DEPLOY.md # Supabase 部署文档
├── LatentSync_DEPLOY.md # LatentSync 部署文档
├── LATENTSYNC_DEPLOY.md # LatentSync 部署文档
├── QWEN3_TTS_DEPLOY.md # 声音克隆部署文档
├── SUBTITLE_DEPLOY.md # 字幕系统部署文档
└── DevLogs/
@@ -206,8 +206,16 @@ ViGent2/Docs/
## 📅 DayN.md 更新规则(日常更新)
### 更新时机
> **边开发边记录,不要等到最后才写。**
- 每完成一个功能/修复后,**立即**追加到 DayN.md
- 避免积攒到对话末尾一次性补写,容易遗漏变更
- `TASK_COMPLETE.md` 同理,重要变更完成后及时同步
### 新建判断 (对话开始前)
1. **回顾进度**:查看 `task_complete.md` 了解当前状态
1. **回顾进度**:查看 `TASK_COMPLETE.md` 了解当前状态
2. **检查日期**:查看最新 `DayN.md`
- **今天 (与当前日期相同)** → 🚨 **绝对禁止创建新文件**,必须**追加**到现有 `DayN.md` 末尾!即使是完全不同的功能模块。
- **之前 (昨天或更早)** → 创建 `Day{N+1}.md`
@@ -263,17 +271,17 @@ ViGent2/Docs/
---
## 📝 task_complete.md 更新规则(仅按需)
## 📝 TASK_COMPLETE.md 更新规则
> ⚠️ **仅当用户明确要求更新 `task_complete.md` 时才更新**
> 与 DayN.md 同步更新,记录重要变更时更新任务总览。
### 更新原则
- **格式一致性**:直接参考 `task_complete.md` 现有格式追加内容。
- **格式一致性**:直接参考 `TASK_COMPLETE.md` 现有格式追加内容。
- **进度更新**:仅在阶段性里程碑时更新进度百分比。
### 🔍 完整性检查清单 (必做)
每次更新 `task_complete.md` 时,必须**逐一检查**以下所有板块:
每次更新 `TASK_COMPLETE.md` 时,必须**逐一检查**以下所有板块:
1. **文件头部 & 导航**
- [ ] `更新时间`:必须是当天日期

View File

@@ -19,19 +19,28 @@ frontend/src/
│ │ │ ├── useHomePersistence.ts # 持久化管理
│ │ │ ├── useBgm.ts
│ │ │ ├── useGeneratedVideos.ts
│ │ │ ├── useGeneratedAudios.ts
│ │ │ ├── useMaterials.ts
│ │ │ ├── useMediaPlayers.ts
│ │ │ ├── useRefAudios.ts
│ │ │ ├── useSavedScripts.ts
│ │ │ ├── useTimelineEditor.ts
│ │ │ └── useTitleSubtitleStyles.ts
│ │ └── ui/ # UI 组件(纯 props + 回调)
│ │ ├── HomePage.tsx
│ │ ├── HomeHeader.tsx
│ │ ├── MaterialSelector.tsx
│ │ ├── ScriptEditor.tsx
│ │ ├── ScriptExtractionModal.tsx
│ │ ├── script-extraction/
│ │ │ └── useScriptExtraction.ts
│ │ ├── TitleSubtitlePanel.tsx
│ │ ├── FloatingStylePreview.tsx
│ │ ├── VoiceSelector.tsx
│ │ ├── RefAudioPanel.tsx
│ │ ├── GeneratedAudiosPanel.tsx
│ │ ├── TimelineEditor.tsx
│ │ ├── ClipTrimmer.tsx
│ │ ├── BgmPanel.tsx
│ │ ├── GenerateActionBar.tsx
│ │ ├── PreviewPanel.tsx
@@ -55,11 +64,11 @@ frontend/src/
│ ├── types/
│ │ ├── user.ts # User 类型定义
│ │ └── publish.ts # 发布相关类型
│ └── contexts/ # 已迁移的 Context
├── contexts/ # 全局 ContextAuth、Task
│ └── contexts/ # 全局 ContextAuth、Task
│ ├── AuthContext.tsx
│ └── TaskContext.tsx
├── components/ # 遗留通用组件
── VideoPreviewModal.tsx
│ └── ScriptExtractionModal.tsx
── VideoPreviewModal.tsx
└── proxy.ts # Next.js middleware路由保护
```
@@ -278,8 +287,8 @@ import { formatDate } from '@/shared/lib/media';
- `shared/lib`通用工具函数media.ts / auth.ts / title.ts
- `shared/hooks`:跨功能通用 hooks
- `shared/types`跨功能实体类型User / PublishVideo 等)
- `contexts/`:全局 ContextAuthContext / TaskContext
- `components/`遗留通用组件VideoPreviewModal
- `shared/contexts`:全局 ContextAuthContext / TaskContext
- `components/`遗留通用组件VideoPreviewModal
## 类型定义规范
@@ -298,6 +307,15 @@ import { formatDate } from '@/shared/lib/media';
- 标题字号 / 字幕字号
- 背景音乐选择 / 音量 / 开关状态
- 素材选择 / 历史作品选择
- 选中配音 ID (`selectedAudioId`)
- 时间轴段信息 (`useTimelineEditor` 的 localStorage)
### 历史文案(独立持久化)
`useSavedScripts` hook 独立管理历史文案的 localStorage 持久化:
- key: `vigent_{storageKey}_savedScripts`
- 仅在用户手动保存/删除时写入 localStorage不使用自动持久化 effect
-`useHomePersistence` 完全独立,互不影响
### 实施规范
- 使用 `storageKey = userId || 'guest'`,按用户隔离。

View File

@@ -17,7 +17,9 @@ ViGent2 的前端界面,采用 Next.js 16 + TailwindCSS 构建。
- **作品预览**: 生成完成后直接播放下载(作品预览 + 历史作品)。
- **预览优化**: 预览视频 `metadata` 预取,首帧加载更快。
- **本地保存**: 文案/标题/偏好由 `useHomePersistence` 统一持久化,刷新后恢复 (Day 14/17)。
- **历史文案**: 手动保存/加载/删除历史文案,独立 localStorage 持久化 (Day 23)。
- **选择持久化**: 首页/发布页作品选择均使用稳定 `id` 持久化,刷新保持用户选择;新视频生成后自动选中最新 (Day 21)。
- **AI 多语言翻译**: 支持 9 种目标语言翻译文案 + 还原原文 (Day 22)。
### 2. 全自动发布 (`/publish`) [Day 7 新增]
- **多平台管理**: 统一管理抖音、微信视频号、B站、小红书账号状态。
@@ -35,8 +37,17 @@ ViGent2 的前端界面,采用 Next.js 16 + TailwindCSS 构建。
- **TTS 模式选择**: EdgeTTS (预设音色) / 声音克隆 (自定义音色) 切换。
- **参考音频管理**: 上传/列表/删除参考音频 (3-20秒 WAV)。
- **一键克隆**: 选择参考音频后自动调用 Qwen3-TTS 服务。
- **多语言支持**: EdgeTTS 10 语言声音列表,声音克隆 language 透传 (Day 22)。
### 4. 字幕与标题 [Day 13 新增]
### 4. 配音前置 + 时间轴编排 [Day 23 新增]
- **配音独立生成**: 先生成配音 → 选中配音 → 再选素材 → 生成视频。
- **配音管理面板**: 生成/试听/改名/删除/选中,异步生成 + 进度轮询。
- **时间轴编辑器**: wavesurfer.js 音频波形 + 色块可视化素材分配,拖拽分割线调整各段时长。
- **素材截取设置**: ClipTrimmer 双手柄 range slider + HTML5 视频预览播放。
- **拖拽排序**: 时间轴色块支持 HTML5 Drag & Drop 调换素材顺序。
- **自定义分配**: 后端 `custom_assignments` 支持用户定义的素材分配方案。
### 5. 字幕与标题 [Day 13 新增]
- **片头标题**: 可选输入,限制 15 字,视频开头显示 3 秒淡入淡出标题。
- **标题同步**: 首页片头标题修改会同步到发布信息标题。
- **逐字高亮字幕**: 卡拉OK效果默认开启可关闭。
@@ -45,16 +56,16 @@ ViGent2 的前端界面,采用 Next.js 16 + TailwindCSS 构建。
- **默认样式**: 标题 90px 站酷快乐体;字幕 60px 经典黄字 + DingTalkJinBuTi (Day 17)。
- **样式持久化**: 标题/字幕样式与字号刷新保留 (Day 17)。
### 5. 背景音乐 [Day 16 新增]
### 6. 背景音乐 [Day 16 新增]
- **试听预览**: 点击试听即选中,音量滑块实时生效。
- **混音控制**: 仅影响 BGM配音保持原音量。
### 6. 账户设置 [Day 15 新增]
### 7. 账户设置 [Day 15 新增]
- **手机号登录**: 11位中国手机号验证登录。
- **账户下拉菜单**: 显示有效期 + 修改密码 + 安全退出。
- **修改密码**: 弹窗输入当前密码与新密码,修改后强制重新登录。
### 7. 文案提取助手 (`ScriptExtractionModal`) [Day 15 新增]
### 8. 文案提取助手 (`ScriptExtractionModal`) [Day 15 新增]
- **多源提取**: 支持文件拖拽上传与 URL 粘贴 (B站/抖音/TikTok)。
- **AI 洗稿**: 集成 GLM-4.7-Flash自动改写为口播文案。
- **一键填入**: 提取结果直接填充至视频生成输入框。
@@ -66,6 +77,7 @@ ViGent2 的前端界面,采用 Next.js 16 + TailwindCSS 构建。
- **样式**: TailwindCSS
- **图标**: Lucide React
- **组件**: 自定义现代化组件 (Glassmorphism 风格)
- **音频波形**: wavesurfer.js (时间轴编辑器)
- **API**: Axios 实例 `@/shared/api/axios` (对接后端 FastAPI :8006)
## 🚀 开发指南

View File

@@ -298,12 +298,20 @@ Response: audio/wav 文件
SoX could not be found!
```
**解决**: 通过 conda 安装 sox
**解决**:
1. 通过 conda 安装 sox
```bash
conda install -y -c conda-forge sox
```
2. 确保启动脚本 `run_qwen_tts.sh` 中已 export conda env bin 到 PATHPM2 启动时系统 PATH 不含 conda 环境目录):
```bash
export PATH="/home/rongye/ProgramFiles/miniconda3/envs/qwen-tts/bin:$PATH"
```
### CUDA 内存不足
Qwen3-TTS 1.7B 通常需要 8-10GB VRAM。如果遇到 OOM
@@ -371,6 +379,7 @@ FOR INSERT TO anon WITH CHECK (bucket_id = 'ref-audios');
| 日期 | 版本 | 说明 |
|------|------|------|
| 2026-02-09 | 1.2.0 | 修复 SoX PATH 问题run_qwen_tts.sh export conda bin每次生成后 empty_cache() |
| 2026-01-30 | 1.1.0 | 明确默认模型升级为 1.7B-Base替换旧版 0.6B 路径 |
---

View File

@@ -15,9 +15,13 @@
原有流程:
文本 → EdgeTTS → 音频 → LatentSync → FFmpeg合成 → 最终视频
新流程:
文本 → EdgeTTS → 音频 ─┬→ LatentSync → 唇形视频 ─┐
└→ faster-whisper → 字幕JSON ─┴→ Remotion合成 → 最终视频
新流程 (单素材):
文本 → EdgeTTS/Qwen3-TTS/预生成配音 → 音频 ─┬→ LatentSync → 唇形视频 ─┐
└→ faster-whisper → 字幕JSON ─┴→ Remotion合成 → 最终视频
新流程 (多素材):
音频 → 多素材按 custom_assignments 拼接 → LatentSync (单次推理) → 唇形视频 ─┐
音频 → faster-whisper → 字幕JSON ─────────────────────────────────────────────┴→ Remotion合成 → 最终视频
```
## 系统要求
@@ -140,7 +144,7 @@ remotion/
| 阶段 | 进度 | 说明 |
|------|------|------|
| 下载素材 | 0% → 5% | 从 Supabase 下载输入视频 |
| TTS 语音生成 | 5% → 25% | EdgeTTS Qwen3-TTS 生成音频 |
| TTS 语音生成 | 5% → 25% | EdgeTTS / Qwen3-TTS / 预生成配音下载 |
| 唇形同步 | 25% → 80% | LatentSync 推理 |
| 字幕对齐 | 80% → 85% | faster-whisper 生成字级别时间戳 |
| Remotion 渲染 | 85% → 95% | 合成字幕和标题 |
@@ -282,4 +286,5 @@ WhisperService(device="cuda:0") # 或 "cuda:1"
| 日期 | 版本 | 说明 |
|------|------|------|
| 2026-01-29 | 1.0.0 | 初始版本,使用 faster-whisper + Remotion 实现逐字高亮字幕和片头标题 |
| 2026-02-10 | 1.1.0 | 更新架构图:多素材 concat-then-infer、预生成配音选项 |
| 2026-01-30 | 1.0.1 | 字幕高亮样式与标题动画优化,视觉表现更清晰 |

View File

@@ -1,8 +1,8 @@
# ViGent2 开发任务清单 (Task Log)
**项目**: ViGent2 数字人口播视频生成系统
**进度**: 100% (Day 21 - 缺陷修复与持久化回归治理)
**更新时间**: 2026-02-08
**进度**: 100% (Day 23 - 配音前置重构 + 素材时间轴编排 + UI 体验优化)
**更新时间**: 2026-02-10
---
@@ -10,18 +10,69 @@
> 这里记录了每一天的核心开发内容与 milestone。
### Day 21: 缺陷修复与持久化回归治理 (Current)
### Day 23: 配音前置重构 + 素材时间轴编排 + UI 体验优化 + 历史文案 (Current)
#### 第一阶段:配音前置
- [x] **配音生成独立化**: 新增 `generated_audios` 后端模块router/schemas/service5 个 API 端点,复用现有 TTSService / voice_clone_service / task_store。
- [x] **配音管理面板**: 前端新增 `useGeneratedAudios` hook + `GeneratedAudiosPanel` 组件,支持生成/试听/改名/删除/选中。
- [x] **UI 面板重排序**: 文案 → 标题字幕 → 配音方式 → 配音列表 → 素材选择 → BGM → 生成视频。
- [x] **素材区门控**: 未选中配音时素材区显示遮罩,选中后显示配音时长 + 素材均分信息。
- [x] **视频生成对接**: workflow.py 新增预生成音频分支(`generated_audio_id`),跳过内联 TTS向后兼容。
- [x] **持久化**: selectedAudioId 加入 useHomePersistence刷新页面恢复选中配音。
#### 第二阶段:素材时间轴编排
- [x] **时间轴编辑器**: 新增 `TimelineEditor` 组件wavesurfer.js 音频波形 + 色块可视化素材分配,拖拽分割线调整各段时长。
- [x] **素材截取设置**: 新增 `ClipTrimmer` 模态框HTML5 视频预览 + 双端滑块设置源视频截取起点/终点。
- [x] **后端自定义分配**: 新增 `CustomAssignment` 模型,`prepare_segment` 支持 `source_start`workflow 多素材/单素材流水线支持 `custom_assignments`
- [x] **循环截取修复**: `stream_loop + source_start` 改为两步处理(先裁剪再循环),确保从截取起点循环而非从视频 0s 开始。
- [x] **MaterialSelector 精简**: 移除旧的时长信息栏和拖拽排序区(功能迁移到 TimelineEditor
#### 第三阶段UI 体验优化 + TTS 稳定性
- [x] **TTS SoX PATH 修复**: `run_qwen_tts.sh` export conda env bin 到 PATH修复 `SoX could not be found!` 警告。
- [x] **TTS 显存管理**: 每次生成后 `torch.cuda.empty_cache()`asyncio.to_thread 避免阻塞事件循环。
- [x] **配音列表按钮统一**: Play/Edit/Delete 按钮右侧同组 hover 显示,与 RefAudioPanel 一致,移除文案摘要。
- [x] **素材区解除配音门控**: 移除 MaterialSelector 的 selectedAudio 遮罩,素材随时可上传管理。
- [x] **时间轴拖拽排序**: TimelineEditor 色块支持 HTML5 Drag & Drop 调换素材顺序。
- [x] **截取设置 Range Slider**: ClipTrimmer 改为单轨道双手柄(紫色起点+粉色终点),替换两个独立滑块。
- [x] **截取设置视频预览**: 视频区域可播放/暂停,从 sourceStart 到 sourceEnd 自动停止,拖拽手柄时实时 seek。
#### 第四阶段:历史文案 + Bug 修复
- [x] **历史文案保存与加载**: 新增 `useSavedScripts` hook手动保存/加载/删除历史文案,独立 localStorage 持久化。
- [x] **时间轴拖拽修复**: `reorderSegments` 从属性交换改为数组移动splice修复拖拽后时长不跟随素材的 Bug。
- [x] **按钮视觉统一**: 文案编辑区 4 个按钮统一为固定高度 `h-7`,移除多余 `<span>` 嵌套。
- [x] **底部栏调整**: "保存文案"按钮移至底部右侧,移除预计时长显示。
### Day 22: 多素材优化 + AI 翻译 + TTS 多语言
- [x] **多素材 Bug 修复**: 6 个高优 Bug边界溢出、单段 fallback、除零、duration 校验、Whisper 兜底、空列表检查)。
- [x] **架构重构**: 多素材从"逐段 LatentSync"重构为"先拼接再推理",推理次数 N→1。
- [x] **前端优化**: payload 安全、进度消息、上传自动选中、Material 接口统一、拖拽修复、素材上限 4 个。
- [x] **AI 多语言翻译**: 新增 `/api/ai/translate` 接口,前端 9 种语言翻译 + 还原原文。
- [x] **TTS 多语言**: EdgeTTS 10 语言声音列表、翻译自动切换声音、声音克隆 language 透传、textLang 持久化。
### Day 21: 缺陷修复 + 浮动预览 + 发布重构 + 架构优化 + 多素材生成
- [x] **Remotion 崩溃容错**: 渲染进程 SIGABRT 退出时检查输出文件,避免误判失败导致标题/字幕丢失。
- [x] **首页作品选择持久化**: 修复 `fetchGeneratedVideos` 无条件覆盖恢复值的问题,新增 `preferVideoId` 参数控制选中逻辑。
- [x] **发布页作品选择持久化**: 根因为签名 URL 不稳定,全面改用 `video.id` 替代 `path` 进行选择/持久化/比较。
- [x] **预取缓存补全**: 首页预取发布页数据时加入 `id` 字段,确保缓存数据可用于持久化匹配。
- [x] **浮动样式预览窗口**: 标题字幕预览改为 `position: fixed` 浮动窗口,固定左上角,滚动时始终可见。
- [x] **移动端适配**: ScriptEditor 按钮换行、预览默认比例改为 9:16 竖屏。
- [x] **多平台发布重构**: 平台配置独立化DOUYIN_*/WEIXIN_*)、用户隔离 Cookie 管理、抖音刷脸验证二维码、微信发布流程优化。
- [x] **前端结构微调**: ScriptExtractionModal 迁移到 features/、contexts 迁移到 shared/contexts/、清理空目录。
- [x] **后端模块分层**: materials/tools/ref_audios 三个模块补全 router+schemas+service 分层。
- [x] **开发规范更新**: BACKEND_DEV.md 新增渐进原则、DOC_RULES.md 取消 TASK_COMPLETE.md 手动触发约束。
- [x] **文档全面更新**: BACKEND_DEV/README、FRONTEND_DEV、DEPLOY_MANUAL、README.md 同步更新。
- [x] **多素材视频生成(多机位效果)**: 支持多选素材 + 拖拽排序,按素材数量均分音频时长(对齐 Whisper 字边界)自动切换机位。逐段 LatentSync + FFmpeg 拼接。前端 @dnd-kit 拖拽排序 UI。
- [x] **字幕开关移除**: 默认启用逐字高亮字幕,移除开关及相关死代码。
- [x] **视频格式扩展**: 上传支持 mkv/webm/flv/wmv/m4v/ts/mts 等常见格式。
- [x] **Watchdog 优化**: 健康检查阈值提高到 5 次,新增重启冷却期 120 秒,避免误重启。
- [x] **多素材 Bug 修复**: 修复标点分句方案对无句末标点文案无效(改为均分方案)、音频时间偏移导致口型不对齐等缺陷。
### Day 20: 代码质量与安全优化
- [x] **功能性修复**: LatentSync 回退逻辑、任务状态接口认证、User 类型统一。
- [x] **性能优化**: N+1 查询修复、视频上传流式处理、httpx 异步替换、GLM 异步包装。
- [x] **安全修复**: 硬编码 Cookie 配置化、日志敏感信息脱敏、ffprobe 安全调用、CORS 配置化。
- [x] **配置优化**: 存储路径环境变量化、Remotion 预编译加速、LatentSync 绝对路径。
- [x] **文档更新**: 更新 Doc_Rules.md 清单,补齐后端与部署文档;更新 SUBTITLE_DEPLOY.md, FRONTEND_DEV.md, implementation_plan.md。
- [x] **文档更新**: 更新 DOC_RULES.md 清单,补齐后端与部署文档;更新 SUBTITLE_DEPLOY.md, FRONTEND_DEV.md, implementation_plan.md。
- [x] **缺陷修复**: 修复 Remotion 路径解析、发布页持久化竞态、首页选中回归、素材闭包陷阱。
### Day 19: 自动发布稳定性与发布体验优化 🚀
@@ -117,6 +168,7 @@
## 🛤️ 后续规划 (Roadmap)
### 🔴 优先待办
- [x] ~~**配音前置重构 — 第二阶段**: 素材片段截取 + 语音时间轴编排~~ ✅ Day 23 已完成
- [ ] **批量生成架构**: 支持 Excel 导入,批量生产视频。
- [ ] **定时任务后台化**: 迁移前端触发的定时发布到后端 APScheduler。
- [ ] **发布任务恢复机制**: 发布任务化 + 状态持久化 + 前端断点恢复,解决刷新后状态丢失。
@@ -134,7 +186,7 @@
| **核心 API** | 100% | ✅ 稳定 |
| **Web UI** | 100% | ✅ 稳定 (移动端适配) |
| **唇形同步** | 100% | ✅ LatentSync 1.6 |
| **TTS 配音** | 100% | ✅ EdgeTTS + Qwen3 |
| **TTS 配音** | 100% | ✅ EdgeTTS + Qwen3 + 配音前置 + 时间轴编排 |
| **自动发布** | 100% | ✅ 抖音/微信视频号/B站/小红书 |
| **用户认证** | 100% | ✅ 手机号 + JWT |
| **部署运维** | 100% | ✅ PM2 + Watchdog |

View File

@@ -17,13 +17,14 @@
### 核心能力
- 🎬 **高清唇形同步** - LatentSync 1.6 驱动512×512 高分辨率 Latent Diffusion 模型。
- 🎙️ **多模态配音** - 支持 **EdgeTTS** (微软超自然语音) 和 **Qwen3-TTS** (3秒极速声音克隆)。
- 🎙️ **多模态配音** - 支持 **EdgeTTS** (微软超自然语音, 10 语言) 和 **Qwen3-TTS** (3秒极速声音克隆)。配音前置工作流:先生成配音 → 选素材 → 生成视频。
- 📝 **智能字幕** - 集成 faster-whisper + Remotion自动生成逐字高亮 (卡拉OK效果) 字幕。
- 🎨 **样式预设** - 标题/字幕样式选择 + 预览 + 字号调节,支持自定义字体库。
- 🖼️ **作品预览一致性** - 标题/字幕预览按素材分辨率缩放,效果更接近成片。
- 💾 **用户偏好持久化** - 首页状态统一恢复/保存,刷新后延续上次配置
- 🎞️ **多素材多机位** - 支持多选素材 + 时间轴编辑器 (wavesurfer.js 波形可视化),拖拽分割线调整时长、拖拽排序切换机位、截取源视频片段
- 💾 **用户偏好持久化** - 首页状态统一恢复/保存,刷新后延续上次配置。历史文案手动保存与加载。
- 🎵 **背景音乐** - 试听 + 音量控制 + 混音,保持配音音量稳定。
- 🤖 **AI 辅助创作** - 内置 GLM-4.7-Flash支持 B站/抖音链接文案提取、AI 洗稿、标题/标签自动生成。
- 🤖 **AI 辅助创作** - 内置 GLM-4.7-Flash支持 B站/抖音链接文案提取、AI 洗稿、标题/标签自动生成、9 语言翻译
### 平台化功能
- 📱 **全自动发布** - 支持抖音/微信视频号/B站/小红书立即发布;扫码登录 + Cookie 持久化。
@@ -40,7 +41,7 @@
| 领域 | 核心技术 | 说明 |
|------|----------|------|
| **前端** | Next.js 16 | TypeScript, TailwindCSS, SWR |
| **前端** | Next.js 16 | TypeScript, TailwindCSS, SWR, wavesurfer.js |
| **后端** | FastAPI | Python 3.10, AsyncIO, PM2 |
| **数据库** | Supabase | PostgreSQL, Storage (本地/S3), Auth |
| **唇形同步** | LatentSync 1.6 | PyTorch 2.5, Diffusers, DeepCache |

View File

@@ -15,6 +15,7 @@ from app.modules.ref_audios.router import router as ref_audios_router
from app.modules.ai.router import router as ai_router
from app.modules.tools.router import router as tools_router
from app.modules.assets.router import router as assets_router
from app.modules.generated_audios.router import router as generated_audios_router
from loguru import logger
import os
@@ -124,6 +125,7 @@ app.include_router(ref_audios_router, prefix="/api/ref-audios", tags=["RefAudios
app.include_router(ai_router) # /api/ai
app.include_router(tools_router, prefix="/api/tools", tags=["Tools"])
app.include_router(assets_router, prefix="/api/assets", tags=["Assets"])
app.include_router(generated_audios_router, prefix="/api/generated-audios", tags=["GeneratedAudios"])
@app.on_event("startup")

View File

@@ -24,6 +24,33 @@ class GenerateMetaResponse(BaseModel):
tags: list[str]
class TranslateRequest(BaseModel):
"""翻译请求"""
text: str
target_lang: str
@router.post("/translate")
async def translate_text(req: TranslateRequest):
"""
AI 翻译文案
将文案翻译为指定目标语言
"""
if not req.text or not req.text.strip():
raise HTTPException(status_code=400, detail="文案不能为空")
if not req.target_lang or not req.target_lang.strip():
raise HTTPException(status_code=400, detail="目标语言不能为空")
try:
logger.info(f"Translating text to {req.target_lang}: {req.text[:50]}...")
translated = await glm_service.translate_text(req.text.strip(), req.target_lang.strip())
return success_response({"translated_text": translated})
except Exception as e:
logger.error(f"Translate failed: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/generate-meta")
async def generate_meta(req: GenerateMetaRequest):
"""

View File

@@ -0,0 +1,77 @@
"""生成配音 API"""
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
import uuid
from loguru import logger
from app.core.deps import get_current_user
from app.core.response import success_response
from app.modules.videos.task_store import create_task, get_task
from app.modules.generated_audios.schemas import GenerateAudioRequest, RenameAudioRequest
from app.modules.generated_audios import service
router = APIRouter()
@router.post("/generate")
async def generate_audio(
req: GenerateAudioRequest,
background_tasks: BackgroundTasks,
user: dict = Depends(get_current_user),
):
"""异步生成配音(返回 task_id"""
task_id = str(uuid.uuid4())
create_task(task_id, user["id"])
background_tasks.add_task(service.generate_audio_task, task_id, req, user["id"])
return success_response({"task_id": task_id})
@router.get("/tasks/{task_id}")
async def get_audio_task(task_id: str, user: dict = Depends(get_current_user)):
"""轮询配音生成进度"""
task = get_task(task_id)
if task.get("status") != "not_found" and task.get("user_id") != user["id"]:
return success_response({"status": "not_found"})
return success_response(task)
@router.get("")
async def list_audios(user: dict = Depends(get_current_user)):
"""列出当前用户所有已生成配音"""
try:
result = await service.list_generated_audios(user["id"])
return success_response(result)
except Exception as e:
logger.error(f"列出配音失败: {e}")
raise HTTPException(status_code=500, detail=f"获取列表失败: {str(e)}")
@router.delete("/{audio_id:path}")
async def delete_audio(audio_id: str, user: dict = Depends(get_current_user)):
"""删除配音"""
try:
await service.delete_generated_audio(audio_id, user["id"])
return success_response(message="删除成功")
except PermissionError as e:
raise HTTPException(status_code=403, detail=str(e))
except Exception as e:
logger.error(f"删除配音失败: {e}")
raise HTTPException(status_code=500, detail=f"删除失败: {str(e)}")
@router.put("/{audio_id:path}")
async def rename_audio(
audio_id: str,
request: RenameAudioRequest,
user: dict = Depends(get_current_user),
):
"""重命名配音"""
try:
result = await service.rename_generated_audio(audio_id, request.new_name, user["id"])
return success_response(result, message="重命名成功")
except PermissionError as e:
raise HTTPException(status_code=403, detail=str(e))
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"重命名配音失败: {e}")
raise HTTPException(status_code=500, detail=f"重命名失败: {str(e)}")

View File

@@ -0,0 +1,30 @@
from pydantic import BaseModel
from typing import Optional, List
class GenerateAudioRequest(BaseModel):
text: str
tts_mode: str = "edgetts"
voice: str = "zh-CN-YunxiNeural"
ref_audio_id: Optional[str] = None
ref_text: Optional[str] = None
language: str = "zh-CN"
class RenameAudioRequest(BaseModel):
new_name: str
class GeneratedAudioItem(BaseModel):
id: str
name: str
path: str
duration_sec: float
text: str
tts_mode: str
language: str
created_at: int
class GeneratedAudioListResponse(BaseModel):
items: List[GeneratedAudioItem]

View File

@@ -0,0 +1,263 @@
"""生成配音 - 业务逻辑"""
import re
import json
import time
import asyncio
import subprocess
import tempfile
import os
from pathlib import Path
from typing import Optional
import httpx
from loguru import logger
from app.services.storage import storage_service
from app.services.tts_service import TTSService
from app.services.voice_clone_service import voice_clone_service
from app.modules.videos.task_store import task_store
from app.modules.generated_audios.schemas import (
GenerateAudioRequest,
GeneratedAudioItem,
GeneratedAudioListResponse,
)
BUCKET = "generated-audios"
def _locale_to_qwen_lang(locale: str) -> str:
mapping = {"zh": "Chinese", "en": "English"}
return mapping.get(locale.split("-")[0], "Auto")
def _get_audio_duration(file_path: str) -> float:
try:
result = subprocess.run(
['ffprobe', '-v', 'quiet', '-show_entries', 'format=duration',
'-of', 'csv=p=0', file_path],
capture_output=True, text=True, timeout=10
)
return float(result.stdout.strip())
except Exception as e:
logger.warning(f"获取音频时长失败: {e}")
return 0.0
async def generate_audio_task(task_id: str, req: GenerateAudioRequest, user_id: str):
"""后台任务:生成配音"""
try:
task_store.update(task_id, {"status": "processing", "progress": 10, "message": "正在生成配音..."})
with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp:
audio_path = tmp.name
try:
if req.tts_mode == "voiceclone":
if not req.ref_audio_id or not req.ref_text:
raise ValueError("声音克隆模式需要提供参考音频和参考文字")
task_store.update(task_id, {"progress": 20, "message": "正在下载参考音频..."})
with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_ref:
ref_local = tmp_ref.name
try:
ref_url = await storage_service.get_signed_url(
bucket="ref-audios", path=req.ref_audio_id
)
timeout = httpx.Timeout(None)
async with httpx.AsyncClient(timeout=timeout) as client:
async with client.stream("GET", ref_url) as resp:
resp.raise_for_status()
with open(ref_local, "wb") as f:
async for chunk in resp.aiter_bytes():
f.write(chunk)
task_store.update(task_id, {"progress": 40, "message": "正在克隆声音 (Qwen3-TTS)..."})
await voice_clone_service.generate_audio(
text=req.text,
ref_audio_path=ref_local,
ref_text=req.ref_text,
output_path=audio_path,
language=_locale_to_qwen_lang(req.language),
)
finally:
if os.path.exists(ref_local):
os.unlink(ref_local)
else:
task_store.update(task_id, {"progress": 30, "message": "正在生成语音 (EdgeTTS)..."})
tts = TTSService()
await tts.generate_audio(req.text, req.voice, audio_path)
task_store.update(task_id, {"progress": 70, "message": "正在上传配音..."})
duration = _get_audio_duration(audio_path)
timestamp = int(time.time())
audio_id = f"{user_id}/{timestamp}_audio.wav"
meta_id = f"{user_id}/{timestamp}_audio.json"
# 生成 display_name
now = time.strftime("%Y%m%d_%H%M", time.localtime(timestamp))
display_name = f"配音_{now}"
with open(audio_path, "rb") as f:
wav_data = f.read()
await storage_service.upload_file(
bucket=BUCKET, path=audio_id,
file_data=wav_data, content_type="audio/wav",
)
metadata = {
"display_name": display_name,
"text": req.text,
"tts_mode": req.tts_mode,
"voice": req.voice if req.tts_mode == "edgetts" else None,
"ref_audio_id": req.ref_audio_id,
"language": req.language,
"duration_sec": duration,
"created_at": timestamp,
}
await storage_service.upload_file(
bucket=BUCKET, path=meta_id,
file_data=json.dumps(metadata, ensure_ascii=False).encode("utf-8"),
content_type="application/json",
)
signed_url = await storage_service.get_signed_url(BUCKET, audio_id)
task_store.update(task_id, {
"status": "completed",
"progress": 100,
"message": f"配音生成完成 ({duration:.1f}s)",
"output": {
"audio_id": audio_id,
"name": display_name,
"path": signed_url,
"duration_sec": duration,
"text": req.text,
"tts_mode": req.tts_mode,
"language": req.language,
"created_at": timestamp,
},
})
finally:
if os.path.exists(audio_path):
os.unlink(audio_path)
except Exception as e:
import traceback
task_store.update(task_id, {
"status": "failed",
"message": f"配音生成失败: {str(e)}",
"error": traceback.format_exc(),
})
logger.error(f"Generate audio failed: {e}")
async def list_generated_audios(user_id: str) -> dict:
"""列出用户的所有已生成配音"""
files = await storage_service.list_files(BUCKET, user_id)
wav_files = [f for f in files if f.get("name", "").endswith("_audio.wav")]
if not wav_files:
return GeneratedAudioListResponse(items=[]).model_dump()
async def fetch_info(f):
name = f.get("name", "")
storage_path = f"{user_id}/{name}"
meta_name = name.replace("_audio.wav", "_audio.json")
meta_path = f"{user_id}/{meta_name}"
display_name = name
text = ""
tts_mode = "edgetts"
language = "zh-CN"
duration_sec = 0.0
created_at = 0
try:
meta_url = await storage_service.get_signed_url(BUCKET, meta_path)
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(meta_url)
if resp.status_code == 200:
meta = resp.json()
display_name = meta.get("display_name", name)
text = meta.get("text", "")
tts_mode = meta.get("tts_mode", "edgetts")
language = meta.get("language", "zh-CN")
duration_sec = meta.get("duration_sec", 0.0)
created_at = meta.get("created_at", 0)
except Exception as e:
logger.debug(f"读取配音 metadata 失败: {e}")
try:
created_at = int(name.split("_")[0])
except:
pass
signed_url = await storage_service.get_signed_url(BUCKET, storage_path)
return GeneratedAudioItem(
id=storage_path,
name=display_name,
path=signed_url,
duration_sec=duration_sec,
text=text,
tts_mode=tts_mode,
language=language,
created_at=created_at,
)
items = await asyncio.gather(*[fetch_info(f) for f in wav_files])
items = sorted(items, key=lambda x: x.created_at, reverse=True)
return GeneratedAudioListResponse(items=items).model_dump()
async def delete_generated_audio(audio_id: str, user_id: str) -> None:
if not audio_id.startswith(f"{user_id}/"):
raise PermissionError("无权删除此文件")
await storage_service.delete_file(BUCKET, audio_id)
meta_path = audio_id.replace("_audio.wav", "_audio.json")
try:
await storage_service.delete_file(BUCKET, meta_path)
except:
pass
async def rename_generated_audio(audio_id: str, new_name: str, user_id: str) -> dict:
if not audio_id.startswith(f"{user_id}/"):
raise PermissionError("无权修改此文件")
new_name = new_name.strip()
if not new_name:
raise ValueError("新名称不能为空")
meta_path = audio_id.replace("_audio.wav", "_audio.json")
try:
meta_url = await storage_service.get_signed_url(BUCKET, meta_path)
async with httpx.AsyncClient() as client:
resp = await client.get(meta_url)
if resp.status_code == 200:
metadata = resp.json()
else:
raise Exception(f"Failed to fetch metadata: {resp.status_code}")
except Exception as e:
logger.warning(f"无法读取配音元数据: {e}, 将创建新的")
metadata = {
"display_name": new_name,
"text": "",
"tts_mode": "edgetts",
"language": "zh-CN",
"duration_sec": 0.0,
"created_at": int(time.time()),
}
metadata["display_name"] = new_name
await storage_service.upload_file(
bucket=BUCKET,
path=meta_path,
file_data=json.dumps(metadata, ensure_ascii=False).encode("utf-8"),
content_type="application/json",
)
return {"name": new_name}

View File

@@ -1,416 +1,62 @@
from fastapi import APIRouter, UploadFile, File, HTTPException, Request, BackgroundTasks, Depends
from app.core.config import settings
from app.core.deps import get_current_user
from app.core.response import success_response
from app.services.storage import storage_service
import re
import time
import traceback
import os
import aiofiles
from pathlib import Path
from loguru import logger
import asyncio
from pydantic import BaseModel
from typing import Optional
import httpx
from fastapi import APIRouter, HTTPException, Request, Depends
from loguru import logger
from app.core.deps import get_current_user
from app.core.response import success_response
from app.modules.materials.schemas import RenameMaterialRequest
from app.modules.materials import service
router = APIRouter()
class RenameMaterialRequest(BaseModel):
new_name: str
def sanitize_filename(filename: str) -> str:
safe_name = re.sub(r'[<>:"/\\|?*]', '_', filename)
if len(safe_name) > 100:
ext = Path(safe_name).suffix
safe_name = safe_name[:100 - len(ext)] + ext
return safe_name
async def process_and_upload(temp_file_path: str, original_filename: str, content_type: str, user_id: str):
"""Background task to strip multipart headers and upload to Supabase"""
try:
logger.info(f"Processing raw upload: {temp_file_path} for user {user_id}")
# 1. Analyze file to find actual video content (strip multipart boundaries)
# This is a simplified manual parser for a SINGLE file upload.
# Structure:
# --boundary
# Content-Disposition: form-data; name="file"; filename="..."
# Content-Type: video/mp4
# \r\n\r\n
# [DATA]
# \r\n--boundary--
# We need to read the first few KB to find the header end
start_offset = 0
end_offset = 0
boundary = b""
file_size = os.path.getsize(temp_file_path)
with open(temp_file_path, 'rb') as f:
# Read first 4KB to find header
head = f.read(4096)
# Find boundary
first_line_end = head.find(b'\r\n')
if first_line_end == -1:
raise Exception("Could not find boundary in multipart body")
boundary = head[:first_line_end] # e.g. --boundary123
logger.info(f"Detected boundary: {boundary}")
# Find end of headers (\r\n\r\n)
header_end = head.find(b'\r\n\r\n')
if header_end == -1:
raise Exception("Could not find end of multipart headers")
start_offset = header_end + 4
logger.info(f"Video data starts at offset: {start_offset}")
# Find end boundary (read from end of file)
# It should be \r\n + boundary + -- + \r\n
# We seek to end-200 bytes
f.seek(max(0, file_size - 200))
tail = f.read()
# The closing boundary is usually --boundary--
# We look for the last occurrence of the boundary
last_boundary_pos = tail.rfind(boundary)
if last_boundary_pos != -1:
# The data ends before \r\n + boundary
# The tail buffer relative position needs to be converted to absolute
end_pos_in_tail = last_boundary_pos
# We also need to check for the preceding \r\n
if end_pos_in_tail >= 2 and tail[end_pos_in_tail-2:end_pos_in_tail] == b'\r\n':
end_pos_in_tail -= 2
# Absolute end offset
end_offset = (file_size - 200) + last_boundary_pos
# Correction for CRLF before boundary
# Actually, simply: read until (file_size - len(tail) + last_boundary_pos) - 2
end_offset = (max(0, file_size - 200) + last_boundary_pos) - 2
else:
logger.warning("Could not find closing boundary, assuming EOF")
end_offset = file_size
logger.info(f"Video data ends at offset: {end_offset}. Total video size: {end_offset - start_offset}")
# 2. Extract and Upload to Supabase
# Since we have the file on disk, we can just pass the file object (seeked) to upload_file?
# Or if upload_file expects bytes/path, checking storage.py...
# It takes `file_data` (bytes) or file-like?
# supabase-py's `upload` method handles parsing if we pass a file object.
# But we need to pass ONLY the video slice.
# So we create a generator or a sliced file object?
# Simpler: Read the slice into memory if < 1GB? Or copy to new temp file?
# Copying to new temp file is safer for memory.
video_path = temp_file_path + "_video.mp4"
with open(temp_file_path, 'rb') as src, open(video_path, 'wb') as dst:
src.seek(start_offset)
# Copy in chunks
bytes_to_copy = end_offset - start_offset
copied = 0
while copied < bytes_to_copy:
chunk_size = min(1024*1024*10, bytes_to_copy - copied) # 10MB chunks
chunk = src.read(chunk_size)
if not chunk:
break
dst.write(chunk)
copied += len(chunk)
logger.info(f"Extracted video content to {video_path}")
# 3. Upload to Supabase with user isolation
timestamp = int(time.time())
safe_name = re.sub(r'[^a-zA-Z0-9._-]', '', original_filename)
# 使用 user_id 作为目录前缀实现隔离
storage_path = f"{user_id}/{timestamp}_{safe_name}"
# Use storage service (this calls Supabase which might do its own http request)
# We read the cleaned video file
with open(video_path, 'rb') as f:
file_content = f.read() # Still reading into memory for simple upload call, but server has 32GB RAM so ok for 500MB
await storage_service.upload_file(
bucket=storage_service.BUCKET_MATERIALS,
path=storage_path,
file_data=file_content,
content_type=content_type
)
logger.info(f"Upload to Supabase complete: {storage_path}")
# Cleanup
os.remove(temp_file_path)
os.remove(video_path)
return storage_path
except Exception as e:
logger.error(f"Background upload processing failed: {e}\n{traceback.format_exc()}")
raise
router = APIRouter()
@router.post("")
async def upload_material(
request: Request,
background_tasks: BackgroundTasks,
current_user: dict = Depends(get_current_user)
):
user_id = current_user["id"]
logger.info(f"ENTERED upload_material (Streaming Mode) for user {user_id}. Headers: {request.headers}")
filename = "unknown_video.mp4" # Fallback
content_type = "video/mp4"
# Try to parse filename from header if possible (unreliable in raw stream)
# We will rely on post-processing or client hint
# Frontend sends standard multipart.
# Create temp file
timestamp = int(time.time())
temp_filename = f"upload_{timestamp}.raw"
temp_path = os.path.join("/tmp", temp_filename) # Use /tmp on Linux
# Ensure /tmp exists (it does) but verify paths
if os.name == 'nt': # Local dev
temp_path = f"d:/tmp/{temp_filename}"
os.makedirs("d:/tmp", exist_ok=True)
logger.info(f"Upload material request from user {user_id}")
try:
total_size = 0
last_log = 0
async with aiofiles.open(temp_path, 'wb') as f:
async for chunk in request.stream():
await f.write(chunk)
total_size += len(chunk)
# Log progress every 20MB
if total_size - last_log > 20 * 1024 * 1024:
logger.info(f"Receiving stream... Processed {total_size / (1024*1024):.2f} MB")
last_log = total_size
logger.info(f"Stream reception complete. Total size: {total_size} bytes. Saved to {temp_path}")
if total_size == 0:
raise HTTPException(400, "Received empty body")
# Attempt to extract filename from the saved file's first bytes?
# Or just accept it as "uploaded_video.mp4" for now to prove it works.
# We can try to regex the header in the file content we just wrote.
# Implemented in background task to return success immediately.
# Wait, if we return immediately, the user's UI might not show the file yet?
# The prompt says "Wait for upload".
# But to avoid User Waiting Timeout, maybe returning early is better?
# NO, user expects the file to be in the list.
# So we Must await the processing.
# But "Processing" (Strip + Upload to Supabase) takes time.
# Receiving took time.
# If we await Supabase upload, does it timeout?
# Supabase upload is outgoing. Usually faster/stable.
# Let's await the processing to ensure "List Materials" shows it.
# We need to extract the filename for the list.
# Quick extract filename from first 4kb
with open(temp_path, 'rb') as f:
head = f.read(4096).decode('utf-8', errors='ignore')
match = re.search(r'filename="([^"]+)"', head)
if match:
filename = match.group(1)
logger.info(f"Extracted filename from body: {filename}")
# Run processing sync (in await)
storage_path = await process_and_upload(temp_path, filename, content_type, user_id)
# Get signed URL (it exists now)
signed_url = await storage_service.get_signed_url(
bucket=storage_service.BUCKET_MATERIALS,
path=storage_path
)
size_mb = total_size / (1024 * 1024) # Approximate (includes headers)
# 从 storage_path 提取显示名
display_name = storage_path.split('/')[-1] # 去掉 user_id 前缀
if '_' in display_name:
parts = display_name.split('_', 1)
if parts[0].isdigit():
display_name = parts[1]
return success_response({
"id": storage_path,
"name": display_name,
"path": signed_url,
"size_mb": size_mb,
"type": "video"
})
result = await service.upload_material(request, user_id)
return success_response(result)
except ValueError as e:
raise HTTPException(400, str(e))
except Exception as e:
error_msg = f"Streaming upload failed: {str(e)}"
detail_msg = f"Exception: {repr(e)}\nArgs: {e.args}\n{traceback.format_exc()}"
logger.error(error_msg + "\n" + detail_msg)
# Write to debug file
try:
with open("debug_upload.log", "a") as logf:
logf.write(f"\n--- Error at {time.ctime()} ---\n")
logf.write(detail_msg)
logf.write("\n-----------------------------\n")
except:
pass
if os.path.exists(temp_path):
try:
os.remove(temp_path)
except:
pass
raise HTTPException(500, f"Upload failed. Check server logs. Error: {str(e)}")
raise HTTPException(500, f"Upload failed. Error: {str(e)}")
@router.get("")
async def list_materials(current_user: dict = Depends(get_current_user)):
user_id = current_user["id"]
try:
# 只列出当前用户目录下的文件
files_obj = await storage_service.list_files(
bucket=storage_service.BUCKET_MATERIALS,
path=user_id
)
semaphore = asyncio.Semaphore(8)
async def build_item(f):
name = f.get('name')
if not name or name == '.emptyFolderPlaceholder':
return None
display_name = name
if '_' in name:
parts = name.split('_', 1)
if parts[0].isdigit():
display_name = parts[1]
full_path = f"{user_id}/{name}"
async with semaphore:
signed_url = await storage_service.get_signed_url(
bucket=storage_service.BUCKET_MATERIALS,
path=full_path
)
metadata = f.get('metadata', {})
size = metadata.get('size', 0)
created_at_str = f.get('created_at', '')
created_at = 0
if created_at_str:
from datetime import datetime
try:
dt = datetime.fromisoformat(created_at_str.replace('Z', '+00:00'))
created_at = int(dt.timestamp())
except Exception:
pass
return {
"id": full_path,
"name": display_name,
"path": signed_url,
"size_mb": size / (1024 * 1024),
"type": "video",
"created_at": created_at
}
tasks = [build_item(f) for f in files_obj]
results = await asyncio.gather(*tasks, return_exceptions=True)
materials = []
for item in results:
if not item:
continue
if isinstance(item, Exception):
logger.warning(f"Material signed url build failed: {item}")
continue
materials.append(item)
materials.sort(key=lambda x: x['id'], reverse=True)
return success_response({"materials": materials})
except Exception as e:
logger.error(f"List materials failed: {e}")
return success_response({"materials": []}, message="获取素材失败")
materials = await service.list_materials(user_id)
return success_response({"materials": materials})
@router.delete("/{material_id:path}")
async def delete_material(material_id: str, current_user: dict = Depends(get_current_user)):
@router.delete("/{material_id:path}")
async def delete_material(material_id: str, current_user: dict = Depends(get_current_user)):
user_id = current_user["id"]
# 验证 material_id 属于当前用户
if not material_id.startswith(f"{user_id}/"):
raise HTTPException(403, "无权删除此素材")
try:
await storage_service.delete_file(
bucket=storage_service.BUCKET_MATERIALS,
path=material_id
)
return success_response(message="素材已删除")
except Exception as e:
raise HTTPException(500, f"删除失败: {str(e)}")
@router.put("/{material_id:path}")
async def rename_material(
material_id: str,
payload: RenameMaterialRequest,
current_user: dict = Depends(get_current_user)
):
user_id = current_user["id"]
if not material_id.startswith(f"{user_id}/"):
raise HTTPException(403, "无权重命名此素材")
new_name_raw = payload.new_name.strip() if payload.new_name else ""
if not new_name_raw:
raise HTTPException(400, "新名称不能为空")
old_name = material_id.split("/", 1)[1]
old_ext = Path(old_name).suffix
base_name = Path(new_name_raw).stem if Path(new_name_raw).suffix else new_name_raw
safe_base = sanitize_filename(base_name).strip()
if not safe_base:
raise HTTPException(400, "新名称无效")
new_filename = f"{safe_base}{old_ext}"
prefix = None
if "_" in old_name:
maybe_prefix, _ = old_name.split("_", 1)
if maybe_prefix.isdigit():
prefix = maybe_prefix
if prefix:
new_filename = f"{prefix}_{new_filename}"
new_path = f"{user_id}/{new_filename}"
try:
if new_path != material_id:
await storage_service.move_file(
bucket=storage_service.BUCKET_MATERIALS,
from_path=material_id,
to_path=new_path
)
signed_url = await storage_service.get_signed_url(
bucket=storage_service.BUCKET_MATERIALS,
path=new_path
)
display_name = new_filename
if "_" in new_filename:
parts = new_filename.split("_", 1)
if parts[0].isdigit():
display_name = parts[1]
return success_response({
"id": new_path,
"name": display_name,
"path": signed_url,
}, message="重命名成功")
except Exception as e:
raise HTTPException(500, f"重命名失败: {str(e)}")
await service.delete_material(material_id, user_id)
return success_response(message="素材已删除")
except PermissionError as e:
raise HTTPException(403, str(e))
except Exception as e:
raise HTTPException(500, f"删除失败: {str(e)}")
@router.put("/{material_id:path}")
async def rename_material(
material_id: str,
payload: RenameMaterialRequest,
current_user: dict = Depends(get_current_user)
):
user_id = current_user["id"]
try:
result = await service.rename_material(material_id, payload.new_name, user_id)
return success_response(result, message="重命名成功")
except PermissionError as e:
raise HTTPException(403, str(e))
except ValueError as e:
raise HTTPException(400, str(e))
except Exception as e:
raise HTTPException(500, f"重命名失败: {str(e)}")

View File

@@ -0,0 +1,14 @@
from pydantic import BaseModel
class RenameMaterialRequest(BaseModel):
new_name: str
class MaterialItem(BaseModel):
id: str
name: str
path: str
size_mb: float
type: str = "video"
created_at: int = 0

View File

@@ -0,0 +1,296 @@
import re
import os
import time
import asyncio
import traceback
import aiofiles
from pathlib import Path
from loguru import logger
from app.services.storage import storage_service
def sanitize_filename(filename: str) -> str:
safe_name = re.sub(r'[<>:"/\\|?*]', '_', filename)
if len(safe_name) > 100:
ext = Path(safe_name).suffix
safe_name = safe_name[:100 - len(ext)] + ext
return safe_name
def _extract_display_name(storage_name: str) -> str:
"""从存储文件名中提取显示名(去掉时间戳前缀)"""
if '_' in storage_name:
parts = storage_name.split('_', 1)
if parts[0].isdigit():
return parts[1]
return storage_name
async def _process_and_upload(temp_file_path: str, original_filename: str, content_type: str, user_id: str) -> str:
"""Strip multipart headers and upload to Supabase, return storage_path"""
try:
logger.info(f"Processing raw upload: {temp_file_path} for user {user_id}")
file_size = os.path.getsize(temp_file_path)
with open(temp_file_path, 'rb') as f:
head = f.read(4096)
first_line_end = head.find(b'\r\n')
if first_line_end == -1:
raise Exception("Could not find boundary in multipart body")
boundary = head[:first_line_end]
logger.info(f"Detected boundary: {boundary}")
header_end = head.find(b'\r\n\r\n')
if header_end == -1:
raise Exception("Could not find end of multipart headers")
start_offset = header_end + 4
logger.info(f"Video data starts at offset: {start_offset}")
f.seek(max(0, file_size - 200))
tail = f.read()
last_boundary_pos = tail.rfind(boundary)
if last_boundary_pos != -1:
end_offset = (max(0, file_size - 200) + last_boundary_pos) - 2
else:
logger.warning("Could not find closing boundary, assuming EOF")
end_offset = file_size
logger.info(f"Video data ends at offset: {end_offset}. Total video size: {end_offset - start_offset}")
video_path = temp_file_path + "_video.mp4"
with open(temp_file_path, 'rb') as src, open(video_path, 'wb') as dst:
src.seek(start_offset)
bytes_to_copy = end_offset - start_offset
copied = 0
while copied < bytes_to_copy:
chunk_size = min(1024 * 1024 * 10, bytes_to_copy - copied)
chunk = src.read(chunk_size)
if not chunk:
break
dst.write(chunk)
copied += len(chunk)
logger.info(f"Extracted video content to {video_path}")
timestamp = int(time.time())
safe_name = re.sub(r'[^a-zA-Z0-9._-]', '', original_filename)
storage_path = f"{user_id}/{timestamp}_{safe_name}"
with open(video_path, 'rb') as f:
file_content = f.read()
await storage_service.upload_file(
bucket=storage_service.BUCKET_MATERIALS,
path=storage_path,
file_data=file_content,
content_type=content_type
)
logger.info(f"Upload to Supabase complete: {storage_path}")
os.remove(temp_file_path)
os.remove(video_path)
return storage_path
except Exception as e:
logger.error(f"Background upload processing failed: {e}\n{traceback.format_exc()}")
raise
async def upload_material(request, user_id: str) -> dict:
"""接收流式上传并存储到 Supabase返回素材信息"""
filename = "unknown_video.mp4"
content_type = "video/mp4"
timestamp = int(time.time())
temp_filename = f"upload_{timestamp}.raw"
temp_path = os.path.join("/tmp", temp_filename)
if os.name == 'nt':
temp_path = f"d:/tmp/{temp_filename}"
os.makedirs("d:/tmp", exist_ok=True)
try:
total_size = 0
last_log = 0
async with aiofiles.open(temp_path, 'wb') as f:
async for chunk in request.stream():
await f.write(chunk)
total_size += len(chunk)
if total_size - last_log > 20 * 1024 * 1024:
logger.info(f"Receiving stream... Processed {total_size / (1024*1024):.2f} MB")
last_log = total_size
logger.info(f"Stream reception complete. Total size: {total_size} bytes. Saved to {temp_path}")
if total_size == 0:
raise ValueError("Received empty body")
with open(temp_path, 'rb') as f:
head = f.read(4096).decode('utf-8', errors='ignore')
match = re.search(r'filename="([^"]+)"', head)
if match:
filename = match.group(1)
logger.info(f"Extracted filename from body: {filename}")
storage_path = await _process_and_upload(temp_path, filename, content_type, user_id)
signed_url = await storage_service.get_signed_url(
bucket=storage_service.BUCKET_MATERIALS,
path=storage_path
)
size_mb = total_size / (1024 * 1024)
display_name = _extract_display_name(storage_path.split('/')[-1])
return {
"id": storage_path,
"name": display_name,
"path": signed_url,
"size_mb": size_mb,
"type": "video"
}
except Exception as e:
error_msg = f"Streaming upload failed: {str(e)}"
detail_msg = f"Exception: {repr(e)}\nArgs: {e.args}\n{traceback.format_exc()}"
logger.error(error_msg + "\n" + detail_msg)
try:
with open("debug_upload.log", "a") as logf:
logf.write(f"\n--- Error at {time.ctime()} ---\n")
logf.write(detail_msg)
logf.write("\n-----------------------------\n")
except:
pass
if os.path.exists(temp_path):
try:
os.remove(temp_path)
except:
pass
raise
async def list_materials(user_id: str) -> list[dict]:
"""列出用户的所有素材"""
try:
files_obj = await storage_service.list_files(
bucket=storage_service.BUCKET_MATERIALS,
path=user_id
)
semaphore = asyncio.Semaphore(8)
async def build_item(f):
name = f.get('name')
if not name or name == '.emptyFolderPlaceholder':
return None
display_name = _extract_display_name(name)
full_path = f"{user_id}/{name}"
async with semaphore:
signed_url = await storage_service.get_signed_url(
bucket=storage_service.BUCKET_MATERIALS,
path=full_path
)
metadata = f.get('metadata', {})
size = metadata.get('size', 0)
created_at_str = f.get('created_at', '')
created_at = 0
if created_at_str:
from datetime import datetime
try:
dt = datetime.fromisoformat(created_at_str.replace('Z', '+00:00'))
created_at = int(dt.timestamp())
except Exception:
pass
return {
"id": full_path,
"name": display_name,
"path": signed_url,
"size_mb": size / (1024 * 1024),
"type": "video",
"created_at": created_at
}
tasks = [build_item(f) for f in files_obj]
results = await asyncio.gather(*tasks, return_exceptions=True)
materials = []
for item in results:
if not item:
continue
if isinstance(item, Exception):
logger.warning(f"Material signed url build failed: {item}")
continue
materials.append(item)
materials.sort(key=lambda x: x['id'], reverse=True)
return materials
except Exception as e:
logger.error(f"List materials failed: {e}")
return []
async def delete_material(material_id: str, user_id: str) -> None:
"""删除素材"""
if not material_id.startswith(f"{user_id}/"):
raise PermissionError("无权删除此素材")
await storage_service.delete_file(
bucket=storage_service.BUCKET_MATERIALS,
path=material_id
)
async def rename_material(material_id: str, new_name_raw: str, user_id: str) -> dict:
"""重命名素材,返回更新后的素材信息"""
if not material_id.startswith(f"{user_id}/"):
raise PermissionError("无权重命名此素材")
new_name_raw = new_name_raw.strip() if new_name_raw else ""
if not new_name_raw:
raise ValueError("新名称不能为空")
old_name = material_id.split("/", 1)[1]
old_ext = Path(old_name).suffix
base_name = Path(new_name_raw).stem if Path(new_name_raw).suffix else new_name_raw
safe_base = sanitize_filename(base_name).strip()
if not safe_base:
raise ValueError("新名称无效")
new_filename = f"{safe_base}{old_ext}"
prefix = None
if "_" in old_name:
maybe_prefix, _ = old_name.split("_", 1)
if maybe_prefix.isdigit():
prefix = maybe_prefix
if prefix:
new_filename = f"{prefix}_{new_filename}"
new_path = f"{user_id}/{new_filename}"
if new_path != material_id:
await storage_service.move_file(
bucket=storage_service.BUCKET_MATERIALS,
from_path=material_id,
to_path=new_path
)
signed_url = await storage_service.get_signed_url(
bucket=storage_service.BUCKET_MATERIALS,
path=new_path
)
display_name = _extract_display_name(new_filename)
return {
"id": new_path,
"name": display_name,
"path": signed_url,
}

View File

@@ -1,83 +1,14 @@
"""
参考音频管理 API
支持上传/列表/删除参考音频,用于 Qwen3-TTS 声音克隆
"""
"""参考音频管理 API"""
from fastapi import APIRouter, UploadFile, File, Form, HTTPException, Depends
from pydantic import BaseModel
from typing import List, Optional
from pathlib import Path
from loguru import logger
import time
import json
import subprocess
import tempfile
import os
import re
from app.core.deps import get_current_user
from app.services.storage import storage_service
from app.core.response import success_response
from app.modules.ref_audios.schemas import RenameRequest
from app.modules.ref_audios import service
router = APIRouter()
# 支持的音频格式
ALLOWED_AUDIO_EXTENSIONS = {'.wav', '.mp3', '.m4a', '.webm', '.ogg', '.flac', '.aac'}
# 参考音频 bucket
BUCKET_REF_AUDIOS = "ref-audios"
class RefAudioResponse(BaseModel):
id: str
name: str
path: str # signed URL for playback
ref_text: str
duration_sec: float
created_at: int
class RefAudioListResponse(BaseModel):
items: List[RefAudioResponse]
def sanitize_filename(filename: str) -> str:
"""清理文件名,移除特殊字符"""
safe_name = re.sub(r'[<>:"/\\|?*\s]', '_', filename)
if len(safe_name) > 50:
ext = Path(safe_name).suffix
safe_name = safe_name[:50 - len(ext)] + ext
return safe_name
def get_audio_duration(file_path: str) -> float:
"""获取音频时长 (秒)"""
try:
result = subprocess.run(
['ffprobe', '-v', 'quiet', '-show_entries', 'format=duration',
'-of', 'csv=p=0', file_path],
capture_output=True, text=True, timeout=10
)
return float(result.stdout.strip())
except Exception as e:
logger.warning(f"获取音频时长失败: {e}")
return 0.0
def convert_to_wav(input_path: str, output_path: str) -> bool:
"""将音频转换为 WAV 格式 (16kHz, mono)"""
try:
subprocess.run([
'ffmpeg', '-y', '-i', input_path,
'-ar', '16000', # 16kHz 采样率
'-ac', '1', # 单声道
'-acodec', 'pcm_s16le', # 16-bit PCM
output_path
], capture_output=True, timeout=60, check=True)
return True
except Exception as e:
logger.error(f"音频转换失败: {e}")
return False
@router.post("")
async def upload_ref_audio(
@@ -85,156 +16,12 @@ async def upload_ref_audio(
ref_text: str = Form(...),
user: dict = Depends(get_current_user)
):
"""
上传参考音频
- file: 音频文件 (支持 wav, mp3, m4a, webm 等)
- ref_text: 参考音频的转写文字 (必填)
"""
user_id = user["id"]
if not file.filename:
raise HTTPException(status_code=400, detail="文件名无效")
filename = file.filename
# 验证文件扩展名
ext = Path(filename).suffix.lower()
if ext not in ALLOWED_AUDIO_EXTENSIONS:
raise HTTPException(
status_code=400,
detail=f"不支持的音频格式: {ext}。支持的格式: {', '.join(ALLOWED_AUDIO_EXTENSIONS)}"
)
# 验证 ref_text
if not ref_text or len(ref_text.strip()) < 2:
raise HTTPException(status_code=400, detail="参考文字不能为空")
"""上传参考音频"""
try:
# 创建临时文件
with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp_input:
content = await file.read()
tmp_input.write(content)
tmp_input_path = tmp_input.name
# 转换为 WAV 格式
tmp_wav_path = tmp_input_path + ".wav"
if ext != '.wav':
if not convert_to_wav(tmp_input_path, tmp_wav_path):
raise HTTPException(status_code=500, detail="音频格式转换失败")
else:
# 即使是 wav 也要标准化格式
convert_to_wav(tmp_input_path, tmp_wav_path)
# 获取音频时长
duration = get_audio_duration(tmp_wav_path)
if duration < 1.0:
raise HTTPException(status_code=400, detail="音频时长过短,至少需要 1 秒")
if duration > 60.0:
raise HTTPException(status_code=400, detail="音频时长过长,最多 60 秒")
# 3. 处理重名逻辑 (Friendly Display Name)
original_name = filename
# 获取用户现有的所有参考音频列表 (为了检查文件名冲突)
# 注意: 这种列表方式在文件极多时性能一般,但考虑到单用户参考音频数量有限,目前可行
existing_files = await storage_service.list_files(BUCKET_REF_AUDIOS, user_id)
existing_names = set()
# 预加载所有现有的 display name
# 这里需要并发请求 metadata 可能会慢,优化: 仅检查 metadata 文件并解析
# 简易方案: 仅在 metadata 中读取 original_filename
# 但 list_files 返回的是 name我们需要 metadata
# 考虑到性能,这里使用一种妥协方案:
# 我们不做全量检查,而是简单的检查:如果用户上传 myvoice.wav
# 我们看看有没有 (timestamp)_myvoice.wav 这种其实并不能准确判断 display name 是否冲突
#
# 正确做法: 应该有个数据库表存 metadata。但目前是无数据库设计。
#
# 改用简单方案:
# 既然我们无法快速获取所有 display name
# 我们暂时只处理 "在新上传时original_filename 保持原样"
# 但用户希望 "如果在列表中看到重复的,自动加(1)"
#
# 鉴于无数据库架构的限制,要在上传时知道"已有的 display name" 成本太高(需遍历下载所有json)。
#
# 💡 替代方案:
# 我们不检查旧的。我们只保证**存储**唯一。
# 对于用户提到的 "新上传的文件名后加个数字" -> 这通常是指 "另存为" 的逻辑。
# 既然用户现在的痛点是 "显示了时间戳太丑",而我已经去掉了时间戳显示。
# 那么如果用户上传两个 "TEST.wav",列表里就会有两个 "TEST.wav" (但时间不同)。
# 这其实是可以接受的。
#
# 但如果用户强求 "自动重命名":
# 我们可以在这里做一个轻量级的 "同名检测"
# 检查有没有 *_{original_name} 的文件存在。
# 如果 storage 里已经有 123_abc.wav, 456_abc.wav
# 我们可以认为 abc.wav 已经存在。
dup_count = 0
search_suffix = f"_{original_name}" # 比如 _test.wav
for f in existing_files:
fname = f.get('name', '')
if fname.endswith(search_suffix):
dup_count += 1
final_display_name = original_name
if dup_count > 0:
name_stem = Path(original_name).stem
name_ext = Path(original_name).suffix
final_display_name = f"{name_stem}({dup_count}){name_ext}"
# 生成存储路径 (唯一ID)
timestamp = int(time.time())
safe_name = sanitize_filename(Path(filename).stem)
storage_path = f"{user_id}/{timestamp}_{safe_name}.wav"
# 上传 WAV 文件到 Supabase
with open(tmp_wav_path, 'rb') as f:
wav_data = f.read()
await storage_service.upload_file(
bucket=BUCKET_REF_AUDIOS,
path=storage_path,
file_data=wav_data,
content_type="audio/wav"
)
# 上传元数据 JSON
metadata = {
"ref_text": ref_text.strip(),
"original_filename": final_display_name, # 这里的名字如果有重复会自动加(1)
"duration_sec": duration,
"created_at": timestamp
}
metadata_path = f"{user_id}/{timestamp}_{safe_name}.json"
await storage_service.upload_file(
bucket=BUCKET_REF_AUDIOS,
path=metadata_path,
file_data=json.dumps(metadata, ensure_ascii=False).encode('utf-8'),
content_type="application/json"
)
# 获取签名 URL
signed_url = await storage_service.get_signed_url(BUCKET_REF_AUDIOS, storage_path)
# 清理临时文件
os.unlink(tmp_input_path)
if os.path.exists(tmp_wav_path):
os.unlink(tmp_wav_path)
return success_response(RefAudioResponse(
id=storage_path,
name=filename,
path=signed_url,
ref_text=ref_text.strip(),
duration_sec=duration,
created_at=timestamp
).model_dump())
except HTTPException:
raise
result = await service.upload_ref_audio(file, ref_text, user["id"])
return success_response(result)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"上传参考音频失败: {e}")
raise HTTPException(status_code=500, detail=f"上传失败: {str(e)}")
@@ -243,81 +30,9 @@ async def upload_ref_audio(
@router.get("")
async def list_ref_audios(user: dict = Depends(get_current_user)):
"""列出当前用户的所有参考音频"""
user_id = user["id"]
try:
# 列出用户目录下的文件
files = await storage_service.list_files(BUCKET_REF_AUDIOS, user_id)
# 过滤出 .wav 文件
wav_files = [f for f in files if f.get("name", "").endswith(".wav")]
if not wav_files:
return success_response(RefAudioListResponse(items=[]).model_dump())
# 并发获取所有 metadata 和签名 URL
async def fetch_audio_info(f):
"""获取单个音频的信息metadata + signed URL"""
name = f.get("name", "")
storage_path = f"{user_id}/{name}"
metadata_name = name.replace(".wav", ".json")
metadata_path = f"{user_id}/{metadata_name}"
ref_text = ""
duration_sec = 0.0
created_at = 0
original_filename = ""
try:
# 获取 metadata 内容
metadata_url = await storage_service.get_signed_url(BUCKET_REF_AUDIOS, metadata_path)
import httpx
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(metadata_url)
if resp.status_code == 200:
metadata = resp.json()
ref_text = metadata.get("ref_text", "")
duration_sec = metadata.get("duration_sec", 0.0)
created_at = metadata.get("created_at", 0)
original_filename = metadata.get("original_filename", "")
except Exception as e:
logger.debug(f"读取 metadata 失败: {e}")
# 从文件名提取时间戳
try:
created_at = int(name.split("_")[0])
except:
pass
# 获取音频签名 URL
signed_url = await storage_service.get_signed_url(BUCKET_REF_AUDIOS, storage_path)
# 优先显示原始文件名 (去掉时间戳前缀)
display_name = original_filename if original_filename else name
# 如果原始文件名丢失,尝试从现有文件名中通过正则去掉时间戳
if not display_name or display_name == name:
# 匹配 "1234567890_filename.wav"
match = re.match(r'^\d+_(.+)$', name)
if match:
display_name = match.group(1)
return RefAudioResponse(
id=storage_path,
name=display_name,
path=signed_url,
ref_text=ref_text,
duration_sec=duration_sec,
created_at=created_at
)
# 使用 asyncio.gather 并发获取所有音频信息
import asyncio
items = await asyncio.gather(*[fetch_audio_info(f) for f in wav_files])
# 按创建时间倒序排列
items = sorted(items, key=lambda x: x.created_at, reverse=True)
return success_response(RefAudioListResponse(items=items).model_dump())
result = await service.list_ref_audios(user["id"])
return success_response(result)
except Exception as e:
logger.error(f"列出参考音频失败: {e}")
raise HTTPException(status_code=500, detail=f"获取列表失败: {str(e)}")
@@ -326,96 +41,30 @@ async def list_ref_audios(user: dict = Depends(get_current_user)):
@router.delete("/{audio_id:path}")
async def delete_ref_audio(audio_id: str, user: dict = Depends(get_current_user)):
"""删除参考音频"""
user_id = user["id"]
# 安全检查:确保只能删除自己的文件
if not audio_id.startswith(f"{user_id}/"):
raise HTTPException(status_code=403, detail="无权删除此文件")
try:
# 删除 WAV 文件
await storage_service.delete_file(BUCKET_REF_AUDIOS, audio_id)
# 删除 metadata JSON
metadata_path = audio_id.replace(".wav", ".json")
try:
await storage_service.delete_file(BUCKET_REF_AUDIOS, metadata_path)
except:
pass # metadata 可能不存在
await service.delete_ref_audio(audio_id, user["id"])
return success_response(message="删除成功")
except PermissionError as e:
raise HTTPException(status_code=403, detail=str(e))
except Exception as e:
logger.error(f"删除参考音频失败: {e}")
raise HTTPException(status_code=500, detail=f"删除失败: {str(e)}")
class RenameRequest(BaseModel):
new_name: str
@router.put("/{audio_id:path}")
async def rename_ref_audio(
audio_id: str,
request: RenameRequest,
user: dict = Depends(get_current_user)
):
"""重命名参考音频 (修改 metadata 中的 display name)"""
user_id = user["id"]
# 安全检查
if not audio_id.startswith(f"{user_id}/"):
raise HTTPException(status_code=403, detail="无权修改此文件")
new_name = request.new_name.strip()
if not new_name:
raise HTTPException(status_code=400, detail="新名称不能为空")
# 确保新名称有后缀 (保留原后缀或添加 .wav)
if not Path(new_name).suffix:
new_name += ".wav"
"""重命名参考音频"""
try:
# 1. 下载现有的 metadata
metadata_path = audio_id.replace(".wav", ".json")
try:
# 获取已有的 JSON
import httpx
metadata_url = await storage_service.get_signed_url(BUCKET_REF_AUDIOS, metadata_path)
if not metadata_url:
# 如果 json 不存在,则需要新建一个基础的
raise Exception("Metadata not found")
async with httpx.AsyncClient() as client:
resp = await client.get(metadata_url)
if resp.status_code == 200:
metadata = resp.json()
else:
raise Exception(f"Failed to fetch metadata: {resp.status_code}")
except Exception as e:
logger.warning(f"无法读取元数据: {e}, 将创建新的元数据")
# 兜底:如果读取失败,构建最小元数据
metadata = {
"ref_text": "", # 可能丢失
"duration_sec": 0.0,
"created_at": int(time.time()),
"original_filename": new_name
}
# 2. 更新 original_filename
metadata["original_filename"] = new_name
# 3. 覆盖上传 metadata
await storage_service.upload_file(
bucket=BUCKET_REF_AUDIOS,
path=metadata_path,
file_data=json.dumps(metadata, ensure_ascii=False).encode('utf-8'),
content_type="application/json"
)
return success_response({"name": new_name}, message="重命名成功")
result = await service.rename_ref_audio(audio_id, request.new_name, user["id"])
return success_response(result, message="重命名成功")
except PermissionError as e:
raise HTTPException(status_code=403, detail=str(e))
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"重命名失败: {e}")
raise HTTPException(status_code=500, detail=f"重命名失败: {str(e)}")

View File

@@ -0,0 +1,19 @@
from pydantic import BaseModel
from typing import List
class RefAudioResponse(BaseModel):
id: str
name: str
path: str
ref_text: str
duration_sec: float
created_at: int
class RefAudioListResponse(BaseModel):
items: List[RefAudioResponse]
class RenameRequest(BaseModel):
new_name: str

View File

@@ -0,0 +1,269 @@
import re
import os
import time
import json
import asyncio
import subprocess
import tempfile
from pathlib import Path
from typing import Optional
import httpx
from loguru import logger
from app.services.storage import storage_service
from app.modules.ref_audios.schemas import RefAudioResponse, RefAudioListResponse
ALLOWED_AUDIO_EXTENSIONS = {'.wav', '.mp3', '.m4a', '.webm', '.ogg', '.flac', '.aac'}
BUCKET_REF_AUDIOS = "ref-audios"
def sanitize_filename(filename: str) -> str:
"""清理文件名,移除特殊字符"""
safe_name = re.sub(r'[<>:"/\\|?*\s]', '_', filename)
if len(safe_name) > 50:
ext = Path(safe_name).suffix
safe_name = safe_name[:50 - len(ext)] + ext
return safe_name
def _get_audio_duration(file_path: str) -> float:
"""获取音频时长 (秒)"""
try:
result = subprocess.run(
['ffprobe', '-v', 'quiet', '-show_entries', 'format=duration',
'-of', 'csv=p=0', file_path],
capture_output=True, text=True, timeout=10
)
return float(result.stdout.strip())
except Exception as e:
logger.warning(f"获取音频时长失败: {e}")
return 0.0
def _convert_to_wav(input_path: str, output_path: str) -> bool:
"""将音频转换为 WAV 格式 (16kHz, mono)"""
try:
subprocess.run([
'ffmpeg', '-y', '-i', input_path,
'-ar', '16000',
'-ac', '1',
'-acodec', 'pcm_s16le',
output_path
], capture_output=True, timeout=60, check=True)
return True
except Exception as e:
logger.error(f"音频转换失败: {e}")
return False
async def upload_ref_audio(file, ref_text: str, user_id: str) -> dict:
"""上传参考音频:转码、获取时长、存储到 Supabase"""
if not file.filename:
raise ValueError("文件名无效")
filename = file.filename
ext = Path(filename).suffix.lower()
if ext not in ALLOWED_AUDIO_EXTENSIONS:
raise ValueError(f"不支持的音频格式: {ext}。支持的格式: {', '.join(ALLOWED_AUDIO_EXTENSIONS)}")
if not ref_text or len(ref_text.strip()) < 2:
raise ValueError("参考文字不能为空")
# 创建临时文件
with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp_input:
content = await file.read()
tmp_input.write(content)
tmp_input_path = tmp_input.name
try:
# 转换为 WAV 格式
tmp_wav_path = tmp_input_path + ".wav"
if not _convert_to_wav(tmp_input_path, tmp_wav_path):
raise RuntimeError("音频格式转换失败")
# 获取音频时长
duration = _get_audio_duration(tmp_wav_path)
if duration < 1.0:
raise ValueError("音频时长过短,至少需要 1 秒")
if duration > 60.0:
raise ValueError("音频时长过长,最多 60 秒")
# 检查重名
existing_files = await storage_service.list_files(BUCKET_REF_AUDIOS, user_id)
dup_count = 0
search_suffix = f"_{filename}"
for f in existing_files:
fname = f.get('name', '')
if fname.endswith(search_suffix):
dup_count += 1
final_display_name = filename
if dup_count > 0:
name_stem = Path(filename).stem
name_ext = Path(filename).suffix
final_display_name = f"{name_stem}({dup_count}){name_ext}"
# 生成存储路径
timestamp = int(time.time())
safe_name = sanitize_filename(Path(filename).stem)
storage_path = f"{user_id}/{timestamp}_{safe_name}.wav"
# 上传 WAV 文件
with open(tmp_wav_path, 'rb') as f:
wav_data = f.read()
await storage_service.upload_file(
bucket=BUCKET_REF_AUDIOS,
path=storage_path,
file_data=wav_data,
content_type="audio/wav"
)
# 上传元数据 JSON
metadata = {
"ref_text": ref_text.strip(),
"original_filename": final_display_name,
"duration_sec": duration,
"created_at": timestamp
}
metadata_path = f"{user_id}/{timestamp}_{safe_name}.json"
await storage_service.upload_file(
bucket=BUCKET_REF_AUDIOS,
path=metadata_path,
file_data=json.dumps(metadata, ensure_ascii=False).encode('utf-8'),
content_type="application/json"
)
# 获取签名 URL
signed_url = await storage_service.get_signed_url(BUCKET_REF_AUDIOS, storage_path)
return RefAudioResponse(
id=storage_path,
name=filename,
path=signed_url,
ref_text=ref_text.strip(),
duration_sec=duration,
created_at=timestamp
).model_dump()
finally:
os.unlink(tmp_input_path)
if os.path.exists(tmp_input_path + ".wav"):
os.unlink(tmp_input_path + ".wav")
async def list_ref_audios(user_id: str) -> dict:
"""列出用户的所有参考音频"""
files = await storage_service.list_files(BUCKET_REF_AUDIOS, user_id)
wav_files = [f for f in files if f.get("name", "").endswith(".wav")]
if not wav_files:
return RefAudioListResponse(items=[]).model_dump()
async def fetch_audio_info(f):
name = f.get("name", "")
storage_path = f"{user_id}/{name}"
metadata_name = name.replace(".wav", ".json")
metadata_path = f"{user_id}/{metadata_name}"
ref_text = ""
duration_sec = 0.0
created_at = 0
original_filename = ""
try:
metadata_url = await storage_service.get_signed_url(BUCKET_REF_AUDIOS, metadata_path)
async with httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(metadata_url)
if resp.status_code == 200:
metadata = resp.json()
ref_text = metadata.get("ref_text", "")
duration_sec = metadata.get("duration_sec", 0.0)
created_at = metadata.get("created_at", 0)
original_filename = metadata.get("original_filename", "")
except Exception as e:
logger.debug(f"读取 metadata 失败: {e}")
try:
created_at = int(name.split("_")[0])
except:
pass
signed_url = await storage_service.get_signed_url(BUCKET_REF_AUDIOS, storage_path)
display_name = original_filename if original_filename else name
if not display_name or display_name == name:
match = re.match(r'^\d+_(.+)$', name)
if match:
display_name = match.group(1)
return RefAudioResponse(
id=storage_path,
name=display_name,
path=signed_url,
ref_text=ref_text,
duration_sec=duration_sec,
created_at=created_at
)
items = await asyncio.gather(*[fetch_audio_info(f) for f in wav_files])
items = sorted(items, key=lambda x: x.created_at, reverse=True)
return RefAudioListResponse(items=items).model_dump()
async def delete_ref_audio(audio_id: str, user_id: str) -> None:
"""删除参考音频及其元数据"""
if not audio_id.startswith(f"{user_id}/"):
raise PermissionError("无权删除此文件")
await storage_service.delete_file(BUCKET_REF_AUDIOS, audio_id)
metadata_path = audio_id.replace(".wav", ".json")
try:
await storage_service.delete_file(BUCKET_REF_AUDIOS, metadata_path)
except:
pass
async def rename_ref_audio(audio_id: str, new_name: str, user_id: str) -> dict:
"""重命名参考音频(修改 metadata 中的 display name"""
if not audio_id.startswith(f"{user_id}/"):
raise PermissionError("无权修改此文件")
new_name = new_name.strip()
if not new_name:
raise ValueError("新名称不能为空")
if not Path(new_name).suffix:
new_name += ".wav"
# 下载现有 metadata
metadata_path = audio_id.replace(".wav", ".json")
try:
metadata_url = await storage_service.get_signed_url(BUCKET_REF_AUDIOS, metadata_path)
async with httpx.AsyncClient() as client:
resp = await client.get(metadata_url)
if resp.status_code == 200:
metadata = resp.json()
else:
raise Exception(f"Failed to fetch metadata: {resp.status_code}")
except Exception as e:
logger.warning(f"无法读取元数据: {e}, 将创建新的元数据")
metadata = {
"ref_text": "",
"duration_sec": 0.0,
"created_at": int(time.time()),
"original_filename": new_name
}
# 更新并覆盖上传
metadata["original_filename"] = new_name
await storage_service.upload_file(
bucket=BUCKET_REF_AUDIOS,
path=metadata_path,
file_data=json.dumps(metadata, ensure_ascii=False).encode('utf-8'),
content_type="application/json"
)
return {"name": new_name}

View File

@@ -1,417 +1,32 @@
from fastapi import APIRouter, UploadFile, File, Form, HTTPException
from typing import Optional, Any, cast
import asyncio
import shutil
import os
import time
from pathlib import Path
from loguru import logger
from typing import Optional
import traceback
import re
import json
import requests
from urllib.parse import unquote
from loguru import logger
from app.services.whisper_service import whisper_service
from app.services.glm_service import glm_service
from app.core.response import success_response
from app.modules.tools import service
router = APIRouter()
@router.post("/extract-script")
async def extract_script_tool(
file: Optional[UploadFile] = File(None),
url: Optional[str] = Form(None),
rewrite: bool = Form(True)
):
"""
独立文案提取工具
支持上传视频/音频 OR 输入视频链接 -> 提取文字 -> (可选) AI洗稿
"""
if not file and not url:
raise HTTPException(400, "必须提供文件或视频链接")
temp_path = None
"""独立文案提取工具"""
try:
timestamp = int(time.time())
temp_dir = Path("/tmp")
if os.name == 'nt':
temp_dir = Path("d:/tmp")
temp_dir.mkdir(parents=True, exist_ok=True)
# 1. 获取/保存文件
loop = asyncio.get_event_loop()
if file:
filename = file.filename
if not filename:
raise HTTPException(400, "文件名无效")
safe_filename = Path(filename).name.replace(" ", "_")
temp_path = temp_dir / f"tool_extract_{timestamp}_{safe_filename}"
# 文件 I/O 放入线程池
await loop.run_in_executor(None, lambda: shutil.copyfileobj(file.file, open(temp_path, "wb")))
logger.info(f"Tool processing upload file: {temp_path}")
else:
if not url:
raise HTTPException(400, "必须提供视频链接")
url_value: str = url
# URL 下载逻辑
# 自动提取文案中的链接 (支持 Douyin/Bilibili 等分享文案)
url_match = re.search(r'https?://[^\s]+', url_value)
if url_match:
extracted_url = url_match.group(0)
logger.info(f"Extracted URL from text: {extracted_url}")
url_value = extracted_url
logger.info(f"Tool downloading URL: {url_value}")
# 封装 yt-dlp 下载函数 (Blocking)
def _download_yt_dlp():
import yt_dlp
logger.info("Attempting download with yt-dlp...")
ydl_opts = {
'format': 'bestaudio/best',
'outtmpl': str(temp_dir / f"tool_download_{timestamp}_%(id)s.%(ext)s"),
'quiet': True,
'no_warnings': True,
'http_headers': {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Referer': 'https://www.douyin.com/',
}
}
with yt_dlp.YoutubeDL() as ydl_raw:
ydl: Any = ydl_raw
ydl.params.update(ydl_opts)
info = ydl.extract_info(url_value, download=True)
if 'requested_downloads' in info:
downloaded_file = info['requested_downloads'][0]['filepath']
else:
ext = info.get('ext', 'mp4')
id = info.get('id')
downloaded_file = str(temp_dir / f"tool_download_{timestamp}_{id}.{ext}")
return Path(downloaded_file)
# 先尝试 yt-dlp (Run in Executor)
try:
temp_path = await loop.run_in_executor(None, _download_yt_dlp)
logger.info(f"yt-dlp downloaded to: {temp_path}")
except Exception as e:
logger.warning(f"yt-dlp download failed: {e}. Trying manual Douyin fallback...")
# 失败则尝试手动解析 (Douyin Fallback)
if "douyin" in url_value:
manual_path = await download_douyin_manual(url_value, temp_dir, timestamp)
if manual_path:
temp_path = manual_path
logger.info(f"Manual Douyin fallback successful: {temp_path}")
else:
raise HTTPException(400, f"视频下载失败。yt-dlp 报错: {str(e)}")
elif "bilibili" in url_value:
manual_path = await download_bilibili_manual(url_value, temp_dir, timestamp)
if manual_path:
temp_path = manual_path
logger.info(f"Manual Bilibili fallback successful: {temp_path}")
else:
raise HTTPException(400, f"视频下载失败。yt-dlp 报错: {str(e)}")
else:
raise HTTPException(400, f"视频下载失败: {str(e)}")
if not temp_path or not temp_path.exists():
raise HTTPException(400, "文件获取失败")
# 1.5 安全转换: 强制转为 WAV (16k)
import subprocess
audio_path = temp_dir / f"extract_audio_{timestamp}.wav"
def _convert_audio():
try:
convert_cmd = [
'ffmpeg',
'-i', str(temp_path),
'-vn', # 忽略视频
'-acodec', 'pcm_s16le',
'-ar', '16000', # Whisper 推荐采样率
'-ac', '1', # 单声道
'-y', # 覆盖
str(audio_path)
]
# 捕获 stderr
subprocess.run(convert_cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return True
except subprocess.CalledProcessError as e:
error_log = e.stderr.decode('utf-8', errors='ignore') if e.stderr else str(e)
logger.error(f"FFmpeg check/convert failed: {error_log}")
# 检查是否为 HTML
head = b""
try:
with open(temp_path, 'rb') as f:
head = f.read(100)
except: pass
if b'<!DOCTYPE html' in head or b'<html' in head:
raise ValueError("HTML_DETECTED")
raise ValueError("CONVERT_FAILED")
# 执行转换 (Run in Executor)
try:
await loop.run_in_executor(None, _convert_audio)
logger.info(f"Converted to WAV: {audio_path}")
target_path = audio_path
except ValueError as ve:
if str(ve) == "HTML_DETECTED":
raise HTTPException(400, "下载的文件是网页而非视频,请重试或手动上传。")
else:
raise HTTPException(400, "下载的文件已损坏或格式无法识别。")
# 2. 提取文案 (Whisper)
script = await whisper_service.transcribe(str(target_path))
# 3. AI 洗稿 (GLM)
rewritten = None
if rewrite:
if script and len(script.strip()) > 0:
logger.info("Rewriting script...")
rewritten = await glm_service.rewrite_script(script)
else:
logger.warning("No script extracted, skipping rewrite")
return success_response({
"original_script": script,
"rewritten_script": rewritten
})
except HTTPException as he:
raise he
result = await service.extract_script(file=file, url=url, rewrite=rewrite)
return success_response(result)
except ValueError as e:
raise HTTPException(400, str(e))
except HTTPException:
raise
except Exception as e:
logger.error(f"Tool extract failed: {e}")
logger.error(traceback.format_exc())
# Friendly error message
msg = str(e)
if "Fresh cookies" in msg:
msg = "下载失败:目标平台开启了反爬验证,请过段时间重试或直接上传视频文件。"
raise HTTPException(500, f"提取失败: {msg}")
finally:
# 清理临时文件
if temp_path and temp_path.exists():
try:
os.remove(temp_path)
logger.info(f"Cleaned up temp file: {temp_path}")
except Exception as e:
logger.warning(f"Failed to cleanup temp file {temp_path}: {e}")
async def download_douyin_manual(url: str, temp_dir: Path, timestamp: int) -> Optional[Path]:
"""
手动下载抖音视频 (Fallback logic - Ported from SuperIPAgent/douyinDownloader)
使用特定的 User Profile URL 和硬编码 Cookie 绕过反爬
"""
import httpx
logger.info(f"[SuperIPAgent] Starting download for: {url}")
try:
# 1. 提取 Modal ID (支持短链跳转)
headers = {
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
}
# 如果是短链或重定向 - 使用异步 httpx
async with httpx.AsyncClient(follow_redirects=True, timeout=10.0) as client:
resp = await client.get(url, headers=headers)
final_url = str(resp.url)
logger.info(f"[SuperIPAgent] Final URL: {final_url}")
modal_id = None
match = re.search(r'/video/(\d+)', final_url)
if match:
modal_id = match.group(1)
if not modal_id:
logger.error("[SuperIPAgent] Could not extract modal_id")
return None
logger.info(f"[SuperIPAgent] Extracted modal_id: {modal_id}")
# 2. 构造特定请求 URL (Copy from SuperIPAgent)
# 使用特定用户的 Profile 页 + modal_id 参数,配合特定 Cookie
target_url = f"https://www.douyin.com/user/MS4wLjABAAAAN_s_hups7LD0N4qnrM3o2gI0vuG3pozNaEolz2_py3cHTTrpVr1Z4dukFD9SOlwY?from_tab_name=main&modal_id={modal_id}"
# 3. 使用配置的 Cookie (从环境变量 DOUYIN_COOKIE 读取)
from app.core.config import settings
if not settings.DOUYIN_COOKIE:
logger.warning("[SuperIPAgent] DOUYIN_COOKIE 未配置,视频下载可能失败")
headers_with_cookie = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"cookie": settings.DOUYIN_COOKIE,
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
}
logger.info(f"[SuperIPAgent] Requesting page with Cookie...")
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.get(target_url, headers=headers_with_cookie)
# 4. 解析 RENDER_DATA
content_match = re.findall(r'<script id="RENDER_DATA" type="application/json">(.*?)</script>', response.text)
if not content_match:
# 尝试解码后再查找?或者结构变了
# 再尝试找 SSR_HYDRATED_DATA
if "SSR_HYDRATED_DATA" in response.text:
content_match = re.findall(r'<script id="SSR_HYDRATED_DATA" type="application/json">(.*?)</script>', response.text)
if not content_match:
logger.error(f"[SuperIPAgent] Could not find RENDER_DATA in page (len={len(response.text)})")
return None
content = unquote(content_match[0])
try:
data = json.loads(content)
except:
logger.error("[SuperIPAgent] JSON decode failed")
return None
# 5. 提取视频流
video_url = None
try:
# 路径通常是: app -> videoDetail -> video -> bitRateList -> playAddr -> src
if "app" in data and "videoDetail" in data["app"]:
info = data["app"]["videoDetail"]["video"]
if "bitRateList" in info and info["bitRateList"]:
video_url = info["bitRateList"][0]["playAddr"][0]["src"]
elif "playAddr" in info and info["playAddr"]:
video_url = info["playAddr"][0]["src"]
except Exception as e:
logger.error(f"[SuperIPAgent] Path extraction failed: {e}")
if not video_url:
logger.error("[SuperIPAgent] No video_url found")
return None
if video_url.startswith("//"):
video_url = "https:" + video_url
logger.info(f"[SuperIPAgent] Found video URL: {video_url[:50]}...")
# 6. 下载 (带 Header) - 使用异步 httpx
temp_path = temp_dir / f"douyin_manual_{timestamp}.mp4"
download_headers = {
'Referer': 'https://www.douyin.com/',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
}
async with httpx.AsyncClient(timeout=60.0) as client:
async with client.stream("GET", video_url, headers=download_headers) as dl_resp:
if dl_resp.status_code == 200:
with open(temp_path, 'wb') as f:
async for chunk in dl_resp.aiter_bytes(chunk_size=8192):
f.write(chunk)
logger.info(f"[SuperIPAgent] Downloaded successfully: {temp_path}")
return temp_path
else:
logger.error(f"[SuperIPAgent] Download failed: {dl_resp.status_code}")
return None
except Exception as e:
logger.error(f"[SuperIPAgent] Logic failed: {e}")
return None
async def download_bilibili_manual(url: str, temp_dir: Path, timestamp: int) -> Optional[Path]:
"""
手动下载 Bilibili 视频 (Fallback logic - Playwright Version)
B站通常音视频分离这里只提取音频即可因为只需要文案
"""
from playwright.async_api import async_playwright
logger.info(f"[Playwright] Starting Bilibili download for: {url}")
playwright = None
browser = None
try:
playwright = await async_playwright().start()
# Launch browser (ensure chromium is installed: playwright install chromium)
browser = await playwright.chromium.launch(headless=True, args=['--no-sandbox', '--disable-setuid-sandbox'])
# Mobile User Agent often gives single stream?
# But Bilibili mobile web is tricky. Desktop is fine.
context = await browser.new_context(
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
page = await context.new_page()
# Intercept audio responses?
# Bilibili streams are usually .m4s
# But finding the initial state is easier.
logger.info("[Playwright] Navigating to Bilibili...")
await page.goto(url, timeout=45000)
# Wait for video element (triggers loading)
try:
await page.wait_for_selector('video', timeout=15000)
except:
logger.warning("[Playwright] Video selector timeout")
# 1. Try extracting from __playinfo__
# window.__playinfo__ contains dash streams
playinfo = await page.evaluate("window.__playinfo__")
audio_url = None
if playinfo and "data" in playinfo and "dash" in playinfo["data"]:
dash = playinfo["data"]["dash"]
if "audio" in dash and dash["audio"]:
audio_url = dash["audio"][0]["baseUrl"]
logger.info(f"[Playwright] Found audio stream in __playinfo__: {audio_url[:50]}...")
# 2. If playinfo fails, try extracting video src (sometimes it's a blob, which we can't fetch easily without interception)
# But interception is complex. Let's try requests with Referer if we have URL.
if not audio_url:
logger.warning("[Playwright] Could not find audio in __playinfo__")
return None
# Download the audio stream
temp_path = temp_dir / f"bilibili_audio_{timestamp}.m4s" # usually m4s
try:
api_request = context.request
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Referer": "https://www.bilibili.com/"
}
logger.info(f"[Playwright] Downloading audio stream...")
response = await api_request.get(audio_url, headers=headers)
if response.status == 200:
body = await response.body()
with open(temp_path, 'wb') as f:
f.write(body)
logger.info(f"[Playwright] Downloaded successfully: {temp_path}")
return temp_path
else:
logger.error(f"[Playwright] API Request failed: {response.status}")
return None
except Exception as e:
logger.error(f"[Playwright] Download logic error: {e}")
return None
except Exception as e:
logger.error(f"[Playwright] Bilibili download failed: {e}")
return None
finally:
if browser:
await browser.close()
if playwright:
await playwright.stop()

View File

@@ -0,0 +1,7 @@
from pydantic import BaseModel
from typing import Optional
class ExtractScriptResponse(BaseModel):
original_script: Optional[str] = None
rewritten_script: Optional[str] = None

View File

@@ -0,0 +1,355 @@
import asyncio
import os
import re
import json
import time
import shutil
import subprocess
import traceback
from pathlib import Path
from typing import Optional, Any
from urllib.parse import unquote
import httpx
from loguru import logger
from app.services.whisper_service import whisper_service
from app.services.glm_service import glm_service
async def extract_script(file=None, url: Optional[str] = None, rewrite: bool = True) -> dict:
"""
文案提取:上传文件或视频链接 -> Whisper 转写 -> (可选) GLM 洗稿
"""
if not file and not url:
raise ValueError("必须提供文件或视频链接")
temp_path = None
try:
timestamp = int(time.time())
temp_dir = Path("/tmp")
if os.name == 'nt':
temp_dir = Path("d:/tmp")
temp_dir.mkdir(parents=True, exist_ok=True)
loop = asyncio.get_event_loop()
# 1. 获取/保存文件
if file:
filename = file.filename
if not filename:
raise ValueError("文件名无效")
safe_filename = Path(filename).name.replace(" ", "_")
temp_path = temp_dir / f"tool_extract_{timestamp}_{safe_filename}"
await loop.run_in_executor(None, lambda: shutil.copyfileobj(file.file, open(temp_path, "wb")))
logger.info(f"Tool processing upload file: {temp_path}")
else:
temp_path = await _download_video(url, temp_dir, timestamp)
if not temp_path or not temp_path.exists():
raise ValueError("文件获取失败")
# 1.5 安全转换: 强制转为 WAV (16k)
audio_path = temp_dir / f"extract_audio_{timestamp}.wav"
try:
await loop.run_in_executor(None, lambda: _convert_to_wav(temp_path, audio_path))
logger.info(f"Converted to WAV: {audio_path}")
except ValueError as ve:
if str(ve) == "HTML_DETECTED":
raise ValueError("下载的文件是网页而非视频,请重试或手动上传。")
else:
raise ValueError("下载的文件已损坏或格式无法识别。")
# 2. 提取文案 (Whisper)
script = await whisper_service.transcribe(str(audio_path))
# 3. AI 洗稿 (GLM)
rewritten = None
if rewrite and script and len(script.strip()) > 0:
logger.info("Rewriting script...")
rewritten = await glm_service.rewrite_script(script)
return {
"original_script": script,
"rewritten_script": rewritten
}
finally:
if temp_path and temp_path.exists():
try:
os.remove(temp_path)
logger.info(f"Cleaned up temp file: {temp_path}")
except Exception as e:
logger.warning(f"Failed to cleanup temp file {temp_path}: {e}")
def _convert_to_wav(input_path: Path, output_path: Path) -> None:
"""FFmpeg 转换为 16k WAV"""
try:
convert_cmd = [
'ffmpeg',
'-i', str(input_path),
'-vn',
'-acodec', 'pcm_s16le',
'-ar', '16000',
'-ac', '1',
'-y',
str(output_path)
]
subprocess.run(convert_cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
error_log = e.stderr.decode('utf-8', errors='ignore') if e.stderr else str(e)
logger.error(f"FFmpeg check/convert failed: {error_log}")
head = b""
try:
with open(input_path, 'rb') as f:
head = f.read(100)
except:
pass
if b'<!DOCTYPE html' in head or b'<html' in head:
raise ValueError("HTML_DETECTED")
raise ValueError("CONVERT_FAILED")
async def _download_video(url: str, temp_dir: Path, timestamp: int) -> Path:
"""下载视频yt-dlp 优先,失败回退手动解析)"""
url_value = url
url_match = re.search(r'https?://[^\s]+', url_value)
if url_match:
extracted_url = url_match.group(0)
logger.info(f"Extracted URL from text: {extracted_url}")
url_value = extracted_url
logger.info(f"Tool downloading URL: {url_value}")
loop = asyncio.get_event_loop()
# 先尝试 yt-dlp
try:
temp_path = await loop.run_in_executor(None, lambda: _download_yt_dlp(url_value, temp_dir, timestamp))
logger.info(f"yt-dlp downloaded to: {temp_path}")
return temp_path
except Exception as e:
logger.warning(f"yt-dlp download failed: {e}. Trying manual fallback...")
if "douyin" in url_value:
manual_path = await _download_douyin_manual(url_value, temp_dir, timestamp)
if manual_path:
return manual_path
raise ValueError(f"视频下载失败。yt-dlp 报错: {str(e)}")
elif "bilibili" in url_value:
manual_path = await _download_bilibili_manual(url_value, temp_dir, timestamp)
if manual_path:
return manual_path
raise ValueError(f"视频下载失败。yt-dlp 报错: {str(e)}")
else:
raise ValueError(f"视频下载失败: {str(e)}")
def _download_yt_dlp(url_value: str, temp_dir: Path, timestamp: int) -> Path:
"""yt-dlp 下载(阻塞调用,应在线程池中运行)"""
import yt_dlp
logger.info("Attempting download with yt-dlp...")
ydl_opts = {
'format': 'bestaudio/best',
'outtmpl': str(temp_dir / f"tool_download_{timestamp}_%(id)s.%(ext)s"),
'quiet': True,
'no_warnings': True,
'http_headers': {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Referer': 'https://www.douyin.com/',
}
}
with yt_dlp.YoutubeDL() as ydl_raw:
ydl: Any = ydl_raw
ydl.params.update(ydl_opts)
info = ydl.extract_info(url_value, download=True)
if 'requested_downloads' in info:
downloaded_file = info['requested_downloads'][0]['filepath']
else:
ext = info.get('ext', 'mp4')
id = info.get('id')
downloaded_file = str(temp_dir / f"tool_download_{timestamp}_{id}.{ext}")
return Path(downloaded_file)
async def _download_douyin_manual(url: str, temp_dir: Path, timestamp: int) -> Optional[Path]:
"""手动下载抖音视频 (Fallback)"""
logger.info(f"[SuperIPAgent] Starting download for: {url}")
try:
headers = {
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
}
async with httpx.AsyncClient(follow_redirects=True, timeout=10.0) as client:
resp = await client.get(url, headers=headers)
final_url = str(resp.url)
logger.info(f"[SuperIPAgent] Final URL: {final_url}")
modal_id = None
match = re.search(r'/video/(\d+)', final_url)
if match:
modal_id = match.group(1)
if not modal_id:
logger.error("[SuperIPAgent] Could not extract modal_id")
return None
logger.info(f"[SuperIPAgent] Extracted modal_id: {modal_id}")
target_url = f"https://www.douyin.com/user/MS4wLjABAAAAN_s_hups7LD0N4qnrM3o2gI0vuG3pozNaEolz2_py3cHTTrpVr1Z4dukFD9SOlwY?from_tab_name=main&modal_id={modal_id}"
from app.core.config import settings
if not settings.DOUYIN_COOKIE:
logger.warning("[SuperIPAgent] DOUYIN_COOKIE 未配置,视频下载可能失败")
headers_with_cookie = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"cookie": settings.DOUYIN_COOKIE,
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
}
logger.info(f"[SuperIPAgent] Requesting page with Cookie...")
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.get(target_url, headers=headers_with_cookie)
content_match = re.findall(r'<script id="RENDER_DATA" type="application/json">(.*?)</script>', response.text)
if not content_match:
if "SSR_HYDRATED_DATA" in response.text:
content_match = re.findall(r'<script id="SSR_HYDRATED_DATA" type="application/json">(.*?)</script>', response.text)
if not content_match:
logger.error(f"[SuperIPAgent] Could not find RENDER_DATA in page (len={len(response.text)})")
return None
content = unquote(content_match[0])
try:
data = json.loads(content)
except:
logger.error("[SuperIPAgent] JSON decode failed")
return None
video_url = None
try:
if "app" in data and "videoDetail" in data["app"]:
info = data["app"]["videoDetail"]["video"]
if "bitRateList" in info and info["bitRateList"]:
video_url = info["bitRateList"][0]["playAddr"][0]["src"]
elif "playAddr" in info and info["playAddr"]:
video_url = info["playAddr"][0]["src"]
except Exception as e:
logger.error(f"[SuperIPAgent] Path extraction failed: {e}")
if not video_url:
logger.error("[SuperIPAgent] No video_url found")
return None
if video_url.startswith("//"):
video_url = "https:" + video_url
logger.info(f"[SuperIPAgent] Found video URL: {video_url[:50]}...")
temp_path = temp_dir / f"douyin_manual_{timestamp}.mp4"
download_headers = {
'Referer': 'https://www.douyin.com/',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
}
async with httpx.AsyncClient(timeout=60.0) as client:
async with client.stream("GET", video_url, headers=download_headers) as dl_resp:
if dl_resp.status_code == 200:
with open(temp_path, 'wb') as f:
async for chunk in dl_resp.aiter_bytes(chunk_size=8192):
f.write(chunk)
logger.info(f"[SuperIPAgent] Downloaded successfully: {temp_path}")
return temp_path
else:
logger.error(f"[SuperIPAgent] Download failed: {dl_resp.status_code}")
return None
except Exception as e:
logger.error(f"[SuperIPAgent] Logic failed: {e}")
return None
async def _download_bilibili_manual(url: str, temp_dir: Path, timestamp: int) -> Optional[Path]:
"""手动下载 Bilibili 视频 (Playwright Fallback)"""
from playwright.async_api import async_playwright
logger.info(f"[Playwright] Starting Bilibili download for: {url}")
playwright = None
browser = None
try:
playwright = await async_playwright().start()
browser = await playwright.chromium.launch(headless=True, args=['--no-sandbox', '--disable-setuid-sandbox'])
context = await browser.new_context(
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
page = await context.new_page()
logger.info("[Playwright] Navigating to Bilibili...")
await page.goto(url, timeout=45000)
try:
await page.wait_for_selector('video', timeout=15000)
except:
logger.warning("[Playwright] Video selector timeout")
playinfo = await page.evaluate("window.__playinfo__")
audio_url = None
if playinfo and "data" in playinfo and "dash" in playinfo["data"]:
dash = playinfo["data"]["dash"]
if "audio" in dash and dash["audio"]:
audio_url = dash["audio"][0]["baseUrl"]
logger.info(f"[Playwright] Found audio stream in __playinfo__: {audio_url[:50]}...")
if not audio_url:
logger.warning("[Playwright] Could not find audio in __playinfo__")
return None
temp_path = temp_dir / f"bilibili_audio_{timestamp}.m4s"
try:
api_request = context.request
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Referer": "https://www.bilibili.com/"
}
logger.info(f"[Playwright] Downloading audio stream...")
response = await api_request.get(audio_url, headers=headers)
if response.status == 200:
body = await response.body()
with open(temp_path, 'wb') as f:
f.write(body)
logger.info(f"[Playwright] Downloaded successfully: {temp_path}")
return temp_path
else:
logger.error(f"[Playwright] API Request failed: {response.status}")
return None
except Exception as e:
logger.error(f"[Playwright] Download logic error: {e}")
return None
except Exception as e:
logger.error(f"[Playwright] Bilibili download failed: {e}")
return None
finally:
if browser:
await browser.close()
if playwright:
await playwright.stop()

View File

@@ -1,14 +1,24 @@
from pydantic import BaseModel
from typing import Optional
from typing import Optional, List
class CustomAssignment(BaseModel):
material_path: str
start: float # 音频时间轴起点
end: float # 音频时间轴终点
source_start: float = 0.0 # 源视频截取起点
class GenerateRequest(BaseModel):
text: str
voice: str = "zh-CN-YunxiNeural"
material_path: str
material_paths: Optional[List[str]] = None
tts_mode: str = "edgetts"
ref_audio_id: Optional[str] = None
ref_text: Optional[str] = None
language: str = "zh-CN"
generated_audio_id: Optional[str] = None # 预生成配音 ID存在时跳过内联 TTS
title: Optional[str] = None
enable_subtitles: bool = True
subtitle_style_id: Optional[str] = None
@@ -19,3 +29,4 @@ class GenerateRequest(BaseModel):
subtitle_bottom_margin: Optional[int] = None
bgm_id: Optional[str] = None
bgm_volume: Optional[float] = 0.2
custom_assignments: Optional[List[CustomAssignment]] = None

View File

@@ -1,4 +1,4 @@
from typing import Optional, Any
from typing import Optional, Any, List
from pathlib import Path
import time
import traceback
@@ -24,6 +24,17 @@ from .schemas import GenerateRequest
from .task_store import task_store
def _locale_to_whisper_lang(locale: str) -> str:
"""'en-US''en', 'zh-CN''zh'"""
return locale.split("-")[0] if "-" in locale else locale
def _locale_to_qwen_lang(locale: str) -> str:
"""'zh-CN''Chinese', 'en-US''English', 其他 → 'Auto'"""
mapping = {"zh": "Chinese", "en": "English"}
return mapping.get(locale.split("-")[0], "Auto")
_lipsync_service: Optional[LipSyncService] = None
_lipsync_ready: Optional[bool] = None
_lipsync_last_check: float = 0
@@ -79,26 +90,140 @@ def _update_task(task_id: str, **updates: Any) -> None:
task_store.update(task_id, updates)
# ── 多素材辅助函数 ──
def _split_equal(segments: List[dict], material_paths: List[str]) -> List[dict]:
"""按素材数量均分音频时长,对齐到最近的 Whisper 字边界。
Args:
segments: Whisper 产出的 segment 列表, 每个包含 words (字级时间戳)
material_paths: 素材路径列表
Returns:
[{"material_path": "...", "start": 0.0, "end": 5.2, "index": 0}, ...]
"""
# 展平所有 Whisper 字符
all_chars: List[dict] = []
for seg in segments:
for w in seg.get("words", []):
all_chars.append(w)
n = len(material_paths)
if not all_chars or n == 0:
return [{"material_path": material_paths[0] if material_paths else "",
"start": 0.0, "end": 99999.0, "index": 0}]
# 素材数不能超过字符数,否则边界会重复
if n > len(all_chars):
logger.warning(f"[MultiMat] 素材数({n}) > 字符数({len(all_chars)}),裁剪为 {len(all_chars)}")
n = len(all_chars)
total_start = all_chars[0]["start"]
total_end = all_chars[-1]["end"]
seg_dur = (total_end - total_start) / n
# 计算 N-1 个分割点,对齐到最近的字边界
boundaries = [0] # 第一段从第 0 个字开始
for i in range(1, n):
target_time = total_start + i * seg_dur
# 找到 start 时间最接近 target_time 的字
best_idx = boundaries[-1] + 1 # 至少比上一个边界后移 1
best_diff = float("inf")
for j in range(boundaries[-1] + 1, len(all_chars)):
diff = abs(all_chars[j]["start"] - target_time)
if diff < best_diff:
best_diff = diff
best_idx = j
elif diff > best_diff:
break # 时间递增,差值开始变大后可以停了
boundaries.append(min(best_idx, len(all_chars) - 1))
boundaries.append(len(all_chars)) # 最后一段到末尾
# 按边界生成分配结果
assignments: List[dict] = []
for i in range(n):
s_idx = boundaries[i]
e_idx = boundaries[i + 1]
if s_idx >= len(all_chars) or s_idx >= e_idx:
continue
assignments.append({
"material_path": material_paths[i],
"start": all_chars[s_idx]["start"],
"end": all_chars[e_idx - 1]["end"],
"text": "".join(c["word"] for c in all_chars[s_idx:e_idx]),
"index": len(assignments),
})
if not assignments:
return [{"material_path": material_paths[0], "start": 0.0, "end": 99999.0, "index": 0}]
logger.info(f"[MultiMat] 均分 {len(all_chars)} 字为 {len(assignments)}")
for a in assignments:
dur = a["end"] - a["start"]
logger.info(f"{a['index']}: [{a['start']:.2f}-{a['end']:.2f}s] ({dur:.1f}s) {a['text'][:20]}")
return assignments
async def process_video_generation(task_id: str, req: GenerateRequest, user_id: str):
temp_files = []
try:
start_time = time.time()
# ── 确定素材列表 ──
material_paths: List[str] = []
if req.material_paths and len(req.material_paths) > 1:
material_paths = req.material_paths
else:
material_paths = [req.material_path]
is_multi = len(material_paths) > 1
_update_task(task_id, status="processing", progress=5, message="正在下载素材...")
temp_dir = settings.UPLOAD_DIR / "temp"
temp_dir.mkdir(parents=True, exist_ok=True)
input_material_path = temp_dir / f"{task_id}_input.mp4"
temp_files.append(input_material_path)
await _download_material(req.material_path, input_material_path)
# 单素材模式:下载主素材
if not is_multi:
input_material_path = temp_dir / f"{task_id}_input.mp4"
temp_files.append(input_material_path)
await _download_material(material_paths[0], input_material_path)
_update_task(task_id, message="正在生成语音...", progress=10)
audio_path = temp_dir / f"{task_id}_audio.wav"
temp_files.append(audio_path)
if req.tts_mode == "voiceclone":
if req.generated_audio_id:
# 新流程:使用预生成的配音
_update_task(task_id, message="正在下载配音...", progress=12)
audio_url = await storage_service.get_signed_url(
bucket="generated-audios",
path=req.generated_audio_id,
)
await _download_material(audio_url, audio_path)
# 从元数据获取 language
meta_path = req.generated_audio_id.replace("_audio.wav", "_audio.json")
try:
meta_url = await storage_service.get_signed_url(
bucket="generated-audios", path=meta_path,
)
import httpx as _httpx
async with _httpx.AsyncClient(timeout=5.0) as client:
resp = await client.get(meta_url)
if resp.status_code == 200:
meta = resp.json()
req.language = meta.get("language", req.language)
if not req.text.strip():
req.text = meta.get("text", req.text)
except Exception as e:
logger.warning(f"读取配音元数据失败: {e}")
elif req.tts_mode == "voiceclone":
if not req.ref_audio_id or not req.ref_text:
raise ValueError("声音克隆模式需要提供参考音频和参考文字")
@@ -119,7 +244,7 @@ async def process_video_generation(task_id: str, req: GenerateRequest, user_id:
ref_audio_path=str(ref_audio_local),
ref_text=req.ref_text,
output_path=str(audio_path),
language="Chinese"
language=_locale_to_qwen_lang(req.language)
)
else:
_update_task(task_id, message="正在生成语音 (EdgeTTS)...")
@@ -128,52 +253,237 @@ async def process_video_generation(task_id: str, req: GenerateRequest, user_id:
tts_time = time.time() - start_time
print(f"[Pipeline] TTS completed in {tts_time:.1f}s")
_update_task(task_id, progress=25)
_update_task(task_id, message="正在合成唇形 (LatentSync)...", progress=30)
lipsync = _get_lipsync_service()
lipsync_video_path = temp_dir / f"{task_id}_lipsync.mp4"
temp_files.append(lipsync_video_path)
lipsync_start = time.time()
is_ready = await _check_lipsync_ready()
if is_ready:
print(f"[LipSync] Starting LatentSync inference...")
_update_task(task_id, progress=35, message="正在运行 LatentSync 推理...")
await lipsync.generate(str(input_material_path), str(audio_path), str(lipsync_video_path))
else:
print(f"[LipSync] LatentSync not ready, copying original video")
_update_task(task_id, message="唇形同步不可用,使用原始视频...")
import shutil
shutil.copy(str(input_material_path), lipsync_video_path)
lipsync_time = time.time() - lipsync_start
print(f"[Pipeline] LipSync completed in {lipsync_time:.1f}s")
_update_task(task_id, progress=80)
video = VideoService()
captions_path = None
if req.enable_subtitles:
_update_task(task_id, message="正在生成字幕 (Whisper)...", progress=82)
captions_path = temp_dir / f"{task_id}_captions.json"
temp_files.append(captions_path)
if is_multi:
# ══════════════════════════════════════
# 多素材流水线
# ══════════════════════════════════════
_update_task(task_id, progress=12, message="正在分配素材...")
try:
await whisper_service.align(
audio_path=str(audio_path),
text=req.text,
output_path=str(captions_path)
if req.custom_assignments:
# 用户自定义分配,跳过 Whisper 均分
assignments = [
{
"material_path": a.material_path,
"start": a.start,
"end": a.end,
"source_start": a.source_start,
"index": i,
}
for i, a in enumerate(req.custom_assignments)
]
# 仍然需要 Whisper 生成字幕(如果启用)
captions_path = temp_dir / f"{task_id}_captions.json"
temp_files.append(captions_path)
if req.enable_subtitles:
_update_task(task_id, message="正在生成字幕 (Whisper)...")
try:
await whisper_service.align(
audio_path=str(audio_path),
text=req.text,
output_path=str(captions_path),
language=_locale_to_whisper_lang(req.language),
)
print(f"[Pipeline] Whisper alignment completed (custom assignments)")
except Exception as e:
logger.warning(f"Whisper alignment failed: {e}")
captions_path = None
else:
captions_path = None
else:
# 原有逻辑Whisper → _split_equal
_update_task(task_id, message="正在生成字幕 (Whisper)...")
captions_path = temp_dir / f"{task_id}_captions.json"
temp_files.append(captions_path)
try:
captions_data = await whisper_service.align(
audio_path=str(audio_path),
text=req.text,
output_path=str(captions_path),
language=_locale_to_whisper_lang(req.language),
)
print(f"[Pipeline] Whisper alignment completed (multi-material)")
except Exception as e:
logger.warning(f"Whisper alignment failed: {e}")
captions_data = None
captions_path = None
_update_task(task_id, progress=15, message="正在分配素材...")
if captions_data and captions_data.get("segments"):
assignments = _split_equal(captions_data["segments"], material_paths)
else:
# Whisper 失败 → 按时长均分(不依赖字符对齐)
logger.warning("[MultiMat] Whisper 无数据,按时长均分")
audio_dur = video._get_duration(str(audio_path))
if audio_dur <= 0:
audio_dur = 30.0 # 安全兜底
seg_dur = audio_dur / len(material_paths)
assignments = [
{"material_path": material_paths[i], "start": i * seg_dur,
"end": (i + 1) * seg_dur, "index": i}
for i in range(len(material_paths))
]
# 扩展段覆盖完整音频范围首段从0开始末段到音频结尾
audio_duration = video._get_duration(str(audio_path))
if assignments and audio_duration > 0:
assignments[0]["start"] = 0.0
assignments[-1]["end"] = audio_duration
num_segments = len(assignments)
print(f"[Pipeline] Multi-material: {num_segments} segments, {len(material_paths)} materials")
if num_segments == 0:
raise RuntimeError("Multi-material: no valid segments after splitting")
lipsync_start = time.time()
# ── 第一步:下载所有素材并检测分辨率 ──
material_locals: List[Path] = []
resolutions = []
for i, assignment in enumerate(assignments):
material_local = temp_dir / f"{task_id}_material_{i}.mp4"
temp_files.append(material_local)
await _download_material(assignment["material_path"], material_local)
material_locals.append(material_local)
resolutions.append(video.get_resolution(str(material_local)))
# 分辨率不一致时,统一到第一个素材的分辨率
base_res = resolutions[0] if resolutions else (0, 0)
need_scale = any(r != base_res for r in resolutions) and base_res[0] > 0
if need_scale:
logger.info(f"[MultiMat] 素材分辨率不一致,统一到 {base_res[0]}x{base_res[1]}")
# ── 第二步:裁剪每段素材到对应时长 ──
prepared_segments: List[Path] = []
for i, assignment in enumerate(assignments):
seg_progress = 15 + int((i / num_segments) * 30) # 15% → 45%
seg_dur = assignment["end"] - assignment["start"]
_update_task(
task_id,
progress=seg_progress,
message=f"正在准备素材 {i+1}/{num_segments}..."
)
print(f"[Pipeline] Whisper alignment completed")
except Exception as e:
logger.warning(f"Whisper alignment failed, skipping subtitles: {e}")
prepared_path = temp_dir / f"{task_id}_prepared_{i}.mp4"
temp_files.append(prepared_path)
video.prepare_segment(
str(material_locals[i]), seg_dur, str(prepared_path),
target_resolution=base_res if need_scale else None,
source_start=assignment.get("source_start", 0.0),
)
prepared_segments.append(prepared_path)
# ── 第二步:拼接所有素材片段 ──
_update_task(task_id, progress=50, message="正在拼接素材片段...")
concat_path = temp_dir / f"{task_id}_concat.mp4"
temp_files.append(concat_path)
video.concat_videos(
[str(p) for p in prepared_segments],
str(concat_path)
)
# ── 第三步:一次 LatentSync 推理 ──
is_ready = await _check_lipsync_ready()
if is_ready:
_update_task(task_id, progress=55, message="正在合成唇形 (LatentSync)...")
print(f"[LipSync] Multi-material: single LatentSync on concatenated video")
try:
await lipsync.generate(str(concat_path), str(audio_path), str(lipsync_video_path))
except Exception as e:
logger.warning(f"[LipSync] Failed, fallback to concat without lipsync: {e}")
import shutil
shutil.copy(str(concat_path), str(lipsync_video_path))
else:
print(f"[LipSync] Not ready, using concatenated video without lipsync")
import shutil
shutil.copy(str(concat_path), str(lipsync_video_path))
lipsync_time = time.time() - lipsync_start
print(f"[Pipeline] Multi-material prepare + concat + LipSync completed in {lipsync_time:.1f}s")
_update_task(task_id, progress=80)
# 如果用户关闭了字幕,清除 captions_pathWhisper 仅用于句子切分)
if not req.enable_subtitles:
captions_path = None
else:
# ══════════════════════════════════════
# 单素材流水线(原有逻辑)
# ══════════════════════════════════════
# 单素材 + source_start先截取片段
single_source_start = 0.0
if req.custom_assignments and len(req.custom_assignments) == 1:
single_source_start = req.custom_assignments[0].source_start
if single_source_start > 0:
_update_task(task_id, progress=20, message="正在截取素材片段...")
audio_dur = video._get_duration(str(audio_path))
if audio_dur <= 0:
audio_dur = 30.0
trimmed_path = temp_dir / f"{task_id}_trimmed.mp4"
temp_files.append(trimmed_path)
video.prepare_segment(
str(input_material_path), audio_dur, str(trimmed_path),
source_start=single_source_start,
)
input_material_path = trimmed_path
_update_task(task_id, progress=25)
_update_task(task_id, message="正在合成唇形 (LatentSync)...", progress=30)
lipsync_start = time.time()
is_ready = await _check_lipsync_ready()
if is_ready:
print(f"[LipSync] Starting LatentSync inference...")
_update_task(task_id, progress=35, message="正在运行 LatentSync 推理...")
await lipsync.generate(str(input_material_path), str(audio_path), str(lipsync_video_path))
else:
print(f"[LipSync] LatentSync not ready, copying original video")
_update_task(task_id, message="唇形同步不可用,使用原始视频...")
import shutil
shutil.copy(str(input_material_path), lipsync_video_path)
lipsync_time = time.time() - lipsync_start
print(f"[Pipeline] LipSync completed in {lipsync_time:.1f}s")
_update_task(task_id, progress=80)
# 单素材模式Whisper 在 LatentSync 之后
if req.enable_subtitles:
_update_task(task_id, message="正在生成字幕 (Whisper)...", progress=82)
captions_path = temp_dir / f"{task_id}_captions.json"
temp_files.append(captions_path)
try:
await whisper_service.align(
audio_path=str(audio_path),
text=req.text,
output_path=str(captions_path),
language=_locale_to_whisper_lang(req.language),
)
print(f"[Pipeline] Whisper alignment completed")
except Exception as e:
logger.warning(f"Whisper alignment failed, skipping subtitles: {e}")
captions_path = None
_update_task(task_id, progress=85)
video = VideoService()
final_audio_path = audio_path
if req.bgm_id:
_update_task(task_id, message="正在合成背景音乐...", progress=86)

View File

@@ -43,6 +43,7 @@ class GLMService:
要求:
1. 标题要简洁有力能吸引观众点击不超过10个字
2. 标签要与内容相关便于搜索和推荐只要3个
3. 标题和标签必须使用与口播文案相同的语言(如文案是英文就用英文,日文就用日文)
请严格按以下JSON格式返回不要包含其他内容
{{"title": "标题", "tags": ["标签1", "标签2", "标签3"]}}"""
@@ -120,6 +121,49 @@ class GLMService:
async def translate_text(self, text: str, target_lang: str) -> str:
"""
将文案翻译为指定语言
Args:
text: 原始文案
target_lang: 目标语言(如 English, 日本語 等)
Returns:
翻译后的文案
"""
prompt = f"""请将以下文案翻译为{target_lang}
原文:
{text}
要求:
1. 只返回翻译后的文案,不要添加任何解释或说明
2. 保持原文的语气和风格
3. 翻译要自然流畅,符合目标语言的表达习惯"""
try:
client = self._get_client()
logger.info(f"Using GLM to translate text to {target_lang}")
import asyncio
response = await asyncio.to_thread(
client.chat.completions.create,
model=settings.GLM_MODEL,
messages=[{"role": "user", "content": prompt}],
thinking={"type": "disabled"},
max_tokens=2000,
temperature=0.3
)
content = response.choices[0].message.content
logger.info("GLM translation completed")
return content.strip()
except Exception as e:
logger.error(f"GLM translate error: {e}")
raise Exception(f"AI 翻译失败: {str(e)}")
def _parse_json_response(self, content: str) -> dict:
"""解析 GLM 返回的 JSON 内容"""
# 尝试直接解析

View File

@@ -20,12 +20,13 @@ class StorageService:
self.BUCKET_MATERIALS = "materials"
self.BUCKET_OUTPUTS = "outputs"
self.BUCKET_REF_AUDIOS = "ref-audios"
self.BUCKET_GENERATED_AUDIOS = "generated-audios"
# 确保所有 bucket 存在
self._ensure_buckets()
def _ensure_buckets(self):
"""确保所有必需的 bucket 存在"""
buckets = [self.BUCKET_MATERIALS, self.BUCKET_OUTPUTS, self.BUCKET_REF_AUDIOS]
buckets = [self.BUCKET_MATERIALS, self.BUCKET_OUTPUTS, self.BUCKET_REF_AUDIOS, self.BUCKET_GENERATED_AUDIOS]
try:
existing = self.supabase.storage.list_buckets()
existing_names = {b.name for b in existing} if existing else set()

View File

@@ -138,3 +138,145 @@ class VideoService:
return output_path
else:
raise RuntimeError("FFmpeg composition failed")
def concat_videos(self, video_paths: list, output_path: str) -> str:
"""使用 FFmpeg concat demuxer 拼接多个视频片段"""
if not video_paths:
raise ValueError("No video segments to concat")
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
# 生成 concat list 文件
list_path = Path(output_path).parent / f"{Path(output_path).stem}_concat.txt"
with open(list_path, "w", encoding="utf-8") as f:
for vp in video_paths:
f.write(f"file '{vp}'\n")
cmd = [
"ffmpeg", "-y",
"-f", "concat",
"-safe", "0",
"-i", str(list_path),
"-c", "copy",
output_path,
]
try:
if self._run_ffmpeg(cmd):
return output_path
else:
raise RuntimeError("FFmpeg concat failed")
finally:
try:
list_path.unlink(missing_ok=True)
except Exception:
pass
def split_audio(self, audio_path: str, start: float, end: float, output_path: str) -> str:
"""用 FFmpeg 按时间范围切分音频"""
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
duration = end - start
if duration <= 0:
raise ValueError(f"Invalid audio split range: start={start}, end={end}, duration={duration}")
cmd = [
"ffmpeg", "-y",
"-ss", str(start),
"-t", str(duration),
"-i", audio_path,
"-c", "copy",
output_path,
]
if self._run_ffmpeg(cmd):
return output_path
raise RuntimeError(f"FFmpeg audio split failed: {start}-{end}")
def get_resolution(self, file_path: str) -> tuple:
"""获取视频分辨率,返回 (width, height)"""
cmd = [
'ffprobe', '-v', 'error',
'-select_streams', 'v:0',
'-show_entries', 'stream=width,height',
'-of', 'csv=p=0',
file_path
]
try:
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
parts = result.stdout.strip().split(',')
return (int(parts[0]), int(parts[1]))
except Exception:
return (0, 0)
def prepare_segment(self, video_path: str, target_duration: float, output_path: str,
target_resolution: tuple = None, source_start: float = 0.0) -> str:
"""将素材视频裁剪或循环到指定时长(无音频)。
target_resolution: (width, height) 如需统一分辨率则传入,否则保持原分辨率。
source_start: 源视频截取起点(秒),默认 0。
"""
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
video_dur = self._get_duration(video_path)
if video_dur <= 0:
video_dur = target_duration
# 可用时长 = 从 source_start 到视频结尾
available = max(video_dur - source_start, 0.1)
needs_loop = target_duration > available
needs_scale = target_resolution is not None
# 当需要循环且有 source_start 时,先裁剪出片段,再循环裁剪后的文件
# 避免 stream_loop 循环整个视频(而不是从 source_start 开始的片段)
actual_input = video_path
trim_temp = None
if needs_loop and source_start > 0:
trim_temp = str(Path(output_path).parent / (Path(output_path).stem + "_trim_tmp.mp4"))
trim_cmd = [
"ffmpeg", "-y",
"-ss", str(source_start),
"-i", video_path,
"-t", str(available),
"-an",
"-c:v", "libx264", "-preset", "fast", "-crf", "18",
trim_temp,
]
if not self._run_ffmpeg(trim_cmd):
raise RuntimeError(f"FFmpeg trim for loop failed: {video_path}")
actual_input = trim_temp
source_start = 0.0 # 已裁剪,不需要再 seek
# 重新计算循环次数(基于裁剪后文件)
available = self._get_duration(trim_temp) or available
loop_count = int(target_duration / available) + 1 if needs_loop else 0
cmd = ["ffmpeg", "-y"]
if needs_loop:
cmd.extend(["-stream_loop", str(loop_count)])
if source_start > 0:
cmd.extend(["-ss", str(source_start)])
cmd.extend(["-i", actual_input, "-t", str(target_duration), "-an"])
if needs_scale:
w, h = target_resolution
cmd.extend(["-vf", f"scale={w}:{h}:force_original_aspect_ratio=decrease,pad={w}:{h}:(ow-iw)/2:(oh-ih)/2"])
# 需要循环、缩放或指定起点时必须重编码,否则用 stream copy 保持原画质
if needs_loop or needs_scale or source_start > 0:
cmd.extend(["-c:v", "libx264", "-preset", "fast", "-crf", "18"])
else:
cmd.extend(["-c:v", "copy"])
cmd.append(output_path)
try:
if self._run_ffmpeg(cmd):
return output_path
raise RuntimeError(f"FFmpeg prepare_segment failed: {video_path}")
finally:
# 清理裁剪临时文件
if trim_temp:
try:
Path(trim_temp).unlink(missing_ok=True)
except Exception:
pass

View File

@@ -48,7 +48,7 @@ class VoiceCloneService:
"""
# 使用锁确保串行执行,避免 GPU 显存溢出
async with self._lock:
logger.info(f"🎤 Voice Clone: {text[:30]}...")
logger.info(f"🎤 Voice Clone: {text[:30]}... (language={language})")
Path(output_path).parent.mkdir(parents=True, exist_ok=True)
# 读取参考音频

View File

@@ -20,16 +20,23 @@ MAX_CHARS_PER_LINE = 12
def split_word_to_chars(word: str, start: float, end: float) -> list:
"""
将词拆分成单个字符,时间戳线性插值
将词拆分成单个字符,时间戳线性插值
保留英文词前的空格Whisper 输出如 " Hello"),用于正确重建英文字幕。
Args:
word: 词文本
word: 词文本(可能含前导空格)
start: 词开始时间
end: 词结束时间
Returns:
单字符列表,每个包含 word/start/end
"""
# 保留前导空格(英文 Whisper 输出常见 " Hello" 形式)
leading_space = ""
if word and not word[0].strip():
leading_space = " "
word = word.lstrip()
tokens = []
ascii_buffer = ""
@@ -54,7 +61,8 @@ def split_word_to_chars(word: str, start: float, end: float) -> list:
return []
if len(tokens) == 1:
return [{"word": tokens[0], "start": start, "end": end}]
w = leading_space + tokens[0] if leading_space else tokens[0]
return [{"word": w, "start": start, "end": end}]
# 线性插值时间戳
duration = end - start
@@ -64,8 +72,11 @@ def split_word_to_chars(word: str, start: float, end: float) -> list:
for i, token in enumerate(tokens):
token_start = start + i * token_duration
token_end = start + (i + 1) * token_duration
w = token
if i == 0 and leading_space:
w = leading_space + w
result.append({
"word": token,
"word": w,
"start": round(token_start, 3),
"end": round(token_end, 3)
})
@@ -108,7 +119,7 @@ def split_segment_to_lines(words: List[dict], max_chars: int = MAX_CHARS_PER_LIN
if should_break and current_words:
segments.append({
"text": current_text,
"text": current_text.strip(),
"start": current_words[0]["start"],
"end": current_words[-1]["end"],
"words": current_words.copy()
@@ -119,7 +130,7 @@ def split_segment_to_lines(words: List[dict], max_chars: int = MAX_CHARS_PER_LIN
# 处理剩余的字
if current_words:
segments.append({
"text": current_text,
"text": current_text.strip(),
"start": current_words[0]["start"],
"end": current_words[-1]["end"],
"words": current_words.copy()
@@ -162,7 +173,8 @@ class WhisperService:
self,
audio_path: str,
text: str,
output_path: Optional[str] = None
output_path: Optional[str] = None,
language: str = "zh",
) -> dict:
"""
对音频进行转录,生成字级别时间戳
@@ -171,12 +183,16 @@ class WhisperService:
audio_path: 音频文件路径
text: 原始文本(用于参考,但实际使用 whisper 转录结果)
output_path: 可选,输出 JSON 文件路径
language: 语言代码 (zh/en 等)
Returns:
包含字级别时间戳的字典
"""
import asyncio
# 英文等西文需要更大的每行字数
max_chars = 40 if language != "zh" else MAX_CHARS_PER_LINE
def _do_transcribe():
model = self._load_model()
@@ -185,7 +201,7 @@ class WhisperService:
# 转录并获取字级别时间戳
segments_iter, info = model.transcribe(
audio_path,
language="zh",
language=language,
word_timestamps=True, # 启用字级别时间戳
vad_filter=True, # 启用 VAD 过滤静音
)
@@ -198,9 +214,10 @@ class WhisperService:
all_words = []
if segment.words:
for word_info in segment.words:
word_text = word_info.word.strip()
if word_text:
word_text = word_info.word
if word_text.strip():
# 将词拆分成单字,时间戳线性插值
# 保留前导空格用于英文词间距
chars = split_word_to_chars(
word_text,
word_info.start,
@@ -210,7 +227,7 @@ class WhisperService:
# 将长段落按标点和字数拆分成多行
if all_words:
line_segments = split_segment_to_lines(all_words, MAX_CHARS_PER_LINE)
line_segments = split_segment_to_lines(all_words, max_chars)
all_segments.extend(line_segments)
logger.info(f"Generated {len(all_segments)} subtitle segments")

View File

@@ -23,20 +23,28 @@ SERVICES = [
"name": "vigent2-qwen-tts",
"url": "http://localhost:8009/health",
"failures": 0,
"threshold": 3,
"threshold": 5, # 连续5次失败才重启5×30s = 2.5分钟容忍期)
"timeout": 10.0,
"restart_cmd": ["pm2", "restart", "vigent2-qwen-tts"]
"restart_cmd": ["pm2", "restart", "vigent2-qwen-tts"],
"cooldown_until": 0, # 重启后的冷却截止时间戳
"cooldown_sec": 120, # 重启后等待120秒再开始检查
}
]
async def check_service(service):
"""检查单个服务健康状态"""
# 冷却期内跳过检查
now = time.time()
if now < service.get("cooldown_until", 0):
remaining = int(service["cooldown_until"] - now)
logger.debug(f"⏳ 服务 {service['name']} 冷却中,剩余 {remaining}s")
return True
try:
timeout = service.get("timeout", 10.0)
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.get(service["url"])
if response.status_code == 200:
# 成功
if service["failures"] > 0:
logger.info(f"✅ 服务 {service['name']} 已恢复正常")
service["failures"] = 0
@@ -45,35 +53,36 @@ async def check_service(service):
logger.warning(f"⚠️ 服务 {service['name']} 返回状态码 {response.status_code}")
except Exception as e:
logger.warning(f"⚠️ 无法连接服务 {service['name']}: {str(e)}")
# 失败处理
service["failures"] += 1
logger.warning(f"❌ 服务 {service['name']} 连续失败 {service['failures']}/{service['threshold']}")
if service["failures"] >= service['threshold']:
logger.error(f"🚨 服务 {service['name']} 已达到失败阈值,正在重启...")
try:
subprocess.run(service["restart_cmd"], check=True)
logger.info(f"♻️ 服务 {service['name']} 重启命令已发送")
# 重启后给予一段宽限期 (例如 60秒) 不检查,等待服务启动
service["failures"] = 0 # 重置计数
return "restarting"
service["failures"] = 0
# 设置冷却期,等待服务完成启动和模型加载
service["cooldown_until"] = time.time() + service.get("cooldown_sec", 120)
return "restarting"
except Exception as restart_error:
logger.error(f"💥 重启服务 {service['name']} 失败: {restart_error}")
return False
async def main():
logger.info("🛡️ ViGent2 服务看门狗 (Watchdog) 已启动")
# 启动时给所有服务一个初始冷却期,避免服务还没起来就被判定失败
for service in SERVICES:
service["cooldown_until"] = time.time() + 60
while True:
# 并发检查所有服务
for service in SERVICES:
result = await check_service(service)
if result == "restarting":
# 如果有服务重启,额外等待包含启动时间
pass
await check_service(service)
# 每 30 秒检查一次
await asyncio.sleep(30)

View File

@@ -8,6 +8,9 @@
"name": "frontend",
"version": "0.1.0",
"dependencies": {
"@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@supabase/supabase-js": "^2.93.1",
"axios": "^1.13.4",
"lucide-react": "^0.563.0",
@@ -15,7 +18,8 @@
"react": "19.2.3",
"react-dom": "19.2.3",
"sonner": "^2.0.7",
"swr": "^2.3.8"
"swr": "^2.3.8",
"wavesurfer.js": "^7.12.1"
},
"devDependencies": {
"@tailwindcss/postcss": "^4",
@@ -281,6 +285,59 @@
"node": ">=6.9.0"
}
},
"node_modules/@dnd-kit/accessibility": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/@dnd-kit/accessibility/-/accessibility-3.1.1.tgz",
"integrity": "sha512-2P+YgaXF+gRsIihwwY1gCsQSYnu9Zyj2py8kY5fFvUM1qm2WA2u639R6YNVfU4GWr+ZM5mqEsfHZZLoRONbemw==",
"license": "MIT",
"dependencies": {
"tslib": "^2.0.0"
},
"peerDependencies": {
"react": ">=16.8.0"
}
},
"node_modules/@dnd-kit/core": {
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/@dnd-kit/core/-/core-6.3.1.tgz",
"integrity": "sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ==",
"license": "MIT",
"dependencies": {
"@dnd-kit/accessibility": "^3.1.1",
"@dnd-kit/utilities": "^3.2.2",
"tslib": "^2.0.0"
},
"peerDependencies": {
"react": ">=16.8.0",
"react-dom": ">=16.8.0"
}
},
"node_modules/@dnd-kit/sortable": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@dnd-kit/sortable/-/sortable-10.0.0.tgz",
"integrity": "sha512-+xqhmIIzvAYMGfBYYnbKuNicfSsk4RksY2XdmJhT+HAC01nix6fHCztU68jooFiMUB01Ky3F0FyOvhG/BZrWkg==",
"license": "MIT",
"dependencies": {
"@dnd-kit/utilities": "^3.2.2",
"tslib": "^2.0.0"
},
"peerDependencies": {
"@dnd-kit/core": "^6.3.0",
"react": ">=16.8.0"
}
},
"node_modules/@dnd-kit/utilities": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/@dnd-kit/utilities/-/utilities-3.2.2.tgz",
"integrity": "sha512-+MKAJEOfaBe5SmV6t34p80MMKhjvUz0vRrvVJbPT0WElzaOJ/1xs+D+KDv+tD/NE5ujfrChEcshd4fLn0wpiqg==",
"license": "MIT",
"dependencies": {
"tslib": "^2.0.0"
},
"peerDependencies": {
"react": ">=16.8.0"
}
},
"node_modules/@emnapi/core": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz",
@@ -6611,6 +6668,12 @@
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/wavesurfer.js": {
"version": "7.12.1",
"resolved": "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-7.12.1.tgz",
"integrity": "sha512-NswPjVHxk0Q1F/VMRemCPUzSojjuHHisQrBqQiRXg7MVbe3f5vQ6r0rTTXA/a/neC/4hnOEC4YpXca4LpH0SUg==",
"license": "BSD-3-Clause"
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

View File

@@ -9,6 +9,9 @@
"lint": "eslint"
},
"dependencies": {
"@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@supabase/supabase-js": "^2.93.1",
"axios": "^1.13.4",
"lucide-react": "^0.563.0",
@@ -16,7 +19,8 @@
"react": "19.2.3",
"react-dom": "19.2.3",
"sonner": "^2.0.7",
"swr": "^2.3.8"
"swr": "^2.3.8",
"wavesurfer.js": "^7.12.1"
},
"devDependencies": {
"@tailwindcss/postcss": "^4",

View File

@@ -1,8 +1,8 @@
import type { Metadata, Viewport } from "next";
import { Geist, Geist_Mono } from "next/font/google";
import "./globals.css";
import { AuthProvider } from "@/contexts/AuthContext";
import { TaskProvider } from "@/contexts/TaskContext";
import { AuthProvider } from "@/shared/contexts/AuthContext";
import { TaskProvider } from "@/shared/contexts/TaskContext";
import { Toaster } from "sonner";
@@ -46,7 +46,6 @@ export default function RootLayout({
<Toaster
position="top-center"
richColors
closeButton
toastOptions={{
duration: 3000,
className: "text-sm",

View File

@@ -1,7 +1,7 @@
"use client";
import { useState, useEffect, useRef } from "react";
import { useAuth } from "@/contexts/AuthContext";
import { useAuth } from "@/shared/contexts/AuthContext";
import api from "@/shared/api/axios";
import { ApiResponse } from "@/shared/api/types";

View File

@@ -1,6 +1,6 @@
"use client";
import { useTask } from "@/contexts/TaskContext";
import { useTask } from "@/shared/contexts/TaskContext";
import Link from "next/link";
import { usePathname } from "next/navigation";

View File

@@ -0,0 +1,192 @@
import { useCallback, useEffect, useRef, useState } from "react";
import api from "@/shared/api/axios";
import { ApiResponse, unwrap } from "@/shared/api/types";
import { toast } from "sonner";
export interface GeneratedAudio {
id: string;
name: string;
path: string;
duration_sec: number;
text: string;
tts_mode: string;
language: string;
created_at: number;
}
interface AudioTask {
status: string;
progress?: number;
message?: string;
output?: GeneratedAudio & { audio_id: string };
}
interface UseGeneratedAudiosOptions {
selectedAudioId: string | null;
setSelectedAudioId: React.Dispatch<React.SetStateAction<string | null>>;
}
export const useGeneratedAudios = ({
selectedAudioId,
setSelectedAudioId,
}: UseGeneratedAudiosOptions) => {
const [generatedAudios, setGeneratedAudios] = useState<GeneratedAudio[]>([]);
const [selectedAudio, setSelectedAudio] = useState<GeneratedAudio | null>(null);
const [isGeneratingAudio, setIsGeneratingAudio] = useState(false);
const [audioTaskId, setAudioTaskId] = useState<string | null>(null);
const [audioTask, setAudioTask] = useState<AudioTask | null>(null);
const pollRef = useRef<NodeJS.Timeout | null>(null);
const fetchGeneratedAudios = useCallback(async (selectId?: string) => {
try {
const { data: res } = await api.get<ApiResponse<{ items: GeneratedAudio[] }>>(
"/api/generated-audios"
);
const payload = unwrap(res);
const items: GeneratedAudio[] = payload.items || [];
setGeneratedAudios(items);
if (selectId && items.length > 0) {
if (selectId === "__latest__") {
setSelectedAudioId(items[0].id);
setSelectedAudio(items[0]);
} else {
const found = items.find((a) => a.id === selectId);
if (found) {
setSelectedAudioId(found.id);
setSelectedAudio(found);
}
}
}
} catch (error) {
console.error("获取配音列表失败:", error);
}
}, [setSelectedAudioId]);
// Sync selectedAudio when selectedAudioId changes externally (e.g. from persistence)
useEffect(() => {
if (!selectedAudioId || generatedAudios.length === 0) return;
const found = generatedAudios.find((a) => a.id === selectedAudioId);
if (found) {
setSelectedAudio(found);
}
}, [selectedAudioId, generatedAudios]);
const stopPolling = useCallback(() => {
if (pollRef.current) {
clearInterval(pollRef.current);
pollRef.current = null;
}
}, []);
const startPolling = useCallback((taskId: string) => {
stopPolling();
pollRef.current = setInterval(async () => {
try {
const { data: res } = await api.get<ApiResponse<AudioTask>>(
`/api/generated-audios/tasks/${taskId}`
);
const task = unwrap(res);
setAudioTask(task);
if (task.status === "completed") {
stopPolling();
setIsGeneratingAudio(false);
setAudioTaskId(null);
// Refresh list and select the new audio
await fetchGeneratedAudios("__latest__");
toast.success(task.message || "配音生成完成");
} else if (task.status === "failed") {
stopPolling();
setIsGeneratingAudio(false);
setAudioTaskId(null);
toast.error(task.message || "配音生成失败");
} else if (task.status === "not_found") {
stopPolling();
setIsGeneratingAudio(false);
setAudioTaskId(null);
setAudioTask(null);
toast.error("任务已丢失(服务可能已重启),请重新生成");
}
} catch {
// Network error, keep polling
}
}, 1000);
}, [stopPolling, fetchGeneratedAudios]);
// Cleanup on unmount
useEffect(() => {
return () => stopPolling();
}, [stopPolling]);
const generateAudio = useCallback(async (params: {
text: string;
tts_mode: string;
voice?: string;
ref_audio_id?: string;
ref_text?: string;
language: string;
}) => {
setIsGeneratingAudio(true);
setAudioTask({ status: "pending", progress: 0, message: "正在提交..." });
try {
const { data: res } = await api.post<ApiResponse<{ task_id: string }>>(
"/api/generated-audios/generate",
params
);
const { task_id } = unwrap(res);
setAudioTaskId(task_id);
startPolling(task_id);
} catch (err: unknown) {
setIsGeneratingAudio(false);
setAudioTask(null);
const axiosErr = err as { response?: { data?: { message?: string } }; message?: string };
const errorMsg = axiosErr.response?.data?.message || axiosErr.message || String(err);
toast.error(`配音生成失败: ${errorMsg}`);
}
}, [startPolling]);
const deleteAudio = useCallback(async (audioId: string) => {
if (!confirm("确定要删除这个配音吗?")) return;
try {
await api.delete(`/api/generated-audios/${encodeURIComponent(audioId)}`);
if (selectedAudioId === audioId) {
setSelectedAudioId(null);
setSelectedAudio(null);
}
fetchGeneratedAudios();
} catch (error) {
toast.error("删除失败: " + error);
}
}, [fetchGeneratedAudios, selectedAudioId, setSelectedAudioId]);
const renameAudio = useCallback(async (audioId: string, newName: string) => {
try {
await api.put(`/api/generated-audios/${encodeURIComponent(audioId)}`, {
new_name: newName,
});
fetchGeneratedAudios();
} catch (err: unknown) {
toast.error("重命名失败: " + String(err));
}
}, [fetchGeneratedAudios]);
const selectAudio = useCallback((audio: GeneratedAudio) => {
setSelectedAudioId(audio.id);
setSelectedAudio(audio);
}, [setSelectedAudioId]);
return {
generatedAudios,
selectedAudio,
selectedAudioId,
isGeneratingAudio,
audioTask,
fetchGeneratedAudios,
generateAudio,
deleteAudio,
renameAudio,
selectAudio,
};
};

View File

@@ -11,27 +11,81 @@ import {
} from "@/shared/lib/media";
import { clampTitle } from "@/shared/lib/title";
import { useTitleInput } from "@/shared/hooks/useTitleInput";
import { useAuth } from "@/contexts/AuthContext";
import { useTask } from "@/contexts/TaskContext";
import { useAuth } from "@/shared/contexts/AuthContext";
import { useTask } from "@/shared/contexts/TaskContext";
import { toast } from "sonner";
import { usePublishPrefetch } from "@/shared/hooks/usePublishPrefetch";
import { PublishAccount } from "@/shared/types/publish";
import { useBgm } from "@/features/home/model/useBgm";
import { useGeneratedVideos } from "@/features/home/model/useGeneratedVideos";
import { useGeneratedAudios } from "@/features/home/model/useGeneratedAudios";
import { useHomePersistence } from "@/features/home/model/useHomePersistence";
import { useMaterials } from "@/features/home/model/useMaterials";
import { useMediaPlayers } from "@/features/home/model/useMediaPlayers";
import { useRefAudios } from "@/features/home/model/useRefAudios";
import { useTitleSubtitleStyles } from "@/features/home/model/useTitleSubtitleStyles";
import { useTimelineEditor } from "@/features/home/model/useTimelineEditor";
import { useSavedScripts } from "@/features/home/model/useSavedScripts";
import { ApiResponse, unwrap } from "@/shared/api/types";
const VOICES = [
{ id: "zh-CN-YunxiNeural", name: "云溪 (男声-年轻)" },
{ id: "zh-CN-YunjianNeural", name: "云 (男声-新闻)" },
{ id: "zh-CN-YunyangNeural", name: "云 (男声-专业)" },
{ id: "zh-CN-XiaoxiaoNeural", name: "晓晓 (声-活泼)" },
{ id: "zh-CN-XiaoyiNeural", name: "晓 (女声-温柔)" },
];
const VOICES: Record<string, { id: string; name: string }[]> = {
"zh-CN": [
{ id: "zh-CN-YunxiNeural", name: "云 (男声-年轻)" },
{ id: "zh-CN-YunjianNeural", name: "云 (男声-新闻)" },
{ id: "zh-CN-YunyangNeural", name: "云扬 (声-专业)" },
{ id: "zh-CN-XiaoxiaoNeural", name: "晓 (女声-活泼)" },
{ id: "zh-CN-XiaoyiNeural", name: "晓伊 (女声-温柔)" },
],
"en-US": [
{ id: "en-US-GuyNeural", name: "Guy (Male)" },
{ id: "en-US-JennyNeural", name: "Jenny (Female)" },
],
"ja-JP": [
{ id: "ja-JP-KeitaNeural", name: "圭太 (男声)" },
{ id: "ja-JP-NanamiNeural", name: "七海 (女声)" },
],
"ko-KR": [
{ id: "ko-KR-InJoonNeural", name: "인준 (男声)" },
{ id: "ko-KR-SunHiNeural", name: "선히 (女声)" },
],
"fr-FR": [
{ id: "fr-FR-HenriNeural", name: "Henri (Male)" },
{ id: "fr-FR-DeniseNeural", name: "Denise (Female)" },
],
"de-DE": [
{ id: "de-DE-ConradNeural", name: "Conrad (Male)" },
{ id: "de-DE-KatjaNeural", name: "Katja (Female)" },
],
"es-ES": [
{ id: "es-ES-AlvaroNeural", name: "Álvaro (Male)" },
{ id: "es-ES-ElviraNeural", name: "Elvira (Female)" },
],
"ru-RU": [
{ id: "ru-RU-DmitryNeural", name: "Дмитрий (Male)" },
{ id: "ru-RU-SvetlanaNeural", name: "Светлана (Female)" },
],
"it-IT": [
{ id: "it-IT-DiegoNeural", name: "Diego (Male)" },
{ id: "it-IT-ElsaNeural", name: "Elsa (Female)" },
],
"pt-BR": [
{ id: "pt-BR-AntonioNeural", name: "Antonio (Male)" },
{ id: "pt-BR-FranciscaNeural", name: "Francisca (Female)" },
],
};
const LANG_TO_LOCALE: Record<string, string> = {
"中文": "zh-CN",
"English": "en-US",
"日本語": "ja-JP",
"한국어": "ko-KR",
"Français": "fr-FR",
"Deutsch": "de-DE",
"Español": "es-ES",
"Русский": "ru-RU",
"Italiano": "it-IT",
"Português": "pt-BR",
};
@@ -70,22 +124,17 @@ interface RefAudio {
created_at: number;
}
interface Material {
id: string;
name: string;
path: string;
size_mb: number;
scene?: string;
}
import type { Material } from "@/shared/types/material";
export const useHomeController = () => {
const apiBase = getApiBaseUrl();
const [selectedMaterial, setSelectedMaterial] = useState<string>("");
const [selectedMaterials, setSelectedMaterials] = useState<string[]>([]);
const [previewMaterial, setPreviewMaterial] = useState<string | null>(null);
const [text, setText] = useState<string>("");
const [voice, setVoice] = useState<string>("zh-CN-YunxiNeural");
const [textLang, setTextLang] = useState<string>("zh-CN");
// 使用全局任务状态
const { currentTask, isGenerating, startTask } = useTask();
@@ -96,7 +145,6 @@ export const useHomeController = () => {
// 字幕和标题相关状态
const [videoTitle, setVideoTitle] = useState<string>("");
const [enableSubtitles, setEnableSubtitles] = useState<boolean>(true);
const [selectedSubtitleStyleId, setSelectedSubtitleStyleId] = useState<string>("");
const [selectedTitleStyleId, setSelectedTitleStyleId] = useState<string>("");
const [subtitleFontSize, setSubtitleFontSize] = useState<number>(80);
@@ -119,6 +167,13 @@ export const useHomeController = () => {
const [selectedRefAudio, setSelectedRefAudio] = useState<RefAudio | null>(null);
const [refText, setRefText] = useState(FIXED_REF_TEXT);
// 预生成配音选中 ID
const [selectedAudioId, setSelectedAudioId] = useState<string | null>(null);
// ClipTrimmer 模态框状态
const [clipTrimmerOpen, setClipTrimmerOpen] = useState(false);
const [clipTrimmerSegmentId, setClipTrimmerSegmentId] = useState<string | null>(null);
// 音频预览与重命名状态
const [editingAudioId, setEditingAudioId] = useState<string | null>(null);
const [editName, setEditName] = useState("");
@@ -181,8 +236,8 @@ export const useHomeController = () => {
{ new_name: editMaterialName.trim() }
);
const payload = unwrap(res);
if (selectedMaterial === materialId && payload?.id) {
setSelectedMaterial(payload.id);
if (selectedMaterials.includes(materialId) && payload?.id) {
setSelectedMaterials((prev) => prev.map((x) => (x === materialId ? payload.id : x)));
}
setEditingMaterialId(null);
setEditMaterialName("");
@@ -197,6 +252,10 @@ export const useHomeController = () => {
// AI 生成标题标签
const [isGeneratingMeta, setIsGeneratingMeta] = useState(false);
// AI 多语言翻译
const [isTranslating, setIsTranslating] = useState(false);
const [originalText, setOriginalText] = useState<string | null>(null);
// 在线录音相关
const [isRecording, setIsRecording] = useState(false);
const [recordedBlob, setRecordedBlob] = useState<Blob | null>(null);
@@ -226,11 +285,13 @@ export const useHomeController = () => {
uploadError,
setUploadError,
fetchMaterials,
toggleMaterial,
reorderMaterials,
deleteMaterial,
handleUpload,
} = useMaterials({
selectedMaterial,
setSelectedMaterial,
selectedMaterials,
setSelectedMaterials,
});
const {
@@ -296,6 +357,33 @@ export const useHomeController = () => {
resolveMediaUrl,
});
const {
generatedAudios,
selectedAudio,
isGeneratingAudio,
audioTask,
fetchGeneratedAudios,
generateAudio,
deleteAudio,
renameAudio,
selectAudio,
} = useGeneratedAudios({
selectedAudioId,
setSelectedAudioId,
});
const {
segments: timelineSegments,
reorderSegments,
setSourceRange,
toCustomAssignments,
} = useTimelineEditor({
audioDuration: selectedAudio?.duration_sec ?? 0,
materials,
selectedMaterials,
storageKey,
});
useEffect(() => {
if (isAuthLoading || !userId) return;
let active = true;
@@ -338,14 +426,14 @@ export const useHomeController = () => {
setText,
videoTitle,
setVideoTitle,
enableSubtitles,
setEnableSubtitles,
ttsMode,
setTtsMode,
voice,
setVoice,
selectedMaterial,
setSelectedMaterial,
textLang,
setTextLang,
selectedMaterials,
setSelectedMaterials,
selectedSubtitleStyleId,
setSelectedSubtitleStyleId,
selectedTitleStyleId,
@@ -369,8 +457,18 @@ export const useHomeController = () => {
selectedVideoId,
setSelectedVideoId,
selectedRefAudio,
selectedAudioId,
setSelectedAudioId,
});
const { savedScripts, saveScript, deleteScript: deleteSavedScript } = useSavedScripts(storageKey);
const handleSaveScript = () => {
if (!text.trim()) return;
saveScript(text);
toast.success("文案已保存");
};
const syncTitleToPublish = (value: string) => {
if (typeof window !== "undefined") {
localStorage.setItem(`vigent_${storageKey}_publish_title`, value);
@@ -390,6 +488,7 @@ export const useHomeController = () => {
fetchMaterials(),
fetchGeneratedVideos(),
fetchRefAudios(),
fetchGeneratedAudios(),
refreshSubtitleStyles(),
refreshTitleStyles(),
fetchBgmList(),
@@ -410,7 +509,8 @@ export const useHomeController = () => {
}, [isGenerating, currentTask, fetchGeneratedVideos]);
useEffect(() => {
const material = materials.find((item) => item.id === selectedMaterial);
const firstSelected = selectedMaterials[0];
const material = materials.find((item) => item.id === firstSelected);
if (!material?.path) {
setMaterialDimensions(null);
return;
@@ -450,7 +550,7 @@ export const useHomeController = () => {
video.removeEventListener("loadedmetadata", handleLoaded);
video.removeEventListener("error", handleError);
};
}, [materials, selectedMaterial]);
}, [materials, selectedMaterials]);
useEffect(() => {
@@ -485,13 +585,22 @@ export const useHomeController = () => {
}
}, [selectedBgmId, bgmList]);
// 素材列表滚动:跳过首次恢复,仅用户主动操作时滚动
const materialScrollReady = useRef(false);
useEffect(() => {
if (!selectedMaterial) return;
const target = materialItemRefs.current[selectedMaterial];
const firstSelected = selectedMaterials[0];
if (!firstSelected) return;
if (!materialScrollReady.current) {
// 首次有选中素材时标记就绪,但不滚动(避免刷新后整页跳动)
materialScrollReady.current = true;
return;
}
const target = materialItemRefs.current[firstSelected];
if (target) {
target.scrollIntoView({ block: "nearest", behavior: "smooth" });
}
}, [selectedMaterial, materials]);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedMaterials.length]);
// 【修复】历史视频默认选中逻辑
// 当持久化恢复完成,且列表加载完毕,如果没选中任何视频,默认选中第一个
@@ -639,19 +748,87 @@ export const useHomeController = () => {
}
};
// AI 多语言翻译
const handleTranslate = async (targetLang: string) => {
if (!text.trim()) {
toast.error("请先输入口播文案");
return;
}
// 首次翻译时保存原文
if (originalText === null) {
setOriginalText(text);
}
setIsTranslating(true);
try {
const { data: res } = await api.post<ApiResponse<{ translated_text: string }>>(
"/api/ai/translate",
{ text: text.trim(), target_lang: targetLang }
);
const payload = unwrap(res);
setText(payload.translated_text || "");
// 根据翻译目标语言更新 textLang 并自动切换声音
const locale = LANG_TO_LOCALE[targetLang] || "zh-CN";
setTextLang(locale);
if (ttsMode === "edgetts") {
const langVoices = VOICES[locale] || VOICES["zh-CN"];
setVoice(langVoices[0].id);
}
} catch (err: unknown) {
console.error("AI translate failed:", err);
const axiosErr = err as { response?: { data?: { message?: string } }; message?: string };
const errorMsg = axiosErr.response?.data?.message || axiosErr.message || String(err);
toast.error(`AI 翻译失败: ${errorMsg}`);
} finally {
setIsTranslating(false);
}
};
const handleRestoreOriginal = () => {
if (originalText !== null) {
setText(originalText);
setOriginalText(null);
setTextLang("zh-CN");
if (ttsMode === "edgetts") {
setVoice(VOICES["zh-CN"][0].id);
}
}
};
// 生成配音
const handleGenerateAudio = async () => {
if (!text.trim()) {
toast.error("请先输入文案");
return;
}
if (ttsMode === "voiceclone" && !selectedRefAudio) {
toast.error("请选择参考音频");
return;
}
const params = {
text: text.trim(),
tts_mode: ttsMode,
voice: ttsMode === "edgetts" ? voice : undefined,
ref_audio_id: ttsMode === "voiceclone" ? selectedRefAudio!.id : undefined,
ref_text: ttsMode === "voiceclone" ? refText : undefined,
language: textLang,
};
await generateAudio(params);
};
// 生成视频
const handleGenerate = async () => {
if (!selectedMaterial || !text.trim()) {
if (selectedMaterials.length === 0 || !text.trim()) {
toast.error("请先选择素材并填写文案");
return;
}
// 声音克隆模式校验
if (ttsMode === "voiceclone") {
if (!selectedRefAudio) {
toast.error("请选择或上传参考音频");
return;
}
if (!selectedAudio) {
toast.error("请先生成并选中配音");
return;
}
if (enableBgm && !selectedBgmId) {
@@ -663,26 +840,44 @@ export const useHomeController = () => {
try {
// 查找选中的素材对象以获取路径
const materialObj = materials.find((m) => m.id === selectedMaterial);
if (!materialObj) {
const firstMaterialObj = materials.find((m) => m.id === selectedMaterials[0]);
if (!firstMaterialObj) {
toast.error("素材数据异常");
return;
}
// 构建请求参数
// 构建请求参数 - 使用预生成配音
const payload: Record<string, unknown> = {
material_path: materialObj.path,
text: text,
tts_mode: ttsMode,
material_path: firstMaterialObj.path,
text: selectedAudio.text || text,
generated_audio_id: selectedAudio.id,
language: selectedAudio.language || textLang,
title: videoTitle.trim() || undefined,
enable_subtitles: enableSubtitles,
enable_subtitles: true,
};
if (enableSubtitles && selectedSubtitleStyleId) {
// 多素材
if (selectedMaterials.length > 1) {
payload.material_paths = selectedMaterials
.map((id) => materials.find((x) => x.id === id)?.path)
.filter((path): path is string => !!path);
// 发送自定义时间轴分配
const assignments = toCustomAssignments();
if (assignments.length > 0) {
payload.custom_assignments = assignments;
}
}
// 单素材 + 截取起点
if (selectedMaterials.length === 1 && timelineSegments[0]?.sourceStart > 0) {
payload.custom_assignments = toCustomAssignments();
}
if (selectedSubtitleStyleId) {
payload.subtitle_style_id = selectedSubtitleStyleId;
}
if (enableSubtitles && subtitleFontSize) {
if (subtitleFontSize) {
payload.subtitle_font_size = Math.round(subtitleFontSize);
}
@@ -698,22 +893,13 @@ export const useHomeController = () => {
payload.title_top_margin = Math.round(titleTopMargin);
}
if (enableSubtitles) {
payload.subtitle_bottom_margin = Math.round(subtitleBottomMargin);
}
payload.subtitle_bottom_margin = Math.round(subtitleBottomMargin);
if (enableBgm && selectedBgmId) {
payload.bgm_id = selectedBgmId;
payload.bgm_volume = bgmVolume;
}
if (ttsMode === "edgetts") {
payload.voice = voice;
} else {
payload.ref_audio_id = selectedRefAudio!.id;
payload.ref_text = refText;
}
// 创建生成任务
const { data: res } = await api.post<ApiResponse<{ task_id: string }>>(
"/api/videos/generate",
@@ -774,8 +960,8 @@ export const useHomeController = () => {
fetchMaterials,
deleteMaterial,
handleUpload,
selectedMaterial,
setSelectedMaterial,
selectedMaterials,
toggleMaterial,
handlePreviewMaterial,
editingMaterialId,
editMaterialName,
@@ -789,6 +975,13 @@ export const useHomeController = () => {
setExtractModalOpen,
handleGenerateMeta,
isGeneratingMeta,
handleTranslate,
isTranslating,
originalText,
handleRestoreOriginal,
savedScripts,
handleSaveScript,
deleteSavedScript,
showStylePreview,
setShowStylePreview,
videoTitle,
@@ -809,17 +1002,16 @@ export const useHomeController = () => {
setTitleTopMargin,
subtitleBottomMargin,
setSubtitleBottomMargin,
enableSubtitles,
setEnableSubtitles,
resolveAssetUrl,
getFontFormat,
buildTextShadow,
materialDimensions,
ttsMode,
setTtsMode,
voices: VOICES,
voices: VOICES[textLang] || VOICES["zh-CN"],
voice,
setVoice,
textLang,
refAudios,
selectedRefAudio,
handleSelectRefAudio,
@@ -870,5 +1062,22 @@ export const useHomeController = () => {
fetchGeneratedVideos,
registerVideoRef,
formatDate,
generatedAudios,
selectedAudio,
selectedAudioId,
isGeneratingAudio,
audioTask,
fetchGeneratedAudios,
handleGenerateAudio,
deleteAudio,
renameAudio,
selectAudio,
timelineSegments,
reorderSegments,
setSourceRange,
clipTrimmerOpen,
setClipTrimmerOpen,
clipTrimmerSegmentId,
setClipTrimmerSegmentId,
};
};

View File

@@ -17,14 +17,14 @@ interface UseHomePersistenceOptions {
setText: React.Dispatch<React.SetStateAction<string>>;
videoTitle: string;
setVideoTitle: React.Dispatch<React.SetStateAction<string>>;
enableSubtitles: boolean;
setEnableSubtitles: React.Dispatch<React.SetStateAction<boolean>>;
ttsMode: 'edgetts' | 'voiceclone';
setTtsMode: React.Dispatch<React.SetStateAction<'edgetts' | 'voiceclone'>>;
voice: string;
setVoice: React.Dispatch<React.SetStateAction<string>>;
selectedMaterial: string;
setSelectedMaterial: React.Dispatch<React.SetStateAction<string>>;
textLang: string;
setTextLang: React.Dispatch<React.SetStateAction<string>>;
selectedMaterials: string[];
setSelectedMaterials: React.Dispatch<React.SetStateAction<string[]>>;
selectedSubtitleStyleId: string;
setSelectedSubtitleStyleId: React.Dispatch<React.SetStateAction<string>>;
selectedTitleStyleId: string;
@@ -48,6 +48,8 @@ interface UseHomePersistenceOptions {
selectedVideoId: string | null;
setSelectedVideoId: React.Dispatch<React.SetStateAction<string | null>>;
selectedRefAudio: RefAudio | null;
selectedAudioId: string | null;
setSelectedAudioId: React.Dispatch<React.SetStateAction<string | null>>;
}
export const useHomePersistence = ({
@@ -57,14 +59,14 @@ export const useHomePersistence = ({
setText,
videoTitle,
setVideoTitle,
enableSubtitles,
setEnableSubtitles,
ttsMode,
setTtsMode,
voice,
setVoice,
selectedMaterial,
setSelectedMaterial,
textLang,
setTextLang,
selectedMaterials,
setSelectedMaterials,
selectedSubtitleStyleId,
setSelectedSubtitleStyleId,
selectedTitleStyleId,
@@ -88,6 +90,8 @@ export const useHomePersistence = ({
selectedVideoId,
setSelectedVideoId,
selectedRefAudio,
selectedAudioId,
setSelectedAudioId,
}: UseHomePersistenceOptions) => {
const [isRestored, setIsRestored] = useState(false);
@@ -96,9 +100,9 @@ export const useHomePersistence = ({
const savedText = localStorage.getItem(`vigent_${storageKey}_text`);
const savedTitle = localStorage.getItem(`vigent_${storageKey}_title`);
const savedSubtitles = localStorage.getItem(`vigent_${storageKey}_subtitles`);
const savedTtsMode = localStorage.getItem(`vigent_${storageKey}_ttsMode`);
const savedVoice = localStorage.getItem(`vigent_${storageKey}_voice`);
const savedTextLang = localStorage.getItem(`vigent_${storageKey}_textLang`);
const savedMaterial = localStorage.getItem(`vigent_${storageKey}_material`);
const savedSubtitleStyle = localStorage.getItem(`vigent_${storageKey}_subtitleStyle`);
const savedTitleStyle = localStorage.getItem(`vigent_${storageKey}_titleStyle`);
@@ -106,6 +110,7 @@ export const useHomePersistence = ({
const savedTitleFontSize = localStorage.getItem(`vigent_${storageKey}_titleFontSize`);
const savedBgmId = localStorage.getItem(`vigent_${storageKey}_bgmId`);
const savedSelectedVideoId = localStorage.getItem(`vigent_${storageKey}_selectedVideoId`);
const savedSelectedAudioId = localStorage.getItem(`vigent_${storageKey}_selectedAudioId`);
const savedBgmVolume = localStorage.getItem(`vigent_${storageKey}_bgmVolume`);
const savedEnableBgm = localStorage.getItem(`vigent_${storageKey}_enableBgm`);
const savedTitleTopMargin = localStorage.getItem(`vigent_${storageKey}_titleTopMargin`);
@@ -113,11 +118,23 @@ export const useHomePersistence = ({
setText(savedText || "大家好,欢迎来到我的频道,今天给大家分享一些有趣的内容。");
setVideoTitle(savedTitle ? clampTitle(savedTitle) : "");
setEnableSubtitles(savedSubtitles !== null ? savedSubtitles === 'true' : true);
setTtsMode((savedTtsMode as 'edgetts' | 'voiceclone') || 'edgetts');
setVoice(savedVoice || "zh-CN-YunxiNeural");
if (savedTextLang) setTextLang(savedTextLang);
if (savedMaterial) setSelectedMaterial(savedMaterial);
if (savedMaterial) {
try {
const parsed = JSON.parse(savedMaterial);
if (Array.isArray(parsed)) {
setSelectedMaterials(parsed);
} else {
setSelectedMaterials([savedMaterial]);
}
} catch {
// 旧格式: 单字符串
setSelectedMaterials([savedMaterial]);
}
}
if (savedSubtitleStyle) setSelectedSubtitleStyleId(savedSubtitleStyle);
if (savedTitleStyle) setSelectedTitleStyleId(savedTitleStyle);
@@ -141,6 +158,7 @@ export const useHomePersistence = ({
if (savedBgmVolume) setBgmVolume(parseFloat(savedBgmVolume));
if (savedEnableBgm !== null) setEnableBgm(savedEnableBgm === 'true');
if (savedSelectedVideoId) setSelectedVideoId(savedSelectedVideoId);
if (savedSelectedAudioId) setSelectedAudioId(savedSelectedAudioId);
if (savedTitleTopMargin) {
const parsed = parseInt(savedTitleTopMargin, 10);
@@ -157,15 +175,16 @@ export const useHomePersistence = ({
isAuthLoading,
setBgmVolume,
setEnableBgm,
setEnableSubtitles,
setSelectedBgmId,
setSelectedMaterial,
setSelectedMaterials,
setSelectedSubtitleStyleId,
setSelectedTitleStyleId,
setSelectedVideoId,
setSelectedAudioId,
setSubtitleFontSize,
setSubtitleSizeLocked,
setText,
setTextLang,
setTitleFontSize,
setTitleSizeLocked,
setTitleTopMargin,
@@ -192,10 +211,6 @@ export const useHomePersistence = ({
return () => clearTimeout(timeout);
}, [videoTitle, storageKey, isRestored]);
useEffect(() => {
if (isRestored) localStorage.setItem(`vigent_${storageKey}_subtitles`, String(enableSubtitles));
}, [enableSubtitles, storageKey, isRestored]);
useEffect(() => {
if (isRestored) localStorage.setItem(`vigent_${storageKey}_ttsMode`, ttsMode);
}, [ttsMode, storageKey, isRestored]);
@@ -205,10 +220,14 @@ export const useHomePersistence = ({
}, [voice, storageKey, isRestored]);
useEffect(() => {
if (isRestored && selectedMaterial) {
localStorage.setItem(`vigent_${storageKey}_material`, selectedMaterial);
if (isRestored) localStorage.setItem(`vigent_${storageKey}_textLang`, textLang);
}, [textLang, storageKey, isRestored]);
useEffect(() => {
if (isRestored && selectedMaterials.length > 0) {
localStorage.setItem(`vigent_${storageKey}_material`, JSON.stringify(selectedMaterials));
}
}, [selectedMaterial, storageKey, isRestored]);
}, [selectedMaterials, storageKey, isRestored]);
useEffect(() => {
if (isRestored && selectedSubtitleStyleId) {
@@ -275,6 +294,15 @@ export const useHomePersistence = ({
}
}, [selectedVideoId, storageKey, isRestored]);
useEffect(() => {
if (!isRestored) return;
if (selectedAudioId) {
localStorage.setItem(`vigent_${storageKey}_selectedAudioId`, selectedAudioId);
} else {
localStorage.removeItem(`vigent_${storageKey}_selectedAudioId`);
}
}, [selectedAudioId, storageKey, isRestored]);
useEffect(() => {
if (isRestored && selectedRefAudio) {
localStorage.setItem(`vigent_${storageKey}_refAudioId`, selectedRefAudio.id);

View File

@@ -2,23 +2,44 @@ import { useCallback, useState } from "react";
import api from "@/shared/api/axios";
import { ApiResponse, unwrap } from "@/shared/api/types";
import { toast } from "sonner";
import { resolveMediaUrl } from "@/shared/lib/media";
import type { Material } from "@/shared/types/material";
interface Material {
id: string;
name: string;
scene: string;
size_mb: number;
path: string;
/** Probe video duration from a URL using <video> element */
function probeVideoDuration(url: string): Promise<number> {
return new Promise((resolve) => {
const video = document.createElement("video");
video.preload = "metadata";
video.crossOrigin = "anonymous";
const cleanup = () => {
video.removeEventListener("loadedmetadata", onMeta);
video.removeEventListener("error", onError);
video.src = "";
};
const onMeta = () => {
const dur = video.duration;
cleanup();
resolve(Number.isFinite(dur) ? dur : 0);
};
const onError = () => {
cleanup();
resolve(0);
};
video.addEventListener("loadedmetadata", onMeta);
video.addEventListener("error", onError);
video.src = url;
video.load();
});
}
interface UseMaterialsOptions {
selectedMaterial: string;
setSelectedMaterial: React.Dispatch<React.SetStateAction<string>>;
selectedMaterials: string[];
setSelectedMaterials: React.Dispatch<React.SetStateAction<string[]>>;
}
export const useMaterials = ({
selectedMaterial,
setSelectedMaterial,
selectedMaterials,
setSelectedMaterials,
}: UseMaterialsOptions) => {
const [materials, setMaterials] = useState<Material[]>([]);
const [fetchError, setFetchError] = useState<string | null>(null);
@@ -41,12 +62,25 @@ export const useMaterials = ({
setMaterials(nextMaterials);
setLastMaterialCount(nextMaterials.length);
setSelectedMaterial((prev) => {
// 如果当前选中的素材在列表中依然存在,保持选中
const exists = nextMaterials.some((item) => item.id === prev);
if (exists) return prev;
// Probe video durations in background
if (nextMaterials.length > 0) {
Promise.all(
nextMaterials.map(async (m) => {
const url = resolveMediaUrl(m.path);
if (!url) return m;
const dur = await probeVideoDuration(url);
return { ...m, duration_sec: dur };
})
).then((enriched) => setMaterials(enriched));
}
setSelectedMaterials((prev) => {
// 保留已选中且仍存在的
const existingIds = new Set(nextMaterials.map((m) => m.id));
const kept = prev.filter((id) => existingIds.has(id));
if (kept.length > 0) return kept;
// 否则默认选中第一个
return nextMaterials[0]?.id || "";
return nextMaterials[0]?.id ? [nextMaterials[0].id] : [];
});
} catch (error) {
console.error("获取素材失败:", error);
@@ -54,29 +88,58 @@ export const useMaterials = ({
} finally {
setIsFetching(false);
}
}, [setSelectedMaterial]);
}, [setSelectedMaterials]);
const MAX_MATERIALS = 4;
const toggleMaterial = useCallback((id: string) => {
setSelectedMaterials((prev) => {
if (prev.includes(id)) {
// 不能取消最后一个
if (prev.length <= 1) return prev;
return prev.filter((x) => x !== id);
}
if (prev.length >= MAX_MATERIALS) return prev;
return [...prev, id];
});
}, [setSelectedMaterials]);
const reorderMaterials = useCallback((activeId: string, overId: string) => {
setSelectedMaterials((prev) => {
const oldIndex = prev.indexOf(activeId);
const newIndex = prev.indexOf(overId);
if (oldIndex === -1 || newIndex === -1) return prev;
const next = [...prev];
next.splice(oldIndex, 1);
next.splice(newIndex, 0, activeId);
return next;
});
}, [setSelectedMaterials]);
const deleteMaterial = useCallback(async (materialId: string) => {
if (!confirm("确定要删除这个素材吗?")) return;
try {
await api.delete(`/api/materials/${materialId}`);
fetchMaterials();
if (selectedMaterial === materialId) {
setSelectedMaterial("");
if (selectedMaterials.includes(materialId)) {
setSelectedMaterials((prev) => {
const next = prev.filter((id) => id !== materialId);
return next.length > 0 ? next : [];
});
}
} catch (error) {
toast.error("删除失败: " + error);
}
}, [fetchMaterials, selectedMaterial, setSelectedMaterial]);
}, [fetchMaterials, selectedMaterials, setSelectedMaterials]);
const handleUpload = useCallback(async (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
if (!file) return;
const validTypes = ['.mp4', '.mov', '.avi'];
const validTypes = ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.flv', '.wmv', '.m4v', '.ts', '.mts'];
const ext = file.name.toLowerCase().slice(file.name.lastIndexOf('.'));
if (!validTypes.includes(ext)) {
setUploadError('支持 MP4、MOV、AVI 格式');
setUploadError('支持的视频格式');
return;
}
@@ -100,7 +163,34 @@ export const useMaterials = ({
setUploadProgress(100);
setIsUploading(false);
fetchMaterials();
// 上传后重新拉列表并自动选中新素材
const { data: res } = await api.get<ApiResponse<{ materials: Material[] }>>(
`/api/materials?t=${new Date().getTime()}`
);
const payload = unwrap(res);
const nextMaterials = payload.materials || [];
setMaterials(nextMaterials);
setLastMaterialCount(nextMaterials.length);
// Probe video durations in background
if (nextMaterials.length > 0) {
Promise.all(
nextMaterials.map(async (m) => {
const url = resolveMediaUrl(m.path);
if (!url) return m;
const dur = await probeVideoDuration(url);
return { ...m, duration_sec: dur };
})
).then((enriched) => setMaterials(enriched));
}
// 找出新增的素材 ID 并自动选中
const oldIds = new Set(materials.map((m) => m.id));
const newIds = nextMaterials.filter((m) => !oldIds.has(m.id)).map((m) => m.id);
if (newIds.length > 0) {
setSelectedMaterials((prev) => [...prev, ...newIds]);
}
} catch (err: unknown) {
console.error("Upload failed:", err);
setIsUploading(false);
@@ -122,6 +212,8 @@ export const useMaterials = ({
uploadError,
setUploadError,
fetchMaterials,
toggleMaterial,
reorderMaterials,
deleteMaterial,
handleUpload,
};

View File

@@ -0,0 +1,51 @@
import { useState, useEffect, useRef } from "react";
export interface SavedScript {
id: string;
name: string;
content: string;
savedAt: number;
}
export function useSavedScripts(storageKey: string) {
const lsKey = `vigent_${storageKey}_savedScripts`;
const lsKeyRef = useRef(lsKey);
lsKeyRef.current = lsKey;
const [savedScripts, setSavedScripts] = useState<SavedScript[]>([]);
// Re-read from localStorage whenever lsKey changes (e.g. guest → userId)
useEffect(() => {
try {
const raw = localStorage.getItem(lsKey);
setSavedScripts(raw ? JSON.parse(raw) : []);
} catch {
setSavedScripts([]);
}
}, [lsKey]);
const saveScript = (content: string) => {
const name = content.slice(0, 15).replace(/\n/g, " ") || "未命名";
const entry: SavedScript = {
id: Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
name,
content,
savedAt: Date.now(),
};
setSavedScripts((prev) => {
const next = [entry, ...prev];
localStorage.setItem(lsKeyRef.current, JSON.stringify(next));
return next;
});
};
const deleteScript = (id: string) => {
setSavedScripts((prev) => {
const next = prev.filter((s) => s.id !== id);
localStorage.setItem(lsKeyRef.current, JSON.stringify(next));
return next;
});
};
return { savedScripts, saveScript, deleteScript };
}

View File

@@ -0,0 +1,246 @@
import { useCallback, useEffect, useRef, useState } from "react";
import type { Material } from "@/shared/types/material";
export interface TimelineSegment {
id: string;
materialId: string;
materialName: string;
start: number;
end: number;
sourceStart: number;
sourceEnd: number;
color: string;
}
export interface CustomAssignment {
material_path: string;
start: number;
end: number;
source_start: number;
}
const COLORS = ["#8b5cf6", "#ec4899", "#06b6d4", "#f59e0b", "#10b981", "#f97316"];
/** Serializable subset for localStorage */
interface SegmentSnapshot {
materialId: string;
start: number;
end: number;
sourceStart: number;
sourceEnd: number;
}
/** Get effective duration of a segment (clipped range or full material duration) */
function getEffectiveDuration(
seg: { sourceStart: number; sourceEnd: number; materialId: string },
mats: Material[]
): number {
if (seg.sourceEnd > seg.sourceStart) return seg.sourceEnd - seg.sourceStart;
const mat = mats.find((m) => m.id === seg.materialId);
return mat?.duration_sec ?? 0;
}
/**
* Recalculate segment start/end positions based on effective durations.
* - Segments placed sequentially by effective duration
* - Segments exceeding audioDuration keep their positions (overflow, start >= duration)
* - Last visible segment is capped/extended to exactly audioDuration (loop fill)
*/
function recalcPositions(
segs: TimelineSegment[],
mats: Material[],
duration: number
): TimelineSegment[] {
if (segs.length === 0 || duration <= 0) return segs;
const fallbackDur = duration / segs.length;
let cursor = 0;
const result = segs.map((seg) => {
const effDur = getEffectiveDuration(seg, mats);
const dur = effDur > 0 ? effDur : fallbackDur;
const newSeg = { ...seg, start: cursor, end: cursor + dur };
cursor += dur;
return newSeg;
});
// Find last segment that starts before audioDuration
let lastVisibleIdx = -1;
for (let i = result.length - 1; i >= 0; i--) {
if (result[i].start < duration) {
lastVisibleIdx = i;
break;
}
}
// Cap/extend last visible segment to exactly audioDuration
if (lastVisibleIdx >= 0) {
result[lastVisibleIdx] = { ...result[lastVisibleIdx], end: duration };
}
return result;
}
interface UseTimelineEditorOptions {
audioDuration: number;
materials: Material[];
selectedMaterials: string[];
storageKey?: string;
}
export const useTimelineEditor = ({
audioDuration,
materials,
selectedMaterials,
storageKey,
}: UseTimelineEditorOptions) => {
const [segments, setSegments] = useState<TimelineSegment[]>([]);
const prevKey = useRef("");
const restoredRef = useRef(false);
// Refs for stable callbacks (avoid recreating on every materials/duration change)
const materialsRef = useRef(materials);
materialsRef.current = materials;
const audioDurationRef = useRef(audioDuration);
audioDurationRef.current = audioDuration;
// Build a durationsKey so segments re-init when material durations become available
const durationsKey = selectedMaterials
.map((id) => materials.find((m) => m.id === id)?.duration_sec ?? 0)
.join(",");
// Build a cache key from materials + duration
const cacheKey = `${selectedMaterials.join(",")}_${audioDuration.toFixed(1)}`;
const lsKey = storageKey ? `vigent_${storageKey}_timeline` : null;
const initSegments = useCallback(() => {
if (selectedMaterials.length === 0 || audioDuration <= 0) {
setSegments([]);
return;
}
// Try restore from localStorage
if (lsKey) {
try {
const raw = localStorage.getItem(lsKey);
if (raw) {
const saved = JSON.parse(raw) as { key: string; segments: SegmentSnapshot[] };
if (saved.key === cacheKey && saved.segments.length === selectedMaterials.length) {
const allMatch = saved.segments.every(
(s, i) => s.materialId === selectedMaterials[i] || saved.segments.some((ss) => ss.materialId === selectedMaterials[i])
);
if (allMatch) {
const restored: TimelineSegment[] = saved.segments.map((s, i) => {
const mat = materials.find((m) => m.id === s.materialId);
return {
id: `seg-${i}-${Date.now()}`,
materialId: s.materialId,
materialName: mat?.scene || mat?.name || s.materialId,
start: 0,
end: 0,
sourceStart: s.sourceStart,
sourceEnd: s.sourceEnd,
color: COLORS[i % COLORS.length],
};
});
setSegments(recalcPositions(restored, materials, audioDuration));
restoredRef.current = true;
return;
}
}
}
} catch {
// ignore parse errors
}
}
// Create fresh segments — positions derived by recalcPositions
const newSegments: TimelineSegment[] = selectedMaterials.map((matId, i) => {
const mat = materials.find((m) => m.id === matId);
return {
id: `seg-${i}-${Date.now()}`,
materialId: matId,
materialName: mat?.scene || mat?.name || matId,
start: 0,
end: 0,
sourceStart: 0,
sourceEnd: 0,
color: COLORS[i % COLORS.length],
};
});
setSegments(recalcPositions(newSegments, materials, audioDuration));
}, [audioDuration, materials, selectedMaterials, lsKey, cacheKey]);
// Auto-init when selectedMaterials, audioDuration, or material durations change
useEffect(() => {
const key = `${selectedMaterials.join(",")}_${audioDuration}_${durationsKey}`;
if (key !== prevKey.current) {
prevKey.current = key;
initSegments();
}
}, [selectedMaterials, audioDuration, durationsKey, initSegments]);
// Persist segments to localStorage on change (debounced)
useEffect(() => {
if (!lsKey || segments.length === 0) return;
const timeout = setTimeout(() => {
const snapshots: SegmentSnapshot[] = segments.map((s) => ({
materialId: s.materialId,
start: s.start,
end: s.end,
sourceStart: s.sourceStart,
sourceEnd: s.sourceEnd,
}));
localStorage.setItem(lsKey, JSON.stringify({ key: cacheKey, segments: snapshots }));
}, 300);
return () => clearTimeout(timeout);
}, [segments, lsKey, cacheKey]);
const reorderSegments = useCallback(
(fromIdx: number, toIdx: number) => {
setSegments((prev) => {
if (fromIdx < 0 || toIdx < 0 || fromIdx >= prev.length || toIdx >= prev.length) return prev;
if (fromIdx === toIdx) return prev;
const next = [...prev];
// Move the segment: remove from old position, insert at new position
const [moved] = next.splice(fromIdx, 1);
next.splice(toIdx, 0, moved);
return recalcPositions(next, materialsRef.current, audioDurationRef.current);
});
},
[]
);
const setSourceRange = useCallback(
(id: string, sourceStart: number, sourceEnd: number) => {
setSegments((prev) => {
const updated = prev.map((s) => (s.id === id ? { ...s, sourceStart, sourceEnd } : s));
return recalcPositions(updated, materialsRef.current, audioDurationRef.current);
});
},
[]
);
const toCustomAssignments = useCallback((): CustomAssignment[] => {
const duration = audioDurationRef.current;
return segments
.filter((seg) => seg.start < duration)
.map((seg) => {
const mat = materialsRef.current.find((m) => m.id === seg.materialId);
return {
material_path: mat?.path || seg.materialId,
start: seg.start,
end: seg.end,
source_start: seg.sourceStart,
};
});
}, [segments]);
return {
segments,
initSegments,
reorderSegments,
setSourceRange,
toCustomAssignments,
};
};

View File

@@ -0,0 +1,293 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { X, Play, Pause } from "lucide-react";
import type { TimelineSegment } from "@/features/home/model/useTimelineEditor";
interface ClipTrimmerProps {
isOpen: boolean;
segment: TimelineSegment | null;
materialUrl: string | null;
onConfirm: (sourceStart: number, sourceEnd: number) => void;
onClose: () => void;
}
function formatSec(sec: number): string {
const m = Math.floor(sec / 60);
const s = sec % 60;
return `${String(m).padStart(2, "0")}:${s.toFixed(1).padStart(4, "0")}`;
}
export function ClipTrimmer({
isOpen,
segment,
materialUrl,
onConfirm,
onClose,
}: ClipTrimmerProps) {
const videoRef = useRef<HTMLVideoElement>(null);
const trackRef = useRef<HTMLDivElement>(null);
const [duration, setDuration] = useState(0);
const [sourceStart, setSourceStart] = useState(0);
const [sourceEnd, setSourceEnd] = useState(0);
const [currentTime, setCurrentTime] = useState(0);
const [isPlaying, setIsPlaying] = useState(false);
const [dragging, setDragging] = useState<"start" | "end" | null>(null);
const animRef = useRef<number>(0);
// Reset state when segment changes
useEffect(() => {
if (segment && isOpen) {
setSourceStart(segment.sourceStart);
setSourceEnd(segment.sourceEnd);
setCurrentTime(segment.sourceStart);
setIsPlaying(false);
}
}, [segment, isOpen]);
// Track currentTime during playback
useEffect(() => {
if (!isPlaying || !videoRef.current) return;
const tick = () => {
if (!videoRef.current) return;
const t = videoRef.current.currentTime;
const end = sourceEnd || duration;
if (t >= end) {
videoRef.current.pause();
videoRef.current.currentTime = sourceStart;
setCurrentTime(sourceStart);
setIsPlaying(false);
return;
}
setCurrentTime(t);
animRef.current = requestAnimationFrame(tick);
};
animRef.current = requestAnimationFrame(tick);
return () => cancelAnimationFrame(animRef.current);
}, [isPlaying, sourceStart, sourceEnd, duration]);
// Seek video when not playing and currentTime changes
useEffect(() => {
if (videoRef.current && !isPlaying) {
videoRef.current.currentTime = currentTime;
}
}, [currentTime, isPlaying]);
const handleLoadedMetadata = useCallback(() => {
if (videoRef.current) {
const dur = videoRef.current.duration;
setDuration(dur);
if (sourceEnd === 0) {
setSourceEnd(dur);
}
}
}, [sourceEnd]);
const togglePlay = useCallback(() => {
if (!videoRef.current || duration === 0) return;
if (isPlaying) {
videoRef.current.pause();
setIsPlaying(false);
} else {
const end = sourceEnd || duration;
if (videoRef.current.currentTime >= end || videoRef.current.currentTime < sourceStart) {
videoRef.current.currentTime = sourceStart;
setCurrentTime(sourceStart);
}
videoRef.current.play().catch(() => {});
setIsPlaying(true);
}
}, [isPlaying, sourceStart, sourceEnd, duration]);
// --- Dual-handle slider logic ---
const getPositionFromEvent = useCallback(
(clientX: number) => {
if (!trackRef.current || duration === 0) return 0;
const rect = trackRef.current.getBoundingClientRect();
const ratio = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width));
return ratio * duration;
},
[duration]
);
const handleThumbPointerDown = useCallback(
(which: "start" | "end", e: React.PointerEvent) => {
e.preventDefault();
e.stopPropagation();
setDragging(which);
(e.target as HTMLElement).setPointerCapture(e.pointerId);
},
[]
);
const handleTrackPointerMove = useCallback(
(e: React.PointerEvent) => {
if (!dragging) return;
const pos = getPositionFromEvent(e.clientX);
const minGap = 0.5;
if (dragging === "start") {
const clamped = Math.max(0, Math.min(pos, (sourceEnd || duration) - minGap));
setSourceStart(clamped);
setCurrentTime(clamped);
} else {
const clamped = Math.min(duration, Math.max(pos, sourceStart + minGap));
setSourceEnd(clamped);
}
},
[dragging, getPositionFromEvent, sourceStart, sourceEnd, duration]
);
const handleTrackPointerUp = useCallback(() => {
setDragging(null);
}, []);
const handleConfirm = () => {
onConfirm(sourceStart, sourceEnd >= duration ? 0 : sourceEnd);
};
if (!isOpen || !segment) return null;
const assignedDur = segment.end - segment.start;
const effectiveEnd = sourceEnd || duration;
const clipDur = effectiveEnd - sourceStart;
const startPct = duration > 0 ? (sourceStart / duration) * 100 : 0;
const endPct = duration > 0 ? (effectiveEnd / duration) * 100 : 100;
const playheadPct = duration > 0 ? (currentTime / duration) * 100 : 0;
return (
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 backdrop-blur-sm" onClick={onClose}>
<div
className="bg-gray-900 border border-white/10 rounded-2xl w-full max-w-lg mx-4 overflow-hidden"
onClick={(e) => e.stopPropagation()}
>
{/* Header */}
<div className="flex items-center justify-between px-5 py-3 border-b border-white/10">
<h3 className="text-white font-semibold text-sm">
- {segment.materialName}
</h3>
<button onClick={onClose} className="text-gray-400 hover:text-white">
<X className="h-4 w-4" />
</button>
</div>
{/* Video preview */}
<div className="px-5 pt-4">
<div className="relative bg-black rounded-lg overflow-hidden aspect-video group">
{materialUrl ? (
<video
ref={videoRef}
src={materialUrl}
className="w-full h-full object-contain"
onLoadedMetadata={handleLoadedMetadata}
onEnded={() => setIsPlaying(false)}
preload="auto"
muted
/>
) : (
<div className="flex items-center justify-center h-full text-gray-500 text-sm">
</div>
)}
{/* Play/Pause overlay */}
{materialUrl && (
<button
onClick={togglePlay}
className="absolute inset-0 flex items-center justify-center bg-black/0 hover:bg-black/30 transition-colors"
>
<div className={`p-3 rounded-full bg-black/60 text-white transition-opacity ${isPlaying ? "opacity-0 group-hover:opacity-100" : "opacity-100"}`}>
{isPlaying ? <Pause className="h-6 w-6" /> : <Play className="h-6 w-6" />}
</div>
</button>
)}
<div className="absolute bottom-2 right-2 bg-black/70 text-white text-[10px] px-2 py-0.5 rounded pointer-events-none">
{formatSec(currentTime)}
</div>
</div>
</div>
{/* Dual-handle range slider */}
<div className="px-5 py-4 space-y-3">
<div className="text-xs text-gray-400 flex justify-between">
<span>: {duration > 0 ? formatSec(duration) : "加载中..."}</span>
</div>
{/* Custom range track */}
<div
ref={trackRef}
className="relative h-8 cursor-pointer select-none touch-none"
onPointerMove={handleTrackPointerMove}
onPointerUp={handleTrackPointerUp}
onPointerLeave={handleTrackPointerUp}
>
{/* Background track */}
<div className="absolute top-1/2 -translate-y-1/2 left-0 right-0 h-2 bg-white/10 rounded-full" />
{/* Selected range */}
<div
className="absolute top-1/2 -translate-y-1/2 h-2 rounded-full"
style={{
left: `${startPct}%`,
width: `${endPct - startPct}%`,
backgroundColor: segment.color + "88",
}}
/>
{/* Playhead indicator */}
{duration > 0 && (
<div
className="absolute top-1/2 -translate-y-1/2 w-0.5 h-4 bg-white/60 rounded-full pointer-events-none"
style={{ left: `${playheadPct}%` }}
/>
)}
{/* Start thumb */}
<div
onPointerDown={(e) => handleThumbPointerDown("start", e)}
className="absolute top-1/2 -translate-y-1/2 -translate-x-1/2 w-4 h-4 rounded-full bg-purple-500 border-2 border-white shadow-lg cursor-grab active:cursor-grabbing hover:scale-110 transition-transform z-10"
style={{ left: `${startPct}%` }}
title={`起点: ${formatSec(sourceStart)}`}
/>
{/* End thumb */}
<div
onPointerDown={(e) => handleThumbPointerDown("end", e)}
className="absolute top-1/2 -translate-y-1/2 -translate-x-1/2 w-4 h-4 rounded-full bg-pink-500 border-2 border-white shadow-lg cursor-grab active:cursor-grabbing hover:scale-110 transition-transform z-10"
style={{ left: `${endPct}%` }}
title={`终点: ${formatSec(effectiveEnd)}`}
/>
</div>
{/* Time labels */}
<div className="flex justify-between text-xs text-gray-400">
<span className="text-purple-400">{formatSec(sourceStart)}</span>
<span className="text-pink-400">{formatSec(effectiveEnd)}</span>
</div>
{/* Info */}
<div className="text-[11px] text-gray-500 flex items-center gap-2 flex-wrap">
<span>: {clipDur.toFixed(1)}s</span>
<span className="text-gray-600">|</span>
<span>: {assignedDur.toFixed(1)}s</span>
{clipDur < assignedDur && <span className="text-amber-500">()</span>}
{clipDur > assignedDur && <span className="text-cyan-500">()</span>}
</div>
</div>
{/* Actions */}
<div className="flex justify-end gap-2 px-5 pb-4">
<button
onClick={onClose}
className="px-4 py-1.5 text-xs bg-white/10 hover:bg-white/20 rounded-lg text-gray-300 transition-colors"
>
</button>
<button
onClick={handleConfirm}
className="px-4 py-1.5 text-xs bg-gradient-to-r from-purple-600 to-pink-600 hover:from-purple-700 hover:to-pink-700 text-white rounded-lg transition-colors"
>
</button>
</div>
</div>
</div>
);
}

View File

@@ -4,6 +4,7 @@ interface GenerateActionBarProps {
isGenerating: boolean;
progress: number;
disabled: boolean;
materialCount?: number;
onGenerate: () => void;
}
@@ -11,43 +12,51 @@ export function GenerateActionBar({
isGenerating,
progress,
disabled,
materialCount = 1,
onGenerate,
}: GenerateActionBarProps) {
return (
<button
onClick={onGenerate}
disabled={disabled}
className={`w-full py-4 rounded-xl font-bold text-lg transition-all ${disabled
? "bg-gray-600 cursor-not-allowed text-gray-400"
: "bg-gradient-to-r from-purple-600 to-pink-600 hover:from-purple-700 hover:to-pink-700 text-white shadow-lg hover:shadow-purple-500/25"
}`}
>
{isGenerating ? (
<span className="flex items-center justify-center gap-3">
<svg className="animate-spin h-5 w-5" viewBox="0 0 24 24">
<circle
className="opacity-25"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="4"
fill="none"
/>
<path
className="opacity-75"
fill="currentColor"
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"
/>
</svg>
... {progress}%
</span>
) : (
<span className="flex items-center justify-center gap-2">
<Rocket className="h-5 w-5" />
</span>
<div>
<button
onClick={onGenerate}
disabled={disabled}
className={`w-full py-4 rounded-xl font-bold text-lg transition-all ${disabled
? "bg-gray-600 cursor-not-allowed text-gray-400"
: "bg-gradient-to-r from-purple-600 to-pink-600 hover:from-purple-700 hover:to-pink-700 text-white shadow-lg hover:shadow-purple-500/25"
}`}
>
{isGenerating ? (
<span className="flex items-center justify-center gap-3">
<svg className="animate-spin h-5 w-5" viewBox="0 0 24 24">
<circle
className="opacity-25"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="4"
fill="none"
/>
<path
className="opacity-75"
fill="currentColor"
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"
/>
</svg>
... {progress}%
</span>
) : (
<span className="flex items-center justify-center gap-2">
<Rocket className="h-5 w-5" />
</span>
)}
</button>
{!isGenerating && materialCount >= 2 && (
<p className="text-xs text-gray-400 text-center mt-1.5">
({materialCount} )
</p>
)}
</button>
</div>
);
}

View File

@@ -0,0 +1,224 @@
import { useState, useRef, useCallback, useEffect } from "react";
import { Play, Pause, Pencil, Trash2, Check, X, RefreshCw, Mic } from "lucide-react";
import type { GeneratedAudio } from "@/features/home/model/useGeneratedAudios";
interface AudioTask {
status: string;
progress?: number;
message?: string;
}
interface GeneratedAudiosPanelProps {
generatedAudios: GeneratedAudio[];
selectedAudioId: string | null;
isGeneratingAudio: boolean;
audioTask: AudioTask | null;
onGenerateAudio: () => void;
onRefresh: () => void;
onSelectAudio: (audio: GeneratedAudio) => void;
onDeleteAudio: (id: string) => void;
onRenameAudio: (id: string, newName: string) => void;
hasText: boolean;
}
export function GeneratedAudiosPanel({
generatedAudios,
selectedAudioId,
isGeneratingAudio,
audioTask,
onGenerateAudio,
onRefresh,
onSelectAudio,
onDeleteAudio,
onRenameAudio,
hasText,
}: GeneratedAudiosPanelProps) {
const [editingId, setEditingId] = useState<string | null>(null);
const [editName, setEditName] = useState("");
const [playingId, setPlayingId] = useState<string | null>(null);
const audioRef = useRef<HTMLAudioElement | null>(null);
const stopPlaying = useCallback(() => {
if (audioRef.current) {
audioRef.current.pause();
audioRef.current.currentTime = 0;
audioRef.current = null;
}
setPlayingId(null);
}, []);
// Cleanup on unmount
useEffect(() => {
return () => {
if (audioRef.current) {
audioRef.current.pause();
audioRef.current = null;
}
};
}, []);
const togglePlay = (audio: GeneratedAudio, e: React.MouseEvent) => {
e.stopPropagation();
if (playingId === audio.id) {
stopPlaying();
return;
}
stopPlaying();
const player = new Audio(audio.path);
player.onended = () => setPlayingId(null);
player.play().catch(() => {});
audioRef.current = player;
setPlayingId(audio.id);
};
const startEditing = (audio: GeneratedAudio, e: React.MouseEvent) => {
e.stopPropagation();
setEditingId(audio.id);
setEditName(audio.name);
};
const saveEditing = (audioId: string, e: React.MouseEvent) => {
e.stopPropagation();
if (!editName.trim()) return;
onRenameAudio(audioId, editName.trim());
setEditingId(null);
setEditName("");
};
const cancelEditing = (e: React.MouseEvent) => {
e.stopPropagation();
setEditingId(null);
setEditName("");
};
return (
<div className="bg-white/5 rounded-2xl p-4 sm:p-6 border border-white/10 backdrop-blur-sm">
<div className="flex justify-between items-center gap-2 mb-4">
<h2 className="text-base sm:text-lg font-semibold text-white flex items-center gap-2 whitespace-nowrap">
<Mic className="h-4 w-4 text-purple-400" />
</h2>
<div className="flex gap-1.5">
<button
onClick={onGenerateAudio}
disabled={isGeneratingAudio || !hasText}
className={`px-2 py-1 text-xs rounded transition-all whitespace-nowrap flex items-center gap-1 ${
isGeneratingAudio || !hasText
? "bg-gray-600 cursor-not-allowed text-gray-400"
: "bg-gradient-to-r from-purple-600 to-pink-600 hover:from-purple-700 hover:to-pink-700 text-white"
}`}
>
<Mic className="h-3.5 w-3.5" />
</button>
<button
onClick={onRefresh}
className="px-2 py-1 text-xs bg-white/10 hover:bg-white/20 rounded text-gray-300 whitespace-nowrap flex items-center gap-1"
>
<RefreshCw className="h-3.5 w-3.5" />
</button>
</div>
</div>
{/* 生成进度 */}
{isGeneratingAudio && audioTask && (
<div className="mb-4 p-3 bg-purple-500/10 rounded-xl border border-purple-500/30">
<div className="flex justify-between text-sm text-purple-300 mb-2">
<span>{audioTask.message || "生成中..."}</span>
<span>{audioTask.progress || 0}%</span>
</div>
<div className="h-2 bg-black/30 rounded-full overflow-hidden">
<div
className="h-full bg-gradient-to-r from-purple-500 to-pink-500 transition-all duration-300"
style={{ width: `${audioTask.progress || 0}%` }}
/>
</div>
</div>
)}
{/* 配音列表 */}
{generatedAudios.length === 0 ? (
<div className="text-center py-6 text-gray-400">
<p className="text-sm"></p>
<p className="text-xs mt-1 text-gray-500"></p>
</div>
) : (
<div className="space-y-2 max-h-48 sm:max-h-56 overflow-y-auto hide-scrollbar">
{generatedAudios.map((audio) => {
const isSelected = selectedAudioId === audio.id;
return (
<div
key={audio.id}
onClick={() => onSelectAudio(audio)}
className={`p-3 rounded-lg border transition-all cursor-pointer flex items-center justify-between group ${
isSelected
? "border-purple-500 bg-purple-500/20"
: "border-white/10 bg-white/5 hover:border-white/30"
}`}
>
{editingId === audio.id ? (
<div className="flex-1 flex items-center gap-2" onClick={(e) => e.stopPropagation()}>
<input
value={editName}
onChange={(e) => setEditName(e.target.value)}
className="flex-1 bg-black/40 border border-white/20 rounded-md px-2 py-1 text-xs text-white"
autoFocus
onKeyDown={(e) => {
if (e.key === "Enter") saveEditing(audio.id, e as unknown as React.MouseEvent);
if (e.key === "Escape") cancelEditing(e as unknown as React.MouseEvent);
}}
/>
<button onClick={(e) => saveEditing(audio.id, e)} className="p-1 text-green-400 hover:text-green-300" title="保存">
<Check className="h-4 w-4" />
</button>
<button onClick={cancelEditing} className="p-1 text-gray-400 hover:text-white" title="取消">
<X className="h-4 w-4" />
</button>
</div>
) : (
<>
<div className="min-w-0 flex-1">
<div className="text-white text-sm truncate">{audio.name}</div>
<div className="text-gray-400 text-xs">{audio.duration_sec.toFixed(1)}s</div>
</div>
<div className="flex items-center gap-1 pl-2 opacity-0 group-hover:opacity-100 transition-opacity">
<button
onClick={(e) => togglePlay(audio, e)}
className="p-1 text-gray-500 hover:text-purple-400 transition-colors"
title={playingId === audio.id ? "暂停" : "播放"}
>
{playingId === audio.id ? (
<Pause className="h-3.5 w-3.5" />
) : (
<Play className="h-3.5 w-3.5" />
)}
</button>
<button
onClick={(e) => startEditing(audio, e)}
className="p-1 text-gray-500 hover:text-white transition-colors"
title="重命名"
>
<Pencil className="h-3.5 w-3.5" />
</button>
<button
onClick={(e) => {
e.stopPropagation();
onDeleteAudio(audio.id);
}}
className="p-1 text-gray-500 hover:text-red-400 transition-colors"
title="删除"
>
<Trash2 className="h-3.5 w-3.5" />
</button>
</div>
</>
)}
</div>
);
})}
</div>
)}
</div>
);
}

View File

@@ -1,20 +1,24 @@
"use client";
import { useEffect } from "react";
import { useEffect, useMemo } from "react";
import { useRouter } from "next/navigation";
import VideoPreviewModal from "@/components/VideoPreviewModal";
import ScriptExtractionModal from "@/components/ScriptExtractionModal";
import ScriptExtractionModal from "./ScriptExtractionModal";
import { useHomeController } from "@/features/home/model/useHomeController";
import { resolveMediaUrl } from "@/shared/lib/media";
import { BgmPanel } from "@/features/home/ui/BgmPanel";
import { GenerateActionBar } from "@/features/home/ui/GenerateActionBar";
import { HistoryList } from "@/features/home/ui/HistoryList";
import { HomeHeader } from "@/features/home/ui/HomeHeader";
import { MaterialSelector } from "@/features/home/ui/MaterialSelector";
import { TimelineEditor } from "@/features/home/ui/TimelineEditor";
import { ClipTrimmer } from "@/features/home/ui/ClipTrimmer";
import { PreviewPanel } from "@/features/home/ui/PreviewPanel";
import { RefAudioPanel } from "@/features/home/ui/RefAudioPanel";
import { ScriptEditor } from "@/features/home/ui/ScriptEditor";
import { TitleSubtitlePanel } from "@/features/home/ui/TitleSubtitlePanel";
import { VoiceSelector } from "@/features/home/ui/VoiceSelector";
import { GeneratedAudiosPanel } from "@/features/home/ui/GeneratedAudiosPanel";
export function HomePage() {
const router = useRouter();
@@ -34,8 +38,8 @@ export function HomePage() {
fetchMaterials,
deleteMaterial,
handleUpload,
selectedMaterial,
setSelectedMaterial,
selectedMaterials,
toggleMaterial,
handlePreviewMaterial,
editingMaterialId,
editMaterialName,
@@ -49,6 +53,13 @@ export function HomePage() {
setExtractModalOpen,
handleGenerateMeta,
isGeneratingMeta,
handleTranslate,
isTranslating,
originalText,
handleRestoreOriginal,
savedScripts,
handleSaveScript,
deleteSavedScript,
showStylePreview,
setShowStylePreview,
videoTitle,
@@ -69,8 +80,6 @@ export function HomePage() {
setTitleTopMargin,
subtitleBottomMargin,
setSubtitleBottomMargin,
enableSubtitles,
setEnableSubtitles,
resolveAssetUrl,
getFontFormat,
buildTextShadow,
@@ -130,12 +139,40 @@ export function HomePage() {
fetchGeneratedVideos,
registerVideoRef,
formatDate,
generatedAudios,
selectedAudio,
selectedAudioId,
isGeneratingAudio,
audioTask,
fetchGeneratedAudios,
handleGenerateAudio,
deleteAudio,
renameAudio,
selectAudio,
timelineSegments,
reorderSegments,
setSourceRange,
clipTrimmerOpen,
setClipTrimmerOpen,
clipTrimmerSegmentId,
setClipTrimmerSegmentId,
} = useHomeController();
useEffect(() => {
router.prefetch("/publish");
}, [router]);
const clipTrimmerSegment = useMemo(
() => timelineSegments.find((s) => s.id === clipTrimmerSegmentId) ?? null,
[timelineSegments, clipTrimmerSegmentId]
);
const clipTrimmerMaterialUrl = useMemo(() => {
if (!clipTrimmerSegment) return null;
const mat = materials.find((m) => m.id === clipTrimmerSegment.materialId);
return mat?.path ? resolveMediaUrl(mat.path) : null;
}, [clipTrimmerSegment, materials]);
return (
<div className="min-h-dvh">
<HomeHeader />
@@ -144,42 +181,24 @@ export function HomePage() {
<div className="grid grid-cols-1 lg:grid-cols-2 gap-8">
{/* 左侧: 输入区域 */}
<div className="space-y-6">
{/* 素材选择 */}
<MaterialSelector
materials={materials}
selectedMaterial={selectedMaterial}
isFetching={isFetching}
lastMaterialCount={lastMaterialCount}
editingMaterialId={editingMaterialId}
editMaterialName={editMaterialName}
isUploading={isUploading}
uploadProgress={uploadProgress}
uploadError={uploadError}
fetchError={fetchError}
apiBase={apiBase}
onUploadChange={handleUpload}
onRefresh={fetchMaterials}
onSelectMaterial={setSelectedMaterial}
onPreviewMaterial={handlePreviewMaterial}
onStartEditing={startMaterialEditing}
onEditNameChange={setEditMaterialName}
onSaveEditing={saveMaterialEditing}
onCancelEditing={cancelMaterialEditing}
onDeleteMaterial={deleteMaterial}
onClearUploadError={() => setUploadError(null)}
registerMaterialRef={registerMaterialRef}
/>
{/* 文案输入 */}
{/* 1. 文案输入 */}
<ScriptEditor
text={text}
onChangeText={setText}
onOpenExtractModal={() => setExtractModalOpen(true)}
onGenerateMeta={handleGenerateMeta}
isGeneratingMeta={isGeneratingMeta}
onTranslate={handleTranslate}
isTranslating={isTranslating}
hasOriginalText={originalText !== null}
onRestoreOriginal={handleRestoreOriginal}
savedScripts={savedScripts}
onSaveScript={handleSaveScript}
onLoadScript={setText}
onDeleteScript={deleteSavedScript}
/>
{/* 标题和字幕设置 */}
{/* 2. 标题和字幕设置 */}
<TitleSubtitlePanel
showStylePreview={showStylePreview}
onTogglePreview={() => setShowStylePreview((prev) => !prev)}
@@ -207,8 +226,6 @@ export function HomePage() {
onTitleTopMarginChange={setTitleTopMargin}
subtitleBottomMargin={subtitleBottomMargin}
onSubtitleBottomMarginChange={setSubtitleBottomMargin}
enableSubtitles={enableSubtitles}
onToggleSubtitles={setEnableSubtitles}
resolveAssetUrl={resolveAssetUrl}
getFontFormat={getFontFormat}
buildTextShadow={buildTextShadow}
@@ -216,7 +233,7 @@ export function HomePage() {
previewBaseHeight={materialDimensions?.height || 1920}
/>
{/* 配音方式选择 */}
{/* 3. 配音方式选择 */}
<VoiceSelector
ttsMode={ttsMode}
onSelectTtsMode={setTtsMode}
@@ -254,7 +271,69 @@ export function HomePage() {
)}
/>
{/* 背景音乐 */}
{/* 4. 配音列表 */}
<GeneratedAudiosPanel
generatedAudios={generatedAudios}
selectedAudioId={selectedAudioId}
isGeneratingAudio={isGeneratingAudio}
audioTask={audioTask}
onGenerateAudio={handleGenerateAudio}
onRefresh={() => fetchGeneratedAudios()}
onSelectAudio={selectAudio}
onDeleteAudio={deleteAudio}
onRenameAudio={renameAudio}
hasText={!!text.trim()}
/>
{/* 5. 视频素材 */}
<MaterialSelector
materials={materials}
selectedMaterials={selectedMaterials}
isFetching={isFetching}
lastMaterialCount={lastMaterialCount}
editingMaterialId={editingMaterialId}
editMaterialName={editMaterialName}
isUploading={isUploading}
uploadProgress={uploadProgress}
uploadError={uploadError}
fetchError={fetchError}
apiBase={apiBase}
onUploadChange={handleUpload}
onRefresh={fetchMaterials}
onToggleMaterial={toggleMaterial}
onPreviewMaterial={handlePreviewMaterial}
onStartEditing={startMaterialEditing}
onEditNameChange={setEditMaterialName}
onSaveEditing={saveMaterialEditing}
onCancelEditing={cancelMaterialEditing}
onDeleteMaterial={deleteMaterial}
onClearUploadError={() => setUploadError(null)}
registerMaterialRef={registerMaterialRef}
/>
{/* 5.5 时间轴编辑器 — 未选配音/素材时模糊遮挡 */}
<div className="relative">
{(!selectedAudio || selectedMaterials.length === 0) && (
<div className="absolute inset-0 bg-black/50 backdrop-blur-sm rounded-2xl flex items-center justify-center z-10">
<p className="text-gray-400">
{!selectedAudio ? "请先生成并选中配音" : "请先选择素材"}
</p>
</div>
)}
<TimelineEditor
audioDuration={selectedAudio?.duration_sec ?? 0}
audioUrl={selectedAudio ? (resolveMediaUrl(selectedAudio.path) || "") : ""}
segments={timelineSegments}
materials={materials}
onReorderSegment={reorderSegments}
onClickSegment={(seg) => {
setClipTrimmerSegmentId(seg.id);
setClipTrimmerOpen(true);
}}
/>
</div>
{/* 6. 背景音乐 */}
<BgmPanel
bgmList={bgmList}
bgmLoading={bgmLoading}
@@ -272,11 +351,12 @@ export function HomePage() {
registerBgmItemRef={registerBgmItemRef}
/>
{/* 生成按钮 */}
{/* 7. 生成按钮 */}
<GenerateActionBar
isGenerating={isGenerating}
progress={currentTask?.progress || 0}
disabled={isGenerating || !selectedMaterial || (ttsMode === "voiceclone" && !selectedRefAudio)}
materialCount={selectedMaterials.length}
disabled={isGenerating || selectedMaterials.length === 0 || !selectedAudio}
onGenerate={handleGenerate}
/>
</div>
@@ -312,6 +392,19 @@ export function HomePage() {
onClose={() => setExtractModalOpen(false)}
onApply={(nextText) => setText(nextText)}
/>
<ClipTrimmer
isOpen={clipTrimmerOpen}
segment={clipTrimmerSegment}
materialUrl={clipTrimmerMaterialUrl}
onConfirm={(sourceStart, sourceEnd) => {
if (clipTrimmerSegmentId) {
setSourceRange(clipTrimmerSegmentId, sourceStart, sourceEnd);
}
setClipTrimmerOpen(false);
}}
onClose={() => setClipTrimmerOpen(false)}
/>
</div>
);
}

View File

@@ -1,17 +1,10 @@
import type { ChangeEvent, MouseEvent } from "react";
import { type ChangeEvent, type MouseEvent } from "react";
import { Upload, RefreshCw, Eye, Trash2, X, Pencil, Check } from "lucide-react";
interface Material {
id: string;
name: string;
scene: string;
size_mb: number;
path: string;
}
import type { Material } from "@/shared/types/material";
interface MaterialSelectorProps {
materials: Material[];
selectedMaterial: string;
selectedMaterials: string[];
isFetching: boolean;
lastMaterialCount: number;
editingMaterialId: string | null;
@@ -23,7 +16,7 @@ interface MaterialSelectorProps {
apiBase: string;
onUploadChange: (event: ChangeEvent<HTMLInputElement>) => void;
onRefresh: () => void;
onSelectMaterial: (id: string) => void;
onToggleMaterial: (id: string) => void;
onPreviewMaterial: (path: string) => void;
onStartEditing: (material: Material, event: MouseEvent) => void;
onEditNameChange: (value: string) => void;
@@ -36,7 +29,7 @@ interface MaterialSelectorProps {
export function MaterialSelector({
materials,
selectedMaterial,
selectedMaterials,
isFetching,
lastMaterialCount,
editingMaterialId,
@@ -48,7 +41,7 @@ export function MaterialSelector({
apiBase,
onUploadChange,
onRefresh,
onSelectMaterial,
onToggleMaterial,
onPreviewMaterial,
onStartEditing,
onEditNameChange,
@@ -58,20 +51,23 @@ export function MaterialSelector({
onClearUploadError,
registerMaterialRef,
}: MaterialSelectorProps) {
const selectedSet = new Set(selectedMaterials);
const isFull = selectedMaterials.length >= 4;
return (
<div className="bg-white/5 rounded-2xl p-4 sm:p-6 border border-white/10 backdrop-blur-sm">
<div className="flex justify-between items-center gap-2 mb-4">
<h2 className="text-base sm:text-lg font-semibold text-white flex items-center gap-2 whitespace-nowrap">
📹
<span className="ml-1 text-[11px] sm:text-xs text-gray-400/90 font-normal">
()
(4)
</span>
</h2>
<div className="flex gap-1.5">
<input
type="file"
id="video-upload"
accept=".mp4,.mov,.avi"
accept="video/*"
onChange={onUploadChange}
className="hidden"
/>
@@ -126,7 +122,7 @@ export function MaterialSelector({
API: {apiBase}/api/materials/
</div>
) : isFetching && materials.length === 0 ? (
<div className="space-y-2 max-h-64 overflow-y-auto hide-scrollbar" style={{ contentVisibility: 'auto' }}>
<div className="space-y-2 max-h-48 sm:max-h-64 overflow-y-auto hide-scrollbar" style={{ contentVisibility: 'auto' }}>
{Array.from({ length: Math.min(4, Math.max(1, lastMaterialCount || 1)) }).map((_, index) => (
<div
key={`material-skeleton-${index}`}
@@ -147,82 +143,98 @@ export function MaterialSelector({
</div>
) : (
<div
className="space-y-2 max-h-64 overflow-y-auto hide-scrollbar"
className="space-y-2 max-h-48 sm:max-h-64 overflow-y-auto hide-scrollbar"
style={{ contentVisibility: 'auto' }}
>
{materials.map((m) => (
<div
key={m.id}
ref={(el) => registerMaterialRef(m.id, el)}
className={`p-3 rounded-lg border transition-all flex items-center justify-between group ${selectedMaterial === m.id
? "border-purple-500 bg-purple-500/20"
: "border-white/10 bg-white/5 hover:border-white/30"
}`}
>
{editingMaterialId === m.id ? (
<div className="flex-1 flex items-center gap-2" onClick={(e) => e.stopPropagation()}>
<input
value={editMaterialName}
onChange={(e) => onEditNameChange(e.target.value)}
className="flex-1 bg-black/40 border border-white/20 rounded-md px-2 py-1 text-xs text-white"
autoFocus
/>
<button
onClick={(e) => onSaveEditing(m.id, e)}
className="p-1 text-green-400 hover:text-green-300"
title="保存"
>
<Check className="h-4 w-4" />
</button>
<button
onClick={onCancelEditing}
className="p-1 text-gray-400 hover:text-white"
title="取消"
>
<X className="h-4 w-4" />
</button>
</div>
) : (
<button onClick={() => onSelectMaterial(m.id)} className="flex-1 text-left">
<div className="text-white text-sm truncate">{m.scene || m.name}</div>
<div className="text-gray-400 text-xs">{m.size_mb.toFixed(1)} MB</div>
</button>
)}
<div className="flex items-center gap-2 pl-2">
<button
onClick={(e) => {
e.stopPropagation();
if (m.path) {
onPreviewMaterial(m.path);
}
}}
className="p-1 text-gray-500 hover:text-white opacity-0 group-hover:opacity-100 transition-opacity"
title="预览视频"
>
<Eye className="h-4 w-4" />
</button>
{editingMaterialId !== m.id && (
<button
onClick={(e) => onStartEditing(m, e)}
className="p-1 text-gray-500 hover:text-white opacity-0 group-hover:opacity-100 transition-opacity"
title="重命名"
>
<Pencil className="h-4 w-4" />
{materials.map((m) => {
const isSelected = selectedSet.has(m.id);
return (
<div
key={m.id}
ref={(el) => registerMaterialRef(m.id, el)}
className={`p-3 rounded-lg border transition-all flex items-center justify-between group ${isSelected
? "border-purple-500 bg-purple-500/20"
: isFull
? "border-white/5 bg-white/[0.02] opacity-50 cursor-not-allowed"
: "border-white/10 bg-white/5 hover:border-white/30"
}`}
>
{editingMaterialId === m.id ? (
<div className="flex-1 flex items-center gap-2" onClick={(e) => e.stopPropagation()}>
<input
value={editMaterialName}
onChange={(e) => onEditNameChange(e.target.value)}
className="flex-1 bg-black/40 border border-white/20 rounded-md px-2 py-1 text-xs text-white"
autoFocus
/>
<button
onClick={(e) => onSaveEditing(m.id, e)}
className="p-1 text-green-400 hover:text-green-300"
title="保存"
>
<Check className="h-4 w-4" />
</button>
<button
onClick={onCancelEditing}
className="p-1 text-gray-400 hover:text-white"
title="取消"
>
<X className="h-4 w-4" />
</button>
</div>
) : (
<button onClick={() => onToggleMaterial(m.id)} className="flex-1 text-left flex items-center gap-2">
{/* 复选框 */}
<span
className={`flex-shrink-0 w-4 h-4 rounded border flex items-center justify-center text-[10px] ${isSelected
? "border-purple-500 bg-purple-500 text-white"
: "border-white/30 text-transparent"
}`}
>
{isSelected ? "✓" : ""}
</span>
<div className="min-w-0">
<div className="text-white text-sm truncate">{m.scene || m.name}</div>
<div className="text-gray-400 text-xs">{m.size_mb.toFixed(1)} MB</div>
</div>
</button>
)}
<button
onClick={(e) => {
e.stopPropagation();
onDeleteMaterial(m.id);
}}
className="p-1 text-gray-500 hover:text-red-400 opacity-0 group-hover:opacity-100 transition-opacity"
title="删除素材"
>
<Trash2 className="h-4 w-4" />
</button>
<div className="flex items-center gap-2 pl-2">
<button
onClick={(e) => {
e.stopPropagation();
if (m.path) {
onPreviewMaterial(m.path);
}
}}
className="p-1 text-gray-500 hover:text-white opacity-0 group-hover:opacity-100 transition-opacity"
title="预览视频"
>
<Eye className="h-4 w-4" />
</button>
{editingMaterialId !== m.id && (
<button
onClick={(e) => onStartEditing(m, e)}
className="p-1 text-gray-500 hover:text-white opacity-0 group-hover:opacity-100 transition-opacity"
title="重命名"
>
<Pencil className="h-4 w-4" />
</button>
)}
<button
onClick={(e) => {
e.stopPropagation();
onDeleteMaterial(m.id);
}}
className="p-1 text-gray-500 hover:text-red-400 opacity-0 group-hover:opacity-100 transition-opacity"
title="删除素材"
>
<Trash2 className="h-4 w-4" />
</button>
</div>
</div>
</div>
))}
);
})}
</div>
)}
</div>

View File

@@ -1,4 +1,18 @@
import { FileText, Loader2, Sparkles } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { FileText, History, Languages, Loader2, RotateCcw, Save, Sparkles, Trash2 } from "lucide-react";
import type { SavedScript } from "@/features/home/model/useSavedScripts";
const LANGUAGES = [
{ code: "English", label: "英语 English" },
{ code: "日本語", label: "日语 日本語" },
{ code: "한국어", label: "韩语 한국어" },
{ code: "Français", label: "法语 Français" },
{ code: "Deutsch", label: "德语 Deutsch" },
{ code: "Español", label: "西班牙语 Español" },
{ code: "Русский", label: "俄语 Русский" },
{ code: "Italiano", label: "意大利语 Italiano" },
{ code: "Português", label: "葡萄牙语 Português" },
];
interface ScriptEditorProps {
text: string;
@@ -6,6 +20,14 @@ interface ScriptEditorProps {
onOpenExtractModal: () => void;
onGenerateMeta: () => void;
isGeneratingMeta: boolean;
onTranslate: (targetLang: string) => void;
isTranslating: boolean;
hasOriginalText: boolean;
onRestoreOriginal: () => void;
savedScripts: SavedScript[];
onSaveScript: () => void;
onLoadScript: (content: string) => void;
onDeleteScript: (id: string) => void;
}
export function ScriptEditor({
@@ -14,39 +36,176 @@ export function ScriptEditor({
onOpenExtractModal,
onGenerateMeta,
isGeneratingMeta,
onTranslate,
isTranslating,
hasOriginalText,
onRestoreOriginal,
savedScripts,
onSaveScript,
onLoadScript,
onDeleteScript,
}: ScriptEditorProps) {
const [showLangMenu, setShowLangMenu] = useState(false);
const langMenuRef = useRef<HTMLDivElement>(null);
const [showHistoryMenu, setShowHistoryMenu] = useState(false);
const historyMenuRef = useRef<HTMLDivElement>(null);
useEffect(() => {
if (!showLangMenu) return;
const handleClickOutside = (e: MouseEvent) => {
if (langMenuRef.current && !langMenuRef.current.contains(e.target as Node)) {
setShowLangMenu(false);
}
};
document.addEventListener("mousedown", handleClickOutside);
return () => document.removeEventListener("mousedown", handleClickOutside);
}, [showLangMenu]);
useEffect(() => {
if (!showHistoryMenu) return;
const handleClickOutside = (e: MouseEvent) => {
if (historyMenuRef.current && !historyMenuRef.current.contains(e.target as Node)) {
setShowHistoryMenu(false);
}
};
document.addEventListener("mousedown", handleClickOutside);
return () => document.removeEventListener("mousedown", handleClickOutside);
}, [showHistoryMenu]);
const handleSelectLang = (langCode: string) => {
setShowLangMenu(false);
onTranslate(langCode);
};
const formatDate = (ts: number) => {
const d = new Date(ts);
return `${(d.getMonth() + 1).toString().padStart(2, "0")}-${d.getDate().toString().padStart(2, "0")} ${d.getHours().toString().padStart(2, "0")}:${d.getMinutes().toString().padStart(2, "0")}`;
};
return (
<div className="bg-white/5 rounded-2xl p-4 sm:p-6 border border-white/10 backdrop-blur-sm">
<div className="flex flex-wrap justify-between items-center gap-2 mb-4">
<h2 className="text-base sm:text-lg font-semibold text-white flex items-center gap-2 whitespace-nowrap">
<div className="relative z-10 bg-white/5 rounded-2xl p-4 sm:p-6 border border-white/10 backdrop-blur-sm">
<div className="mb-4 space-y-3">
<h2 className="text-base sm:text-lg font-semibold text-white flex items-center gap-2">
</h2>
<div className="flex gap-2 flex-shrink-0">
<div className="flex gap-2 flex-wrap justify-end items-center">
{/* 历史文案 */}
<div className="relative" ref={historyMenuRef}>
<button
onClick={() => setShowHistoryMenu((prev) => !prev)}
className="h-7 px-2.5 text-xs rounded transition-all whitespace-nowrap bg-gray-600 hover:bg-gray-500 text-white inline-flex items-center gap-1"
>
<History className="h-3.5 w-3.5" />
</button>
{showHistoryMenu && (
<div className="absolute left-0 top-full mt-1 z-50 bg-gray-800 border border-white/10 rounded-lg shadow-xl py-1 min-w-[220px] max-h-[280px] overflow-y-auto">
{savedScripts.length === 0 ? (
<div className="px-3 py-3 text-xs text-gray-500 text-center"></div>
) : (
savedScripts.map((script) => (
<div
key={script.id}
className="flex items-center gap-1 px-3 py-1.5 hover:bg-white/10 transition-colors group"
>
<button
onClick={() => {
onLoadScript(script.content);
setShowHistoryMenu(false);
}}
className="flex-1 text-left min-w-0"
>
<div className="text-xs text-gray-200 truncate">{script.name}</div>
<div className="text-[10px] text-gray-500">{formatDate(script.savedAt)}</div>
</button>
<button
onClick={(e) => {
e.stopPropagation();
onDeleteScript(script.id);
}}
className="opacity-0 group-hover:opacity-100 p-1 text-gray-500 hover:text-red-400 transition-all shrink-0"
>
<Trash2 className="h-3 w-3" />
</button>
</div>
))
)}
</div>
)}
</div>
<button
onClick={onOpenExtractModal}
className="px-2 py-1 text-xs rounded transition-all whitespace-nowrap bg-purple-600 hover:bg-purple-700 text-white flex items-center gap-1"
className="h-7 px-2.5 text-xs rounded transition-all whitespace-nowrap bg-purple-600 hover:bg-purple-700 text-white inline-flex items-center gap-1"
>
<FileText className="h-3.5 w-3.5" />
</button>
<div className="relative" ref={langMenuRef}>
<button
onClick={() => setShowLangMenu((prev) => !prev)}
disabled={isTranslating || !text.trim()}
className={`h-7 px-2.5 text-xs rounded transition-all whitespace-nowrap inline-flex items-center gap-1 ${
isTranslating || !text.trim()
? "bg-gray-600 cursor-not-allowed text-gray-400"
: "bg-gradient-to-r from-emerald-600 to-teal-600 hover:from-emerald-700 hover:to-teal-700 text-white"
}`}
>
{isTranslating ? (
<>
<Loader2 className="h-3.5 w-3.5 animate-spin" />
...
</>
) : (
<>
<Languages className="h-3.5 w-3.5" />
AI多语言
</>
)}
</button>
{showLangMenu && (
<div className="absolute right-0 top-full mt-1 z-50 bg-gray-800 border border-white/10 rounded-lg shadow-xl py-1 min-w-[160px]">
{hasOriginalText && (
<>
<button
onClick={() => { setShowLangMenu(false); onRestoreOriginal(); }}
className="w-full text-left px-3 py-1.5 text-xs text-amber-400 hover:bg-white/10 transition-colors flex items-center gap-1"
>
<RotateCcw className="h-3 w-3" />
</button>
<div className="border-t border-white/10 my-1" />
</>
)}
{LANGUAGES.map((lang) => (
<button
key={lang.code}
onClick={() => handleSelectLang(lang.code)}
className="w-full text-left px-3 py-1.5 text-xs text-gray-200 hover:bg-white/10 transition-colors"
>
{lang.label}
</button>
))}
</div>
)}
</div>
<button
onClick={onGenerateMeta}
disabled={isGeneratingMeta || !text.trim()}
className={`px-2 py-1 text-xs rounded transition-all whitespace-nowrap ${isGeneratingMeta || !text.trim()
className={`h-7 px-2.5 text-xs rounded transition-all whitespace-nowrap inline-flex items-center gap-1 ${isGeneratingMeta || !text.trim()
? "bg-gray-600 cursor-not-allowed text-gray-400"
: "bg-gradient-to-r from-blue-600 to-cyan-600 hover:from-blue-700 hover:to-cyan-700 text-white"
}`}
>
{isGeneratingMeta ? (
<span className="flex items-center gap-1">
<>
<Loader2 className="h-3.5 w-3.5 animate-spin" />
...
</span>
</>
) : (
<span className="flex items-center gap-1">
<>
<Sparkles className="h-3.5 w-3.5" />
AI生成标题标签
</span>
</>
)}
</button>
</div>
@@ -57,9 +216,20 @@ export function ScriptEditor({
placeholder="请输入你想说的话..."
className="w-full h-40 bg-black/30 border border-white/10 rounded-xl p-4 text-white placeholder-gray-500 resize-none focus:outline-none focus:border-purple-500 transition-colors hide-scrollbar"
/>
<div className="flex justify-between mt-2 text-sm text-gray-400">
<div className="flex items-center justify-between mt-2 text-sm text-gray-400">
<span>{text.length} </span>
<span>: ~{Math.ceil(text.length / 4)} </span>
<button
onClick={onSaveScript}
disabled={!text.trim()}
className={`px-2.5 py-1 text-xs rounded transition-all flex items-center gap-1 ${
!text.trim()
? "bg-gray-700 cursor-not-allowed text-gray-500"
: "bg-amber-600/80 hover:bg-amber-600 text-white"
}`}
>
<Save className="h-3 w-3" />
</button>
</div>
</div>
);

View File

@@ -0,0 +1,283 @@
import { useEffect, useRef, useCallback, useState } from "react";
import WaveSurfer from "wavesurfer.js";
import type { TimelineSegment } from "@/features/home/model/useTimelineEditor";
import type { Material } from "@/shared/types/material";
interface TimelineEditorProps {
audioDuration: number;
audioUrl: string;
segments: TimelineSegment[];
materials: Material[];
onReorderSegment: (fromIdx: number, toIdx: number) => void;
onClickSegment: (segment: TimelineSegment) => void;
}
function formatTime(sec: number): string {
const m = Math.floor(sec / 60);
const s = sec % 60;
return `${String(m).padStart(2, "0")}:${s.toFixed(1).padStart(4, "0")}`;
}
export function TimelineEditor({
audioDuration,
audioUrl,
segments,
materials,
onReorderSegment,
onClickSegment,
}: TimelineEditorProps) {
const waveRef = useRef<HTMLDivElement>(null);
const wsRef = useRef<WaveSurfer | null>(null);
const [waveReady, setWaveReady] = useState(false);
const [isPlaying, setIsPlaying] = useState(false);
// Refs for high-frequency DOM updates (avoid 60fps re-renders)
const playheadRef = useRef<HTMLDivElement>(null);
const timeRef = useRef<HTMLSpanElement>(null);
const audioDurationRef = useRef(audioDuration);
audioDurationRef.current = audioDuration;
// Drag-to-reorder state
const [dragFromIdx, setDragFromIdx] = useState<number | null>(null);
const [dragOverIdx, setDragOverIdx] = useState<number | null>(null);
// Create / recreate wavesurfer when audioUrl changes
useEffect(() => {
if (!waveRef.current || !audioUrl) return;
// Destroy previous instance
if (wsRef.current) {
wsRef.current.destroy();
wsRef.current = null;
}
const ws = WaveSurfer.create({
container: waveRef.current,
height: 56,
waveColor: "#6d28d9",
progressColor: "#a855f7",
barWidth: 2,
barGap: 1,
barRadius: 2,
cursorWidth: 1,
cursorColor: "#e879f9",
interact: true,
normalize: true,
});
// Click waveform → seek + auto-play
ws.on("interaction", () => ws.play());
ws.on("play", () => setIsPlaying(true));
ws.on("pause", () => setIsPlaying(false));
ws.on("finish", () => {
setIsPlaying(false);
if (playheadRef.current) playheadRef.current.style.display = "none";
});
// High-frequency: update playhead + time via refs (no React re-render)
ws.on("timeupdate", (time: number) => {
const dur = audioDurationRef.current;
if (playheadRef.current && dur > 0) {
playheadRef.current.style.left = `${(time / dur) * 100}%`;
playheadRef.current.style.display = "block";
}
if (timeRef.current) {
timeRef.current.textContent = formatTime(time);
}
});
ws.load(audioUrl);
wsRef.current = ws;
return () => {
ws.destroy();
wsRef.current = null;
setIsPlaying(false);
if (playheadRef.current) playheadRef.current.style.display = "none";
if (timeRef.current) timeRef.current.textContent = formatTime(0);
};
}, [audioUrl, waveReady]);
// Callback ref to detect when waveRef div mounts
const waveCallbackRef = useCallback((node: HTMLDivElement | null) => {
(waveRef as React.MutableRefObject<HTMLDivElement | null>).current = node;
setWaveReady(!!node);
}, []);
const handlePlayPause = useCallback(() => {
wsRef.current?.playPause();
}, []);
// Drag-to-reorder handlers
const handleDragStart = useCallback((idx: number, e: React.DragEvent) => {
setDragFromIdx(idx);
e.dataTransfer.effectAllowed = "move";
e.dataTransfer.setData("text/plain", String(idx));
}, []);
const handleDragOver = useCallback((idx: number, e: React.DragEvent) => {
e.preventDefault();
e.dataTransfer.dropEffect = "move";
setDragOverIdx(idx);
}, []);
const handleDragLeave = useCallback(() => {
setDragOverIdx(null);
}, []);
const handleDrop = useCallback((toIdx: number, e: React.DragEvent) => {
e.preventDefault();
const fromIdx = parseInt(e.dataTransfer.getData("text/plain"), 10);
if (!isNaN(fromIdx) && fromIdx !== toIdx) {
onReorderSegment(fromIdx, toIdx);
}
setDragFromIdx(null);
setDragOverIdx(null);
}, [onReorderSegment]);
const handleDragEnd = useCallback(() => {
setDragFromIdx(null);
setDragOverIdx(null);
}, []);
// Filter visible vs overflow segments
const visibleSegments = segments.filter((s) => s.start < audioDuration);
const overflowSegments = segments.filter((s) => s.start >= audioDuration);
const hasSegments = visibleSegments.length > 0;
return (
<div className="bg-white/5 rounded-2xl p-4 sm:p-6 border border-white/10 backdrop-blur-sm">
<div className="flex items-center justify-between mb-3">
<h2 className="text-base sm:text-lg font-semibold text-white flex items-center gap-2">
🎞
</h2>
{audioUrl && (
<div className="flex items-center gap-2 text-xs text-gray-400">
<button
onClick={handlePlayPause}
className="w-7 h-7 flex items-center justify-center rounded-full bg-white/10 hover:bg-white/20 text-white transition-colors"
title={isPlaying ? "暂停" : "播放"}
>
{isPlaying ? "⏸" : "▶"}
</button>
<span ref={timeRef} className="tabular-nums">00:00.0</span>
<span className="text-gray-600">/</span>
<span className="tabular-nums">{formatTime(audioDuration)}</span>
</div>
)}
</div>
{/* Waveform — always rendered so ref stays mounted */}
<div className="relative mb-1">
<div ref={waveCallbackRef} className="rounded-lg overflow-hidden bg-black/20 cursor-pointer" style={{ minHeight: 56 }} />
</div>
{/* Segment blocks or empty placeholder */}
{hasSegments ? (
<>
<div className="relative h-14 flex select-none">
{/* Playhead — syncs with audio playback */}
<div
ref={playheadRef}
className="absolute top-0 h-full w-0.5 bg-fuchsia-400 z-10 pointer-events-none"
style={{ display: "none", left: "0%" }}
/>
{visibleSegments.map((seg, i) => {
const left = (seg.start / audioDuration) * 100;
const width = ((seg.end - seg.start) / audioDuration) * 100;
const segDur = seg.end - seg.start;
const isDragTarget = dragOverIdx === i && dragFromIdx !== i;
// Compute loop portion for the last visible segment
const isLastVisible = i === visibleSegments.length - 1;
let loopPercent = 0;
if (isLastVisible && audioDuration > 0) {
const mat = materials.find((m) => m.id === seg.materialId);
const matDur = mat?.duration_sec ?? 0;
const effDur = (seg.sourceEnd > seg.sourceStart)
? (seg.sourceEnd - seg.sourceStart)
: matDur;
if (effDur > 0 && segDur > effDur + 0.1) {
loopPercent = ((segDur - effDur) / segDur) * 100;
}
}
return (
<div key={seg.id} className="absolute top-0 h-full" style={{ left: `${left}%`, width: `${width}%` }}>
<button
draggable
onDragStart={(e) => handleDragStart(i, e)}
onDragOver={(e) => handleDragOver(i, e)}
onDragLeave={handleDragLeave}
onDrop={(e) => handleDrop(i, e)}
onDragEnd={handleDragEnd}
onClick={() => onClickSegment(seg)}
className={`relative w-full h-full rounded-lg flex flex-col items-center justify-center overflow-hidden cursor-grab active:cursor-grabbing transition-all border ${
isDragTarget
? "ring-2 ring-purple-400 border-purple-400 scale-[1.02]"
: dragFromIdx === i
? "opacity-50 border-white/10"
: "hover:opacity-90 border-white/10"
}`}
style={{ backgroundColor: seg.color + "33", borderColor: isDragTarget ? undefined : seg.color + "66" }}
title={`拖拽可调换顺序 · 点击设置截取范围\n${seg.materialName}\n${segDur.toFixed(1)}s${loopPercent > 0 ? ` (含循环 ${(segDur * loopPercent / 100).toFixed(1)}s)` : ""}`}
>
<span className="text-[11px] text-white/90 truncate max-w-full px-1 leading-tight z-[1]">
{seg.materialName}
</span>
<span className="text-[10px] text-white/60 leading-tight z-[1]">
{segDur.toFixed(1)}s
</span>
{seg.sourceStart > 0 && (
<span className="text-[9px] text-amber-400/80 leading-tight z-[1]">
{seg.sourceStart.toFixed(1)}s
</span>
)}
{/* Loop fill stripe overlay */}
{loopPercent > 0 && (
<div
className="absolute top-0 right-0 h-full pointer-events-none flex items-center justify-center"
style={{
width: `${loopPercent}%`,
background: `repeating-linear-gradient(-45deg, transparent, transparent 3px, rgba(255,255,255,0.07) 3px, rgba(255,255,255,0.07) 6px)`,
borderLeft: "1px dashed rgba(255,255,255,0.25)",
}}
>
<span className="text-[9px] text-white/30"></span>
</div>
)}
</button>
</div>
);
})}
</div>
{/* Overflow segments — shown as gray chips */}
{overflowSegments.length > 0 && (
<div className="flex flex-wrap items-center gap-1.5 mt-1.5">
<span className="text-[10px] text-gray-500">使:</span>
{overflowSegments.map((seg) => (
<span
key={seg.id}
className="text-[10px] text-gray-500 bg-white/5 border border-white/10 rounded px-1.5 py-0.5"
>
{seg.materialName}
</span>
))}
</div>
)}
<p className="text-[10px] text-gray-500 mt-1.5">
· ·
</p>
</>
) : (
<>
<div className="h-14 bg-white/5 rounded-lg" />
<p className="text-[10px] text-gray-500 mt-1.5">
</p>
</>
)}
</div>
);
}

View File

@@ -52,8 +52,6 @@ interface TitleSubtitlePanelProps {
onTitleTopMarginChange: (value: number) => void;
subtitleBottomMargin: number;
onSubtitleBottomMarginChange: (value: number) => void;
enableSubtitles: boolean;
onToggleSubtitles: (value: boolean) => void;
resolveAssetUrl: (path?: string | null) => string | null;
getFontFormat: (fontFile?: string) => string;
buildTextShadow: (color: string, size: number) => string;
@@ -82,8 +80,6 @@ export function TitleSubtitlePanel({
onTitleTopMarginChange,
subtitleBottomMargin,
onSubtitleBottomMarginChange,
enableSubtitles,
onToggleSubtitles,
resolveAssetUrl,
getFontFormat,
buildTextShadow,
@@ -117,7 +113,7 @@ export function TitleSubtitlePanel({
subtitleFontSize={subtitleFontSize}
titleTopMargin={titleTopMargin}
subtitleBottomMargin={subtitleBottomMargin}
enableSubtitles={enableSubtitles}
enableSubtitles={true}
resolveAssetUrl={resolveAssetUrl}
getFontFormat={getFontFormat}
buildTextShadow={buildTextShadow}
@@ -186,7 +182,7 @@ export function TitleSubtitlePanel({
</div>
)}
{enableSubtitles && subtitleStyles.length > 0 && (
{subtitleStyles.length > 0 && (
<div className="mt-4">
<label className="text-sm text-gray-300 mb-2 block"></label>
<div className="grid grid-cols-2 gap-2">
@@ -232,22 +228,6 @@ export function TitleSubtitlePanel({
</div>
</div>
)}
<div className="mt-4 pt-4 border-t border-white/10 flex items-center justify-between">
<div>
<span className="text-sm text-gray-300"></span>
<p className="text-xs text-gray-500 mt-1">OK效果字幕</p>
</div>
<label className="relative inline-flex items-center cursor-pointer">
<input
type="checkbox"
checked={enableSubtitles}
onChange={(e) => onToggleSubtitles(e.target.checked)}
className="sr-only peer"
/>
<div className="w-11 h-6 bg-gray-600 peer-focus:outline-none rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-purple-600"></div>
</label>
</div>
</div>
);
}

View File

@@ -5,8 +5,8 @@ import { ApiResponse, unwrap } from "@/shared/api/types";
import { formatDate, getApiBaseUrl, isAbsoluteUrl, resolveMediaUrl } from "@/shared/lib/media";
import { clampTitle } from "@/shared/lib/title";
import { useTitleInput } from "@/shared/hooks/useTitleInput";
import { useAuth } from "@/contexts/AuthContext";
import { useTask } from "@/contexts/TaskContext";
import { useAuth } from "@/shared/contexts/AuthContext";
import { useTask } from "@/shared/contexts/TaskContext";
import { toast } from "sonner";
import { usePublishPrefetch } from "@/shared/hooks/usePublishPrefetch";
import {

View File

@@ -0,0 +1,8 @@
export interface Material {
id: string;
name: string;
path: string;
size_mb: number;
scene?: string;
duration_sec?: number;
}

View File

@@ -120,6 +120,7 @@ async def generate(
if not _model_loaded:
raise HTTPException(status_code=503, detail="Model not loaded")
import torch
import soundfile as sf
# 保存上传的参考音频到临时文件
@@ -132,18 +133,25 @@ async def generate(
output_path = tempfile.mktemp(suffix=".wav")
try:
print(f"🎤 Generating: {text[:30]}...")
print(f"🎤 Generating: {text[:50]}... ({len(text)} chars)")
print(f"📝 Ref text: {ref_text[:50]}...")
print(f"🌐 Language: {language}")
start = time.time()
wavs, sr = _model.generate_voice_clone(
# 在线程池中运行,避免阻塞事件循环导致健康检查超时
import asyncio
wavs, sr = await asyncio.to_thread(
_model.generate_voice_clone,
text=text,
language=language,
ref_audio=ref_audio_path,
ref_text=ref_text,
)
# 释放 CUDA 缓存,防止显存碎片累积
torch.cuda.empty_cache()
sf.write(output_path, wavs[0], sr)
duration = len(wavs[0]) / sr
@@ -154,11 +162,17 @@ async def generate(
output_path,
media_type="audio/wav",
filename="output.wav",
background=None # 让客户端下载完再删除
background=None
)
except Exception as e:
print(f"❌ Generation failed: {e}")
# 释放 CUDA 缓存
try:
import torch
torch.cuda.empty_cache()
except:
pass
raise HTTPException(status_code=500, detail=str(e))
finally:
# 清理参考音频临时文件

View File

@@ -5,5 +5,7 @@
cd /home/rongye/ProgramFiles/ViGent2/models/Qwen3-TTS
# 使用 qwen-tts conda 环境的 Python
/home/rongye/ProgramFiles/miniconda3/envs/qwen-tts/bin/python qwen_tts_server.py
# 确保 conda env 的 bin 目录在 PATH 中,让 sox 等工具可被找到
export PATH="/home/rongye/ProgramFiles/miniconda3/envs/qwen-tts/bin:$PATH"
python qwen_tts_server.py