feat: Add real-time interaction and TikTok streaming

package.json

@@ -29,8 +29,10 @@
     "pinia": "^2.2.6",
     "tdesign-icons-vue-next": "^0.3.3",
     "tdesign-vue-next": "^1.10.3",
+    "tiktok-live-connector": "^2.0.7-beta1",
     "vue-i18n": "^10.0.5",
-    "vue-router": "^4.4.5"
+    "vue-router": "^4.4.5",
+    "ws": "^8.18.3"
   },
   "devDependencies": {
     "@electron-toolkit/eslint-config": "^1.0.2",

src/main/api/tts.js

@@ -2,10 +2,11 @@ import request from './request.js'
 import { serviceUrl } from '../config/config.js'
 import log from '../logger.js'
 
-export function makeAudio(param) {
+export function makeAudio(param, config = {}) {
   log.debug('~ makeAudio ~ param:', JSON.stringify(param))
   return request.post(`${serviceUrl.tts}/v1/invoke`, param, {
-    responseType: 'arraybuffer'
+    responseType: 'arraybuffer',
+    ...config
   })
 }
 

src/main/dao/f2f-model.js

@@ -1,11 +1,11 @@
 import { connect } from '../db/index.js'
 
-export function insert({ modelName, videoPath, audioPath, voiceId }) {
+export function insert({ modelName, videoPath, audioPath, voiceId, isImage, coverPath }) {
   const db = connect()
   const stmt = db.prepare(
-    'INSERT INTO f2f_model (name, video_path, audio_path, voice_id, created_at) VALUES (?, ?, ?, ?, ?)'
+    'INSERT INTO f2f_model (name, video_path, audio_path, voice_id, is_image, cover_path, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
   )
-  const info = stmt.run(modelName, videoPath, audioPath, voiceId, Date.now())
+  const info = stmt.run(modelName, videoPath, audioPath, voiceId, isImage, coverPath, Date.now())
   return info.lastInsertRowid
 }
 

src/main/db/sql.js

@@ -65,5 +65,14 @@ export default [
         script: `alter table video
                     add voice_id integer;
                 `
+    },
+    {
+        version: 4,
+        script: `
+            alter table f2f_model
+                add is_image integer default 0;
+            alter table f2f_model
+                add cover_path text;
+        `
     }
 ]

src/main/index.js

@@ -6,6 +6,7 @@ import { initDB } from './db/index.js'
 import { registerHandler } from './service/index.js'
 import { init as initInterval } from './interval/interval.js'
 import { registerWebHandles } from './handlers'
+import { init as initRealtime } from './service/realtime.js'
 initDB()
 initInterval()
 
@@ -65,6 +66,7 @@ app.whenReady().then(() => {
 
   // 注册主进程服务
   registerHandler()
+  initRealtime()
 
   const mainWindow = createWindow()
 

src/main/service/model.js

@@ -16,44 +16,55 @@ const MODEL_NAME = 'model'
  * @param {string} videoPath 模特视频路径
  * @returns
  */
-async function addModel(modelName, videoPath) {
+async function addModel(modelName, filePath, isImage = false) {
   if (!fs.existsSync(assetPath.model)) {
     fs.mkdirSync(assetPath.model, {
       recursive: true
     })
   }
-  // copy video to model video path
-  const extname = path.extname(videoPath)
-  const modelFileName = dayjs().format('YYYYMMDDHHmmssSSS') + extname
-  const modelPath = path.join(assetPath.model, modelFileName)
 
-  await toH264(videoPath, modelPath)
+  if (isImage) {
+    const extname = path.extname(filePath)
+    const modelFileName = dayjs().format('YYYYMMDDHHmmssSSS') + extname
+    const modelPath = path.join(assetPath.model, modelFileName)
+    fs.copyFileSync(filePath, modelPath)
+    const relativeModelPath = path.relative(assetPath.model, modelPath)
+    const id = insert({ modelName, coverPath: relativeModelPath, isImage: 1 })
+    return id
+  } else {
+    // copy video to model video path
+    const extname = path.extname(filePath)
+    const modelFileName = dayjs().format('YYYYMMDDHHmmssSSS') + extname
+    const modelPath = path.join(assetPath.model, modelFileName)
 
-  // 用ffmpeg分离音频
-  if (!fs.existsSync(assetPath.ttsTrain)) {
-    fs.mkdirSync(assetPath.ttsTrain, {
-      recursive: true
-    })
-  }
-  const audioPath = path.join(assetPath.ttsTrain, modelFileName.replace(extname, '.wav'))
-  return extractAudio(modelPath, audioPath).then(() => {
-    // 训练语音模型
-    const relativeAudioPath = path.relative(assetPath.ttsRoot, audioPath)
-    if (process.env.NODE_ENV === 'development') {
-      // TODO 写死调试
-      return trainVoice('origin_audio/test.wav', 'zh')
-    } else {
-      return trainVoice(relativeAudioPath, 'zh')
+    await toH264(filePath, modelPath)
+
+    // 用ffmpeg分离音频
+    if (!fs.existsSync(assetPath.ttsTrain)) {
+      fs.mkdirSync(assetPath.ttsTrain, {
+        recursive: true
+      })
     }
-  }).then((voiceId)=>{
-    // 插入模特信息
-    const relativeModelPath = path.relative(assetPath.model, modelPath)
-    const relativeAudioPath = path.relative(assetPath.ttsRoot, audioPath)
+    const audioPath = path.join(assetPath.ttsTrain, modelFileName.replace(extname, '.wav'))
+    return extractAudio(modelPath, audioPath).then(() => {
+      // 训练语音模型
+      const relativeAudioPath = path.relative(assetPath.ttsRoot, audioPath)
+      if (process.env.NODE_ENV === 'development') {
+        // TODO 写死调试
+        return trainVoice('origin_audio/test.wav', 'zh')
+      } else {
+        return trainVoice(relativeAudioPath, 'zh')
+      }
+    }).then((voiceId)=>{
+      // 插入模特信息
+      const relativeModelPath = path.relative(assetPath.model, modelPath)
+      const relativeAudioPath = path.relative(assetPath.ttsRoot, audioPath)
 
-    // insert model info to db
-    const id = insert({ modelName, videoPath: relativeModelPath, audioPath: relativeAudioPath, voiceId })
-    return id
-  })
+      // insert model info to db
+      const id = insert({ modelName, videoPath: relativeModelPath, audioPath: relativeAudioPath, voiceId, isImage: 0 })
+      return id
+    })
+  }
 }
 
 function page({ page, pageSize, name = '' }) {
@@ -101,8 +112,8 @@ function countModel(name = '') {
 }
 
 export function init() {
-  ipcMain.handle(MODEL_NAME + '/addModel', (event, ...args) => {
-    return addModel(...args)
+  ipcMain.handle(MODEL_NAME + '/addModel', (event, modelName, filePath, isImage) => {
+    return addModel(modelName, filePath, isImage)
   })
   ipcMain.handle(MODEL_NAME + '/page', (event, ...args) => {
     return page(...args)