diff --git a/packages/i18n/src/locales/en/settings.yaml b/packages/i18n/src/locales/en/settings.yaml index ea352a5f..8af7636f 100644 --- a/packages/i18n/src/locales/en/settings.yaml +++ b/packages/i18n/src/locales/en/settings.yaml @@ -437,6 +437,9 @@ pages: xai: description: x.ai title: xAI + transcriptions: + playground: + title: Transcription Playground title: Providers scene: description: Configure the environment where the character lives diff --git a/packages/i18n/src/locales/zh-Hans/settings.yaml b/packages/i18n/src/locales/zh-Hans/settings.yaml index 46279788..787406b6 100644 --- a/packages/i18n/src/locales/zh-Hans/settings.yaml +++ b/packages/i18n/src/locales/zh-Hans/settings.yaml @@ -415,6 +415,9 @@ pages: xai: description: X.AI title: xAI + transcriptions: + playground: + title: 实验平台 title: 服务来源 scene: description: 配置角色所在环境 diff --git a/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue b/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue index 96776ae9..d62349a2 100644 --- a/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue +++ b/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue @@ -2,11 +2,12 @@ import type { GenerateTranscriptionResult } from '@xsai/generate-transcription' import { FieldRange, FieldSelect } from '@proj-airi/ui' -import { useDevicesList } from '@vueuse/core' +import { until, useDevicesList, useUserMedia } from '@vueuse/core' import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue' +import { useI18n } from 'vue-i18n' import { useAudioRecorder } from '../../../composables/audio/audio-recorder' -import { LevelMeter, ThresholdMeter } from '../../Gadgets' +import { LevelMeter, TestDummyMarker, ThresholdMeter } from '../../Gadgets' import { Button } from '../../Misc' const props = defineProps<{ @@ -16,25 +17,24 @@ const props = defineProps<{ apiKeyConfigured?: boolean }>() +const { t } = useI18n() const devices = useDevicesList({ constraints: { audio: true }, requestPermissions: true }) - +const selectedAudioInput = ref(devices.audioInputs.value[0]?.deviceId || '') +const deviceConstraints = computed(() => ({ audio: { deviceId: { exact: selectedAudioInput.value } } })) +const { stream, stop, start, enabled } = useUserMedia({ constraints: deviceConstraints, enabled: false, autoSwitch: false }) const audioInputs = computed(() => devices.audioInputs.value) -const selectedAudioInput = ref(devices.audioInputs.value[0]?.deviceId || '') const speakingThreshold = ref(25) // 0-100 (for volume-based fallback) - -const errorMessage = ref('') - -const volumeLevel = ref(0) // 0-100 - const isMonitoring = ref(false) const isSpeaking = ref(false) +const errorMessage = ref('') + const audioContext = ref() -const mediaStream = ref() const analyser = ref() const dataArray = ref() const animationFrame = ref() +const volumeLevel = ref(0) // 0-100 const audios = ref([]) const audioCleanups = ref<(() => void)[]>([]) @@ -47,32 +47,21 @@ const audioURLs = computed(() => { }) const transcriptions = ref([]) -const { startRecord, stopRecord } = useAudioRecorder(mediaStream) +const { startRecord, stopRecord, onStop } = useAudioRecorder(stream) // Audio monitoring async function setupAudioMonitoring() { try { - if (!selectedAudioInput.value) { - console.warn('No audio input device selected') - return - } - // Clean up existing connections await stopAudioMonitoring() - // Get user media with selected device - mediaStream.value = await navigator.mediaDevices.getUserMedia({ - audio: { - deviceId: selectedAudioInput.value, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - }) + enabled.value = true + await start() + await until(stream).toBeTruthy() // Create audio context audioContext.value = new AudioContext() - const source = audioContext.value.createMediaStreamSource(mediaStream.value) + const source = audioContext.value.createMediaStreamSource(stream.value!) // Create analyser for volume detection analyser.value = audioContext.value.createAnalyser() @@ -103,9 +92,9 @@ async function stopAudioMonitoring() { } // Stop media stream - if (mediaStream.value) { - mediaStream.value.getTracks().forEach(track => track.stop()) - mediaStream.value = undefined + if (stream.value) { + stream.value.getTracks().forEach(track => track.stop()) + stream.value = undefined } // Close audio context @@ -114,6 +103,9 @@ async function stopAudioMonitoring() { audioContext.value = undefined } + await stopRecord() + await stop() + analyser.value = undefined dataArray.value = undefined volumeLevel.value = 0 @@ -152,7 +144,7 @@ watch(selectedAudioInput, async () => { watch(audioInputs, () => { if (!selectedAudioInput.value && audioInputs.value.length > 0) { - selectedAudioInput.value = audioInputs.value[0]?.deviceId + selectedAudioInput.value = audioInputs.value.find(input => input.deviceId === 'default')?.deviceId || audioInputs.value[0].deviceId } }) @@ -165,11 +157,22 @@ async function toggleMonitoring() { } else { await stopAudioMonitoring() - const audio = await stopRecord() - audios.value.push(audio) - const res = await props.generateTranscription(new File([audio], 'audio.wav', { type: 'audio/wav' })) - transcriptions.value.push(res.text) + onStop(async (recording) => { + try { + if (recording) { + audios.value.push(recording) + const res = await props.generateTranscription(new File([recording], 'recording.wav')) + transcriptions.value.push(res.text) + } + } + catch (err) { + errorMessage.value = err instanceof Error ? err.message : String(err) + console.error('Error generating transcription:', errorMessage.value) + } + }) + + await stopRecord() isMonitoring.value = false } @@ -187,7 +190,7 @@ const speakingIndicatorClass = computed(() => { onMounted(() => { devices.ensurePermissions().then(() => nextTick()).then(() => { if (audioInputs.value.length > 0 && !selectedAudioInput.value) { - selectedAudioInput.value = audioInputs.value[0]?.deviceId + selectedAudioInput.value = audioInputs.value.find(input => input.deviceId === 'default')?.deviceId || audioInputs.value[0].deviceId } }) }) @@ -199,6 +202,15 @@ onUnmounted(() => { diff --git a/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionProviderSettings.vue b/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionProviderSettings.vue index 9c6aa80e..ff92000d 100644 --- a/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionProviderSettings.vue +++ b/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionProviderSettings.vue @@ -10,6 +10,7 @@ import { ProviderBaseUrlInput, ProviderBasicSettings, ProviderSettingsContainer, + ProviderSettingsLayout2, } from '.' import { useProvidersStore } from '../../../stores' diff --git a/packages/stage-ui/src/composables/audio/audio-recorder.ts b/packages/stage-ui/src/composables/audio/audio-recorder.ts index 0c3fdbe2..0c9c5a0b 100644 --- a/packages/stage-ui/src/composables/audio/audio-recorder.ts +++ b/packages/stage-ui/src/composables/audio/audio-recorder.ts @@ -1,49 +1,96 @@ import type { MaybeRefOrGetter } from 'vue' import { until } from '@vueuse/core' -import { ref, toRef } from 'vue' +import { ref, shallowRef, toRef, watch } from 'vue' + +export function useDownload(url: string, fileName: string) { + const link = document.createElement('a') + link.href = url + link.download = fileName + document.body.appendChild(link) + link.click() + document.body.removeChild(link) +} export function useAudioRecorder( media: MaybeRefOrGetter, - start: () => Promise = () => Promise.resolve(), ) { const audioRecorder = ref() const mediaRef = toRef(media) + const recordingChunk = shallowRef([]) + const recording = ref() + + const onStopHooks = ref Promise>>([]) async function startRecord() { - await start() await until(mediaRef).toBeTruthy() - if (!mediaRef.value) { - console.error('No media media available') return } + recording.value = undefined + recordingChunk.value = [] + audioRecorder.value = new MediaRecorder(mediaRef.value) audioRecorder.value.start() + + audioRecorder.value.onerror = (event) => { + console.error('Error recording audio:', event) + } + + audioRecorder.value.onstop = () => { + if (recordingChunk.value.length > 0) { + const blob = new Blob(recordingChunk.value, { type: audioRecorder.value?.mimeType }) + recording.value = blob + + if (blob.size > 0) { + useDownload(URL.createObjectURL(blob), `recording-${Date.now()}.webm`) + } + } + else { + recording.value = undefined + } + + for (const hook of onStopHooks.value) { + hook(recording.value) + } + } + + audioRecorder.value.ondataavailable = (event) => { + recordingChunk.value.push(event.data) + } } - function stopRecord() { - return new Promise((resolve, reject) => { - if (!audioRecorder.value) { - return - } + function onStop(callback: (recording: Blob | undefined) => Promise) { + onStopHooks.value.push(callback) + } - audioRecorder.value.onerror = (event) => { - console.error('Error recording audio:', event) - reject(event) - } - audioRecorder.value.ondataavailable = (event) => { - const audioBlob = event.data - resolve(audioBlob) - } + async function stopRecord() { + if (!audioRecorder.value) { + return [] + } + audioRecorder.value?.stop() + return recordingChunk.value + } + + watch(mediaRef, () => { + if (audioRecorder.value && audioRecorder.value.state === 'recording') { audioRecorder.value.stop() - }) - } + } + + audioRecorder.value = undefined + + if (mediaRef.value && mediaRef.value.active) { + startRecord() + } + }) return { startRecord, stopRecord, + onStop, + recordingChunk, + recording, } }