feat: add WaveAudioPlayer with waveform visualization and authenticated audio fetch (#10158)

## Summary

Add a waveform-based audio player component (`WaveAudioPlayer`)
replacing the native `<audio>` element, with authenticated API fetch for
cloud audio playback.

## Changes

- **What**:
- Add `useWaveAudioPlayer` composable with waveform visualization from
audio data (Web Audio API `decodeAudioData`), playback controls, and
seek support
- Add `WaveAudioPlayer.vue` component with compact (inline waveform +
time) and expanded (full transport controls) variants
- Replace native `<audio>` in `MediaAudioTop.vue` and `ResultAudio.vue`
with `WaveAudioPlayer`
- Use `api.fetchApi()` instead of bare `fetch()` to include Firebase JWT
auth headers, fixing 401 errors in cloud environments
  - Add Storybook stories and unit tests

## Review Focus

- The audio URL is fetched via `api.fetchApi()` with auth headers,
converted to a Blob URL, then passed to the native `<audio>` element.
This avoids 401 Unauthorized in cloud environments where `/api/view`
requires authentication.
- URL-to-route extraction logic (`url.includes(apiBase)`) handles both
full API URLs and relative paths.


[screen-capture.webm](https://github.com/user-attachments/assets/44e61812-0391-4b47-a199-92927e75f8b4)

┆Issue is synchronized with this [Notion
page](https://www.notion.so/PR-10158-feat-add-WaveAudioPlayer-with-waveform-visualization-and-authenticated-audio-fetch-3266d73d365081beab3fc6274c39fcd4)
by [Unito](https://www.unito.io)

---------

Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Alexander Brown <drjkl@comfy.org>
This commit is contained in:
Jin Yi
2026-03-21 08:48:28 +09:00
committed by GitHub
parent 2b51babbcd
commit 7d9fa2bfc5
11 changed files with 651 additions and 29 deletions

View File

@@ -631,3 +631,10 @@ export function isPreviewableMediaType(mediaType: MediaType): boolean {
mediaType === '3D'
)
}
export function formatTime(seconds: number): string {
if (isNaN(seconds) || seconds === 0) return '0:00'
const mins = Math.floor(seconds / 60)
const secs = Math.floor(seconds % 60)
return `${mins}:${secs.toString().padStart(2, '0')}`
}

View File

@@ -0,0 +1,60 @@
import type { Meta, StoryObj } from '@storybook/vue3-vite'
import WaveAudioPlayer from './WaveAudioPlayer.vue'
const meta: Meta<typeof WaveAudioPlayer> = {
title: 'Components/Audio/WaveAudioPlayer',
component: WaveAudioPlayer,
tags: ['autodocs'],
parameters: { layout: 'centered' }
}
export default meta
type Story = StoryObj<typeof meta>
export const Default: Story = {
args: {
src: '/assets/audio/sample.wav',
barCount: 40,
height: 32
},
decorators: [
(story) => ({
components: { story },
template:
'<div class="w-80 rounded-lg bg-base-background p-4"><story /></div>'
})
]
}
export const BottomAligned: Story = {
args: {
src: '/assets/audio/sample.wav',
barCount: 40,
height: 48,
align: 'bottom'
},
decorators: [
(story) => ({
components: { story },
template:
'<div class="w-80 rounded-lg bg-base-background p-4"><story /></div>'
})
]
}
export const Expanded: Story = {
args: {
src: '/assets/audio/sample.wav',
variant: 'expanded',
barCount: 80,
height: 120
},
decorators: [
(story) => ({
components: { story },
template:
'<div class="w-[600px] rounded-2xl bg-base-background/80 p-8 backdrop-blur-sm"><story /></div>'
})
]
}

View File

@@ -0,0 +1,221 @@
<template>
<!-- Compact: [] [waveform] [time] -->
<div
v-if="variant === 'compact'"
:class="
cn('flex w-full gap-2', align === 'center' ? 'items-center' : 'items-end')
"
@pointerdown.stop
@click.stop
>
<Button
variant="textonly"
size="icon-sm"
class="size-7 shrink-0 rounded-full bg-muted-foreground/15 hover:bg-muted-foreground/25"
:aria-label="isPlaying ? $t('g.pause') : $t('g.play')"
:loading="loading"
@click.stop="togglePlayPause"
>
<i
v-if="!isPlaying"
class="ml-0.5 icon-[lucide--play] size-3 text-base-foreground"
/>
<i v-else class="icon-[lucide--pause] size-3 text-base-foreground" />
</Button>
<div
:ref="(el) => (waveformRef = el as HTMLElement)"
:class="
cn(
'flex min-w-0 flex-1 cursor-pointer gap-px',
align === 'center' ? 'items-center' : 'items-end'
)
"
:style="{ height: height + 'px' }"
@click="handleWaveformClick"
>
<div
v-for="(bar, index) in bars"
:key="index"
:class="
cn(
'min-h-0.5 flex-1 rounded-full',
loading
? 'bg-muted-foreground/20'
: index <= playedBarIndex
? 'bg-base-foreground'
: 'bg-muted-foreground/40'
)
"
:style="{ height: (bar.height / 100) * height + 'px' }"
/>
</div>
<span class="shrink-0 text-xs text-muted-foreground tabular-nums">
{{ formattedCurrentTime }} / {{ formattedDuration }}
</span>
</div>
<!-- Expanded: waveform / progress bar + times / transport -->
<div v-else class="flex w-full flex-col gap-4" @pointerdown.stop @click.stop>
<div
class="flex w-full items-center gap-0.5"
:style="{ height: height + 'px' }"
>
<div
v-for="(bar, index) in bars"
:key="index"
:class="
cn(
'min-h-0.5 flex-1 rounded-full',
loading ? 'bg-muted-foreground/20' : 'bg-base-foreground'
)
"
:style="{ height: (bar.height / 100) * height + 'px' }"
/>
</div>
<div class="flex flex-col gap-1">
<div
ref="progressRef"
class="relative h-1 w-full cursor-pointer rounded-full bg-muted-foreground/20"
@click="handleProgressClick"
>
<div
class="absolute top-0 left-0 h-full rounded-full bg-base-foreground"
:style="{ width: progressRatio + '%' }"
/>
</div>
<div
class="flex justify-between text-xs text-muted-foreground tabular-nums"
>
<span>{{ formattedCurrentTime }}</span>
<span>{{ formattedDuration }}</span>
</div>
</div>
<div class="flex items-center gap-2">
<div class="w-20" />
<div class="flex flex-1 items-center justify-center gap-2">
<Button
variant="textonly"
size="icon-sm"
class="size-8 rounded-full"
:aria-label="$t('g.skipToStart')"
:disabled="loading"
@click="seekToStart"
>
<i class="icon-[lucide--skip-back] size-4 text-base-foreground" />
</Button>
<Button
variant="textonly"
size="icon-sm"
class="size-10 rounded-full bg-muted-foreground/15 hover:bg-muted-foreground/25"
:aria-label="isPlaying ? $t('g.pause') : $t('g.play')"
:loading="loading"
@click="togglePlayPause"
>
<i
v-if="!isPlaying"
class="ml-0.5 icon-[lucide--play] size-5 text-base-foreground"
/>
<i v-else class="icon-[lucide--pause] size-5 text-base-foreground" />
</Button>
<Button
variant="textonly"
size="icon-sm"
class="size-8 rounded-full"
:aria-label="$t('g.skipToEnd')"
:disabled="loading"
@click="seekToEnd"
>
<i class="icon-[lucide--skip-forward] size-4 text-base-foreground" />
</Button>
</div>
<div class="flex w-20 items-center gap-1">
<Button
variant="textonly"
size="icon-sm"
class="size-8 shrink-0 rounded-full"
:aria-label="$t('g.volume')"
:disabled="loading"
@click="toggleMute"
>
<i :class="cn(volumeIcon, 'size-4 text-base-foreground')" />
</Button>
<Slider
:model-value="[volume * 100]"
:min="0"
:max="100"
:step="1"
class="flex-1"
@update:model-value="(v) => (volume = (v?.[0] ?? 100) / 100)"
/>
</div>
</div>
</div>
<audio
:ref="(el) => (audioRef = el as HTMLAudioElement)"
:src="audioSrc"
preload="metadata"
class="hidden"
/>
</template>
<script setup lang="ts">
import { ref, toRef } from 'vue'
import Button from '@/components/ui/button/Button.vue'
import Slider from '@/components/ui/slider/Slider.vue'
import { useWaveAudioPlayer } from '@/composables/useWaveAudioPlayer'
import { cn } from '@/utils/tailwindUtil'
const {
src,
barCount = 40,
height = 32,
align = 'center',
variant = 'compact'
} = defineProps<{
src: string
barCount?: number
height?: number
align?: 'center' | 'bottom'
variant?: 'compact' | 'expanded'
}>()
const progressRef = ref<HTMLElement>()
const {
audioRef,
waveformRef,
audioSrc,
bars,
loading,
isPlaying,
playedBarIndex,
progressRatio,
formattedCurrentTime,
formattedDuration,
togglePlayPause,
seekToStart,
seekToEnd,
volume,
volumeIcon,
toggleMute,
seekToRatio,
handleWaveformClick
} = useWaveAudioPlayer({
src: toRef(() => src),
barCount
})
function handleProgressClick(event: MouseEvent) {
if (!progressRef.value) return
const rect = progressRef.value.getBoundingClientRect()
seekToRatio((event.clientX - rect.left) / rect.width)
}
</script>

View File

@@ -1,19 +1,21 @@
<template>
<audio controls width="100%" height="100%">
<source :src="url" :type="htmlAudioType" />
{{ $t('g.audioFailedToLoad') }}
</audio>
<div
class="m-auto w-[min(90vw,42rem)] rounded-2xl bg-base-background/80 p-8 backdrop-blur-sm"
>
<WaveAudioPlayer
:src="result.url"
variant="expanded"
:height="120"
:bar-count="80"
/>
</div>
</template>
<script setup lang="ts">
import { computed } from 'vue'
import WaveAudioPlayer from '@/components/common/WaveAudioPlayer.vue'
import type { ResultItemImpl } from '@/stores/queueStore'
const { result } = defineProps<{
defineProps<{
result: ResultItemImpl
}>()
const url = computed(() => result.url)
const htmlAudioType = computed(() => result.htmlAudioType)
</script>

View File

@@ -0,0 +1,130 @@
import { ref } from 'vue'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { useWaveAudioPlayer } from './useWaveAudioPlayer'
vi.mock('@vueuse/core', async (importOriginal) => {
const actual = await importOriginal<Record<string, unknown>>()
return {
...actual,
useMediaControls: () => ({
playing: ref(false),
currentTime: ref(0),
duration: ref(0)
})
}
})
const mockFetchApi = vi.fn()
const originalAudioContext = globalThis.AudioContext
afterEach(() => {
globalThis.AudioContext = originalAudioContext
mockFetchApi.mockReset()
})
vi.mock('@/scripts/api', () => ({
api: {
apiURL: (route: string) => '/api' + route,
fetchApi: (...args: unknown[]) => mockFetchApi(...args)
}
}))
describe('useWaveAudioPlayer', () => {
it('initializes with default bar count', () => {
const src = ref('')
const { bars } = useWaveAudioPlayer({ src })
expect(bars.value).toHaveLength(40)
})
it('initializes with custom bar count', () => {
const src = ref('')
const { bars } = useWaveAudioPlayer({ src, barCount: 20 })
expect(bars.value).toHaveLength(20)
})
it('returns playedBarIndex as -1 when duration is 0', () => {
const src = ref('')
const { playedBarIndex } = useWaveAudioPlayer({ src })
expect(playedBarIndex.value).toBe(-1)
})
it('generates bars with heights between 10 and 70', () => {
const src = ref('')
const { bars } = useWaveAudioPlayer({ src })
for (const bar of bars.value) {
expect(bar.height).toBeGreaterThanOrEqual(10)
expect(bar.height).toBeLessThanOrEqual(70)
}
})
it('starts in paused state', () => {
const src = ref('')
const { isPlaying } = useWaveAudioPlayer({ src })
expect(isPlaying.value).toBe(false)
})
it('shows 0:00 for formatted times initially', () => {
const src = ref('')
const { formattedCurrentTime, formattedDuration } = useWaveAudioPlayer({
src
})
expect(formattedCurrentTime.value).toBe('0:00')
expect(formattedDuration.value).toBe('0:00')
})
it('fetches and decodes audio when src changes', async () => {
const mockAudioBuffer = {
getChannelData: vi.fn(() => new Float32Array(80))
}
const mockDecodeAudioData = vi.fn(() => Promise.resolve(mockAudioBuffer))
const mockClose = vi.fn().mockResolvedValue(undefined)
globalThis.AudioContext = class {
decodeAudioData = mockDecodeAudioData
close = mockClose
} as unknown as typeof AudioContext
mockFetchApi.mockResolvedValue({
ok: true,
arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)),
headers: { get: () => 'audio/wav' }
})
const src = ref('/api/view?filename=audio.wav&type=output')
const { bars, loading } = useWaveAudioPlayer({ src, barCount: 10 })
await vi.waitFor(() => {
expect(loading.value).toBe(false)
})
expect(mockFetchApi).toHaveBeenCalledWith(
'/view?filename=audio.wav&type=output'
)
expect(mockDecodeAudioData).toHaveBeenCalled()
expect(bars.value).toHaveLength(10)
})
it('clears blobUrl and shows placeholder bars when fetch fails', async () => {
mockFetchApi.mockRejectedValue(new Error('Network error'))
const src = ref('/api/view?filename=audio.wav&type=output')
const { bars, loading, audioSrc } = useWaveAudioPlayer({
src,
barCount: 10
})
await vi.waitFor(() => {
expect(loading.value).toBe(false)
})
expect(bars.value).toHaveLength(10)
expect(audioSrc.value).toBe('/api/view?filename=audio.wav&type=output')
})
it('does not call decodeAudioSource when src is empty', () => {
const src = ref('')
useWaveAudioPlayer({ src })
expect(mockFetchApi).not.toHaveBeenCalled()
})
})

View File

@@ -0,0 +1,205 @@
import { useMediaControls, whenever } from '@vueuse/core'
import { computed, onUnmounted, ref } from 'vue'
import type { Ref } from 'vue'
import { api } from '@/scripts/api'
import { formatTime } from '@/utils/formatUtil'
interface WaveformBar {
height: number
}
interface UseWaveAudioPlayerOptions {
src: Ref<string>
barCount?: number
}
export function useWaveAudioPlayer(options: UseWaveAudioPlayerOptions) {
const { src, barCount = 40 } = options
const audioRef = ref<HTMLAudioElement>()
const waveformRef = ref<HTMLElement>()
const blobUrl = ref<string>()
const loading = ref(false)
let decodeRequestId = 0
const bars = ref<WaveformBar[]>(generatePlaceholderBars())
const { playing, currentTime, duration, volume, muted } =
useMediaControls(audioRef)
const playedBarIndex = computed(() => {
if (duration.value === 0) return -1
return Math.floor((currentTime.value / duration.value) * barCount) - 1
})
const formattedCurrentTime = computed(() => formatTime(currentTime.value))
const formattedDuration = computed(() => formatTime(duration.value))
const audioSrc = computed(() =>
src.value ? (blobUrl.value ?? src.value) : ''
)
function generatePlaceholderBars(): WaveformBar[] {
return Array.from({ length: barCount }, () => ({
height: Math.random() * 60 + 10
}))
}
function generateBarsFromBuffer(buffer: AudioBuffer) {
const channelData = buffer.getChannelData(0)
if (channelData.length === 0) {
bars.value = generatePlaceholderBars()
return
}
const averages: number[] = []
for (let i = 0; i < barCount; i++) {
const start = Math.floor((i * channelData.length) / barCount)
const end = Math.max(
start + 1,
Math.floor(((i + 1) * channelData.length) / barCount)
)
let sum = 0
for (let j = start; j < end && j < channelData.length; j++) {
sum += Math.abs(channelData[j])
}
averages.push(sum / (end - start))
}
const peak = Math.max(...averages) || 1
bars.value = averages.map((avg) => ({
height: Math.max(8, (avg / peak) * 100)
}))
}
async function decodeAudioSource(url: string) {
const requestId = ++decodeRequestId
loading.value = true
let ctx: AudioContext | undefined
try {
const apiBase = api.apiURL('/')
const route = url.includes(apiBase)
? url.slice(url.indexOf(apiBase) + api.apiURL('').length)
: url
const response = await api.fetchApi(route)
if (requestId !== decodeRequestId) return
if (!response.ok) {
throw new Error(`Failed to fetch audio (${response.status})`)
}
const arrayBuffer = await response.arrayBuffer()
if (requestId !== decodeRequestId) return
const blob = new Blob([arrayBuffer.slice(0)], {
type: response.headers.get('content-type') ?? 'audio/wav'
})
if (blobUrl.value) URL.revokeObjectURL(blobUrl.value)
blobUrl.value = URL.createObjectURL(blob)
ctx = new AudioContext()
const audioBuffer = await ctx.decodeAudioData(arrayBuffer)
if (requestId !== decodeRequestId) return
generateBarsFromBuffer(audioBuffer)
} catch {
if (requestId === decodeRequestId) {
if (blobUrl.value) {
URL.revokeObjectURL(blobUrl.value)
blobUrl.value = undefined
}
bars.value = generatePlaceholderBars()
}
} finally {
await ctx?.close()
if (requestId === decodeRequestId) {
loading.value = false
}
}
}
const progressRatio = computed(() => {
if (duration.value === 0) return 0
return (currentTime.value / duration.value) * 100
})
function togglePlayPause() {
playing.value = !playing.value
}
function seekToStart() {
currentTime.value = 0
}
function seekToEnd() {
currentTime.value = duration.value
playing.value = false
}
function seekToRatio(ratio: number) {
const clamped = Math.max(0, Math.min(1, ratio))
currentTime.value = clamped * duration.value
}
function toggleMute() {
muted.value = !muted.value
}
const volumeIcon = computed(() => {
if (muted.value || volume.value === 0) return 'icon-[lucide--volume-x]'
if (volume.value < 0.5) return 'icon-[lucide--volume-1]'
return 'icon-[lucide--volume-2]'
})
function handleWaveformClick(event: MouseEvent) {
if (!waveformRef.value || duration.value === 0) return
const rect = waveformRef.value.getBoundingClientRect()
const ratio = Math.max(
0,
Math.min(1, (event.clientX - rect.left) / rect.width)
)
currentTime.value = ratio * duration.value
if (!playing.value) {
playing.value = true
}
}
whenever(
src,
(url) => {
playing.value = false
currentTime.value = 0
void decodeAudioSource(url)
},
{ immediate: true }
)
onUnmounted(() => {
decodeRequestId += 1
audioRef.value?.pause()
if (blobUrl.value) {
URL.revokeObjectURL(blobUrl.value)
blobUrl.value = undefined
}
})
return {
audioRef,
waveformRef,
audioSrc,
bars,
loading,
isPlaying: playing,
playedBarIndex,
progressRatio,
formattedCurrentTime,
formattedDuration,
togglePlayPause,
seekToStart,
seekToEnd,
volume,
volumeIcon,
toggleMute,
seekToRatio,
handleWaveformClick
}
}

View File

@@ -343,9 +343,13 @@
"frameNodes": "Frame Nodes",
"listening": "Listening...",
"ready": "Ready",
"play": "Play",
"pause": "Pause",
"playPause": "Play/Pause",
"playRecording": "Play Recording",
"playing": "Playing",
"skipToStart": "Skip to Start",
"skipToEnd": "Skip to End",
"stopPlayback": "Stop Playback",
"playbackSpeed": "Playback Speed",
"volume": "Volume",

View File

@@ -8,16 +8,20 @@
$t('assetBrowser.media.audioPlaceholder')
}}</span>
</div>
<audio
controls
class="absolute bottom-0 left-0 w-full p-2"
:src="asset.src"
@click.stop
/>
<div class="absolute bottom-0 left-0 w-full p-2">
<WaveAudioPlayer
:src="asset.src"
:bar-count="40"
:height="32"
align="bottom"
/>
</div>
</div>
</template>
<script setup lang="ts">
import WaveAudioPlayer from '@/components/common/WaveAudioPlayer.vue'
import type { AssetMeta } from '../schemas/mediaAssetSchema'
const { asset } = defineProps<{

View File

@@ -96,7 +96,7 @@ import { useAudioService } from '@/services/audioService'
import { useAudioPlayback } from '../composables/audio/useAudioPlayback'
import { useAudioRecorder } from '../composables/audio/useAudioRecorder'
import { useAudioWaveform } from '../composables/audio/useAudioWaveform'
import { formatTime } from '../utils/audioUtils'
import { formatTime } from '@/utils/formatUtil'
const { t } = useI18n()

View File

@@ -156,7 +156,7 @@ import { downloadFile } from '@/base/common/downloadUtil'
import Button from '@/components/ui/button/Button.vue'
import { cn } from '@/utils/tailwindUtil'
import { formatTime } from '../../utils/audioUtils'
import { formatTime } from '@/utils/formatUtil'
const { t } = useI18n()
const toast = useToast()

View File

@@ -1,17 +1,6 @@
import type { ResultItemType } from '@/schemas/apiSchema'
import { app } from '@/scripts/app'
/**
* Format time in MM:SS format
*/
export function formatTime(seconds: number): string {
if (isNaN(seconds) || seconds === 0) return '0:00'
const mins = Math.floor(seconds / 60)
const secs = Math.floor(seconds % 60)
return `${mins}:${secs.toString().padStart(2, '0')}`
}
export function getResourceURL(
subfolder: string,
filename: string,