tts: 新增真人声音配音模式(录音/上传声音样本,用自己的声音合成语音)

This commit is contained in:
2570050763@qq.com 2026-09-18 13:10:14 +08:00
parent 12eef4b854
commit 14bc7598bc
4 changed files with 371 additions and 10 deletions

View File

@ -377,6 +377,19 @@ button:disabled {
transition: border-color 180ms ease, background-color 180ms ease, transform 180ms ease;
}
.voice-clone-panel {
border: 1px solid rgba(0, 209, 230, 0.3);
border-radius: 0.85rem;
background: rgba(0, 209, 230, 0.05);
padding: 1rem;
transition: border-color 180ms ease, background-color 180ms ease;
}
.voice-clone-panel.is-ready {
border-color: rgba(0, 209, 230, 0.7);
background: rgba(0, 209, 230, 0.09);
}
.assist-card:hover {
border-color: rgba(0, 209, 230, 0.65);
background: rgba(0, 209, 230, 0.06);

65
app/tts-clone/route.ts Normal file
View File

@ -0,0 +1,65 @@
import { NextResponse } from "next/server";
export const runtime = "nodejs";
const formatMime: Record<string, string> = {
wav: "audio/wav",
mp3: "audio/mpeg",
flac: "audio/flac",
opus: "audio/opus",
aac: "audio/aac",
pcm: "application/octet-stream",
};
// 真人声音配音(声音克隆):
// 前端把「文本 + 客户声音样本(data URL)」发到这里,服务端再调用配置的声音克隆引擎,返回音频流。
// 生产环境需在服务端配置 TTS_CLONE_URL 指向 TTS 引擎的克隆接口;
// 未配置时返回 501前端会提示「真人配音引擎尚未接通请先选择内置音色」。
export async function POST(request: Request) {
let body: { text?: unknown; reference_audio?: unknown; response_format?: unknown; speed?: unknown };
try {
body = await request.json();
} catch {
return NextResponse.json({ error: { message: "请求体不是合法 JSON" } }, { status: 400 });
}
const text = typeof body.text === "string" ? body.text.trim() : "";
const referenceAudio = typeof body.reference_audio === "string" ? body.reference_audio : "";
const format = typeof body.response_format === "string" && body.response_format ? body.response_format : "wav";
const speed = Number(body.speed) || 1;
if (!text) return NextResponse.json({ error: { message: "请输入需要合成的文本" } }, { status: 400 });
if (!referenceAudio.startsWith("data:audio/")) {
return NextResponse.json({ error: { message: "请先录制或上传声音样本" } }, { status: 400 });
}
if (referenceAudio.length > 12 * 1024 * 1024) {
return NextResponse.json({ error: { message: "声音样本过大,请控制在 8 MB 以内" } }, { status: 400 });
}
const cloneUrl = process.env.TTS_CLONE_URL;
if (!cloneUrl) {
return NextResponse.json(
{ error: { message: "真人配音引擎尚未接通,请先选择内置音色" } },
{ status: 501 },
);
}
try {
const upstream = await fetch(cloneUrl, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ input: text, reference_audio: referenceAudio, response_format: format, speed }),
});
if (!upstream.ok) {
const errBody = (await upstream.json().catch(() => ({}))) as { error?: { message?: string } };
throw new Error(errBody?.error?.message ?? `配音引擎返回 ${upstream.status}`);
}
const audio = Buffer.from(await upstream.arrayBuffer());
return new NextResponse(audio, {
headers: { "Content-Type": formatMime[format] ?? "application/octet-stream" },
});
} catch (error) {
const message = error instanceof Error ? error.message : "真人语音合成失败";
return NextResponse.json({ error: { message } }, { status: 502 });
}
}

View File

@ -121,8 +121,12 @@ export default function TtsPage() {
const [pauseError, setPauseError] = useState("");
const [correctionMessage, setCorrectionMessage] = useState("");
const [helperMessage, setHelperMessage] = useState("");
const [myVoice, setMyVoice] = useState<{ name: string; dataUrl: string } | null>(null);
const [recording, setRecording] = useState(false);
const textareaRef = useRef<HTMLDivElement>(null);
const fileInputRef = useRef<HTMLInputElement>(null);
const recorderRef = useRef<MediaRecorder | null>(null);
const voiceFileRef = useRef<HTMLInputElement>(null);
useEffect(() => {
let cancelled = false;
@ -367,19 +371,33 @@ export default function TtsPage() {
async function createSpeech() {
setError("");
if (!text.trim()) return setError("请输入需要合成的文本");
if (!voiceId) return setError("请选择一个音色");
if (!myVoice && !voiceId) return setError("请选择一个音色");
setBusy(true);
try {
const response = await fetch(apiUrl("/v1/audio/speech"), {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ input: normalizeText(text), voice: voiceId, response_format: format, speed }),
});
if (!response.ok) {
const body = await response.json().catch(() => ({}));
throw new Error(body.error?.message ?? `合成失败(${response.status}`);
let blob: Blob;
if (myVoice) {
const response = await fetch("/tts-clone", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ text: normalizeText(text), reference_audio: myVoice.dataUrl, response_format: format, speed }),
});
if (!response.ok) {
const body = await response.json().catch(() => ({}));
throw new Error(body.error?.message ?? `真人语音合成失败(${response.status}`);
}
blob = await response.blob();
} else {
const response = await fetch(apiUrl("/v1/audio/speech"), {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ input: normalizeText(text), voice: voiceId, response_format: format, speed }),
});
if (!response.ok) {
const body = await response.json().catch(() => ({}));
throw new Error(body.error?.message ?? `合成失败(${response.status}`);
}
blob = await response.blob();
}
const blob = await response.blob();
const url = URL.createObjectURL(blob);
setResult({ url, format, fileName: `kaotings-tts-${new Date().toISOString().replace(/[:.]/g, "-")}.${format}` });
} catch (speechError) {
@ -389,6 +407,57 @@ export default function TtsPage() {
}
}
async function startRecording() {
if (recording) return;
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const preferred = ["audio/webm", "audio/mp4", "audio/ogg"].find((mime) => MediaRecorder.isTypeSupported(mime));
const recorder = new MediaRecorder(stream, preferred ? { mimeType: preferred } : undefined);
const chunks: Blob[] = [];
recorder.ondataavailable = (event) => { if (event.data.size > 0) chunks.push(event.data); };
recorder.onstop = () => {
stream.getTracks().forEach((track) => track.stop());
const blob = new Blob(chunks, { type: recorder.mimeType || "audio/webm" });
const reader = new FileReader();
reader.onload = () => {
setMyVoice({ dataUrl: String(reader.result), name: `我的声音 ${new Date().toLocaleTimeString("zh-CN", { hour12: false })}` });
setRecording(false);
};
reader.onerror = () => setRecording(false);
reader.readAsDataURL(blob);
};
recorderRef.current = recorder;
recorder.start();
setRecording(true);
} catch {
setHelperMessage("无法访问麦克风,请允许浏览器使用麦克风,或直接上传录好的音频文件");
}
}
function stopRecording() {
if (recorderRef.current && recorderRef.current.state !== "inactive") recorderRef.current.stop();
}
function handleVoiceFile(file: File | undefined) {
if (!file) return;
const isAudio = file.type.startsWith("audio/") || /\.(wav|mp3|m4a|ogg|webm|aac)$/i.test(file.name);
if (!isAudio) {
setHelperMessage("请上传音频文件wav / mp3 / m4a 等)");
return;
}
if (file.size > 8 * 1024 * 1024) {
setHelperMessage("声音样本不能超过 8 MB建议 1060 秒清晰人声");
return;
}
const reader = new FileReader();
reader.onload = () => setMyVoice({ dataUrl: String(reader.result), name: file.name });
reader.readAsDataURL(file);
}
function clearMyVoice() {
setMyVoice(null);
}
async function handleOcr(file: File | undefined) {
if (!file) return;
if (!file.type.startsWith("image/")) {
@ -512,6 +581,25 @@ export default function TtsPage() {
</div>
<input accept="image/*" className="hidden" onChange={(event) => void handleOcr(event.target.files?.[0])} ref={fileInputRef} type="file" />
{helperMessage ? <p className="mt-4 rounded-lg border border-cyan/20 bg-cyan/5 px-3 py-2 text-xs text-cyan">{helperMessage}</p> : null}
<section aria-label="真人声音配音模式" className={`voice-clone-panel mt-6 ${myVoice ? "is-ready" : ""}`}>
<div className="flex flex-wrap items-center gap-x-4 gap-y-3">
<div className="min-w-0 flex-1">
<p className="font-semibold"><span aria-hidden="true">🎙</span> </p>
<p className="mt-1 text-xs leading-5 text-muted"></p>
</div>
<div className="flex flex-wrap items-center gap-2">
{recording ? <button className="button-secondary" onClick={stopRecording} type="button"> </button> : <button className="button-secondary" onClick={() => { void startRecording(); }} type="button"> </button>}
<button className="button-ghost min-h-9 px-3 py-1.5 text-xs" onClick={() => voiceFileRef.current?.click()} type="button"></button>
{myVoice ? <button className="text-xs text-danger hover:underline" onClick={clearMyVoice} type="button"></button> : null}
</div>
</div>
{myVoice ? <div className="mt-4 flex flex-wrap items-center gap-3">
<span className="rounded-full border border-cyan/30 bg-cyan/10 px-3 py-1 text-xs font-semibold text-cyan"> {myVoice.name} 使</span>
<audio className="h-9 w-full max-w-96" controls preload="metadata" src={myVoice.dataUrl} />
</div> : null}
{recording ? <p className="mt-3 text-xs text-cyan"> 1060 </p> : null}
<input accept="audio/*,.wav,.mp3,.m4a" className="hidden" onChange={(event) => { handleVoiceFile(event.target.files?.[0]); event.target.value = ""; }} ref={voiceFileRef} type="file" />
</section>
</section>
<aside className="space-y-6">

View File

@ -0,0 +1,195 @@
// 本地演示用 mock TTS 服务:实现 AI Lab 页面依赖的 /v1/audio/* 接口。
// 启动node services/mock-tts/server.mjs配合 .env.local 里 TTS_MOCK_UPSTREAM=http://127.0.0.1:8100
// 真实上游为外部 TTS 引擎,需要服务端凭据;此服务仅用于无引擎环境下演示与联调。
import { createServer } from "node:http";
const PORT = Number(process.env.MOCK_TTS_PORT ?? 8100);
const SAMPLE_RATE = 22050;
const voices = [
{
id: "xiaoman", name: "小满", desc: "温暖亲和的女声,适合品牌宣传与产品介绍。",
category: "通用女声", gender: "女声", genre: "温和", accent: "普通话", style: "亲切",
styles: ["亲切", "自然"], preview: "/v1/previews/xiaoman.wav", range: [240, 400],
},
{
id: "qingluo", name: "青落", desc: "清亮利落的女声,适合新闻播报与知识讲解。",
category: "通用女声", gender: "女声", genre: "专业", accent: "普通话", style: "专业",
styles: ["专业", "清晰"], preview: "/v1/previews/qingluo.wav", range: [260, 430],
},
{
id: "tianxin", name: "甜芯", desc: "甜美活泼的女声,适合促销活动与社媒内容。",
category: "通用女声", gender: "女声", genre: "甜美", accent: "台湾腔", style: "活泼",
styles: ["活泼", "甜美"], preview: "/v1/previews/tianxin.wav", range: [280, 460],
},
{
id: "mufan", name: "沐帆", desc: "低沉沉稳的男声,适合纪录片旁白与高端品牌。",
category: "通用男声", gender: "男声", genre: "磁性", accent: "普通话", style: "沉稳",
styles: ["沉稳", "磁性"], preview: "/v1/previews/mufan.wav", range: [110, 200],
},
{
id: "haoran", name: "浩然", desc: "爽朗有力的男声,适合广告口号与活动主持。",
category: "通用男声", gender: "男声", genre: "活力", accent: "普通话", style: "有力",
styles: ["有力", "活力"], preview: "/v1/previews/haoran.wav", range: [130, 230],
},
{
id: "awan", name: "阿万", desc: "带粤语口音的男声,适合大湾区本地化内容。",
category: "通用男声", gender: "男声", genre: "温和", accent: "粤语", style: "自然",
styles: ["自然"], preview: "/v1/previews/awan.wav", range: [120, 210],
},
{
id: "doudou", name: "豆豆", desc: "元气满满的童声,适合儿歌故事与亲子内容。",
category: "童声", gender: "童声", genre: "活泼", accent: "普通话", style: "可爱",
styles: ["可爱", "活泼"], preview: "/v1/previews/doudou.wav", range: [300, 480],
},
{
id: "dongbei-ge", name: "东北哥", desc: "幽默豪爽的东北男声,适合短视频与段子配音。",
category: "情景演绎", gender: "男声", genre: "活力", accent: "东北腔", style: "幽默",
styles: ["幽默", "豪爽"], preview: "/v1/previews/dongbei-ge.wav", range: [115, 205],
},
];
const facets = {
gender: [...new Set(voices.map((v) => v.gender))],
genre: [...new Set(voices.map((v) => v.genre))],
accent: [...new Set(voices.map((v) => v.accent))],
style: [...new Set(voices.map((v) => v.style))],
};
function json(res, status, body) {
res.writeHead(status, { "Content-Type": "application/json; charset=utf-8", "Access-Control-Allow-Origin": "*" });
res.end(JSON.stringify(body));
}
// 极简 WAV 合成:按字符生成短音,<pause:ms> 生成静音。仅作演示,非真实语音。
function synthesizeWav(text, voiceId, speed) {
const voice = voices.find((v) => v.id === voiceId) ?? voices[0];
const rate = Math.min(4, Math.max(0.25, Number(speed) || 1));
const segments = [];
for (const part of String(text ?? "").split(/(<pause:\d+>)/)) {
const pause = part.match(/^<pause:(\d+)>$/);
if (pause) segments.push({ pauseMs: Math.min(10000, Math.max(50, Number(pause[1]))) });
else if (part) segments.push({ chars: [...part] });
}
const samples = [];
for (const segment of segments) {
if (segment.pauseMs) {
const silence = Math.floor((SAMPLE_RATE * segment.pauseMs) / 1000 / rate);
samples.push(...new Array(silence).fill(0));
continue;
}
for (const [index, ch] of segment.chars.entries()) {
if (/\s/.test(ch)) {
samples.push(...new Array(Math.floor(SAMPLE_RATE * 0.06 / rate)).fill(0));
continue;
}
// 以字符与音色做种子生成 160320Hz 的基频,让不同文本/音色听感有区分
const seed = (ch.codePointAt(0) * 31 + voice.id.charCodeAt(0) * 7) % 1000 / 1000;
const freq = voice.range[0] + seed * (voice.range[1] - voice.range[0]);
const duration = 0.13 + ((index % 3) * 0.02);
const count = Math.floor((SAMPLE_RATE * duration) / rate);
for (let i = 0; i < count; i += 1) {
const t = i / SAMPLE_RATE;
const envelope = Math.min(1, i / (count * 0.15), (count - i) / (count * 0.25));
const value = Math.sin(2 * Math.PI * freq * t) * 0.55 + Math.sin(2 * Math.PI * freq * 2 * t) * 0.18;
samples.push(Math.round(Math.max(-1, Math.min(1, value * envelope)) * 32000));
}
samples.push(...new Array(Math.floor(SAMPLE_RATE * 0.025 / rate)).fill(0));
}
}
const capped = samples.slice(0, SAMPLE_RATE * 30);
const buffer = Buffer.alloc(44 + capped.length * 2);
buffer.write("RIFF", 0);
buffer.writeUInt32LE(36 + capped.length * 2, 4);
buffer.write("WAVE", 8);
buffer.write("fmt ", 12);
buffer.writeUInt32LE(16, 16);
buffer.writeUInt16LE(1, 20); // PCM
buffer.writeUInt16LE(1, 22); // mono
buffer.writeUInt32LE(SAMPLE_RATE, 24);
buffer.writeUInt32LE(SAMPLE_RATE * 2, 28);
buffer.writeUInt16LE(2, 32);
buffer.writeUInt16LE(16, 34);
buffer.write("data", 36);
buffer.writeUInt32LE(capped.length * 2, 40);
capped.forEach((value, i) => buffer.writeInt16LE(value, 44 + i * 2));
return buffer;
}
function voiceById(id) {
return voices.find((v) => v.id === id);
}
const server = createServer((req, res) => {
const url = new URL(req.url, `http://127.0.0.1:${PORT}`);
if (req.method === "OPTIONS") {
res.writeHead(204, {
"Access-Control-Allow-Origin": "*",
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
"Access-Control-Allow-Headers": "Content-Type",
});
return res.end();
}
if (req.method === "GET" && url.pathname === "/v1/audio/config") {
return json(res, 200, {
model: "qwen3-tts-mock",
formats: ["wav"],
default_format: "wav",
speed: { min: 0.25, max: 4, default: 1 },
pause: {
supported: true, min_ms: 50, max_ms: 10000,
presets: [{ ms: 500, label: "0.5s" }, { ms: 1000, label: "1s" }, { ms: 2000, label: "2s" }],
},
});
}
if (req.method === "GET" && url.pathname === "/v1/audio/voices") {
return json(res, 200, {
categories: [...new Set(voices.map((v) => v.category))],
facets,
voices: voices.map(({ range, ...voice }) => voice),
});
}
const preview = req.method === "GET" && url.pathname.match(/^\/v1\/previews\/([\w-]+)\.wav$/);
if (preview) {
const voice = voiceById(preview[1]);
if (!voice) return json(res, 404, { error: { message: "音色不存在" } });
const audio = synthesizeWav(`你好,我是${voice.name}`, voice.id, 1);
res.writeHead(200, { "Content-Type": "audio/wav", "Content-Length": audio.length, "Access-Control-Allow-Origin": "*" });
return res.end(audio);
}
if (req.method === "POST" && url.pathname === "/v1/audio/speech") {
let raw = "";
req.on("data", (chunk) => { raw += chunk; if (raw.length > 1_000_000) req.destroy(); });
req.on("end", () => {
let body = {};
try { body = JSON.parse(raw || "{}"); } catch { return json(res, 400, { error: { message: "请求体不是合法 JSON" } }); }
if (!voiceById(body.voice)) return json(res, 400, { error: { message: "音色不存在或未提供" } });
if (!String(body.input ?? "").trim()) return json(res, 400, { error: { message: "文本不能为空" } });
const audio = synthesizeWav(body.input, body.voice, body.speed);
res.writeHead(200, { "Content-Type": "audio/wav", "Content-Length": audio.length, "Access-Control-Allow-Origin": "*" });
res.end(audio);
});
return;
}
if (req.method === "POST" && url.pathname === "/v1/audio/clone-speech") {
// 真人声音配音 mock校验文本与声音样本后返回演示音频非真实克隆
let raw = "";
req.on("data", (chunk) => { raw += chunk; if (raw.length > 16_000_000) req.destroy(); });
req.on("end", () => {
let body = {};
try { body = JSON.parse(raw || "{}"); } catch { return json(res, 400, { error: { message: "请求体不是合法 JSON" } }); }
if (!String(body.input ?? "").trim()) return json(res, 400, { error: { message: "文本不能为空" } });
if (!String(body.reference_audio ?? "").startsWith("data:audio/")) return json(res, 400, { error: { message: "未提供声音样本 reference_audio" } });
const audio = synthesizeWav(body.input, "xiaoman", body.speed);
res.writeHead(200, { "Content-Type": "audio/wav", "Content-Length": audio.length, "Access-Control-Allow-Origin": "*" });
res.end(audio);
});
return;
}
if (req.method === "GET" && url.pathname === "/healthz") return json(res, 200, { status: "ok" });
json(res, 404, { error: { message: `mock-tts 未实现 ${req.method} ${url.pathname}` } });
});
server.listen(PORT, "127.0.0.1", () => {
console.log(`[mock-tts] listening on http://127.0.0.1:${PORT}`);
});