Añade TTS con Clonación de Voz a Mastra con Fish Audio

User (Studio chat / workflow)
    │
    ▼
agent  (LLM + instructions)
    │
    └─ fish_tts  ──► POST api.fish.audio/v1/tts
                          │
                          ▼
                     workspace/.../narration.wav
                          │
                          ▼ (optional)
                     hyperframes transcribe → transcript.json
                     composition <audio class="clip">

FISH_API_KEY=your-fish-audio-key
FISH_VOICE_ID=your-cloned-voice-model-id
# TTS model default for fish_tts (override per call with model: still works)
FISH_TTS_MODEL=s2.1-pro-free

POST https://api.fish.audio/v1/tts
Authorization: Bearer YOUR_API_KEY
Content-Type: application/json
model: s2.1-pro-free   # or s2.1-pro | s2-pro | s1

{
  "text": "[confident] Docker makes deployment simple. [break] [excited] This tool makes it even easier.",
  "reference_id": "YOUR_VOICE_MODEL_ID",
  "format": "wav",
  "prosody": {
    "speed": 1.0,
    "volume": 0,
    "normalize_loudness": true
  },
  "temperature": 0.7,
  "top_p": 0.7,
  "chunk_length": 300,
  "normalize": true,
  "latency": "normal",
  "max_new_tokens": 1024,
  "repetition_penalty": 1.2,
  "condition_on_previous_chunks": true
}

{
  "text": "Mastra agents call Kubernetes jobs.",
  "reference_id": "YOUR_VOICE_MODEL_ID",
  "pronunciation_dictionary": [
    {
      "items": [
        { "key": "Kubernetes", "value": "K UW2 B ER0 N EH1 T IY0 Z", "case_sensitive": false }
      ]
    }
  ]
}

import { createTool } from "@mastra/core/tools";
import { z } from "zod";
import { resolve, isAbsolute, dirname } from "node:path";
import { mkdirSync, writeFileSync } from "node:fs";

const FISH_API_BASE = "https://api.fish.audio";
const DEFAULT_VOICE_ID = process.env.FISH_VOICE_ID ?? "";
// Default TTS model from env (s2.1-pro-free for dev; s2.1-pro for production SLA)
const DEFAULT_MODEL = process.env.FISH_TTS_MODEL ?? "s2.1-pro-free";
// Resolve relative output paths against your workspace root.
const WORKSPACE_PATH = process.env.WORKSPACE_PATH ?? `${process.cwd()}/workspace`;

function getApiKey(): string {
  const apiKey = process.env.FISH_API_KEY;
  if (!apiKey) {
    throw new Error(
      "FISH_API_KEY is not set. Add it to .env to enable Fish Audio TTS.",
    );
  }
  return apiKey;
}

export function createFishTts(basePath: string = WORKSPACE_PATH) {
  return createTool({
    id: "fish_tts",
    description: `Generate narration audio from text using Fish Audio S2.1 Pro TTS with a cloned voice.

EMOTION CONTROL (S2.1 Pro): Add emotion/style cues in [square brackets] directly in the text. Examples:
  [happy] What a great tool!
  [excited] And the best part? It's free!
  [calm] Let me walk you through the setup.
  [confident] This is the best option on the market.
  [curious] But what about performance?
  [sarcastic] Oh sure, because THAT always works.
  [whispering] Here's a secret most people miss.
  [laughing] Ha ha, yeah, I tried that too.
  [sighing] Another config file... sigh.

Combine emotions: [sad][whispering] for a quiet sad tone, [excited][laughing] for joyful energy.
Use [break] for a short pause, [long-break] for a longer pause.
Place emotion cues at the START of the sentence they should affect.
Natural language descriptions also work: [warm and friendly], [slightly annoyed], [very energetic].

For technical narration, [confident] or [calm] works best as the base tone. Use [excited] sparingly for highlights. Avoid [happy] for informational content — it sounds salesy.`,
    inputSchema: z.object({
      text: z
        .string()
        .max(10000)
        .describe(
          "Narration text. Supports [emotion] tags like [happy], [excited], [calm], [whispering], [break]. Place cues at sentence starts.",
        ),
      outputPath: z
        .string()
        .describe(
          "Path to save the audio. Relative paths resolve against the workspace. Extension must match format (.wav, .mp3, .opus).",
        ),
      speed: z
        .number()
        .min(0.5)
        .max(2)
        .optional()
        .describe("Speech speed. 1.0 = normal. Default 1.0."),
      volume: z
        .number()
        .min(-20)
        .max(20)
        .optional()
        .describe("Volume in dB. 0 = no change. Default 0."),
      voiceId: z
        .string()
        .optional()
        .describe("Fish Audio voice model ID. Defaults to FISH_VOICE_ID."),
      model: z
        .enum(["s2.1-pro-free", "s2.1-pro", "s2-pro", "s1"])
        .optional()
        .describe(
          "TTS model. Defaults to FISH_TTS_MODEL env (or s2.1-pro-free). Use s2.1-pro for production SLA.",
        ),
      format: z
        .enum(["wav", "mp3", "opus"])
        .optional()
        .describe("Output format. Default wav (good for video pipelines)."),
      temperature: z
        .number()
        .min(0)
        .max(1)
        .optional()
        .describe("Expressiveness. Higher = more varied. Default 0.7."),
      latency: z
        .enum(["normal", "balanced", "low"])
        .optional()
        .describe("normal = best quality (default). low = fastest."),
    }),
    outputSchema: z.object({
      success: z.boolean(),
      path: z.string().optional(),
      bytesWritten: z.number().optional(),
      format: z.string().optional(),
      error: z.string().optional(),
    }),
    execute: async (input) => {
      const apiKey = getApiKey();
      const format = input.format ?? "wav";
      const model = input.model ?? DEFAULT_MODEL;
      const voiceId = input.voiceId ?? DEFAULT_VOICE_ID;
      const speed = input.speed ?? 1.0;
      const volume = input.volume ?? 0;
      const temperature = input.temperature ?? 0.7;
      const latency = input.latency ?? "normal";

      if (!voiceId) {
        return {
          success: false,
          error:
            "No voice ID. Set FISH_VOICE_ID or pass voiceId. Clone a voice at Fish Audio first.",
        };
      }

      const absPath = isAbsolute(input.outputPath)
        ? input.outputPath
        : resolve(basePath, input.outputPath);

      try {
        mkdirSync(dirname(absPath), { recursive: true });
      } catch {
        // Directory may already exist
      }

      const body = {
        text: input.text,
        reference_id: voiceId,
        format,
        prosody: {
          speed,
          volume,
          normalize_loudness: true,
        },
        temperature,
        top_p: 0.7,
        chunk_length: 300,
        normalize: true,
        latency,
        max_new_tokens: 1024,
        repetition_penalty: 1.2,
        condition_on_previous_chunks: true,
      };

      try {
        const res = await fetch(`${FISH_API_BASE}/v1/tts`, {
          method: "POST",
          headers: {
            Authorization: `Bearer ${apiKey}`,
            "Content-Type": "application/json",
            model,
          },
          body: JSON.stringify(body),
        });

        if (!res.ok) {
          let errMsg = `Fish Audio TTS failed (HTTP ${res.status})`;
          try {
            const errData = await res.json();
            errMsg = errData?.message || errMsg;
          } catch {
            // not JSON
          }
          return { success: false, error: errMsg };
        }

        const audioBuffer = Buffer.from(await res.arrayBuffer());
        if (audioBuffer.length === 0) {
          return {
            success: false,
            error: "Fish Audio TTS returned empty audio data.",
          };
        }

        writeFileSync(absPath, audioBuffer);

        return {
          success: true,
          path: absPath,
          bytesWritten: audioBuffer.length,
          format,
        };
      } catch (error) {
        return {
          success: false,
          error: `Fish Audio TTS failed: ${
            error instanceof Error ? error.message : "Unknown error"
          }`,
        };
      }
    },
  });
}

// Default instance against workspace root
export const fishTts = createFishTts();

import { createFishTts } from "../tools/fish-audio-tts";

// Paths like "my-video/assets/narration.wav" land under hyperframes/
const fishTts = createFishTts("/absolute/path/to/workspace/hyperframes");

import { Agent } from "@mastra/core/agent";
import { fishTts } from "../tools/fish-audio-tts";

const AGENT_MODEL =
  process.env.AGENT_MODEL ?? "google/gemini-2.5-flash";

export const voiceAgent = new Agent({
  id: "voice-agent",
  name: "Voice Agent",
  instructions: () => {
    const iso = new Date().toISOString().split("T")[0];
    return `TODAY IS ${iso}.

You generate voiceover with the fish_tts tool using the user's cloned voice.

## Rules
- Always use fish_tts for audio. Never invent a local file path without calling the tool.
- Write narration WITH [emotion] tags. Base technical tone: [confident] or [calm].
- Highlights: [excited]. Problems: [frustrated]. Pauses: [break] or [long-break].
- Place emotion tags at the START of the sentence they affect. One primary emotion per sentence.
- Default speed 1.0. Default format wav.
- After success, report the local path and bytes written.
- If FISH_API_KEY or FISH_VOICE_ID is missing, say so clearly.
- TTS model defaults to FISH_TTS_MODEL (usually s2.1-pro-free); only pass model when you need a different one.`;
  },
  model: AGENT_MODEL,
  tools: {
    fishTts,
  },
  defaultOptions: { maxSteps: 10 },
});

4b. VOICEOVER — Generate narration BEFORE building so the timeline can be sized to it.
- Finalize the narration script from research.
- Call fish_tts with text + output path like <project>/assets/narration.wav
- Keep speed at 1.0
- Embed emotion tags in the script, e.g.:
  "[confident] Docker makes deployment simple. [break] [excited] But this new tool? It makes Docker look complicated."
- After TTS: transcribe for word-level timestamps (e.g. hyperframes transcribe)
- Drop audio into the composition as <audio class="clip">

TTS: Always use fish_tts (cloned user voice).
Do NOT use Kokoro, hyperframes tts, HeyGen, or ElevenLabs for narration.

import { Mastra } from "@mastra/core/mastra";
import { assistant } from "./agents/assistant";
import { voiceAgent } from "./agents/voice-agent";
// or: import { videoCreator } from "./agents/video-creator";

export const mastra = new Mastra({
  agents: {
    assistant,
    voiceAgent,
    // videoCreator,
  },
  // storage, logger, server — same as your existing app
});

// ENABLE_VIDEO=false to skip
if (process.env.ENABLE_VIDEO !== "false") {
  agents.videoCreator = videoCreator;
}

# .env must include:
# FISH_API_KEY=...
# FISH_VOICE_ID=...
# FISH_TTS_MODEL=s2.1-pro-free   # or s2.1-pro for production
# OPENROUTER_API_KEY=...   (or your LLM provider)
# AGENT_MODEL=google/gemini-2.5-flash

bun run dev
# or: npm run dev

Generate a 20-second voiceover for a blog intro about Mastra + Fish Audio.
Save as voice/demo-narration.wav.
Use [confident] for the base tone and [excited] once when mentioning voice cloning.

[confident] If you already have a Mastra agent for research and coding, voice is the next unlock.
[break]
[calm] Clone your voice on Fish Audio in about fifteen seconds, then expose a single fish_tts tool.
[excited] From there, every video can sound like you — without sitting in front of a mic again.

npx hyperframes transcribe assets/narration.wav -d .
# → transcript.json with word-level timestamps