{"schema_version":1,"generated_at":"2026-09-13T20:11:48.771256+00:00","collections":[{"id":"omni","name":"Omni","description":"Multimodal understanding & speech dialogue","repo":"awesome-omni-architectures","url":"https://github.com/kadirnar/awesome-omni-architectures","revision":"8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc","asOf":"2026-09-13","count":144,"categories":[{"id":"omni:omni","name":"Omni & audio-visual","count":44},{"id":"omni:speech","name":"Speech dialogue","count":42},{"id":"omni:audio","name":"Audio understanding","count":45},{"id":"omni:asr","name":"Speech recognition","count":9},{"id":"omni:pipeline","name":"Cascaded systems","count":4}],"creditsUrl":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/CREDITS.md"},{"id":"tts","name":"Text to speech","description":"Voice synthesis, from classics to speech LMs","repo":"awesome-tts-architectures","url":"https://github.com/kadirnar/awesome-tts-architectures","revision":"caa36857411c1b5c5b5a491c18855a4a27fe04ad","asOf":"2026-09-13","count":380,"categories":[{"id":"tts:concatenative","name":"Concatenative","count":1},{"id":"tts:autoregressive","name":"Autoregressive acoustic","count":11},{"id":"tts:parallel","name":"Parallel acoustic","count":25},{"id":"tts:flow-vae","name":"Flows & variational TTS","count":15},{"id":"tts:diffusion","name":"Diffusion & flow matching","count":90},{"id":"tts:token-lm","name":"Speech-token LMs","count":191},{"id":"tts:continuous-lm","name":"Continuous speech LMs","count":32},{"id":"tts:compact","name":"Compact & on-device","count":11},{"id":"tts:api","name":"Commercial TTS","count":4}],"creditsUrl":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/CREDITS.md"},{"id":"codec","name":"Codecs","description":"Neural audio codecs, VAEs & continuous latents","repo":"awesome-codec-architectures","url":"https://github.com/kadirnar/awesome-codec-architectures","revision":"59a1baab61a719c55d8df2150665d2243076c116","asOf":"2026-09-13","count":92,"categories":[{"id":"codec:general","name":"General audio codecs","count":9},{"id":"codec:speech","name":"Speech codecs","count":18},{"id":"codec:semantic","name":"Semantic & acoustic codecs","count":26},{"id":"codec:disentangled","name":"Disentangled codecs","count":4},{"id":"codec:music","name":"Music codecs","count":2},{"id":"codec:continuous","name":"VAE & continuous codecs","count":33}],"creditsUrl":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/CREDITS.md"}],"models":[{"id":"omni:anygpt","slug":"anygpt","name":"AnyGPT","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands and generates text, images, speech and music using a shared token vocabulary.","architecture":"Discrete multimodal token modeling on a language backbone","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["AnyGPT-chat"],"inputs":["T","I","S","M"],"outputs":["T","I","S","M"],"interaction":"generation","specs":{},"date":"2024-02-19","dateKind":"paper","links":{"code":"https://github.com/OpenMOSS/AnyGPT","weights":"https://huggingface.co/OpenMOSS-Team/AnyGPT-chat","paper":"https://arxiv.org/abs/2402.12226","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#anygpt"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0 + Llama 2 terms","codeUrl":"https://github.com/OpenMOSS/AnyGPT/blob/main/README.md","weightsUrl":"https://huggingface.co/OpenMOSS-Team/AnyGPT-chat/blob/c8999624b077b3837f32c3948f0f9e90bd67b860/README.md","notes":"Base-model community terms also apply; repository-wide code licensing is not stated.","components":[]},"sources":[{"kind":"paper","title":"AnyGPT: Unified Multimodal LLM with Discrete Sequence Modeling","url":"https://arxiv.org/abs/2402.12226","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Llama-2-7b-hf/blob/main/LICENSE.txt","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/OpenMOSS/AnyGPT","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"OpenMOSS-Team/AnyGPT-chat — checkpoint and usage","url":"https://huggingface.co/OpenMOSS-Team/AnyGPT-chat","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/OpenMOSS/AnyGPT/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/OpenMOSS-Team/AnyGPT-chat/blob/c8999624b077b3837f32c3948f0f9e90bd67b860/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--anygpt.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2402.12226","originUrl":"https://arxiv.org/html/2402.12226v5/model1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/anygpt.png","sha256":"17f0d591b3600bca99bfcb7e7009d738da9cd4e4ab71cdf74012116e9c356dd1"}},{"id":"omni:audio-flamingo","slug":"audio-flamingo","name":"Audio Flamingo","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers questions about sounds and music and supports audio-grounded dialogue and examples.","architecture":"Audio embeddings connected to a language model through gated cross-attention","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["audio-flamingo"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2024-02-02","dateKind":"paper","links":{"code":"https://github.com/NVIDIA/audio-flamingo/tree/legacy_audio_flamingo_1","weights":"https://huggingface.co/nvidia/audio-flamingo","paper":"https://arxiv.org/abs/2402.01831","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#audio-flamingo"},"license":{"status":"restricted","code":"MIT","weights":"OPT-IML noncommercial license","codeUrl":"https://github.com/NVIDIA/audio-flamingo/tree/legacy_audio_flamingo_1","weightsUrl":"https://huggingface.co/nvidia/audio-flamingo/blob/9e8c1516b93869ce7326556c9fb2c3fac84db77d/README.md","notes":"Noncommercial terms apply to model weights and relevant bundled components.","components":[]},"sources":[{"kind":"paper","title":"Audio Flamingo: A Novel Audio Language Model with Few-Shot Learning and Dialogue Abilities","url":"https://arxiv.org/abs/2402.01831","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/NVIDIA/audio-flamingo/tree/legacy_audio_flamingo_1","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/audio-flamingo — checkpoint and usage","url":"https://huggingface.co/nvidia/audio-flamingo","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/audio-flamingo/blob/9e8c1516b93869ce7326556c9fb2c3fac84db77d/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--audio-flamingo.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2402.01831","originUrl":"https://arxiv.org/html/2402.01831v3/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/audio-flamingo.png","sha256":"319f8a0a66c49c16dd3ec61628147f19272a3362b4f4fb5b87a5917dd0d63a54"}},{"id":"omni:audio-flamingo-2","slug":"audio-flamingo-2","name":"Audio Flamingo 2","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Reasons about environmental sounds and music, including long recordings.","architecture":"Audio encoder and language model trained with audio-skill curricula","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["audio-flamingo-2"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-03-06","dateKind":"paper","links":{"code":"https://github.com/NVIDIA/audio-flamingo/tree/audio_flamingo_2","weights":"https://huggingface.co/nvidia/audio-flamingo-2","paper":"https://arxiv.org/abs/2503.03983","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#audio-flamingo-2"},"license":{"status":"restricted","code":"MIT","weights":"NVIDIA OneWay Noncommercial","codeUrl":"https://github.com/NVIDIA/audio-flamingo/tree/audio_flamingo_2","weightsUrl":"https://huggingface.co/nvidia/audio-flamingo-2/blob/3a7f4aea17e72c7367940affea6749069a7095fc/README.md","notes":"Noncommercial terms apply to model weights and relevant bundled components.","components":[]},"sources":[{"kind":"paper","title":"Audio Flamingo 2: An Audio-Language Model with Long-Audio Understanding and Expert Reasoning Abilities","url":"https://arxiv.org/abs/2503.03983","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/NVIDIA/audio-flamingo/tree/audio_flamingo_2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/audio-flamingo-2 — checkpoint and usage","url":"https://huggingface.co/nvidia/audio-flamingo-2","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/audio-flamingo-2/blob/3a7f4aea17e72c7367940affea6749069a7095fc/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--audio-flamingo-2.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2503.03983","originUrl":"https://arxiv.org/html/2503.03983v1/af2_training_pipeline_latest.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/audio-flamingo-2.png","sha256":"159cd9a89f5b3fe49782c9e94c54df7432f47e0dfa4c8e75f80d6f72b75b178c"}},{"id":"omni:audio-flamingo-3","slug":"audio-flamingo-3","name":"Audio Flamingo 3 Chat","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Discusses speech, sounds and music across multiple recordings and can reply with streaming speech.","architecture":"AF-Whisper encoder, language model and streaming speech module","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["audio-flamingo-3-chat"],"inputs":["T","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-07-10","dateKind":"paper","links":{"code":"https://github.com/NVIDIA/audio-flamingo/tree/audio_flamingo_3","weights":"https://huggingface.co/nvidia/audio-flamingo-3-chat","paper":"https://arxiv.org/abs/2507.08128","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#audio-flamingo-3"},"license":{"status":"restricted","code":"MIT","weights":"NVIDIA OneWay Noncommercial","codeUrl":"https://github.com/NVIDIA/audio-flamingo/tree/audio_flamingo_3","weightsUrl":"https://huggingface.co/nvidia/audio-flamingo-3-chat/blob/f5101f44de138aed8527791212be2e070224fb86/README.md","notes":"Noncommercial terms apply to model weights and relevant bundled components.","components":[]},"sources":[{"kind":"paper","title":"Audio Flamingo 3: Advancing Audio Intelligence with Fully Open Large Audio Language Models","url":"https://arxiv.org/abs/2507.08128","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/NVIDIA/audio-flamingo/tree/audio_flamingo_3","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/audio-flamingo-3-chat — checkpoint and usage","url":"https://huggingface.co/nvidia/audio-flamingo-3-chat","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/audio-flamingo-3-chat/blob/f5101f44de138aed8527791212be2e070224fb86/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--audio-flamingo-3.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2507.08128","originUrl":"https://arxiv.org/html/2507.08128v2/af3_main_diagram.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/audio-flamingo-3.png","sha256":"6aed0d1df9781878565ea602ea600b9582f846c12c00080ce0259058340c5671"}},{"id":"omni:audio-flamingo-next","slug":"audio-flamingo-next","name":"Audio Flamingo Next","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Analyzes long audio recordings and reasons about speech, sounds and music.","architecture":"Extended-context audio-language modeling","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["audio-flamingo-next-hf"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-04-13","dateKind":"paper","links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/audioflamingo3","weights":"https://huggingface.co/nvidia/audio-flamingo-next-hf","paper":"https://arxiv.org/abs/2604.10905","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#audio-flamingo-next"},"license":{"status":"restricted","code":"Apache-2.0","weights":"NVIDIA OneWay Noncommercial","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/nvidia/audio-flamingo-next-hf/blob/5634886e2615c2f587dcf8b93c6edfe9907930ca/README.md","notes":"Noncommercial terms apply to model weights and relevant bundled components.","components":[]},"sources":[{"kind":"paper","title":"Audio Flamingo Next: Next-Generation Open Audio-Language Models for Speech, Sound, and Music","url":"https://arxiv.org/abs/2604.10905","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"model-card","title":"nvidia/audio-flamingo-next-hf — checkpoint and usage","url":"https://huggingface.co/nvidia/audio-flamingo-next-hf","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/audio-flamingo-next-hf/blob/5634886e2615c2f587dcf8b93c6edfe9907930ca/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/audioflamingo3","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--audio-flamingo-next.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2604.10905","originUrl":"https://arxiv.org/html/2604.10905v1/AF-next-final_compressed.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/audio-flamingo-next.png","sha256":"0f0d5227daf67a9b8a52b7010076e3d2b187dff6d4d24c336fb6637cba4566e8"}},{"id":"omni:audio-interaction","slug":"audio-interaction","name":"Audio-Interaction","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Monitors an ongoing audio stream and decides when a text response or proactive intervention is useful.","architecture":"Streaming perceive-decide-respond modeling with asynchronous inference","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["AudioInteraction"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-06-03","dateKind":"paper","links":{"code":"https://github.com/xzf-thu/Audio-Interaction","weights":"https://huggingface.co/zhifeixie/AudioInteraction","paper":"https://arxiv.org/abs/2606.05121","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#audio-interaction"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0","codeUrl":"https://github.com/xzf-thu/Audio-Interaction/blob/main/README.md","weightsUrl":"https://huggingface.co/zhifeixie/AudioInteraction/blob/a7cd702ba1452c3d65c270af83079afa385dfee2/README.md","notes":"Public code and weights were located, but a complete release license was not stated.","components":[]},"sources":[{"kind":"paper","title":"Audio Interaction Model","url":"https://arxiv.org/abs/2606.05121","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/xzf-thu/Audio-Interaction","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"zhifeixie/AudioInteraction — checkpoint and usage","url":"https://huggingface.co/zhifeixie/AudioInteraction","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/xzf-thu/Audio-Interaction/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/zhifeixie/AudioInteraction/blob/a7cd702ba1452c3d65c270af83079afa385dfee2/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--audio-interaction.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2606.05121","originUrl":"https://arxiv.org/html/2606.05121v2/soundfkow.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/audio-interaction.png","sha256":"385349e78f7e152f87fa488b086ded0d4a3f24bbd009ef681d84963b61723a9c"}},{"id":"omni:audio-reasoner","slug":"audio-reasoner","name":"Audio-Reasoner","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Reasons about speech, music and environmental sounds to answer complex audio questions.","architecture":"Reasoning-focused audio-language post-training","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Audio-Reasoner"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-03-04","dateKind":"paper","links":{"code":"https://github.com/xzf-thu/Audio-Reasoner","weights":"https://huggingface.co/zhifeixie/Audio-Reasoner","paper":"https://arxiv.org/abs/2503.02318","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#audio-reasoner"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/xzf-thu/Audio-Reasoner/blob/main/LICENSE","weightsUrl":"https://huggingface.co/zhifeixie/Audio-Reasoner/blob/f38198da84d4e02623a83fa1d005ad31f1d6a6a7/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Audio-Reasoner: Improving Reasoning Capability in Large Audio Language Models","url":"https://arxiv.org/abs/2503.02318","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/xzf-thu/Audio-Reasoner","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"zhifeixie/Audio-Reasoner — checkpoint and usage","url":"https://huggingface.co/zhifeixie/Audio-Reasoner","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/xzf-thu/Audio-Reasoner/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/zhifeixie/Audio-Reasoner/blob/f38198da84d4e02623a83fa1d005ad31f1d6a6a7/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--audio-reasoner.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2503.02318","originUrl":"https://arxiv.org/html/2503.02318v2/figure3-data_preparation-v3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/audio-reasoner.png","sha256":"6340e2ed3baf0cdf6de32e22bb3fefbe6db6b9c62982e573abac35450ed7491f"}},{"id":"omni:audio-visual-flamingo","slug":"audio-visual-flamingo","name":"Audio-Visual Flamingo","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Explains long videos by reasoning jointly about visual events and their soundtracks.","architecture":"Temporally interleaved audiovisual encoders and language decoder","notes":"The public checkpoint returns text. The paper figure additionally depicts a streaming TTS module at the system level; it is not counted as native checkpoint speech output.","variants":["nemotron-labs-audio-visual-flamingo-hf"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-07-17","dateKind":"paper","links":{"code":"https://github.com/lashahub/transformers/tree/add_AudioVisualFlamingo","weights":"https://huggingface.co/nvidia/nemotron-labs-audio-visual-flamingo-hf","paper":"https://arxiv.org/abs/2607.16107","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#audio-visual-flamingo"},"license":{"status":"restricted","code":"Apache-2.0","weights":"NVIDIA OneWay Noncommercial","codeUrl":"https://github.com/lashahub/transformers/blob/add_AudioVisualFlamingo/LICENSE","weightsUrl":"https://huggingface.co/nvidia/nemotron-labs-audio-visual-flamingo-hf/blob/4e40cc207574b8d95628f44691f75b01e012e46c/README.md","notes":"Noncommercial terms apply to model weights and relevant bundled components.","components":[]},"sources":[{"kind":"paper","title":"Audio-Visual Flamingo: Open Audio-Visual Intelligence for Long and Complex Videos","url":"https://arxiv.org/abs/2607.16107","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/lashahub/transformers/tree/add_AudioVisualFlamingo","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/nemotron-labs-audio-visual-flamingo-hf — checkpoint and usage","url":"https://huggingface.co/nvidia/nemotron-labs-audio-visual-flamingo-hf","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/lashahub/transformers/blob/add_AudioVisualFlamingo/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/nemotron-labs-audio-visual-flamingo-hf/blob/4e40cc207574b8d95628f44691f75b01e012e46c/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--audio-visual-flamingo.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2607.16107","originUrl":"https://arxiv.org/html/2607.16107v1/temp_3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/audio-visual-flamingo.png","sha256":"6de76ec88e6b05c62b7e7a8ed328eb9ba7adbbfe5b82847737dca0ed9e20d52a"}},{"id":"omni:azeros","slug":"azeros","name":"AzeroS","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Follows spoken instructions using speech adaptation learned without curated instruction-answer pairs.","architecture":"Self-generated instruction-free speech-language alignment","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["azeros"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-12-31","dateKind":"paper","links":{"code":"https://github.com/AudenAI/Auden","weights":"https://huggingface.co/AudenAI/azeros","paper":"https://arxiv.org/abs/2601.06086","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#azeros"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/AudenAI/Auden/blob/main/LICENSE","weightsUrl":"https://huggingface.co/AudenAI/azeros/blob/e9392d32639e7079cb0e24f7373207ee658a8baf/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"AzeroS: Extending LLM to Speech with Self-Generated Instruction-Free Tuning","url":"https://arxiv.org/abs/2601.06086","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/AudenAI/Auden","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"AudenAI/azeros — checkpoint and usage","url":"https://huggingface.co/AudenAI/azeros","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/AudenAI/Auden/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/AudenAI/azeros/blob/e9392d32639e7079cb0e24f7373207ee658a8baf/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--azeros.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.06086","originUrl":"https://arxiv.org/html/2601.06086v1/images/azeros_framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/azeros.png","sha256":"982c7bd3f47cfe3b5688828df29dfd7f5a880f5c8bc720dd893360e46e24c56a"}},{"id":"omni:baichuan-audio","slug":"baichuan-audio","name":"Baichuan-Audio","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Follows spoken instructions and generates text and expressive speech responses.","architecture":"Audio-text language model with aligned semantic and acoustic tokens","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Baichuan-Audio-Instruct"],"inputs":["T","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-02-24","dateKind":"paper","links":{"code":"https://github.com/baichuan-inc/Baichuan-Audio","weights":"https://huggingface.co/baichuan-inc/Baichuan-Audio-Instruct","paper":"https://arxiv.org/abs/2502.17239","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#baichuan-audio"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/baichuan-inc/Baichuan-Audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/baichuan-inc/Baichuan-Audio-Instruct/blob/6761be848f9bd91847cc647ef70a47b3266ffc36/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Baichuan-Audio: A Unified Framework for End-to-End Speech Interaction","url":"https://arxiv.org/abs/2502.17239","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/baichuan-inc/Baichuan-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"baichuan-inc/Baichuan-Audio-Instruct — checkpoint and usage","url":"https://huggingface.co/baichuan-inc/Baichuan-Audio-Instruct","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/baichuan-inc/Baichuan-Audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/baichuan-inc/Baichuan-Audio-Instruct/blob/6761be848f9bd91847cc647ef70a47b3266ffc36/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--baichuan-audio.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.17239","originUrl":"https://arxiv.org/html/2502.17239v1/audio-llm.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/baichuan-audio.png","sha256":"adc3acab06264800a622c0b3b1c2aa6cf5905f757221efc9c76317d8c131726c"}},{"id":"omni:baichuan-omni-1-5","slug":"baichuan-omni-1-5","name":"Baichuan-Omni 1.5","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands images, video and sound and answers through text or speech.","architecture":"Unified multimodal backbone with speech generation","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Baichuan-Omni-1d5"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-01-26","dateKind":"paper","links":{"code":"https://github.com/baichuan-inc/Baichuan-Omni-1.5","weights":"https://huggingface.co/baichuan-inc/Baichuan-Omni-1d5","paper":"https://arxiv.org/abs/2501.15368","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#baichuan-omni-1-5"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Baichuan model terms","codeUrl":"https://github.com/baichuan-inc/Baichuan-Omni-1.5/blob/main/LICENSE","weightsUrl":"https://huggingface.co/baichuan-inc/Baichuan-Omni-1d5/blob/0b86202f48ec5e273e1aef3b67caf0f4e7cca1b0/README.md","notes":"The model card adds commercial-use conditions beyond its Apache metadata.","components":[]},"sources":[{"kind":"paper","title":"Baichuan-Omni-1.5 Technical Report","url":"https://arxiv.org/abs/2501.15368","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/baichuan-inc/Baichuan-Omni-1.5","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"baichuan-inc/Baichuan-Omni-1d5 — checkpoint and usage","url":"https://huggingface.co/baichuan-inc/Baichuan-Omni-1d5","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/baichuan-inc/Baichuan-Omni-1.5/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/baichuan-inc/Baichuan-Omni-1d5/blob/0b86202f48ec5e273e1aef3b67caf0f4e7cca1b0/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--baichuan-omni-1-5.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2501.15368","originUrl":"https://arxiv.org/html/2501.15368v1/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/baichuan-omni-1-5.png","sha256":"0b72fb6eadbb9665d2573a964508979daa0dc36f9cebaf8b9b4d8d695934fef7"}},{"id":"omni:blsp-emo","slug":"blsp-emo","name":"BLSP-Emo","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Understands vocal emotion and writes empathetic responses to spoken requests.","architecture":"Emotion-aware speech-language alignment with Whisper and Qwen","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["blsp-emo"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/cwang621/blsp-emo","weights":"https://huggingface.co/cwang621/blsp-emo","paper":"https://arxiv.org/abs/2406.03872","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#blsp-emo"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Apache-2.0 + Qwen terms","codeUrl":"https://github.com/cwang621/blsp-emo/blob/main/LICENSE","weightsUrl":"https://huggingface.co/cwang621/blsp-emo/blob/ec0ec614e453a7d3be242482d25cdc7697bc0bf8/README.md","notes":"The Qwen-7B-Chat backbone retains its custom license.","components":[]},"sources":[{"kind":"paper","title":"BLSP-Emo: Towards Empathetic Large Speech-Language Models","url":"https://arxiv.org/abs/2406.03872","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/cwang621/blsp-emo","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"cwang621/blsp-emo — checkpoint and usage","url":"https://huggingface.co/cwang621/blsp-emo","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/cwang621/blsp-emo/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/cwang621/blsp-emo/blob/ec0ec614e453a7d3be242482d25cdc7697bc0bf8/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--blsp-emo.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2406.03872","originUrl":"https://arxiv.org/html/2406.03872v1/emotion_blsp_v4.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/blsp-emo.png","sha256":"a69a051c4e24b94872354e0e9cf24a6b6b2455de4da0bb0dc4278a6b1eaa4553"}},{"id":"omni:br-voice-reasoner","slug":"br-voice-reasoner","name":"BR-Voice-Reasoner","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers spoken knowledge and reasoning questions using a speech-adapted Qwen3-Omni Thinker.","architecture":"On-policy distillation of audio-language projection modules","notes":"Distills audio-language projections of Qwen3-Omni Thinking. Visual inputs are inherited from the backbone; the reported evaluation focuses on speech.","variants":["BR-Voice-Reasoner"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/qwen3_omni_moe","weights":"https://huggingface.co/brgroup/BR-Voice-Reasoner","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#br-voice-reasoner"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/brgroup/BR-Voice-Reasoner/blob/3254a7c6a4377c943952f43d50b284cd43fca0d6/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/qwen3_omni_moe","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"brgroup/BR-Voice-Reasoner — checkpoint and usage","url":"https://huggingface.co/brgroup/BR-Voice-Reasoner","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/brgroup/BR-Voice-Reasoner/blob/3254a7c6a4377c943952f43d50b284cd43fca0d6/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--br-voice-reasoner.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/brgroup/BR-Voice-Reasoner","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/br-voice-reasoner.svg","sha256":"9011b578d28d4f4ed852339058ca108a0968dca25659d1cba97511c9a0686c16"}},{"id":"omni:canary-qwen","slug":"canary-qwen","name":"Canary-Qwen","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Transcribes English speech and offers a separate text-language mode for follow-up processing.","architecture":"FastConformer audio encoder connected to Qwen2.5","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["canary-qwen-2.5b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/NVIDIA-NeMo/Speech","weights":"https://huggingface.co/nvidia/canary-qwen-2.5b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#canary-qwen"},"license":{"status":"open","code":"Apache-2.0","weights":"CC-BY-4.0","codeUrl":"https://github.com/NVIDIA-NeMo/Speech/blob/main/LICENSE","weightsUrl":"https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/NVIDIA-NeMo/Speech","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/canary-qwen-2.5b — checkpoint and usage","url":"https://huggingface.co/nvidia/canary-qwen-2.5b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/NVIDIA-NeMo/Speech/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/canary-qwen-2.5b/blob/b1469e1bba1cfe140205529c79c434ca47180960/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--canary-qwen.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/nvidia/canary-qwen-2.5b","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/canary-qwen.svg","sha256":"3d0a9a61d97345d917d62f4b826b4064f86fde342ba45db8a37c2d0f5397cd53"}},{"id":"omni:covo-audio","slug":"covo-audio","name":"Covo-Audio Chat","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Understands spoken requests and generates context-aware, expressive voice responses.","architecture":"Continuous-audio encoder and unified speech-language decoder","notes":"The selected checkpoint is Covo-Audio-Chat. The paper also describes a distinct Chat-FD full-duplex variant, whose behavior is not assumed for this checkpoint.","variants":["Covo-Audio-Chat"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/Tencent/Covo-Audio","weights":"https://huggingface.co/tencent/Covo-Audio-Chat","paper":"https://arxiv.org/abs/2602.09823","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#covo-audio"},"license":{"status":"restricted","code":"Tencent research license","weights":"Tencent research license","codeUrl":"https://github.com/Tencent/Covo-Audio/blob/main/LICENSE","weightsUrl":"https://github.com/Tencent/Covo-Audio/blob/main/LICENSE","notes":"Academic/research use only; commercial and production use are excluded.","components":[]},"sources":[{"kind":"paper","title":"Covo-Audio Technical Report","url":"https://arxiv.org/abs/2602.09823","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/Tencent/Covo-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"tencent/Covo-Audio-Chat — checkpoint and usage","url":"https://huggingface.co/tencent/Covo-Audio-Chat","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/Tencent/Covo-Audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--covo-audio.webp","kind":"source","label":"Figure 2, PDF p. 4","sourceUrl":"https://arxiv.org/abs/2602.09823","originUrl":"https://arxiv.org/pdf/2602.09823","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/covo-audio.png","sha256":"623ecedbbb816a52db0938d57e8cd5e403cc06c96caee3515e0361006628abf9"}},{"id":"omni:desta2","slug":"desta2","name":"DeSTA2","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Describes and answers questions about audio using a language model aligned with speech representations.","architecture":"Self-generated cross-modal alignment of speech and language representations","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["DeSTA2-8B-beta"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2024-09-30","dateKind":"paper","links":{"code":"https://github.com/kehanlu/DeSTA2","weights":"https://huggingface.co/DeSTA-ntu/DeSTA2-8B-beta","paper":"https://arxiv.org/abs/2409.20007","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#desta2"},"license":{"status":"unclear","code":"Not stated","weights":"Llama community / component terms","codeUrl":"https://github.com/kehanlu/DeSTA2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/DeSTA-ntu/DeSTA2-8B-beta/blob/c8ddfdca7ca208c07c4be2c0c8beceaf12bd6076/README.md","notes":"Base-model and dataset terms apply; no unrestricted standalone weight license is stated.","components":[]},"sources":[{"kind":"paper","title":"DeSTA2: Developing Instruction-Following Speech Language Model Without Speech Instruction-Tuning Data","url":"https://arxiv.org/abs/2409.20007","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/kehanlu/DeSTA2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"DeSTA-ntu/DeSTA2-8B-beta — checkpoint and usage","url":"https://huggingface.co/DeSTA-ntu/DeSTA2-8B-beta","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/kehanlu/DeSTA2/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/DeSTA-ntu/DeSTA2-8B-beta/blob/c8ddfdca7ca208c07c4be2c0c8beceaf12bd6076/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--desta2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2409.20007","originUrl":"https://arxiv.org/html/2409.20007v2/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/desta2.png","sha256":"609f98b2a04a90da4075727a7affa587b70d9adc23a9c04f1b3a3f0b36f6cdfc"}},{"id":"omni:desta2-5-audio","slug":"desta2-5-audio","name":"DeSTA2.5-Audio","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Performs general audio understanding and follows spoken instructions without an external transcript pipeline.","architecture":"Self-generated cross-modal alignment of audio encoders and Llama","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["DeSTA2.5-Audio-Llama-3.1-8B"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-07-03","dateKind":"paper","links":{"code":"https://github.com/kehanlu/DeSTA2.5-Audio","weights":"https://huggingface.co/DeSTA-ntu/DeSTA2.5-Audio-Llama-3.1-8B","paper":"https://arxiv.org/abs/2507.02768","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#desta2-5-audio"},"license":{"status":"unclear","code":"Not stated","weights":"Llama community / component terms","codeUrl":"https://github.com/kehanlu/DeSTA2.5-Audio/blob/main/README.md","weightsUrl":"https://huggingface.co/DeSTA-ntu/DeSTA2.5-Audio-Llama-3.1-8B/blob/1427f8725d35a0c9da0c7c54b1ffcc157a5948af/README.md","notes":"Base-model and dataset terms apply; no unrestricted standalone weight license is stated.","components":[]},"sources":[{"kind":"paper","title":"DeSTA2.5-Audio: Toward General-Purpose Large Audio Language Model with Self-Generated Cross-Modal Alignment","url":"https://arxiv.org/abs/2507.02768","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/kehanlu/DeSTA2.5-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"DeSTA-ntu/DeSTA2.5-Audio-Llama-3.1-8B — checkpoint and usage","url":"https://huggingface.co/DeSTA-ntu/DeSTA2.5-Audio-Llama-3.1-8B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/kehanlu/DeSTA2.5-Audio/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/DeSTA-ntu/DeSTA2.5-Audio-Llama-3.1-8B/blob/1427f8725d35a0c9da0c7c54b1ffcc157a5948af/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--desta2-5-audio.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2507.02768","originUrl":"https://arxiv.org/html/2507.02768v2/method.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/desta2-5-audio.png","sha256":"f6d39e9772d1976307517ec1348d97f7068c7cfd4fdda853ed28d945d5b4e8d7"}},{"id":"omni:dialects2s","slug":"dialects2s","name":"DialectS2S","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Understands and speaks low-resource Chinese dialects in end-to-end voice conversations.","architecture":"Dialect-adapted OpenS2S speech-language model","notes":"Uses the separately released GLM-4-Voice decoder for speech generation; consult that component’s terms as well. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["DialectS2S"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/CASIA-LM/OpenS2S","weights":"https://huggingface.co/CASIA-LM/DialectS2S","paper":"https://arxiv.org/abs/2507.05177","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#dialects2s"},"license":{"status":"open","code":"Apache-2.0","weights":"MIT","codeUrl":"https://github.com/CASIA-LM/OpenS2S/blob/main/README.md","weightsUrl":"https://huggingface.co/CASIA-LM/DialectS2S/blob/938dc520d99941b9cf8634b8161823adf1a5694e/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/CASIA-LM/OpenS2S","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"CASIA-LM/DialectS2S — checkpoint and usage","url":"https://huggingface.co/CASIA-LM/DialectS2S","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/CASIA-LM/OpenS2S/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/CASIA-LM/DialectS2S/blob/938dc520d99941b9cf8634b8161823adf1a5694e/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"paper","title":"OpenS2S: Advancing Fully Open-Source End-to-End Empathetic Large Speech Language Model","url":"https://arxiv.org/abs/2507.05177","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/omni--dialects2s.webp","kind":"source","label":"Family figure — Figure 1","sourceUrl":"https://arxiv.org/abs/2507.05177","originUrl":"https://arxiv.org/html/2507.05177v3/Architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/opens2s.png","sha256":"6238ceff3e92d5914cf1d88091f826d64f4b6dca3bbdcbce6c764a20e590e85a"}},{"id":"omni:diffa","slug":"diffa","name":"DIFFA","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers audio questions using diffusion-based text generation instead of left-to-right decoding.","architecture":"Audio adapter connected to a diffusion language model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["DIFFA"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-07-24","dateKind":"paper","links":{"code":"https://github.com/NKU-HLT/DIFFA","weights":"https://huggingface.co/zhoujiaming777/DIFFA","paper":"https://arxiv.org/abs/2507.18452","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#diffa"},"license":{"status":"unclear","code":"Not stated","weights":"CC-BY-NC-SA-4.0","codeUrl":"https://github.com/NKU-HLT/DIFFA/blob/main/README.md","weightsUrl":"https://huggingface.co/zhoujiaming777/DIFFA/blob/e91d27d9bd13beb6fabec0174eefb4cd438887cd/README.md","notes":"The checkpoint uses a noncommercial share-alike license; repository code terms are not stated.","components":[]},"sources":[{"kind":"paper","title":"DIFFA: Large Language Diffusion Models Can Listen and Understand","url":"https://arxiv.org/abs/2507.18452","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/NKU-HLT/DIFFA","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"zhoujiaming777/DIFFA — checkpoint and usage","url":"https://huggingface.co/zhoujiaming777/DIFFA","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/NKU-HLT/DIFFA/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/zhoujiaming777/DIFFA/blob/e91d27d9bd13beb6fabec0174eefb4cd438887cd/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--diffa.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2507.18452","originUrl":"https://arxiv.org/html/2507.18452v3/p1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/diffa.png","sha256":"5ab58ddfd1a6edc5575763bd880842f50d442ce8f8c6af0dd0fee7a3606bdea7"}},{"id":"omni:diffa-2","slug":"diffa-2","name":"DIFFA-2","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Understands speech, sound and music with a diffusion language model and richer acoustic representations.","architecture":"Whisper encoder with semantic and acoustic adapters into a diffusion decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["DIFFA-2"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-01-30","dateKind":"paper","links":{"code":"https://github.com/NKU-HLT/DIFFA","weights":"https://huggingface.co/zhoujiaming777/DIFFA-2","paper":"https://arxiv.org/abs/2601.23161","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#diffa-2"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/NKU-HLT/DIFFA/blob/main/README.md","weightsUrl":"https://huggingface.co/zhoujiaming777/DIFFA-2/blob/8aea6122bafb25578cb49d142c6ac1f8e83e3dea/README.md","notes":"No release-specific code or weight license was located; do not assume DIFFA 1 terms apply.","components":[]},"sources":[{"kind":"paper","title":"DIFFA-2: A Practical Diffusion Large Language Model for General Audio Understanding","url":"https://arxiv.org/abs/2601.23161","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/NKU-HLT/DIFFA","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"zhoujiaming777/DIFFA-2 — checkpoint and usage","url":"https://huggingface.co/zhoujiaming777/DIFFA-2","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/NKU-HLT/DIFFA/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/zhoujiaming777/DIFFA-2/blob/8aea6122bafb25578cb49d142c6ac1f8e83e3dea/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--diffa-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.23161","originUrl":"https://arxiv.org/html/2601.23161v1/p1_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/diffa-2.png","sha256":"52831c8bc520a2ed75b66921372a0a0f8cca10d3e6cc409f155299f42ada3ec6"}},{"id":"omni:diva","slug":"diva","name":"DiVA","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers spoken questions by transferring a text assistant's behavior into an audio-input model.","architecture":"Self-supervised speech-to-text representation distillation","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["DiVA-llama-3-v0-8b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://huggingface.co/WillHeld/DiVA-llama-3-v0-8b/tree/6e761b15ebde6ccf9b9e91d47a4ae42d79c066ca","weights":"https://huggingface.co/WillHeld/DiVA-llama-3-v0-8b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#diva"},"license":{"status":"restricted","code":"MPL-2.0","weights":"MPL-2.0 + Llama 3 terms","codeUrl":"https://huggingface.co/WillHeld/DiVA-llama-3-v0-8b/blob/6e761b15ebde6ccf9b9e91d47a4ae42d79c066ca/README.md","weightsUrl":"https://huggingface.co/WillHeld/DiVA-llama-3-v0-8b/blob/6e761b15ebde6ccf9b9e91d47a4ae42d79c066ca/README.md","notes":"The Llama 3 backbone retains its community license.","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/WillHeld/DiVA-llama-3-v0-8b/tree/6e761b15ebde6ccf9b9e91d47a4ae42d79c066ca","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"WillHeld/DiVA-llama-3-v0-8b — checkpoint and usage","url":"https://huggingface.co/WillHeld/DiVA-llama-3-v0-8b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/WillHeld/DiVA-llama-3-v0-8b/blob/6e761b15ebde6ccf9b9e91d47a4ae42d79c066ca/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--diva.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/WillHeld/DiVA-llama-3-v0-8b","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/diva.svg","sha256":"bef564a28edce7c10b263c6e62ff08c60f9ff8c2f523995269c7112f62d635d9"}},{"id":"omni:duplex-cascade","slug":"duplex-cascade","name":"DuplexCascade","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Coordinates streaming transcription, language generation and speech synthesis for interruptible voice conversations.","architecture":"Micro-turn language modeling with streaming ASR and TTS modules","notes":"A fine-tuned Qwen2 dialogue controller plus separately deployed Kyutai ASR and TTS modules. The public checkpoint has an access gate requiring acceptance of publisher terms.","variants":["DuplexCascade"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/sbintuitions/DuplexCascade","weights":"https://huggingface.co/sbintuitions/DuplexCascade","paper":"https://arxiv.org/abs/2603.09180","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#duplex-cascade"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/sbintuitions/DuplexCascade/blob/main/LICENSE","weightsUrl":"https://huggingface.co/sbintuitions/DuplexCascade/blob/31c038ece2f006a28722dd60d1df3868fbb2cc42/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Release documentation and license evidence","url":"https://arxiv.org/abs/2603.09180","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/sbintuitions/DuplexCascade","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"sbintuitions/DuplexCascade — checkpoint and usage","url":"https://huggingface.co/sbintuitions/DuplexCascade","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/sbintuitions/DuplexCascade/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/sbintuitions/DuplexCascade/blob/31c038ece2f006a28722dd60d1df3868fbb2cc42/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--duplex-cascade.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.09180","originUrl":"https://arxiv.org/html/2603.09180v1/proposed.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/duplex-cascade.png","sha256":"eaa7297ba3357dd3803c4f7c1c8da555519ebe0fc77c4c926fd30c2759c83bbf"}},{"id":"omni:duplexmamba","slug":"duplexmamba","name":"DuplexMamba","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Reads incoming speech incrementally and supports responsive text generation during speech interaction.","architecture":"Streaming speech adaptation of a Mamba backbone","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["DuplexMamba"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-02-16","dateKind":"paper","links":{"code":"https://github.com/khfs/DuplexMamba","weights":"https://huggingface.co/Xiangyu1/DuplexMamba","paper":"https://arxiv.org/abs/2502.11123","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#duplexmamba"},"license":{"status":"open","code":"GPL-3.0","weights":"Apache-2.0","codeUrl":"https://github.com/khfs/DuplexMamba/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Xiangyu1/DuplexMamba/blob/3fd0fb94963adfaf950a21e0ed062e59e02f2d1a/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"DuplexMamba: Enhancing Real-time Speech Conversations with Duplex and Streaming Capabilities","url":"https://arxiv.org/abs/2502.11123","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/khfs/DuplexMamba","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Xiangyu1/DuplexMamba — checkpoint and usage","url":"https://huggingface.co/Xiangyu1/DuplexMamba","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/khfs/DuplexMamba/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/Xiangyu1/DuplexMamba/blob/3fd0fb94963adfaf950a21e0ed062e59e02f2d1a/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--duplexmamba.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.11123","originUrl":"https://arxiv.org/html/2502.11123v3/DuplexMamba4.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/duplexmamba.png","sha256":"3cdbed1c45afd361117b85575ea38ecf70f589b4b17fe8b641ae6a2219d5284b"}},{"id":"omni:emova","slug":"emova","name":"EMOVA (Qwen2.5-7B)","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Discusses images and spoken requests, replying with text and expressive speech.","architecture":"Speech-vision-language alignment","notes":"The linked open release uses Qwen2.5-7B for image understanding and expressive speech interaction. The figure describes the EMOVA family; license evidence applies to this Qwen checkpoint.","variants":["emova-qwen-2-5-7b-hf"],"inputs":["T","I","S"],"outputs":["T","S"],"interaction":"not-specified","specs":{},"date":"2024-09-26","dateKind":"paper","links":{"code":"https://github.com/emova-ollm/EMOVA","weights":"https://huggingface.co/Emova-ollm/emova-qwen-2-5-7b-hf","paper":"https://arxiv.org/abs/2409.18042","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#emova"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/emova-ollm/EMOVA/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Emova-ollm/emova-qwen-2-5-7b-hf/blob/6bb14e7aeba16e44d290fa7e0ecd81e863bc6505/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"EMOVA: Empowering Language Models to See, Hear and Speak with Vivid Emotions","url":"https://arxiv.org/abs/2409.18042","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"EMOVA — released implementation","url":"https://github.com/emova-ollm/EMOVA","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Emova-ollm/emova-qwen-2-5-7b-hf — official model card","url":"https://huggingface.co/Emova-ollm/emova-qwen-2-5-7b-hf","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"EMOVA — code license evidence","url":"https://github.com/emova-ollm/EMOVA/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"EMOVA — checkpoint license evidence","url":"https://huggingface.co/Emova-ollm/emova-qwen-2-5-7b-hf/blob/6bb14e7aeba16e44d290fa7e0ecd81e863bc6505/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--emova.webp","kind":"source","label":"Family figure · Figure 2","sourceUrl":"https://arxiv.org/abs/2409.18042","originUrl":"https://arxiv.org/html/2409.18042v4/flowchart.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/emova.png","sha256":"18c87efad31f86a8416af2af47011344ede325a947d291ef8b4491cccaccbf4f"}},{"id":"omni:freeze-omni","slug":"freeze-omni","name":"Freeze-Omni","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Enables streaming spoken dialogue while retaining a frozen text model's language abilities.","architecture":"Streaming speech encoder and decoder around a frozen Qwen backbone","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Freeze-Omni"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2024-11-01","dateKind":"paper","links":{"code":"https://github.com/VITA-MLLM/Freeze-Omni","weights":"https://huggingface.co/VITA-MLLM/Freeze-Omni","paper":"https://arxiv.org/abs/2411.00774","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#freeze-omni"},"license":{"status":"restricted","code":"Tencent research license","weights":"Tencent research license","codeUrl":"https://github.com/VITA-MLLM/Freeze-Omni/blob/main/License.txt","weightsUrl":"https://github.com/VITA-MLLM/Freeze-Omni/blob/main/License.txt","notes":"Academic/research use only; commercial and production use are excluded.","components":[]},"sources":[{"kind":"paper","title":"Freeze-Omni: A Smart and Low Latency Speech-to-speech Dialogue Model with Frozen LLM","url":"https://arxiv.org/abs/2411.00774","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/VITA-MLLM/Freeze-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"VITA-MLLM/Freeze-Omni — checkpoint and usage","url":"https://huggingface.co/VITA-MLLM/Freeze-Omni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/VITA-MLLM/Freeze-Omni/blob/main/License.txt","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--freeze-omni.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2411.00774","originUrl":"https://arxiv.org/html/2411.00774v5/fig1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/freeze-omni.png","sha256":"021c7a86c29ec32812c4f6e4bc59e345388a51cfaca6d4aed62fd6196bb92925"}},{"id":"omni:fun-audio-chat","slug":"fun-audio-chat","name":"Fun-Audio-Chat","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Holds low-latency spoken conversations while responding to vocal style and emotion.","architecture":"Dual-resolution speech representations with a core-code prediction head","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Fun-Audio-Chat-8B"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-12-23","dateKind":"paper","links":{"code":"https://github.com/QwenAudio/Fun-Audio-Chat","weights":"https://huggingface.co/FunAudioLLM/Fun-Audio-Chat-8B","paper":"https://arxiv.org/abs/2512.20156","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#fun-audio-chat"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/QwenAudio/Fun-Audio-Chat/blob/main/LICENSE","weightsUrl":"https://huggingface.co/FunAudioLLM/Fun-Audio-Chat-8B/blob/7bf72dc7c705493f817178a0859efff91e9cf73c/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Fun-Audio-Chat Technical Report","url":"https://arxiv.org/abs/2512.20156","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/QwenAudio/Fun-Audio-Chat","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"FunAudioLLM/Fun-Audio-Chat-8B — checkpoint and usage","url":"https://huggingface.co/FunAudioLLM/Fun-Audio-Chat-8B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/QwenAudio/Fun-Audio-Chat/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/FunAudioLLM/Fun-Audio-Chat-8B/blob/7bf72dc7c705493f817178a0859efff91e9cf73c/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--fun-audio-chat.webp","kind":"source","label":"Figure 2, PDF p. 4","sourceUrl":"https://arxiv.org/abs/2512.20156","originUrl":"https://arxiv.org/pdf/2512.20156v4","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/fun-audio-chat.png","sha256":"d47cb2551adf9dabc7ff4b7ef8f49c4ff60d43840498722acd9238231cde8829"}},{"id":"omni:gemma-3n","slug":"gemma-3n","name":"Gemma 3n E4B","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands text, images, video and audio in a model designed for mobile devices.","architecture":"MatFormer language backbone with visual and audio encoders","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["gemma-3n-E4B-it"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/gemma3n","weights":"https://huggingface.co/google/gemma-3n-E4B-it","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#gemma-3n"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Gemma terms","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/google/gemma-3n-E4B-it/blob/c1221e9c62e34a43ab7ffacd1be0ea71f126ef10/README.md","notes":"The weights use custom Gemma terms.","components":[]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://ai.google.dev/gemma/terms","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/gemma3n","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"google/gemma-3n-E4B-it — checkpoint and usage","url":"https://huggingface.co/google/gemma-3n-E4B-it","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/google/gemma-3n-E4B-it/blob/c1221e9c62e34a43ab7ffacd1be0ea71f126ef10/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--gemma-3n.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/google/gemma-3n-E4B-it","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/gemma-3n.svg","sha256":"44e08bd2ec85de57803a35b395bdaea4c36ff9393408b1d069aa7e2956365bf9"}},{"id":"omni:gemma-4-audio-capable-variants","slug":"gemma-4-audio-capable-variants","name":"Gemma 4 E4B","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers questions about images, video and audio in a compact model designed for local use.","architecture":"Hybrid-attention backbone with vision and audio encoders","notes":"The verified E4B instruction-tuned checkpoint accepts text, images, video frames and audio, and generates text. The family figure also covers other sizes; this entry selects E4B.","variants":["gemma-4-E4B-it"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-07-02","dateKind":"paper","links":{"code":"https://github.com/google-deepmind/gemma","weights":"https://huggingface.co/google/gemma-4-E4B-it","paper":"https://arxiv.org/abs/2607.02770","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#gemma-4-audio-capable-variants"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/google-deepmind/gemma/blob/main/LICENSE","weightsUrl":"https://huggingface.co/google/gemma-4-E4B-it/blob/ee0ef6023621cff504d758262d4e04895a5af4a2/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Gemma 4 Technical Report","url":"https://arxiv.org/abs/2607.02770","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"documentation","title":"Gemma 4 model overview | Google AI for Developers","url":"https://ai.google.dev/gemma/docs/core","reviewed_on":"2026-09-12","review_level":"documentation"},{"kind":"documentation","title":"Audio understanding | Gemma | Google AI for Developers","url":"https://ai.google.dev/gemma/docs/capabilities/audio","reviewed_on":"2026-09-12","review_level":"documentation"},{"kind":"repository","title":"Gemma 4 (audio-capable variants) — released implementation","url":"https://github.com/google-deepmind/gemma","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"google/gemma-4-E4B-it — official model card","url":"https://huggingface.co/google/gemma-4-E4B-it","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Gemma 4 (audio-capable variants) — code license evidence","url":"https://github.com/google-deepmind/gemma/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Gemma 4 (audio-capable variants) — checkpoint license evidence","url":"https://huggingface.co/google/gemma-4-E4B-it/blob/ee0ef6023621cff504d758262d4e04895a5af4a2/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--gemma-4-audio-capable-variants.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://arxiv.org/abs/2607.02770","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/gemma-4-audio-capable-variants.svg","sha256":"d163887bfc622361ba308d5dbd9194e9dd184db92f18f009bb43a2e75fd61874"}},{"id":"omni:glm-4-voice","slug":"glm-4-voice","name":"GLM-4-Voice","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Converses in Chinese and English with controllable emotion, speaking rate and vocal style.","architecture":"GLM backbone with low-rate speech tokens and interleaved decoding","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["glm-4-voice-9b"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2024-12-03","dateKind":"paper","links":{"code":"https://github.com/zai-org/GLM-4-Voice","weights":"https://huggingface.co/THUDM/glm-4-voice-9b","paper":"https://arxiv.org/abs/2412.02612","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#glm-4-voice"},"license":{"status":"restricted","code":"Apache-2.0","weights":"GLM-4 model license","codeUrl":"https://github.com/zai-org/GLM-4-Voice/blob/main/LICENSE","weightsUrl":"https://huggingface.co/THUDM/glm-4-voice-9b/blob/352e5a64063448d5e46394753fb65bf1dbab975b/LICENSE","notes":"Custom GLM model terms apply to the speech checkpoint.","components":[]},"sources":[{"kind":"paper","title":"GLM-4-Voice: Towards Intelligent and Human-Like End-to-End Spoken Chatbot","url":"https://arxiv.org/abs/2412.02612","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/zai-org/GLM-4-Voice","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"THUDM/glm-4-voice-9b — checkpoint and usage","url":"https://huggingface.co/THUDM/glm-4-voice-9b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/zai-org/GLM-4-Voice/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/THUDM/glm-4-voice-9b/blob/352e5a64063448d5e46394753fb65bf1dbab975b/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--glm-4-voice.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2412.02612","originUrl":"https://arxiv.org/html/2412.02612v1/overall-arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/glm-4-voice.png","sha256":"13755ee89fbb3d3c0926441a967cd2a3b40f3f70198d7e89fe77fc41d6a9b939"}},{"id":"omni:glm-asr-nano","slug":"glm-asr-nano","name":"GLM-ASR Nano","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Transcribes speech in English, Mandarin and Chinese dialects, including noisy recordings.","architecture":"Compact audio encoder and GLM language decoder","notes":"The public Nano checkpoint is a transcription model; it does not produce conversational speech.","variants":["GLM-ASR-Nano-2512"],"inputs":["S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/zai-org/GLM-ASR","weights":"https://huggingface.co/zai-org/GLM-ASR-Nano-2512","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#glm-asr-nano"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/zai-org/GLM-ASR/blob/main/LICENSE","weightsUrl":"https://huggingface.co/zai-org/GLM-ASR-Nano-2512/blob/61ba4e0b3309b6656edea3e93e419f7bd5c61957/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/zai-org/GLM-ASR","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"zai-org/GLM-ASR-Nano-2512 — checkpoint and usage","url":"https://huggingface.co/zai-org/GLM-ASR-Nano-2512","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/zai-org/GLM-ASR/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/zai-org/GLM-ASR-Nano-2512/blob/61ba4e0b3309b6656edea3e93e419f7bd5c61957/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--glm-asr-nano.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/zai-org/GLM-ASR-Nano-2512","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/glm-asr-nano.svg","sha256":"8fbc03ea899621b6b48ee78740bf93a66e7549891c108ca72b0599149d976993"}},{"id":"omni:granite-speech-3-3","slug":"granite-speech-3-3","name":"Granite Speech 3.3","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Transcribes and translates speech while retaining a text language-model interface.","architecture":"Speech encoder and projector with modality-specific language adapters","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["granite-speech-3.3-8b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-05-13","dateKind":"paper","links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/granite_speech","weights":"https://huggingface.co/ibm-granite/granite-speech-3.3-8b","paper":"https://arxiv.org/abs/2505.08699","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#granite-speech-3-3"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ibm-granite/granite-speech-3.3-8b/blob/5670dcfae4a296e6993ac53eb25f638c6aaa54d0/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Granite-speech: open-source speech-aware LLMs with strong English ASR capabilities","url":"https://arxiv.org/abs/2505.08699","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/granite_speech","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ibm-granite/granite-speech-3.3-8b — checkpoint and usage","url":"https://huggingface.co/ibm-granite/granite-speech-3.3-8b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/ibm-granite/granite-speech-3.3-8b/blob/5670dcfae4a296e6993ac53eb25f638c6aaa54d0/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--granite-speech-3-3.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.08699","originUrl":"https://arxiv.org/html/2505.08699v2/granite-speech-arch-crop.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/granite-speech-3-3.png","sha256":"f08fb3e28094f3db780e24f119227864c7a5ad1428b04bf8a97e2e88a661ccd5"}},{"id":"omni:granite-speech-4-0","slug":"granite-speech-4-0","name":"Granite Speech 4.0","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Provides compact speech transcription and translation with a 1B language backbone.","architecture":"Compact Granite speech encoder-decoder integration","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["granite-4.0-1b-speech"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/granite_speech","weights":"https://huggingface.co/ibm-granite/granite-4.0-1b-speech","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#granite-speech-4-0"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ibm-granite/granite-4.0-1b-speech/blob/bd87ab862416353633ea431fe49b1614003623c5/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/granite_speech","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ibm-granite/granite-4.0-1b-speech — checkpoint and usage","url":"https://huggingface.co/ibm-granite/granite-4.0-1b-speech","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/ibm-granite/granite-4.0-1b-speech/blob/bd87ab862416353633ea431fe49b1614003623c5/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--granite-speech-4-0.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/ibm-granite/granite-4.0-1b-speech","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/granite-speech-4-0.svg","sha256":"189da8a0a1a5986aabeb82dad37dd34125625ff2e6810c31449627048c943fa6"}},{"id":"omni:granite-speech-4-1","slug":"granite-speech-4-1","name":"Granite Speech 4.1","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Transcribes and translates multilingual speech using an updated speech-language architecture.","architecture":"Dual-head speech encoder connected to a Granite language decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["granite-speech-4.1-2b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/granite_speech","weights":"https://huggingface.co/ibm-granite/granite-speech-4.1-2b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#granite-speech-4-1"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ibm-granite/granite-speech-4.1-2b/blob/de575db64086f84fdc79da4932d1076e965bc546/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/granite_speech","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ibm-granite/granite-speech-4.1-2b — checkpoint and usage","url":"https://huggingface.co/ibm-granite/granite-speech-4.1-2b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/ibm-granite/granite-speech-4.1-2b/blob/de575db64086f84fdc79da4932d1076e965bc546/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--granite-speech-4-1.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/ibm-granite/granite-speech-4.1-2b","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/granite-speech-4-1.svg","sha256":"f81533e6419401cbb050abb5914f6bc544ce76c2ae04f5b9f69f694883d22f00"}},{"id":"omni:hibiki","slug":"hibiki","name":"Hibiki","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Translates French speech into English as it arrives while preserving the speaker's voice.","architecture":"Simultaneous speech translation with delayed multi-stream modeling","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["hibiki-2b-pytorch-bf16"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-02-05","dateKind":"paper","links":{"code":"https://github.com/kyutai-labs/hibiki","weights":"https://huggingface.co/kyutai/hibiki-2b-pytorch-bf16","paper":"https://arxiv.org/abs/2502.03382","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#hibiki"},"license":{"status":"open","code":"Apache-2.0","weights":"CC-BY-4.0","codeUrl":"https://github.com/kyutai-labs/hibiki/blob/main/LICENSE-APACHE","weightsUrl":"https://huggingface.co/kyutai/hibiki-2b-pytorch-bf16/blob/bd71144c96f26040612f6414716f5f48ee4fce69/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"High-Fidelity Simultaneous Speech-To-Speech Translation","url":"https://arxiv.org/abs/2502.03382","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/kyutai-labs/hibiki","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"kyutai/hibiki-2b-pytorch-bf16 — checkpoint and usage","url":"https://huggingface.co/kyutai/hibiki-2b-pytorch-bf16","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/kyutai-labs/hibiki/blob/main/LICENSE-APACHE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/kyutai/hibiki-2b-pytorch-bf16/blob/bd71144c96f26040612f6414716f5f48ee4fce69/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--hibiki.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2502.03382","originUrl":"https://arxiv.org/html/2502.03382v2/text_delays_tokens_v2.drawio.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/hibiki.png","sha256":"8a2a7e23b2d5960879067d8105c85dd6ad02ef09d75de66b98597523e52f3d47"}},{"id":"omni:humanomni","slug":"humanomni","name":"HumanOmni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Interprets human behavior, expressions and interactions from video and audio.","architecture":"Human-centric visual and audio feature fusion","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["HumanOmni-7B"],"inputs":["T","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/HumanMLLM/HumanOmni","weights":"https://huggingface.co/StarJiaxing/HumanOmni-7B","paper":"https://arxiv.org/abs/2501.15111","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#humanomni"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/HumanMLLM/HumanOmni/blob/main/README.md","weightsUrl":"https://huggingface.co/StarJiaxing/HumanOmni-7B/tree/60d6f0c0110b80367c3368eb5c100e1d6d3e63e0","notes":"Public code and weights were located, but a complete release license was not stated.","components":[]},"sources":[{"kind":"paper","title":"HumanOmni: A Large Vision-Speech Language Model for Human-Centric Video Understanding","url":"https://arxiv.org/abs/2501.15111","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/HumanMLLM/HumanOmni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"StarJiaxing/HumanOmni-7B — checkpoint and usage","url":"https://huggingface.co/StarJiaxing/HumanOmni-7B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/HumanMLLM/HumanOmni/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation","url":"https://huggingface.co/StarJiaxing/HumanOmni-7B/tree/60d6f0c0110b80367c3368eb5c100e1d6d3e63e0","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--humanomni.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2501.15111","originUrl":"https://arxiv.org/html/2501.15111v1/method_video_2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/humanomni.png","sha256":"83561b151a4665e3a068ce3adf564a960d5856035fa5621b18cb258d2609d80b"}},{"id":"omni:humanomniv2","slug":"humanomniv2","name":"HumanOmniV2","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Interprets people's actions, emotions and social interactions from video and audio.","architecture":"Context-aware reasoning with reinforcement learning","notes":"Extends the HumanOmni line with multimodal reasoning training.","variants":["HumanOmniV2"],"inputs":["T","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-06-26","dateKind":"paper","links":{"code":"https://github.com/HumanMLLM/HumanOmniV2","weights":"https://huggingface.co/PhilipC/HumanOmniV2","paper":"https://arxiv.org/abs/2506.21277","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#humanomniv2"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/HumanMLLM/HumanOmniV2/blob/main/README.md","weightsUrl":"https://huggingface.co/PhilipC/HumanOmniV2/blob/459fbfe536d9a1e0cda872c339bb24051bd2de3b/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"HumanOmniV2: From Understanding to Omni-Modal Reasoning with Context","url":"https://arxiv.org/abs/2506.21277","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"HumanOmniV2 — released implementation","url":"https://github.com/HumanMLLM/HumanOmniV2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"PhilipC/HumanOmniV2 — official model card","url":"https://huggingface.co/PhilipC/HumanOmniV2","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"HumanOmniV2 — code license evidence","url":"https://github.com/HumanMLLM/HumanOmniV2/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"HumanOmniV2 — checkpoint license evidence","url":"https://huggingface.co/PhilipC/HumanOmniV2/blob/459fbfe536d9a1e0cda872c339bb24051bd2de3b/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--humanomniv2.webp","kind":"source","label":"Figure 4","sourceUrl":"https://arxiv.org/abs/2506.21277","originUrl":"https://arxiv.org/html/2506.21277v1/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/humanomniv2.png","sha256":"6d46b883410ca692c196fff8f42bbbc9d864ad119773b9bcfcdd0cf5ba7a6637"}},{"id":"omni:ichigo","slug":"ichigo","name":"Ichigo","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers spoken questions using audio tokens embedded directly into a Llama conversation.","architecture":"Early fusion of WhisperVQ tokens and Llama 3.1 text tokens","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Ichigo-llama3.1-s-instruct-v0.4"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/janhq/ichigo","weights":"https://huggingface.co/homebrewltd/Ichigo-llama3.1-s-instruct-v0.4","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ichigo"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0 + Llama 3.1 terms","codeUrl":"https://github.com/janhq/ichigo/blob/main/README.md","weightsUrl":"https://huggingface.co/homebrewltd/Ichigo-llama3.1-s-instruct-v0.4/blob/0c788b5781573910b40d9031a7e26940c4753709/README.md","notes":"The Llama 3.1 backbone retains its community license; no standalone repository license was located.","components":[]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/janhq/ichigo","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"homebrewltd/Ichigo-llama3.1-s-instruct-v0.4 — checkpoint and usage","url":"https://huggingface.co/homebrewltd/Ichigo-llama3.1-s-instruct-v0.4","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/janhq/ichigo/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/homebrewltd/Ichigo-llama3.1-s-instruct-v0.4/blob/0c788b5781573910b40d9031a7e26940c4753709/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ichigo.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/homebrewltd/Ichigo-llama3.1-s-instruct-v0.4","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ichigo.svg","sha256":"be4735e2157985d9b271a2b086e68ac0d67abb55190b1df653bd5246badb690e"}},{"id":"omni:kimi-audio","slug":"kimi-audio","name":"Kimi-Audio","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Understands speech, sound and music and supports natural voice conversations.","architecture":"Continuous audio features, parallel text-audio heads and flow-matching detokenizer","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Kimi-Audio-7B-Instruct"],"inputs":["T","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-04-25","dateKind":"paper","links":{"code":"https://github.com/MoonshotAI/Kimi-Audio","weights":"https://huggingface.co/moonshotai/Kimi-Audio-7B-Instruct","paper":"https://arxiv.org/abs/2504.18425","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#kimi-audio"},"license":{"status":"open","code":"MIT / Apache-2.0","weights":"MIT","codeUrl":"https://github.com/MoonshotAI/Kimi-Audio/blob/master/README.md","weightsUrl":"https://huggingface.co/moonshotai/Kimi-Audio-7B-Instruct/blob/9a82a84c37ad9eb1307fb6ed8d7b397862ef9e6b/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Kimi-Audio Technical Report","url":"https://arxiv.org/abs/2504.18425","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/MoonshotAI/Kimi-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"moonshotai/Kimi-Audio-7B-Instruct — checkpoint and usage","url":"https://huggingface.co/moonshotai/Kimi-Audio-7B-Instruct","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/MoonshotAI/Kimi-Audio/blob/master/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/moonshotai/Kimi-Audio-7B-Instruct/blob/9a82a84c37ad9eb1307fb6ed8d7b397862ef9e6b/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--kimi-audio.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2504.18425","originUrl":"https://arxiv.org/html/2504.18425v1/kimia-fig.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/kimi-audio.png","sha256":"d274bbadbde72456fce01f801b4f6afd44b01d71f0f9f1abb200dbff96c48369"}},{"id":"omni:lfg-1","slug":"lfg-1","name":"LFG-1","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers English spoken questions locally on Apple Silicon and also accepts images.","architecture":"Gemma 4 audio encoder and language backbone joined by a trained projector","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["LFG-1"],"inputs":["T","I","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/codemadeio/LFG-1","weights":"https://huggingface.co/glenn2/LFG-1","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#lfg-1"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/codemadeio/LFG-1/blob/main/README.md","weightsUrl":"https://huggingface.co/glenn2/LFG-1/blob/dff7a33e27bcbe8d8656565183258634a5489c81/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/codemadeio/LFG-1","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"glenn2/LFG-1 — checkpoint and usage","url":"https://huggingface.co/glenn2/LFG-1","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/codemadeio/LFG-1/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/glenn2/LFG-1/blob/dff7a33e27bcbe8d8656565183258634a5489c81/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lfg-1.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/glenn2/LFG-1","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lfg-1.svg","sha256":"7c5a2ac25a78ec5a92de2023ed42194751b23e9883bce346392211cad3c968c2"}},{"id":"omni:lfg-2","slug":"lfg-2","name":"LFG-2","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Reasons about English spoken instructions and returns a text answer with an optional thought channel.","architecture":"Frozen Gemma 4 encoder and 31B decoder with a deep audio projector","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["LFG-2"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://huggingface.co/glenn2/LFG-2/tree/8ea5f17fa715e7f4310c081e51d20bb0f9380be0","weights":"https://huggingface.co/glenn2/LFG-2","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#lfg-2"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://huggingface.co/glenn2/LFG-2/blob/8ea5f17fa715e7f4310c081e51d20bb0f9380be0/README.md","weightsUrl":"https://huggingface.co/glenn2/LFG-2/blob/8ea5f17fa715e7f4310c081e51d20bb0f9380be0/README.md","notes":"Code and weights are downloadable, but this release does not state a license.","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/glenn2/LFG-2/tree/8ea5f17fa715e7f4310c081e51d20bb0f9380be0","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"glenn2/LFG-2 — checkpoint and usage","url":"https://huggingface.co/glenn2/LFG-2","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/glenn2/LFG-2/blob/8ea5f17fa715e7f4310c081e51d20bb0f9380be0/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lfg-2.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/glenn2/LFG-2","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lfg-2.svg","sha256":"3456f1002d737b38524860f62b8fa1be9d54ac268898c8b9dde7f000849773e5"}},{"id":"omni:lfg-3","slug":"lfg-3","name":"LFG-3","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers English spoken questions using acoustic features instead of an intermediate transcript.","architecture":"Parakeet encoder and Gemma 4 31B joined by a trained projector","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["LFG-3"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://huggingface.co/glenn2/LFG-3/tree/80c9ef580ae37b2b93fc6c3d70268fbea7062c91","weights":"https://huggingface.co/glenn2/LFG-3","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#lfg-3"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://huggingface.co/glenn2/LFG-3/blob/80c9ef580ae37b2b93fc6c3d70268fbea7062c91/README.md","weightsUrl":"https://huggingface.co/glenn2/LFG-3/blob/80c9ef580ae37b2b93fc6c3d70268fbea7062c91/README.md","notes":"Code and weights are downloadable, but this release does not state a license.","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/glenn2/LFG-3/tree/80c9ef580ae37b2b93fc6c3d70268fbea7062c91","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"glenn2/LFG-3 — checkpoint and usage","url":"https://huggingface.co/glenn2/LFG-3","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/glenn2/LFG-3/blob/80c9ef580ae37b2b93fc6c3d70268fbea7062c91/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lfg-3.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/glenn2/LFG-3","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lfg-3.svg","sha256":"c8e759f074f884714657894df678b43633865007547ecdf410f0f5327abbc462"}},{"id":"omni:lfm2-audio","slug":"lfm2-audio","name":"LFM2-Audio-1.5B","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Transcribes speech, synthesizes speech and supports interleaved voice conversations on local devices.","architecture":"LFM2 hybrid backbone with separate audio input and output pathways","notes":"Runs with the public liquid-audio implementation. The LFM2 paper describes the family; the linked checkpoint card documents this version.","variants":["LFM2-Audio-1.5B"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-11-28","dateKind":"paper","links":{"code":"https://github.com/Liquid4All/liquid-audio","weights":"https://huggingface.co/LiquidAI/LFM2-Audio-1.5B","paper":"https://arxiv.org/abs/2511.23404","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#lfm2-audio"},"license":{"status":"restricted","code":"LFM Open License v1.0","weights":"LFM Open License v1.0","codeUrl":"https://github.com/Liquid4All/liquid-audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/LiquidAI/LFM2-Audio-1.5B/blob/c798aad30dc3cd72e72970beab51326b8443bd94/README.md","notes":"Revenue-based commercial-use limit; see the LFM license.","components":[]},"sources":[{"kind":"paper","title":"LFM2 Technical Report","url":"https://arxiv.org/abs/2511.23404","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/Liquid4All/liquid-audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"LiquidAI/LFM2-Audio-1.5B — checkpoint and usage","url":"https://huggingface.co/LiquidAI/LFM2-Audio-1.5B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/Liquid4All/liquid-audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/LiquidAI/LFM2-Audio-1.5B/blob/c798aad30dc3cd72e72970beab51326b8443bd94/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lfm2-audio.webp","kind":"source","label":"Figure 7","sourceUrl":"https://arxiv.org/abs/2511.23404","originUrl":"https://arxiv.org/html/2511.23404v1/audio_arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lfm2-audio.png","sha256":"197f1607aeeee0fac2853ca579db183c495f601a08dc6ebf95b4584ce1c8f750"}},{"id":"omni:lfm2-5-audio","slug":"lfm2-5-audio","name":"LFM2.5-Audio-1.5B","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Handles transcription, speech synthesis and voice chat with a faster audio decoder for local inference.","architecture":"LFM2 backbone with a lightweight LFM audio detokenizer","notes":"Runs with the public liquid-audio implementation. The LFM2 paper describes the family; the linked checkpoint card documents this version.","variants":["LFM2.5-Audio-1.5B"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/Liquid4All/liquid-audio","weights":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B","paper":"https://arxiv.org/abs/2511.23404","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#lfm2-5-audio"},"license":{"status":"restricted","code":"LFM Open License v1.0","weights":"LFM Open License v1.0","codeUrl":"https://github.com/Liquid4All/liquid-audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B/blob/c362a0625dfe45aa588dce5f0ada28a7e5707628/README.md","notes":"Revenue-based commercial-use limit; see the LFM license.","components":[]},"sources":[{"kind":"paper","title":"LFM2 Technical Report","url":"https://arxiv.org/abs/2511.23404","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/Liquid4All/liquid-audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"LiquidAI/LFM2.5-Audio-1.5B — checkpoint and usage","url":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/Liquid4All/liquid-audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B/blob/c362a0625dfe45aa588dce5f0ada28a7e5707628/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lfm2-5-audio.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lfm2-5-audio.svg","sha256":"c4bef195a023e5d0c71bb028a23d58773ea7f9d0ae45739ad11c6552199fcf01"}},{"id":"omni:lfm2-5-audio-jp","slug":"lfm2-5-audio-jp","name":"LFM2.5-Audio-1.5B-JP","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Brings Japanese transcription, speech synthesis and spoken conversation to the LFM2.5 audio family.","architecture":"Japanese-adapted LFM2.5 audio model","notes":"Runs with the public liquid-audio implementation. The LFM2 paper describes the family; the linked checkpoint card documents this version.","variants":["LFM2.5-Audio-1.5B-JP"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/Liquid4All/liquid-audio","weights":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B-JP","paper":"https://arxiv.org/abs/2511.23404","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#lfm2-5-audio-jp"},"license":{"status":"restricted","code":"LFM Open License v1.0","weights":"LFM Open License v1.0","codeUrl":"https://github.com/Liquid4All/liquid-audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B-JP/blob/6c34b4d590f80563f8cb2939c2ebd7686d952394/README.md","notes":"Revenue-based commercial-use limit; see the LFM license.","components":[]},"sources":[{"kind":"paper","title":"LFM2 Technical Report","url":"https://arxiv.org/abs/2511.23404","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/Liquid4All/liquid-audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"LiquidAI/LFM2.5-Audio-1.5B-JP — checkpoint and usage","url":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B-JP","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/Liquid4All/liquid-audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B-JP/blob/6c34b4d590f80563f8cb2939c2ebd7686d952394/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lfm2-5-audio-jp.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B-JP","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lfm2-5-audio-jp.svg","sha256":"5a52a0470905706c353a8df41964508ce78ab7ebe1a42dee5b77e416f4f13cf5"}},{"id":"omni:llama-omni","slug":"llama-omni","name":"LLaMA-Omni","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Answers spoken requests with synchronized text and streaming speech.","architecture":"Whisper encoder, Llama backbone and non-autoregressive speech decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Llama-3.1-8B-Omni"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2024-09-10","dateKind":"paper","links":{"code":"https://github.com/ictnlp/LLaMA-Omni","weights":"https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni","paper":"https://arxiv.org/abs/2409.06666","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#llama-omni"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Research-only","codeUrl":"https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni/blob/4844429b4f81bc9cd93dcf5b1f0c66b8925fbab8/README.md","weightsUrl":"https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni/blob/4844429b4f81bc9cd93dcf5b1f0c66b8925fbab8/README.md","notes":"The model card excludes commercial use of the weights.","components":[]},"sources":[{"kind":"paper","title":"LLaMA-Omni: Seamless Speech Interaction with Large Language Models","url":"https://arxiv.org/abs/2409.06666","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/ictnlp/LLaMA-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ICTNLP/Llama-3.1-8B-Omni — checkpoint and usage","url":"https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni/blob/4844429b4f81bc9cd93dcf5b1f0c66b8925fbab8/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--llama-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2409.06666","originUrl":"https://arxiv.org/html/2409.06666v2/llamas2s_training.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/llama-omni.png","sha256":"e450a5ebddfb05886e4d0d949c48b1ddb3951cd9b4cc9c50f9eb38d36d9399c0"}},{"id":"omni:llama-omni-2","slug":"llama-omni-2","name":"LLaMA-Omni 2","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Supports multi-turn spoken chat with streaming speech generation.","architecture":"Qwen2.5 backbone with an autoregressive streaming speech decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["LLaMA-Omni2-7B"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-05-05","dateKind":"paper","links":{"code":"https://github.com/ictnlp/LLaMA-Omni2","weights":"https://huggingface.co/ICTNLP/LLaMA-Omni2-7B","paper":"https://arxiv.org/abs/2505.02625","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#llama-omni-2"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Research-only","codeUrl":"https://huggingface.co/ICTNLP/LLaMA-Omni2-7B/blob/50291b44553960219adb145846e13b023dfc1e7e/README.md","weightsUrl":"https://huggingface.co/ICTNLP/LLaMA-Omni2-7B/blob/50291b44553960219adb145846e13b023dfc1e7e/README.md","notes":"The model card excludes commercial use of the weights.","components":[]},"sources":[{"kind":"paper","title":"LLaMA-Omni2: LLM-based Real-time Spoken Chatbot with Autoregressive Streaming Speech Synthesis","url":"https://arxiv.org/abs/2505.02625","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/ictnlp/LLaMA-Omni2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ICTNLP/LLaMA-Omni2-7B — checkpoint and usage","url":"https://huggingface.co/ICTNLP/LLaMA-Omni2-7B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/ICTNLP/LLaMA-Omni2-7B/blob/50291b44553960219adb145846e13b023dfc1e7e/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--llama-omni-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.02625","originUrl":"https://arxiv.org/html/2505.02625v1/llama-omni2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/llama-omni-2.png","sha256":"55a4f918a38314d2b43e72d9da9e2a87e9558c9c3d863a983d9360cebe1c9126"}},{"id":"omni:llasm","slug":"llasm","name":"LLaSM","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers spoken questions and supports instruction following across speech and text.","architecture":"Speech encoder aligned to a Chinese Llama language model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["LLaSM-Cllama2"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2023-08-30","dateKind":"paper","links":{"code":"https://github.com/LinkSoul-AI/LLaSM","weights":"https://huggingface.co/LinkSoul/LLaSM-Cllama2","paper":"https://arxiv.org/abs/2308.15930","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#llasm"},"license":{"status":"restricted","code":"Apache-2.0","weights":"OpenRAIL","codeUrl":"https://github.com/LinkSoul-AI/LLaSM/blob/main/LICENSE","weightsUrl":"https://huggingface.co/LinkSoul/LLaSM-Cllama2/blob/053a36b0e0006a437e74b7294ab5d112d5df5371/README.md","notes":"Custom release or component terms apply; consult the linked license evidence.","components":[]},"sources":[{"kind":"paper","title":"LLaSM: Large Language and Speech Model","url":"https://arxiv.org/abs/2308.15930","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/LinkSoul-AI/LLaSM","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"LinkSoul/LLaSM-Cllama2 — checkpoint and usage","url":"https://huggingface.co/LinkSoul/LLaSM-Cllama2","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/LinkSoul-AI/LLaSM/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/LinkSoul/LLaSM-Cllama2/blob/053a36b0e0006a437e74b7294ab5d112d5df5371/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--llasm.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2308.15930","originUrl":"https://arxiv.org/html/2308.15930v3/Figures/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/llasm.png","sha256":"32c8596f0d49de49edf1a0b053cd9422d3e61e5fc4d51cacce087eb49bdc9a86"}},{"id":"omni:longcat-flash-omni","slug":"longcat-flash-omni","name":"LongCat-Flash-Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands text, images, video and audio and produces text or spoken responses.","architecture":"Mixture-of-experts backbone with perception and speech modules","notes":"Real-time audiovisual interaction built on the LongCat-Flash backbone.","variants":["LongCat-Flash-Omni"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-10-31","dateKind":"paper","links":{"code":"https://github.com/meituan-longcat/LongCat-Flash-Omni","weights":"https://huggingface.co/meituan-longcat/LongCat-Flash-Omni","paper":"https://arxiv.org/abs/2511.00279","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#longcat-flash-omni"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/meituan-longcat/LongCat-Flash-Omni/blob/main/LICENSE","weightsUrl":"https://huggingface.co/meituan-longcat/LongCat-Flash-Omni/blob/f65b9038a559d3624d5e52fad28bbe083eb2341e/LICENSE","notes":"","components":[]},"sources":[{"kind":"paper","title":"LongCat-Flash-Omni Technical Report","url":"https://arxiv.org/abs/2511.00279","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"LongCat-Flash-Omni — released implementation","url":"https://github.com/meituan-longcat/LongCat-Flash-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"meituan-longcat/LongCat-Flash-Omni — official model card","url":"https://huggingface.co/meituan-longcat/LongCat-Flash-Omni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"LongCat-Flash-Omni — code license evidence","url":"https://github.com/meituan-longcat/LongCat-Flash-Omni/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"LongCat-Flash-Omni — checkpoint license evidence","url":"https://huggingface.co/meituan-longcat/LongCat-Flash-Omni/blob/f65b9038a559d3624d5e52fad28bbe083eb2341e/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--longcat-flash-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2511.00279","originUrl":"https://arxiv.org/html/2511.00279v2/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/longcat-flash-omni.png","sha256":"d1103d9dc62dbbf2b0adc2da03518f13760bef67012a14f94e54f27733ebd8dd"}},{"id":"omni:longcat-next","slug":"longcat-next","name":"LongCat-Next","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands and generates text, images and audio within one multimodal model.","architecture":"Unified sparse multimodal modeling","notes":"Models multiple modalities through token sequences; check the report for individual generation paths.","variants":["LongCat-Next"],"inputs":["T","I","A"],"outputs":["T","I","A"],"interaction":"generation","specs":{},"date":"2026-03-29","dateKind":"paper","links":{"code":"https://github.com/meituan-longcat/LongCat-Next","weights":"https://huggingface.co/meituan-longcat/LongCat-Next","paper":"https://arxiv.org/abs/2603.27538","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#longcat-next"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/meituan-longcat/LongCat-Next/blob/main/LICENSE","weightsUrl":"https://huggingface.co/meituan-longcat/LongCat-Next/blob/0cf0631862402ff36366e513e4023d22e7e5c84c/LICENSE","notes":"","components":[]},"sources":[{"kind":"paper","title":"LongCat-Next: Lexicalizing Modalities as Discrete Tokens","url":"https://arxiv.org/abs/2603.27538","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"meituan-longcat/LongCat-Next — official repository","url":"https://github.com/meituan-longcat/LongCat-Next","reviewed_on":"2026-09-12","review_level":"readme"},{"kind":"model-card","title":"meituan-longcat/LongCat-Next — official model card","url":"https://huggingface.co/meituan-longcat/LongCat-Next","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"LongCat-Next — code license evidence","url":"https://github.com/meituan-longcat/LongCat-Next/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"LongCat-Next — checkpoint license evidence","url":"https://huggingface.co/meituan-longcat/LongCat-Next/blob/0cf0631862402ff36366e513e4023d22e7e5c84c/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--longcat-next.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2603.27538","originUrl":"https://arxiv.org/html/2603.27538v1/Overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/longcat-next.png","sha256":"46d01b76dd22e1072b0d03abdcf404f1c974d24c89fdb976f7335c3d7accecb3"}},{"id":"omni:lychee-fd","slug":"lychee-fd","name":"Lychee-FD","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Listens and speaks continuously while deciding when to respond, stop or handle interruptions.","architecture":"Shared lower layers with separate semantic, acoustic and control streams","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Lychee-FD"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/HITsz-TMG/Lychee-FD","weights":"https://huggingface.co/HIT-TMG/Lychee-FD","paper":"https://arxiv.org/abs/2607.06540","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#lychee-fd"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/HITsz-TMG/Lychee-FD/blob/main/LICENSE","weightsUrl":"https://huggingface.co/HIT-TMG/Lychee-FD/blob/ac06087bef70b33d4dfebf63a3eadadb4e024f33/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Hierarchical Acoustic-Semantic Modeling: Modality Separation and Semantic Coherence for Full-Duplex SLMs","url":"https://arxiv.org/abs/2607.06540","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/HITsz-TMG/Lychee-FD","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"HIT-TMG/Lychee-FD — checkpoint and usage","url":"https://huggingface.co/HIT-TMG/Lychee-FD","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/HITsz-TMG/Lychee-FD/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/HIT-TMG/Lychee-FD/blob/ac06087bef70b33d4dfebf63a3eadadb4e024f33/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lychee-fd.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2607.06540","originUrl":"https://arxiv.org/html/2607.06540v4/model_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lychee-fd.png","sha256":"656ff97d65f0ee2dd44c4b1dc8c477fc6328c5e7c9df8e05fdedae6c3c9b77d9"}},{"id":"omni:lyra","slug":"lyra","name":"Lyra-Base","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Handles long speech, audiovisual questions and spoken dialogue with a 9B configuration.","architecture":"Speech-centric modality adapters and multimodal low-rank adaptation","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Lyra_Base_9B"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2024-12-12","dateKind":"paper","links":{"code":"https://github.com/JIA-Lab-research/Lyra","weights":"https://huggingface.co/zszhong/Lyra_Base_9B","paper":"https://arxiv.org/abs/2412.09501","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#lyra"},"license":{"status":"restricted","code":"Apache-2.0","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/JIA-Lab-research/Lyra/blob/main/LICENSE","weightsUrl":"https://github.com/JIA-Lab-research/Lyra/blob/main/README.md","notes":"The repository limits model weights to noncommercial research despite the model-card badge.","components":[]},"sources":[{"kind":"paper","title":"Lyra: An Efficient and Speech-Centric Framework for Omni-Cognition","url":"https://arxiv.org/abs/2412.09501","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/JIA-Lab-research/Lyra","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"zszhong/Lyra_Base_9B — checkpoint and usage","url":"https://huggingface.co/zszhong/Lyra_Base_9B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/JIA-Lab-research/Lyra/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://github.com/JIA-Lab-research/Lyra/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lyra.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2412.09501","originUrl":"https://arxiv.org/html/2412.09501v1/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lyra.png","sha256":"87b2ea8e1ade42753d17f6234d1bb14cc017c9b3ddaf22ac06de84495c147c0c"}},{"id":"omni:lyra-mini","slug":"lyra-mini","name":"Lyra-Mini","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Offers audiovisual understanding and spoken interaction in Lyra's smaller 3B configuration.","architecture":"Speech-centric modality adapters and multimodal low-rank adaptation","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["Lyra_Mini_3B"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/JIA-Lab-research/Lyra","weights":"https://huggingface.co/zszhong/Lyra_Mini_3B","paper":"https://arxiv.org/abs/2412.09501","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#lyra-mini"},"license":{"status":"restricted","code":"Apache-2.0","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/JIA-Lab-research/Lyra/blob/main/LICENSE","weightsUrl":"https://github.com/JIA-Lab-research/Lyra/blob/main/README.md","notes":"The repository limits model weights to noncommercial research despite the model-card badge.","components":[]},"sources":[{"kind":"paper","title":"Lyra: An Efficient and Speech-Centric Framework for Omni-Cognition","url":"https://arxiv.org/abs/2412.09501","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/JIA-Lab-research/Lyra","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"zszhong/Lyra_Mini_3B — checkpoint and usage","url":"https://huggingface.co/zszhong/Lyra_Mini_3B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/JIA-Lab-research/Lyra/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://github.com/JIA-Lab-research/Lyra/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--lyra-mini.webp","kind":"source","label":"Family figure — Figure 2","sourceUrl":"https://arxiv.org/abs/2412.09501","originUrl":"https://arxiv.org/html/2412.09501v1/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/lyra.png","sha256":"87b2ea8e1ade42753d17f6234d1bb14cc017c9b3ddaf22ac06de84495c147c0c"}},{"id":"omni:mair-hub-omni","slug":"mair-hub-omni","name":"Mair-hub-0.5B-Omni","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Provides a small speech-to-speech assistant trained with a reproducible Qwen-Omni-like recipe.","architecture":"Whisper encoder, Thinker, Talker and CosyVoice decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["qwen-omni-like-0.5B-speech2speech-en"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/nvidia-china-sae/mair-hub/tree/main/speech-llm/qwen_omni_like","weights":"https://huggingface.co/yuekai/qwen-omni-like-0.5B-speech2speech-en","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#mair-hub-omni"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Not stated","codeUrl":"https://github.com/nvidia-china-sae/mair-hub/blob/main/LICENSE","weightsUrl":"https://huggingface.co/yuekai/qwen-omni-like-0.5B-speech2speech-en/tree/04e54c0d1cb7263c69fe33445c575264166bb045","notes":"The training recipe is Apache-2.0; the published checkpoint does not state a weight license.","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/nvidia-china-sae/mair-hub/tree/main/speech-llm/qwen_omni_like","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"yuekai/qwen-omni-like-0.5B-speech2speech-en — checkpoint and usage","url":"https://huggingface.co/yuekai/qwen-omni-like-0.5B-speech2speech-en","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/nvidia-china-sae/mair-hub/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation","url":"https://huggingface.co/yuekai/qwen-omni-like-0.5B-speech2speech-en/tree/04e54c0d1cb7263c69fe33445c575264166bb045","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--mair-hub-omni.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/yuekai/qwen-omni-like-0.5B-speech2speech-en","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/mair-hub-omni.svg","sha256":"73cb299e4515f59076745c67fda87d1a037873a619e7f782ede5f637f2ba5393"}},{"id":"omni:megrez-omni","slug":"megrez-omni","name":"Megrez-Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers questions about pictures and audio using a compact 3B language backbone.","architecture":"Visual and audio encoders connected to Megrez","notes":"Megrez-3B-Omni is the multimodal member; Megrez-3B-Instruct is text-only.","variants":["Megrez-3B-Omni"],"inputs":["T","I","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-02-19","dateKind":"paper","links":{"code":"https://github.com/infinigence/Infini-Megrez-Omni","weights":"https://huggingface.co/Infinigence/Megrez-3B-Omni","paper":"https://arxiv.org/abs/2502.15803","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#megrez-omni"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/infinigence/Infini-Megrez-Omni/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Infinigence/Megrez-3B-Omni/blob/2a15ac85df31645708eb0bdf769aa1f92eb44a31/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Megrez-Omni Technical Report","url":"https://arxiv.org/abs/2502.15803","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Megrez-Omni — released implementation","url":"https://github.com/infinigence/Infini-Megrez-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Infinigence/Megrez-3B-Omni — official model card","url":"https://huggingface.co/Infinigence/Megrez-3B-Omni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Megrez-Omni — code license evidence","url":"https://github.com/infinigence/Infini-Megrez-Omni/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Megrez-Omni — checkpoint license evidence","url":"https://huggingface.co/Infinigence/Megrez-3B-Omni/blob/2a15ac85df31645708eb0bdf769aa1f92eb44a31/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--megrez-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2502.15803","originUrl":"https://arxiv.org/html/2502.15803v1/architecture_detail.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/megrez-omni.png","sha256":"d5e9a3b5f9ebad855ee7a6b24f2a35c7accb10a8e731dcd784a806ec2ee72795"}},{"id":"omni:mellow","slug":"mellow","name":"Mellow","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Uses reasoning over audio to answer questions that require more than recognizing a sound.","architecture":"Audio-language model trained for hearing and reasoning","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["mellow"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-03-11","dateKind":"paper","links":{"code":"https://github.com/soham97/mellow","weights":"https://huggingface.co/soham97/mellow","paper":"https://arxiv.org/abs/2503.08540","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#mellow"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/soham97/mellow/blob/main/LICENSE","weightsUrl":"https://huggingface.co/soham97/mellow/blob/83672db0dae28764e283210d5bb732621e903d8a/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Mellow: a small audio language model for reasoning","url":"https://arxiv.org/abs/2503.08540","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/soham97/mellow","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"soham97/mellow — checkpoint and usage","url":"https://huggingface.co/soham97/mellow","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/soham97/mellow/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/soham97/mellow/blob/83672db0dae28764e283210d5bb732621e903d8a/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--mellow.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2503.08540","originUrl":"https://arxiv.org/html/2503.08540v1/fig/mellow-Page-5.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/mellow.png","sha256":"397ff41a19d5c6dc0a2d0d8e1a0c50134ef0e260c2e6400a8472905aaca3ed8c"}},{"id":"omni:meralion-2","slug":"meralion-2","name":"MERaLiON 2","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Understands multilingual Southeast Asian speech, code-switching, emotion and audio scenes.","architecture":"Localized Whisper encoder and Gemma 2 decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["MERaLiON-2-10B"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://huggingface.co/MERaLiON/MERaLiON-2-10B/tree/57bb08fd110af1f1ef5a6b16d53ac240393d4f93","weights":"https://huggingface.co/MERaLiON/MERaLiON-2-10B","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#meralion-2"},"license":{"status":"restricted","code":"MERaLiON public license","weights":"MERaLiON public license","codeUrl":"https://huggingface.co/MERaLiON/MERaLiON-2-10B/blob/57bb08fd110af1f1ef5a6b16d53ac240393d4f93/README.md","weightsUrl":"https://huggingface.co/datasets/MERaLiON/MERaLiON_Public_Licence/blob/main/MERaLiON-Public-Licence-v3.pdf","notes":"Custom MERaLiON terms and incorporated base-model licenses apply.","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/MERaLiON/MERaLiON-2-10B/tree/57bb08fd110af1f1ef5a6b16d53ac240393d4f93","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"MERaLiON/MERaLiON-2-10B — checkpoint and usage","url":"https://huggingface.co/MERaLiON/MERaLiON-2-10B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/MERaLiON/MERaLiON-2-10B/blob/57bb08fd110af1f1ef5a6b16d53ac240393d4f93/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/datasets/MERaLiON/MERaLiON_Public_Licence/blob/main/MERaLiON-Public-Licence-v3.pdf","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--meralion-2.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/MERaLiON/MERaLiON-2-10B","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/meralion-2.svg","sha256":"46557b80eaeefed2d6a09ac553be4a8ffc3888875e3c0d796c346f8adf081463"}},{"id":"omni:meralion-3","slug":"meralion-3","name":"MERaLiON 3","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Understands multilingual Southeast Asian speech, including local dialects and conversational code-switching.","architecture":"Speech encoder connected to a multilingual language decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["MERaLiON-3-10B"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://huggingface.co/MERaLiON/MERaLiON-3-10B/tree/3d5c2f772641b1cfeba35743df3db32a07db8c48","weights":"https://huggingface.co/MERaLiON/MERaLiON-3-10B","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#meralion-3"},"license":{"status":"restricted","code":"MERaLiON public license","weights":"MERaLiON public license","codeUrl":"https://huggingface.co/MERaLiON/MERaLiON-3-10B/blob/3d5c2f772641b1cfeba35743df3db32a07db8c48/README.md","weightsUrl":"https://huggingface.co/datasets/MERaLiON/MERaLiON_Public_Licence/blob/main/MERaLiON-3-Public-Licence.pdf","notes":"Custom MERaLiON terms and incorporated base-model licenses apply.","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/MERaLiON/MERaLiON-3-10B/tree/3d5c2f772641b1cfeba35743df3db32a07db8c48","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"MERaLiON/MERaLiON-3-10B — checkpoint and usage","url":"https://huggingface.co/MERaLiON/MERaLiON-3-10B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/MERaLiON/MERaLiON-3-10B/blob/3d5c2f772641b1cfeba35743df3db32a07db8c48/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/datasets/MERaLiON/MERaLiON_Public_Licence/blob/main/MERaLiON-3-Public-Licence.pdf","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--meralion-3.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/MERaLiON/MERaLiON-3-10B","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/meralion-3.svg","sha256":"91c433755f3259255f5035b83a7979472cf6c6eedc6d912969114e75f8369907"}},{"id":"omni:meralion-3-asr","slug":"meralion-3-asr","name":"MERaLiON 3 ASR","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Transcribes Southeast Asian languages, regional dialects and code-switched speech.","architecture":"ASR-optimized MERaLiON 3 speech-language model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["MERaLiON-3-3B-ASR"],"inputs":["S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://huggingface.co/MERaLiON/MERaLiON-3-3B-ASR/tree/9f226d738dee62c60a2f43b0ab7b9a2585a132c3","weights":"https://huggingface.co/MERaLiON/MERaLiON-3-3B-ASR","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#meralion-3-asr"},"license":{"status":"restricted","code":"MERaLiON public license","weights":"MERaLiON public license","codeUrl":"https://huggingface.co/MERaLiON/MERaLiON-3-3B-ASR/blob/9f226d738dee62c60a2f43b0ab7b9a2585a132c3/README.md","weightsUrl":"https://huggingface.co/datasets/MERaLiON/MERaLiON_Public_Licence/blob/main/MERaLiON-3-Public-Licence.pdf","notes":"Custom MERaLiON terms and incorporated base-model licenses apply.","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/MERaLiON/MERaLiON-3-3B-ASR/tree/9f226d738dee62c60a2f43b0ab7b9a2585a132c3","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"MERaLiON/MERaLiON-3-3B-ASR — checkpoint and usage","url":"https://huggingface.co/MERaLiON/MERaLiON-3-3B-ASR","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/MERaLiON/MERaLiON-3-3B-ASR/blob/9f226d738dee62c60a2f43b0ab7b9a2585a132c3/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/datasets/MERaLiON/MERaLiON_Public_Licence/blob/main/MERaLiON-3-Public-Licence.pdf","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--meralion-3-asr.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/MERaLiON/MERaLiON-3-3B-ASR","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/meralion-3-asr.svg","sha256":"b3d28334f7729c0f5f7f3a160b109a7f2474848af4b174d0a5d0399d83a2044a"}},{"id":"omni:meralion-audiollm","slug":"meralion-audiollm","name":"MERaLiON-AudioLLM","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Transcribes, translates and explains audio, including Singaporean speech and local language usage.","architecture":"Whisper encoder and SEA-LION language decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["MERaLiON-AudioLLM-Whisper-SEA-LION"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2024-12-13","dateKind":"paper","links":{"code":"https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION/tree/df3199c9e110a92b1e78b77ebf5acd7009f25f70","weights":"https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION","paper":"https://arxiv.org/abs/2412.09818","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#meralion-audiollm"},"license":{"status":"restricted","code":"MERaLiON public license","weights":"MERaLiON public license","codeUrl":"https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION/blob/df3199c9e110a92b1e78b77ebf5acd7009f25f70/README.md","weightsUrl":"https://huggingface.co/datasets/MERaLiON/MERaLiON_Public_Licence/blob/main/MERaLiON-Public-Licence-v1.pdf","notes":"Custom MERaLiON terms and incorporated base-model licenses apply.","components":[]},"sources":[{"kind":"paper","title":"MERaLiON-AudioLLM: Bridging Audio and Language with Large Language Models","url":"https://arxiv.org/abs/2412.09818","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION/tree/df3199c9e110a92b1e78b77ebf5acd7009f25f70","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION — checkpoint and usage","url":"https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION/blob/df3199c9e110a92b1e78b77ebf5acd7009f25f70/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/datasets/MERaLiON/MERaLiON_Public_Licence/blob/main/MERaLiON-Public-Licence-v1.pdf","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--meralion-audiollm.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2412.09818","originUrl":"https://arxiv.org/html/2412.09818v3/images/model_architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/meralion-audiollm.png","sha256":"4dfb93882e333a26cf2fa2cc999e29f522e4beee7a0336daf3c82baed7306917"}},{"id":"omni:midashenglm","slug":"midashenglm","name":"MiDashengLM","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Describes speech, environmental sounds and music and answers questions about recordings.","architecture":"Dasheng audio encoder aligned to a language model with general audio captions","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["midashenglm-7b-1021-bf16"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-08-06","dateKind":"paper","links":{"code":"https://huggingface.co/mispeech/midashenglm-7b-1021-bf16/tree/f7b0fdb745fc7997a62443007191b50f1c94702b","weights":"https://huggingface.co/mispeech/midashenglm-7b-1021-bf16","paper":"https://arxiv.org/abs/2508.03983","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#midashenglm"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0","codeUrl":"https://huggingface.co/mispeech/midashenglm-7b-1021-bf16/blob/f7b0fdb745fc7997a62443007191b50f1c94702b/README.md","weightsUrl":"https://huggingface.co/mispeech/midashenglm-7b-1021-bf16/blob/f7b0fdb745fc7997a62443007191b50f1c94702b/README.md","notes":"Public code and weights were located, but a complete release license was not stated.","components":[]},"sources":[{"kind":"paper","title":"MiDashengLM: Efficient Audio Understanding with General Audio Captions","url":"https://arxiv.org/abs/2508.03983","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/mispeech/midashenglm-7b-1021-bf16/tree/f7b0fdb745fc7997a62443007191b50f1c94702b","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"mispeech/midashenglm-7b-1021-bf16 — checkpoint and usage","url":"https://huggingface.co/mispeech/midashenglm-7b-1021-bf16","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/mispeech/midashenglm-7b-1021-bf16/blob/f7b0fdb745fc7997a62443007191b50f1c94702b/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--midashenglm.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.03983","originUrl":"https://arxiv.org/html/2508.03983v3/Framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/midashenglm.png","sha256":"e43d19597050511551b0bf6787127adca15d01be3dfe3ee05ddb80307010894d"}},{"id":"omni:mimo-audio","slug":"mimo-audio","name":"MiMo-Audio","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Understands recordings and conducts spoken conversations using instructions and audio examples.","architecture":"Semantic-acoustic audio tokenizer and a speech-text language model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["MiMo-Audio-7B-Instruct"],"inputs":["T","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-12-29","dateKind":"paper","links":{"code":"https://github.com/XiaomiMiMo/MiMo-Audio","weights":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-7B-Instruct","paper":"https://arxiv.org/abs/2512.23808","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#mimo-audio"},"license":{"status":"open","code":"Apache-2.0","weights":"MIT","codeUrl":"https://github.com/XiaomiMiMo/MiMo-Audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-7B-Instruct/blob/c359441c22c2a1c74be5f99a91e83392680e9cc8/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"MiMo-Audio: Audio Language Models are Few-Shot Learners","url":"https://arxiv.org/abs/2512.23808","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/XiaomiMiMo/MiMo-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"XiaomiMiMo/MiMo-Audio-7B-Instruct — checkpoint and usage","url":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-7B-Instruct","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/XiaomiMiMo/MiMo-Audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-7B-Instruct/blob/c359441c22c2a1c74be5f99a91e83392680e9cc8/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--mimo-audio.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2512.23808","originUrl":"https://arxiv.org/html/2512.23808v1/mimoaudio_new.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/mimo-audio.png","sha256":"ba503fa4a05da9ae18149ae2cb2e6b960b9bc8cc351c78792a413763cb03c225"}},{"id":"omni:mimo-v2-5","slug":"mimo-v2-5","name":"MiMo-V2.5","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers questions and follows instructions involving text, images, videos and audio.","architecture":"Multimodal mixture-of-experts language model","notes":"Native multimodal understanding on the MiMo-V2-Flash backbone.","variants":["MiMo-V2.5"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/modeling_mimo_v2.py","weights":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#mimo-v2-5"},"license":{"status":"open","code":"Apache-2.0","weights":"MIT","codeUrl":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/modeling_mimo_v2.py","weightsUrl":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md","notes":"","components":[]},"sources":[{"kind":"model-card","title":"XiaomiMiMo/MiMo-V2.5 · Hugging Face","url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5","reviewed_on":"2026-09-12","review_level":"model-card"},{"kind":"documentation","title":"MiMo-V2.5 — released implementation","url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/modeling_mimo_v2.py","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"MiMo-V2.5 — checkpoint license evidence","url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/63651580ca774f8504f676040460aed3e1244ac1/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--mimo-v2-5.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5","originUrl":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5/resolve/main/assets/architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/mimo-v2-5.png","sha256":"e46b7bd3626e45858e99686512dafd5fc1b3724803abf37ba261c99e74a16f8b"}},{"id":"omni:ming-flash-omni","slug":"ming-flash-omni","name":"Ming-Flash-Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Combines audiovisual understanding with text, image and audio generation.","architecture":"Multimodal mixture-of-experts perception and generation","notes":"Extends the Ming family with image tasks and joint acoustic generation.","variants":["Ming-flash-omni-Preview"],"inputs":["T","I","V","A"],"outputs":["T","I","A"],"interaction":"generation","specs":{},"date":"2025-10-28","dateKind":"paper","links":{"code":"https://github.com/inclusionAI/Ming","weights":"https://huggingface.co/inclusionAI/Ming-flash-omni-Preview","paper":"https://arxiv.org/abs/2510.24821","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#ming-flash-omni"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/inclusionAI/Ming/blob/main/LICENSE","weightsUrl":"https://huggingface.co/inclusionAI/Ming-flash-omni-Preview/blob/8791771443ae7525ee27140dcf6c8f2830f45888/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Ming-Flash-Omni: A Sparse, Unified Architecture for Multimodal Perception and Generation","url":"https://arxiv.org/abs/2510.24821","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Ming-Flash-Omni — released implementation","url":"https://github.com/inclusionAI/Ming","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"inclusionAI/Ming-flash-omni-Preview — official model card","url":"https://huggingface.co/inclusionAI/Ming-flash-omni-Preview","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Ming-Flash-Omni — code license evidence","url":"https://github.com/inclusionAI/Ming/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Ming-Flash-Omni — checkpoint license evidence","url":"https://huggingface.co/inclusionAI/Ming-flash-omni-Preview/blob/8791771443ae7525ee27140dcf6c8f2830f45888/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ming-flash-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2510.24821","originUrl":"https://arxiv.org/html/2510.24821v3/ming_flash.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ming-flash-omni.png","sha256":"8ca6961a26901c868c4581b61666f5c78f639be4ade9596cbacea1e20785d83d"}},{"id":"omni:ming-flash-omni-2-0","slug":"ming-flash-omni-2-0","name":"Ming-Flash-Omni 2.0","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Interprets mixed audiovisual inputs and generates text, images or expressive audio.","architecture":"Unified multimodal mixture-of-experts architecture","notes":"Supports speech, sound and music synthesis alongside image generation and editing.","variants":["Ming-flash-omni-2.0"],"inputs":["T","I","V","A"],"outputs":["T","I","A"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/inclusionAI/Ming","weights":"https://huggingface.co/inclusionAI/Ming-flash-omni-2.0","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#ming-flash-omni-2-0"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/inclusionAI/Ming/blob/main/LICENSE","weightsUrl":"https://huggingface.co/inclusionAI/Ming-flash-omni-2.0/blob/6a2e1dec07066d20f62a743ac7c34284e4a3932d/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"inclusionAI/Ming — official repository","url":"https://github.com/inclusionAI/Ming","reviewed_on":"2026-09-12","review_level":"readme"},{"kind":"model-card","title":"inclusionAI/Ming-flash-omni-2.0 — official model card","url":"https://huggingface.co/inclusionAI/Ming-flash-omni-2.0","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Ming-Flash-Omni 2.0 — code license evidence","url":"https://github.com/inclusionAI/Ming/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Ming-Flash-Omni 2.0 — checkpoint license evidence","url":"https://huggingface.co/inclusionAI/Ming-flash-omni-2.0/blob/6a2e1dec07066d20f62a743ac7c34284e4a3932d/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ming-flash-omni-2-0.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://github.com/inclusionAI/Ming","originUrl":"https://mdn.alipayobjects.com/huamei_xg7bx2/afts/img/WxgUSrdZVj8AAAAAdJAAAAgADhHHAQFr/original","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ming-flash-omni-2-0.png","sha256":"2ebaa0ef9adbc489c7fb0ea6b8ed0af9b6d26447e97603f81276c7fc0eb5e680"}},{"id":"omni:ming-omni","slug":"ming-omni","name":"Ming-Lite-Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Handles audiovisual questions and creates text, images and spoken responses.","architecture":"Modality-specific encoders and a unified multimodal backbone","notes":"The publicly released Ming-Lite-Omni checkpoint covers multimodal perception, speech and image generation. This entry selects the Lite release from the broader Ming-Omni research family.","variants":["Ming-Lite-Omni"],"inputs":["T","I","V","A"],"outputs":["T","I","S"],"interaction":"generation","specs":{},"date":"2025-06-11","dateKind":"paper","links":{"code":"https://github.com/inclusionAI/Ming","weights":"https://huggingface.co/inclusionAI/Ming-Lite-Omni","paper":"https://arxiv.org/abs/2506.09344","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#ming-omni"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/inclusionAI/Ming/blob/main/LICENSE","weightsUrl":"https://huggingface.co/inclusionAI/Ming-Lite-Omni/blob/ecd74fdae16ecb5a97429f943f262f7715ca2ee6/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Ming-Omni: A Unified Multimodal Model for Perception and Generation","url":"https://arxiv.org/abs/2506.09344","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Ming-Omni — released implementation","url":"https://github.com/inclusionAI/Ming","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"inclusionAI/Ming-Lite-Omni — official model card","url":"https://huggingface.co/inclusionAI/Ming-Lite-Omni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Ming-Omni — code license evidence","url":"https://github.com/inclusionAI/Ming/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Ming-Omni — checkpoint license evidence","url":"https://huggingface.co/inclusionAI/Ming-Lite-Omni/blob/ecd74fdae16ecb5a97429f943f262f7715ca2ee6/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ming-omni.webp","kind":"source","label":"Family figure · Figure 2","sourceUrl":"https://arxiv.org/abs/2506.09344","originUrl":"https://arxiv.org/html/2506.09344v1/Ming_moe_uni_v1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ming-omni.png","sha256":"d36623cb640b4a0018f6e542105abcfb64e1c2e1993e9bac885a6a507eeca5fd"}},{"id":"omni:mini-omni","slug":"mini-omni","name":"Mini-Omni","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Listens to spoken questions and streams text and speech answers together.","architecture":"Parallel text-audio token generation on a small language model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["mini-omni"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2024-08-29","dateKind":"paper","links":{"code":"https://github.com/gpt-omni/mini-omni","weights":"https://huggingface.co/gpt-omni/mini-omni","paper":"https://arxiv.org/abs/2408.16725","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#mini-omni"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/gpt-omni/mini-omni/blob/main/LICENSE","weightsUrl":"https://huggingface.co/gpt-omni/mini-omni/blob/ed19d73eb13401dcd2c9de6fa36c69f74f15ae37/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Mini-Omni: Language Models Can Hear, Talk While Thinking in Streaming","url":"https://arxiv.org/abs/2408.16725","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/gpt-omni/mini-omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"gpt-omni/mini-omni — checkpoint and usage","url":"https://huggingface.co/gpt-omni/mini-omni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/gpt-omni/mini-omni/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/gpt-omni/mini-omni/blob/ed19d73eb13401dcd2c9de6fa36c69f74f15ae37/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--mini-omni.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2408.16725","originUrl":"https://arxiv.org/html/2408.16725v3/figures/jpg/frameworkv3.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/mini-omni.png","sha256":"596fd24e98e8f6fc64629197020eb98e90e4ff966ba437048e38bbdc14e6009d"}},{"id":"omni:mini-omni2","slug":"mini-omni2","name":"Mini-Omni2","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Sees images, follows spoken instructions and streams spoken replies with interruption handling.","architecture":"Parallel text-speech generation with visual and speech encoders","notes":"Adds visual input and interruption-aware duplex interaction to Mini-Omni.","variants":["mini-omni2"],"inputs":["T","I","S"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":"2024-10-15","dateKind":"paper","links":{"code":"https://github.com/gpt-omni/mini-omni2","weights":"https://huggingface.co/gpt-omni/mini-omni2","paper":"https://arxiv.org/abs/2410.11190","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#mini-omni2"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/gpt-omni/mini-omni2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/gpt-omni/mini-omni2/blob/49f474f4c38f80cf716859bb1b1442df2a8dea46/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Mini-Omni2: Towards Open-source GPT-4o with Vision, Speech and Duplex Capabilities","url":"https://arxiv.org/abs/2410.11190","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Mini-Omni2 — released implementation","url":"https://github.com/gpt-omni/mini-omni2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"gpt-omni/mini-omni2 — official model card","url":"https://huggingface.co/gpt-omni/mini-omni2","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Mini-Omni2 — code license evidence","url":"https://github.com/gpt-omni/mini-omni2/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Mini-Omni2 — checkpoint license evidence","url":"https://huggingface.co/gpt-omni/mini-omni2/blob/49f474f4c38f80cf716859bb1b1442df2a8dea46/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--mini-omni2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2410.11190","originUrl":"https://arxiv.org/html/2410.11190v3/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/mini-omni2.png","sha256":"baaa3e7e10e8ff8c45f4605f066f665650c3967227645b4fe76f97ee96d5d738"}},{"id":"omni:minicpm-o-2-6","slug":"minicpm-o-2-6","name":"MiniCPM-o 2.6","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Chats about images, video and audio, with speech output and voice imitation.","architecture":"Compact language backbone with visual and speech modules","notes":"The 2.6 omni release supports speech and audiovisual interaction.","variants":["MiniCPM-o-2_6"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/OpenBMB/MiniCPM-V","weights":"https://huggingface.co/openbmb/MiniCPM-o-2_6","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#minicpm-o-2-6"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/OpenBMB/MiniCPM-V/blob/main/LICENSE","weightsUrl":"https://huggingface.co/openbmb/MiniCPM-o-2_6/blob/06849bfd36da94da1f5a88fa17e8ba63f08a4c4d/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"OpenBMB/MiniCPM-o — official repository","url":"https://github.com/OpenBMB/MiniCPM-o","reviewed_on":"2026-09-12","review_level":"readme"},{"kind":"repository","title":"MiniCPM-o 2.6 — released implementation","url":"https://github.com/OpenBMB/MiniCPM-V","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"openbmb/MiniCPM-o-2_6 — official model card","url":"https://huggingface.co/openbmb/MiniCPM-o-2_6","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"MiniCPM-o 2.6 — code license evidence","url":"https://github.com/OpenBMB/MiniCPM-V/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"MiniCPM-o 2.6 — checkpoint license evidence","url":"https://huggingface.co/openbmb/MiniCPM-o-2_6/blob/06849bfd36da94da1f5a88fa17e8ba63f08a4c4d/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--minicpm-o-2-6.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://github.com/OpenBMB/MiniCPM-o","originUrl":"https://raw.githubusercontent.com/OpenBMB/MiniCPM-o/main/assets/minicpm-o-26-framework-v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/minicpm-o-2-6.png","sha256":"e93bbca2702ca913c4023dc2917b87fa4ede175a058ff248fa682484aaad9a08"}},{"id":"omni:minicpm-o-4-5","slug":"minicpm-o-4-5","name":"MiniCPM-o 4.5","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Supports live audiovisual conversations while listening and speaking at the same time.","architecture":"Streaming multimodal perception and full-duplex speech generation","notes":"Adds simultaneous seeing, listening and speaking with proactive behavior.","variants":["MiniCPM-o-4_5"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":"2026-04-30","dateKind":"paper","links":{"code":"https://github.com/OpenBMB/MiniCPM-V","weights":"https://huggingface.co/openbmb/MiniCPM-o-4_5","paper":"https://arxiv.org/abs/2604.27393","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#minicpm-o-4-5"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/OpenBMB/MiniCPM-V/blob/main/LICENSE","weightsUrl":"https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/503e754207c94da6bb26850b4469f367c9ea3582/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"MiniCPM-o 4.5: Towards Real-Time Full-Duplex Omni-Modal Interaction","url":"https://arxiv.org/abs/2604.27393","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"MiniCPM-o 4.5 — released implementation","url":"https://github.com/OpenBMB/MiniCPM-V","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"openbmb/MiniCPM-o-4_5 — official model card","url":"https://huggingface.co/openbmb/MiniCPM-o-4_5","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"MiniCPM-o 4.5 — code license evidence","url":"https://github.com/OpenBMB/MiniCPM-V/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"MiniCPM-o 4.5 — checkpoint license evidence","url":"https://huggingface.co/openbmb/MiniCPM-o-4_5/blob/503e754207c94da6bb26850b4469f367c9ea3582/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--minicpm-o-4-5.webp","kind":"source","label":"Figure 4","sourceUrl":"https://arxiv.org/abs/2604.27393","originUrl":"https://arxiv.org/html/2604.27393v1/arch.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/minicpm-o-4-5.png","sha256":"534ed336027fb724b16b155c1129242be3faf08c40057783da81d8fd855c5fe7"}},{"id":"omni:minimind-o","slug":"minimind-o","name":"MiniMind-O","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Provides a small, trainable model for image questions and spoken conversations.","architecture":"Lightweight vision-language-speech model","notes":"The released MiniMind-3O checkpoint combines text, vision and speech within a small Thinker–Talker system.","variants":["minimind-3o"],"inputs":["T","I","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2026-05-05","dateKind":"paper","links":{"code":"https://github.com/jingyaogong/minimind-o","weights":"https://huggingface.co/jingyaogong/minimind-3o","paper":"https://arxiv.org/abs/2605.03937","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#minimind-o"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/jingyaogong/minimind-o/blob/master/LICENSE","weightsUrl":"https://huggingface.co/jingyaogong/minimind-3o/blob/ee3febbd08cc5b2bd41c039c825a8934232fee33/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"MiniMind-O Technical Report: An Open Small-Scale Speech-Native Omni Model","url":"https://arxiv.org/abs/2605.03937","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"jingyaogong/minimind-o — official repository","url":"https://github.com/jingyaogong/minimind-o","reviewed_on":"2026-09-12","review_level":"readme"},{"kind":"model-card","title":"jingyaogong/minimind-3o — official model card","url":"https://huggingface.co/jingyaogong/minimind-3o","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"MiniMind-O — code license evidence","url":"https://github.com/jingyaogong/minimind-o/blob/master/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"MiniMind-O — checkpoint license evidence","url":"https://huggingface.co/jingyaogong/minimind-3o/blob/ee3febbd08cc5b2bd41c039c825a8934232fee33/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--minimind-o.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.03937","originUrl":"https://arxiv.org/html/2605.03937v1/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/minimind-o.png","sha256":"2ab5286f915bfb58bba857b438f0c8ae9b9afcadf88b47943f955407a7a66c0a"}},{"id":"omni:mio","slug":"mio","name":"MIO","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands audiovisual content and generates text, images or speech using multimodal tokens.","architecture":"Autoregressive language modeling over discrete modality tokens","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["MIO-7B-Instruct"],"inputs":["T","I","V","A"],"outputs":["T","I","S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/MIO-Team/MIO","weights":"https://huggingface.co/m-a-p/MIO-7B-Instruct","paper":"https://arxiv.org/abs/2409.17692","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#mio"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0 + Yi terms","codeUrl":"https://github.com/MIO-Team/MIO/blob/main/README.md","weightsUrl":"https://huggingface.co/m-a-p/MIO-7B-Instruct/blob/b8f5e48ce79fcf3268dba64b1395ac150c1ad99e/README.md","notes":"The Yi-6B-Chat base retains its license; repository-wide code terms are not stated.","components":[]},"sources":[{"kind":"paper","title":"MIO: A Foundation Model on Multimodal Tokens","url":"https://arxiv.org/abs/2409.17692","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/01-ai/Yi-6B-Chat/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/MIO-Team/MIO","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"m-a-p/MIO-7B-Instruct — checkpoint and usage","url":"https://huggingface.co/m-a-p/MIO-7B-Instruct","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/MIO-Team/MIO/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/m-a-p/MIO-7B-Instruct/blob/b8f5e48ce79fcf3268dba64b1395ac150c1ad99e/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--mio.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2409.17692","originUrl":"https://arxiv.org/html/2409.17692v4/mio-framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/mio.png","sha256":"e286ef3191a3c9261f68180174700e6a7857f617637205f3f77eb5f447e47a7c"}},{"id":"omni:moshi","slug":"moshi","name":"Moshi","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Listens and speaks simultaneously, handling conversational overlap and interruptions.","architecture":"Multi-stream Helium Transformer with Mimi codec and inner-monologue text","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["moshiko-pytorch-bf16"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":"2024-09-17","dateKind":"paper","links":{"code":"https://github.com/kyutai-labs/moshi","weights":"https://huggingface.co/kyutai/moshiko-pytorch-bf16","paper":"https://arxiv.org/abs/2410.00037","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#moshi"},"license":{"status":"open","code":"Apache-2.0","weights":"CC-BY-4.0","codeUrl":"https://github.com/kyutai-labs/moshi/blob/main/LICENSE-APACHE","weightsUrl":"https://huggingface.co/kyutai/moshiko-pytorch-bf16/blob/2bfc9ae6e89079a5cc7ed2a68436010d91a3d289/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Moshi: a speech-text foundation model for real-time dialogue","url":"https://arxiv.org/abs/2410.00037","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/kyutai-labs/moshi","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"kyutai/moshiko-pytorch-bf16 — checkpoint and usage","url":"https://huggingface.co/kyutai/moshiko-pytorch-bf16","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/kyutai-labs/moshi/blob/main/LICENSE-APACHE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/kyutai/moshiko-pytorch-bf16/blob/2bfc9ae6e89079a5cc7ed2a68436010d91a3d289/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--moshi.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2410.00037","originUrl":"https://arxiv.org/html/2410.00037v2/figures/overview_moshi_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/moshi.png","sha256":"bc0a1acd0d46f292ca7ce70bd69ebc3ec4941987ba008303af2ee5ea1bf35bec"}},{"id":"omni:moss-audio","slug":"moss-audio","name":"MOSS-Audio Instruct","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Captions recordings, answers time-specific questions and transcribes speech with timestamps.","architecture":"Dedicated audio encoder with DeepStack feature injection and time markers","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["MOSS-Audio-8B-Instruct"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-06-01","dateKind":"paper","links":{"code":"https://github.com/OpenMOSS/MOSS-Audio","weights":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-8B-Instruct","paper":"https://arxiv.org/abs/2606.01802","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#moss-audio"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0","codeUrl":"https://github.com/OpenMOSS/MOSS-Audio/blob/main/README.md","weightsUrl":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-8B-Instruct/blob/6521a39181b47a18f2d9f4b3acfb5bca7b76b57f/README.md","notes":"Public code and weights were located, but a complete release license was not stated.","components":[]},"sources":[{"kind":"paper","title":"MOSS-Audio Technical Report","url":"https://arxiv.org/abs/2606.01802","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/OpenMOSS/MOSS-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"OpenMOSS-Team/MOSS-Audio-8B-Instruct — checkpoint and usage","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-8B-Instruct","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/OpenMOSS/MOSS-Audio/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-8B-Instruct/blob/6521a39181b47a18f2d9f4b3acfb5bca7b76b57f/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--moss-audio.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2606.01802","originUrl":"https://arxiv.org/html/2606.01802v3/arc_final_half.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/moss-audio.png","sha256":"74561f637b7ff61cf11809e438b336e21f6127b1faa08c0e1c8430a143460012"}},{"id":"omni:moss-audio-thinking","slug":"moss-audio-thinking","name":"MOSS-Audio Thinking","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Reasons about speech, environmental sound and music before producing a text answer.","architecture":"Reasoning-tuned MOSS-Audio with temporal grounding","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["MOSS-Audio-8B-Thinking"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/OpenMOSS/MOSS-Audio","weights":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-8B-Thinking","paper":"https://arxiv.org/abs/2606.01802","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#moss-audio-thinking"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0","codeUrl":"https://github.com/OpenMOSS/MOSS-Audio/blob/main/README.md","weightsUrl":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-8B-Thinking/blob/3d09aad8d7803dd131c8f42d2c09364d1f9367db/README.md","notes":"Public code and weights were located, but a complete release license was not stated.","components":[]},"sources":[{"kind":"paper","title":"MOSS-Audio Technical Report","url":"https://arxiv.org/abs/2606.01802","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/OpenMOSS/MOSS-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"OpenMOSS-Team/MOSS-Audio-8B-Thinking — checkpoint and usage","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-8B-Thinking","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/OpenMOSS/MOSS-Audio/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-8B-Thinking/blob/3d09aad8d7803dd131c8f42d2c09364d1f9367db/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--moss-audio-thinking.webp","kind":"source","label":"Family figure — Figure 2","sourceUrl":"https://arxiv.org/abs/2606.01802","originUrl":"https://arxiv.org/html/2606.01802v3/arc_final_half.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/moss-audio.png","sha256":"74561f637b7ff61cf11809e438b336e21f6127b1faa08c0e1c8430a143460012"}},{"id":"omni:moss-speech","slug":"moss-speech","name":"MOSS-Speech","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Converses directly in Chinese and English speech without first generating a text response.","architecture":"Modality-based layer splitting on a frozen-pretrained language backbone","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["MOSS-Speech"],"inputs":["S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-01","dateKind":"paper","links":{"code":"https://github.com/OpenMOSS/MOSS-Speech","weights":"https://huggingface.co/OpenMOSS-Team/MOSS-Speech","paper":"https://arxiv.org/abs/2510.00499","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#moss-speech"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Not stated","codeUrl":"https://github.com/OpenMOSS/MOSS-Speech/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenMOSS-Team/MOSS-Speech/blob/cff025bb41d8459d59abac0b5e44aba7f659ec9e/README.md","notes":"The repository licenses code, but a release-specific weight license was not located.","components":[]},"sources":[{"kind":"paper","title":"MOSS-Speech: Towards True Speech-to-Speech Models Without Text Guidance","url":"https://arxiv.org/abs/2510.00499","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/OpenMOSS/MOSS-Speech","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"OpenMOSS-Team/MOSS-Speech — checkpoint and usage","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Speech","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/OpenMOSS/MOSS-Speech/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Speech/blob/cff025bb41d8459d59abac0b5e44aba7f659ec9e/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--moss-speech.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2510.00499","originUrl":"https://arxiv.org/html/2510.00499v2/figure/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/moss-speech.png","sha256":"ef9a8124f6eb92805145b45f2d5895656501ffebb17bc2d1a20fa5816a55e1b5"}},{"id":"omni:music-flamingo","slug":"music-flamingo","name":"Music Flamingo","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Explains musical structure, instruments and expressive qualities in recordings.","architecture":"Music-adapted audio-language model with temporal reasoning","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["music-flamingo-2601-hf"],"inputs":["T","M"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-11-13","dateKind":"paper","links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/musicflamingo","weights":"https://huggingface.co/nvidia/music-flamingo-2601-hf","paper":"https://arxiv.org/abs/2511.10289","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#music-flamingo"},"license":{"status":"restricted","code":"Apache-2.0","weights":"NVIDIA OneWay Noncommercial","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/nvidia/music-flamingo-2601-hf/blob/6b5be086d52f65a1e204cb0faf70bf54e2741ecd/README.md","notes":"Noncommercial terms apply to model weights and relevant bundled components.","components":[]},"sources":[{"kind":"paper","title":"Music Flamingo: Scaling Music Understanding in Audio Language Models","url":"https://arxiv.org/abs/2511.10289","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"model-card","title":"nvidia/music-flamingo-2601-hf — checkpoint and usage","url":"https://huggingface.co/nvidia/music-flamingo-2601-hf","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/music-flamingo-2601-hf/blob/6b5be086d52f65a1e204cb0faf70bf54e2741ecd/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/musicflamingo","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--music-flamingo.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2511.10289","originUrl":"https://arxiv.org/html/2511.10289v1/main_figure_v1.drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/music-flamingo.png","sha256":"3db72b0f1cc87044c8c0ab6d4a04a439b0f7fe671ed697a7902af4b691eeb8f9"}},{"id":"omni:nemotron-3-nano-omni","slug":"nemotron-3-nano-omni","name":"Nemotron 3 Nano Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Reasons over documents, images, video and audio and returns text responses.","architecture":"Hybrid Mamba-Transformer mixture-of-experts with multimodal encoders","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-04-27","dateKind":"paper","links":{"code":"https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/tree/e5e9932441de940c9a62185c870ea5bcd4cd24e2","weights":"https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16","paper":"https://arxiv.org/abs/2604.24954","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#nemotron-3-nano-omni"},"license":{"status":"restricted","code":"Apache-2.0","weights":"NVIDIA Open Model Agreement","codeUrl":"https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/blob/e5e9932441de940c9a62185c870ea5bcd4cd24e2/modeling.py","weightsUrl":"https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/blob/e5e9932441de940c9a62185c870ea5bcd4cd24e2/README.md","notes":"Custom model terms apply to the weights.","components":[]},"sources":[{"kind":"paper","title":"Nemotron 3 Nano Omni: Efficient and Open Multimodal Intelligence","url":"https://arxiv.org/abs/2604.24954","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/tree/e5e9932441de940c9a62185c870ea5bcd4cd24e2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 — checkpoint and usage","url":"https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/blob/e5e9932441de940c9a62185c870ea5bcd4cd24e2/modeling.py","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/blob/e5e9932441de940c9a62185c870ea5bcd4cd24e2/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--nemotron-3-nano-omni.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2604.24954","originUrl":"https://arxiv.org/html/2604.24954v2/Nemotron_arch_v3_reduced.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/nemotron-3-nano-omni.png","sha256":"00a7e3d03b7c535bdd837ffb0900f657507e6e5fc7b9fca8442e73bfb98da1ef"}},{"id":"omni:nemotron-audex-2b","slug":"nemotron-audex-2b","name":"Nemotron-Labs-Audex 2B","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Performs audio understanding, speech tasks and sound generation with a smaller dense backbone.","architecture":"Dense decoder with audio adapters and speech and sound token outputs","notes":"The selected checkpoint uses a dense 2B backbone. The shared author figure depicts the 30B MoE variant, not the exact 2B layer layout.","variants":["Nemotron-Labs-Audex-2B"],"inputs":["T","A"],"outputs":["T","A"],"interaction":"generation","specs":{},"date":"2026-07-06","dateKind":"paper","links":{"code":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B/tree/77b7e1a4de899769f22c1bc074db666601c28907","weights":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B","paper":"https://arxiv.org/abs/2607.05196","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#nemotron-audex-2b"},"license":{"status":"unclear","code":"Not stated","weights":"NVIDIA OneWay Noncommercial","codeUrl":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B/blob/77b7e1a4de899769f22c1bc074db666601c28907/README.md","weightsUrl":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B/blob/77b7e1a4de899769f22c1bc074db666601c28907/README.md","notes":"Noncommercial weight license; inspect the included code files for component-specific terms.","components":[]},"sources":[{"kind":"paper","title":"Unified Audio Intelligence Without Regressing on Text Intelligence","url":"https://arxiv.org/abs/2607.05196","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B/tree/77b7e1a4de899769f22c1bc074db666601c28907","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/Nemotron-Labs-Audex-2B — checkpoint and usage","url":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B/blob/77b7e1a4de899769f22c1bc074db666601c28907/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--nemotron-audex-2b.webp","kind":"source","label":"Family figure — 30B variant","sourceUrl":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B","originUrl":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B/resolve/77b7e1a4de899769f22c1bc074db666601c28907/assets/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/nemotron-audex-2b.png","sha256":"a14b22ae65ea5c92a15884d5bef8199d072a07f48f0fff116e60d50e8230c606"}},{"id":"omni:nemotron-audex-30b","slug":"nemotron-audex-30b","name":"Nemotron-Labs-Audex 30B-A3B","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Understands recordings, transcribes and translates speech, and generates speech or general sounds.","architecture":"Hybrid mixture-of-experts decoder with audio adapters and extended token vocabulary","notes":"The selected checkpoint uses the 30B-A3B mixture-of-experts backbone depicted in the author figure. Speech and general-audio generation use separate decoder components.","variants":["Nemotron-Labs-Audex-30B-A3B"],"inputs":["T","A"],"outputs":["T","A"],"interaction":"generation","specs":{},"date":"2026-07-06","dateKind":"paper","links":{"code":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B/tree/4e7e342045736382ddf3e2952c313847a08642b8","weights":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B","paper":"https://arxiv.org/abs/2607.05196","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#nemotron-audex-30b"},"license":{"status":"unclear","code":"Not stated","weights":"NVIDIA OneWay Noncommercial","codeUrl":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B/blob/4e7e342045736382ddf3e2952c313847a08642b8/README.md","weightsUrl":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B/blob/4e7e342045736382ddf3e2952c313847a08642b8/README.md","notes":"Noncommercial weight license; inspect the included code files for component-specific terms.","components":[]},"sources":[{"kind":"paper","title":"Unified Audio Intelligence Without Regressing on Text Intelligence","url":"https://arxiv.org/abs/2607.05196","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B/tree/4e7e342045736382ddf3e2952c313847a08642b8","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/Nemotron-Labs-Audex-30B-A3B — checkpoint and usage","url":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B/blob/4e7e342045736382ddf3e2952c313847a08642b8/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--nemotron-audex-30b.webp","kind":"source","label":"Official family architecture diagram","sourceUrl":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B","originUrl":"https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B/resolve/4e7e342045736382ddf3e2952c313847a08642b8/assets/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/nemotron-audex-30b.png","sha256":"a14b22ae65ea5c92a15884d5bef8199d072a07f48f0fff116e60d50e8230c606"}},{"id":"omni:nemotronlabs-voicechat","slug":"nemotronlabs-voicechat","name":"NemotronLabs VoiceChat 11B","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Holds interruptible, full-duplex voice conversations and can call tools while maintaining the dialogue.","architecture":"FastConformer, Nemotron backbone and streaming speech decoder","notes":"This public 11B release is distinct from the earlier 12B VoiceBench VoiceChat V1 early-access submission.","variants":["NVIDIA-NemotronLabs-VoiceChat-11B"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/NVIDIA-NeMo/Speech/tree/nemotron-labs-voicechat","weights":"https://huggingface.co/nvidia/NVIDIA-NemotronLabs-VoiceChat-11B","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#nemotronlabs-voicechat"},"license":{"status":"open","code":"Apache-2.0","weights":"OpenMDW-1.1","codeUrl":"https://github.com/NVIDIA-NeMo/Speech/blob/nemotron-labs-voicechat/LICENSE","weightsUrl":"https://huggingface.co/nvidia/NVIDIA-NemotronLabs-VoiceChat-11B/blob/a4c40ca5b4fe77db13e9840ca4a2b91becf030c8/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/NVIDIA-NeMo/Speech/tree/nemotron-labs-voicechat","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/NVIDIA-NemotronLabs-VoiceChat-11B — checkpoint and usage","url":"https://huggingface.co/nvidia/NVIDIA-NemotronLabs-VoiceChat-11B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/NVIDIA-NeMo/Speech/blob/nemotron-labs-voicechat/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/NVIDIA-NemotronLabs-VoiceChat-11B/blob/a4c40ca5b4fe77db13e9840ca4a2b91becf030c8/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--nemotronlabs-voicechat.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/nvidia/NVIDIA-NemotronLabs-VoiceChat-11B","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/nemotronlabs-voicechat.svg","sha256":"8755d67030b8a22425f19e529ffbc80123f6bd97a2225d8e20595360dca4ea46"}},{"id":"omni:next-gpt","slug":"next-gpt","name":"NExT-GPT","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Accepts and generates combinations of text, images, video and audio through connected modality models.","architecture":"Language backbone with multimodal encoders and diffusion decoders","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["nextgpt_7b_tiva_v0"],"inputs":["T","I","V","A"],"outputs":["T","I","V","A"],"interaction":"generation","specs":{},"date":"2023-09-11","dateKind":"paper","links":{"code":"https://github.com/NExT-GPT/NExT-GPT","weights":"https://huggingface.co/ChocoWu/nextgpt_7b_tiva_v0","paper":"https://arxiv.org/abs/2309.05519","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#next-gpt"},"license":{"status":"restricted","code":"BSD-3-Clause","weights":"CC-BY-NC-SA-4.0","codeUrl":"https://github.com/NExT-GPT/NExT-GPT/blob/main/LICENSE.md","weightsUrl":"https://huggingface.co/ChocoWu/nextgpt_7b_tiva_v0/blob/f28bea33714c810ef6cb9b9e440edcddb93297a4/README.md","notes":"Noncommercial share-alike weight license; downstream modality decoders retain their own terms.","components":[]},"sources":[{"kind":"paper","title":"NExT-GPT: Any-to-Any Multimodal LLM","url":"https://arxiv.org/abs/2309.05519","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/NExT-GPT/NExT-GPT","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ChocoWu/nextgpt_7b_tiva_v0 — checkpoint and usage","url":"https://huggingface.co/ChocoWu/nextgpt_7b_tiva_v0","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/NExT-GPT/NExT-GPT/blob/main/LICENSE.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/ChocoWu/nextgpt_7b_tiva_v0/blob/f28bea33714c810ef6cb9b9e440edcddb93297a4/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--next-gpt.webp","kind":"source","label":"Figure 1, PDF p. 2","sourceUrl":"https://arxiv.org/abs/2309.05519","originUrl":"https://arxiv.org/pdf/2309.05519v3","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/next-gpt.png","sha256":"6dadfa62a410577f6ce3602b831732d6e55bd80bd255ccc6f91b2659689a594a"}},{"id":"omni:nexus","slug":"nexus","name":"Nexus-O","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Combines image, video and audio understanding with text and speech interaction.","architecture":"Unified multimodal perception and generation","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["NEXUS-O"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-02-26","dateKind":"paper","links":{"code":"https://huggingface.co/HiThink-Research/NEXUS-O/tree/eaeaf3501b1b271c2cfa95920788ebf2f81b220b","weights":"https://huggingface.co/HiThink-Research/NEXUS-O","paper":"https://arxiv.org/abs/2503.01879","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#nexus"},"license":{"status":"restricted","code":"Research-only / CC-BY-NC-4.0","weights":"Research-only / CC-BY-NC-4.0","codeUrl":"https://huggingface.co/HiThink-Research/NEXUS-O/blob/eaeaf3501b1b271c2cfa95920788ebf2f81b220b/README.md","weightsUrl":"https://huggingface.co/HiThink-Research/NEXUS-O/blob/eaeaf3501b1b271c2cfa95920788ebf2f81b220b/README.md","notes":"The model-card body restricts research use despite its Apache metadata.","components":[]},"sources":[{"kind":"paper","title":"Nexus: An Omni-Perceptive And -Interactive Model for Language, Audio, And Vision","url":"https://arxiv.org/abs/2503.01879","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/HiThink-Research/NEXUS-O/tree/eaeaf3501b1b271c2cfa95920788ebf2f81b220b","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"HiThink-Research/NEXUS-O — checkpoint and usage","url":"https://huggingface.co/HiThink-Research/NEXUS-O","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/HiThink-Research/NEXUS-O/blob/eaeaf3501b1b271c2cfa95920788ebf2f81b220b/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--nexus.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2503.01879","originUrl":"https://arxiv.org/html/2503.01879v4/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/nexus.png","sha256":"c8c2c68e542920b8c9a4b63c693a02a00ffe33205d6f3e6d0045f055b120d18b"}},{"id":"omni:ola","slug":"ola","name":"Ola","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers questions that combine images, video, audio and text.","architecture":"Progressive alignment of visual and audio representations","notes":"Uses video as a bridge for joint image, video and audio understanding.","variants":["Ola-7b"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-02-06","dateKind":"paper","links":{"code":"https://github.com/Ola-Omni/Ola","weights":"https://huggingface.co/THUdyh/Ola-7b","paper":"https://arxiv.org/abs/2502.04328","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#ola"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/Ola-Omni/Ola/blob/main/LICENSE","weightsUrl":"https://huggingface.co/THUdyh/Ola-7b/blob/d2fcaa533792480ce748321bea5101fed29d857e/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Ola: Pushing the Frontiers of Omni-Modal Language Model","url":"https://arxiv.org/abs/2502.04328","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Ola — released implementation","url":"https://github.com/Ola-Omni/Ola","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"THUdyh/Ola-7b — official model card","url":"https://huggingface.co/THUdyh/Ola-7b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Ola — code license evidence","url":"https://github.com/Ola-Omni/Ola/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Ola — checkpoint license evidence","url":"https://huggingface.co/THUdyh/Ola-7b/blob/d2fcaa533792480ce748321bea5101fed29d857e/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ola.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2502.04328","originUrl":"https://arxiv.org/html/2502.04328v3/fig0-new2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ola.png","sha256":"08a2b031e173a25de87c5b680a05fe31682b470553f74682d73c9cccb0dc8aea"}},{"id":"omni:omni-autothink","slug":"omni-autothink","name":"Omni-AutoThink","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Adapts how much reasoning it uses before answering audiovisual questions.","architecture":"Adaptive multimodal reasoning policy","notes":"Adjusts reasoning behavior across text, audio and visual tasks.","variants":["Omni-AutoThink"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-12-03","dateKind":"paper","links":{"code":"https://github.com/yangdongchao/Omni-AutoThink","weights":"https://huggingface.co/Dongchao/Omni-AutoThink","paper":"https://arxiv.org/abs/2512.03783","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#omni-autothink"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/yangdongchao/Omni-AutoThink/blob/main/README.md","weightsUrl":"https://huggingface.co/Dongchao/Omni-AutoThink/blob/4acf74dcc5ca543ff34361fa76f5137022ecf774/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Omni-AutoThink: Adaptive Multimodal Reasoning via Reinforcement Learning","url":"https://arxiv.org/abs/2512.03783","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Omni-AutoThink — released implementation","url":"https://github.com/yangdongchao/Omni-AutoThink","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Dongchao/Omni-AutoThink — official model card","url":"https://huggingface.co/Dongchao/Omni-AutoThink","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Omni-AutoThink — code license evidence","url":"https://github.com/yangdongchao/Omni-AutoThink/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Omni-AutoThink — checkpoint license evidence","url":"https://huggingface.co/Dongchao/Omni-AutoThink/blob/4acf74dcc5ca543ff34361fa76f5137022ecf774/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--omni-autothink.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2512.03783","originUrl":"https://arxiv.org/html/2512.03783v2/adapthink_v2.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/omni-autothink.png","sha256":"1bfb0ccf0b1d659bc81a204b81e136f241fbbc1f420499a4710f5c500b315fd3"}},{"id":"omni:omni-r1-audio-reasoning","slug":"omni-r1-audio-reasoning","name":"Omni-R1 (audio reasoning)","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers questions about speech, sounds and music using reinforcement-trained audio reasoning.","architecture":"GRPO fine-tuning of the Qwen2.5-Omni Thinker","notes":"Selects the VGGS-GPT-trained checkpoint, distinct from the two-system Omni-R1 project. Weights are served as a public archive: the recorded HTTP ETag identifies the inspected response but is not a cryptographic checksum or an immutable download URL.","variants":["Omni-R1-VGGS-GPT"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/roudimit/Omni-R1","weights":"https://data.csail.mit.edu/public-release-sls/omni-r1/omni-r1-checkpoint-vggs-gpt.tar.gz","paper":"https://arxiv.org/abs/2505.09439","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#omni-r1-audio-reasoning"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Not stated","codeUrl":"https://github.com/roudimit/Omni-R1/blob/main/LICENSE","weightsUrl":"https://github.com/roudimit/Omni-R1/blob/main/README.md","notes":"Public checkpoint archive; no separate weight-license declaration was located.","components":[]},"sources":[{"kind":"paper","title":"Omni-R1: Do You Really Need Audio to Fine-Tune Your Audio LLM?","url":"https://arxiv.org/abs/2505.09439","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Official Omni-R1 audio-reasoning implementation and checkpoint links","url":"https://github.com/roudimit/Omni-R1","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"documentation","title":"Public checkpoint archive (HTTP 200, Content-Length 17880012800, ETag 429bb5800-63599f2c8bc06)","url":"https://data.csail.mit.edu/public-release-sls/omni-r1/omni-r1-checkpoint-vggs-gpt.tar.gz","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Code license","url":"https://github.com/roudimit/Omni-R1/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Author checkpoint release declaration; standalone weight license not stated","url":"https://github.com/roudimit/Omni-R1/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--omni-r1-audio-reasoning.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/roudimit/Omni-R1","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/omni-r1-audio-reasoning.svg","sha256":"c61e9f688095bb3407cd6aafd97b23ed4300d7ff114ac4c8577ab6ffe03a6509"}},{"id":"omni:omni-r1-two-system-collaboration","slug":"omni-r1-two-system-collaboration","name":"Omni-R1 (two-system collaboration)","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Combines global audiovisual reasoning with detailed visual grounding to answer multimodal questions.","architecture":"Collaborative global reasoning and local grounding","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Omni-R1"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/aim-uofa/Omni-R1","weights":"https://huggingface.co/Haoz0206/Omni-R1","paper":"https://arxiv.org/abs/2505.20256","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#omni-r1-two-system-collaboration"},"license":{"status":"unclear","code":"Not stated","weights":"Academic use","codeUrl":"https://github.com/aim-uofa/Omni-R1/blob/main/README.md","weightsUrl":"https://huggingface.co/Haoz0206/Omni-R1/blob/5a9d5210b8f6d795ebd65adfe0fe726fa61be441/README.md","notes":"The model card asks commercial users to contact the authors.","components":[]},"sources":[{"kind":"paper","title":"Omni-R1: Reinforcement Learning for Omnimodal Reasoning via Two-System Collaboration","url":"https://arxiv.org/abs/2505.20256","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/aim-uofa/Omni-R1","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Haoz0206/Omni-R1 — checkpoint and usage","url":"https://huggingface.co/Haoz0206/Omni-R1","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/aim-uofa/Omni-R1/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/Haoz0206/Omni-R1/blob/5a9d5210b8f6d795ebd65adfe0fe726fa61be441/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--omni-r1-two-system-collaboration.webp","kind":"source","label":"Figure 1, PDF p. 2","sourceUrl":"https://arxiv.org/abs/2505.20256","originUrl":"https://arxiv.org/pdf/2505.20256v1","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/omni-r1-two-system-collaboration.png","sha256":"7e5ffbb4d938afe42527c449de4559ded524573f8eadd28ba63bd3dbb3b64114"}},{"id":"omni:omnivinci","slug":"omnivinci","name":"OmniVinci","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers questions about combined visual and audio content with temporal alignment.","architecture":"Synchronized audio-visual encoding with rotary time embeddings","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["omnivinci"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-10-17","dateKind":"paper","links":{"code":"https://huggingface.co/nvidia/omnivinci/tree/7b3777b1c1f7c4e85f7bdef7b765dee1e76c1b7f","weights":"https://huggingface.co/nvidia/omnivinci","paper":"https://arxiv.org/abs/2510.15870","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#omnivinci"},"license":{"status":"restricted","code":"Apache-2.0","weights":"NVIDIA OneWay Noncommercial","codeUrl":"https://huggingface.co/nvidia/omnivinci/blob/7b3777b1c1f7c4e85f7bdef7b765dee1e76c1b7f/modeling_vila.py","weightsUrl":"https://huggingface.co/nvidia/omnivinci/blob/7b3777b1c1f7c4e85f7bdef7b765dee1e76c1b7f/README.md","notes":"Noncommercial terms apply to model weights and relevant bundled components.","components":[]},"sources":[{"kind":"paper","title":"OmniVinci: Enhancing Architecture and Data for Omni-Modal Understanding LLM","url":"https://arxiv.org/abs/2510.15870","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/nvidia/omnivinci/tree/7b3777b1c1f7c4e85f7bdef7b765dee1e76c1b7f","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/omnivinci — checkpoint and usage","url":"https://huggingface.co/nvidia/omnivinci","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/nvidia/omnivinci/blob/7b3777b1c1f7c4e85f7bdef7b765dee1e76c1b7f/modeling_vila.py","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/omnivinci/blob/7b3777b1c1f7c4e85f7bdef7b765dee1e76c1b7f/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--omnivinci.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2510.15870","originUrl":"https://arxiv.org/html/2510.15870v2/architecture_new.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/omnivinci.png","sha256":"57dbcefbe8eb7861d0bff9aed339847d43a06b4c78b0a03f41dbe17c01cbcc20"}},{"id":"omni:onellm","slug":"onellm","name":"OneLLM","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers questions across images, video and audio through one aligned language model.","architecture":"Universal multimodal encoder with dynamic modality routing","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["OneLLM-7B"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2023-12-06","dateKind":"paper","links":{"code":"https://github.com/csuhan/OneLLM","weights":"https://huggingface.co/csuhan/OneLLM-7B","paper":"https://arxiv.org/abs/2312.03700","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#onellm"},"license":{"status":"restricted","code":"Llama 2 community license","weights":"Llama 2 community license","codeUrl":"https://github.com/csuhan/OneLLM/blob/main/LICENSE_llama2","weightsUrl":"https://github.com/csuhan/OneLLM/blob/main/LICENSE","notes":"Custom base-model community terms apply.","components":[]},"sources":[{"kind":"paper","title":"OneLLM: One Framework to Align All Modalities with Language","url":"https://arxiv.org/abs/2312.03700","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Llama-2-7b-hf/blob/main/LICENSE.txt","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/csuhan/OneLLM","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"csuhan/OneLLM-7B — checkpoint and usage","url":"https://huggingface.co/csuhan/OneLLM-7B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/csuhan/OneLLM/blob/main/LICENSE_llama2","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation","url":"https://github.com/csuhan/OneLLM/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--onellm.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2312.03700","originUrl":"https://arxiv.org/html/2312.03700v2/framework_full.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/onellm.png","sha256":"807934fcfd09a386aec70de59611fa7d39c7f306ac3b363865dac4c46b58a5a7"}},{"id":"omni:openomni","slug":"openomni","name":"OpenOmni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Combines audiovisual understanding with multilingual and emotionally aware spoken responses.","architecture":"Language-pivoted multimodal alignment and speech synthesis","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["OpenOmni"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-01-08","dateKind":"paper","links":{"code":"https://github.com/RainBowLuoCS/OpenOmni","weights":"https://huggingface.co/Tongyi-ConvAI/OpenOmni","paper":"https://arxiv.org/abs/2501.04561","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#openomni"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0","codeUrl":"https://github.com/RainBowLuoCS/OpenOmni/blob/main/README.md","weightsUrl":"https://huggingface.co/Tongyi-ConvAI/OpenOmni/blob/30c84c70cd5e5b09ea9d26a5346fc3789401b4e2/README.md","notes":"Public code and weights were located, but a complete release license was not stated.","components":[]},"sources":[{"kind":"paper","title":"OpenOmni: Advancing Open-Source Omnimodal Large Language Models with Progressive Multimodal Alignment and Real-Time Self-Aware Emotional Speech Synthesis","url":"https://arxiv.org/abs/2501.04561","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/RainBowLuoCS/OpenOmni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Tongyi-ConvAI/OpenOmni — checkpoint and usage","url":"https://huggingface.co/Tongyi-ConvAI/OpenOmni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/RainBowLuoCS/OpenOmni/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/Tongyi-ConvAI/OpenOmni/blob/30c84c70cd5e5b09ea9d26a5346fc3789401b4e2/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--openomni.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2501.04561","originUrl":"https://arxiv.org/html/2501.04561v6/overview-v4.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/openomni.png","sha256":"2aaeefc30388402b16db4cf37c71b01dd94526c9a8b22e9eb5ed405344089fa7"}},{"id":"omni:opens2s","slug":"opens2s","name":"OpenS2S","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Recognizes emotional cues in speech and generates empathetic, streaming spoken replies.","architecture":"Audio-language backbone with interleaved streaming speech decoding","notes":"Uses the separately released GLM-4-Voice decoder for speech generation; consult that component’s terms as well.","variants":["OpenS2S"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-07-07","dateKind":"paper","links":{"code":"https://github.com/CASIA-LM/OpenS2S","weights":"https://huggingface.co/CASIA-LM/OpenS2S","paper":"https://arxiv.org/abs/2507.05177","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#opens2s"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/CASIA-LM/OpenS2S/blob/main/README.md","weightsUrl":"https://huggingface.co/CASIA-LM/OpenS2S/blob/ebea6b542b82759115ac7c9596b9bc60ee045aed/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"OpenS2S: Advancing Fully Open-Source End-to-End Empathetic Large Speech Language Model","url":"https://arxiv.org/abs/2507.05177","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/CASIA-LM/OpenS2S","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"CASIA-LM/OpenS2S — checkpoint and usage","url":"https://huggingface.co/CASIA-LM/OpenS2S","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/CASIA-LM/OpenS2S/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/CASIA-LM/OpenS2S/blob/ebea6b542b82759115ac7c9596b9bc60ee045aed/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--opens2s.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.05177","originUrl":"https://arxiv.org/html/2507.05177v3/Architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/opens2s.png","sha256":"6238ceff3e92d5914cf1d88091f826d64f4b6dca3bbdcbce6c764a20e590e85a"}},{"id":"omni:opens2s-1-5","slug":"opens2s-1-5","name":"OpenS2S 1.5","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Provides the updated OpenS2S checkpoint for expressive, empathetic spoken dialogue.","architecture":"Updated OpenS2S speech understanding and streaming generation","notes":"Uses the separately released GLM-4-Voice decoder for speech generation; consult that component’s terms as well. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["OpenS2S_V1.5"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/CASIA-LM/OpenS2S","weights":"https://huggingface.co/CASIA-LM/OpenS2S_V1.5","paper":"https://arxiv.org/abs/2507.05177","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#opens2s-1-5"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/CASIA-LM/OpenS2S/blob/main/README.md","weightsUrl":"https://huggingface.co/CASIA-LM/OpenS2S_V1.5/blob/5dc1a8a49994ddfcc95d4b6480ffadfa60c33423/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/CASIA-LM/OpenS2S","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"CASIA-LM/OpenS2S_V1.5 — checkpoint and usage","url":"https://huggingface.co/CASIA-LM/OpenS2S_V1.5","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/CASIA-LM/OpenS2S/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/CASIA-LM/OpenS2S_V1.5/blob/5dc1a8a49994ddfcc95d4b6480ffadfa60c33423/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"paper","title":"OpenS2S: Advancing Fully Open-Source End-to-End Empathetic Large Speech Language Model","url":"https://arxiv.org/abs/2507.05177","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/omni--opens2s-1-5.webp","kind":"source","label":"Family figure — Figure 1","sourceUrl":"https://arxiv.org/abs/2507.05177","originUrl":"https://arxiv.org/html/2507.05177v3/Architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/opens2s.png","sha256":"6238ceff3e92d5914cf1d88091f826d64f4b6dca3bbdcbce6c764a20e590e85a"}},{"id":"omni:osum","slug":"osum","name":"OSUM","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Recognizes words, timestamps, vocal events, emotions and speaking style from speech.","architecture":"Whisper encoder, audio adapter and Qwen language model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["OSUM"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-01-23","dateKind":"paper","links":{"code":"https://github.com/ASLP-lab/OSUM","weights":"https://huggingface.co/ASLP-lab/OSUM","paper":"https://arxiv.org/abs/2501.13306","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#osum"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/ASLP-lab/OSUM/blob/main/LICENSE.txt","weightsUrl":"https://huggingface.co/ASLP-lab/OSUM/blob/ec725b96c96a1e94578d924b5a9e0285549f5aa0/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"OSUM: Advancing Open Speech Understanding Models with Limited Resources in Academia","url":"https://arxiv.org/abs/2501.13306","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/ASLP-lab/OSUM","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ASLP-lab/OSUM — checkpoint and usage","url":"https://huggingface.co/ASLP-lab/OSUM","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/ASLP-lab/OSUM/blob/main/LICENSE.txt","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/ASLP-lab/OSUM/blob/ec725b96c96a1e94578d924b5a9e0285549f5aa0/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--osum.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2501.13306","originUrl":"https://arxiv.org/html/2501.13306v2/system.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/osum.png","sha256":"73c6a17d05eff280fda0070a459010f37507f1292695784642b0a7ae5e39b28e"}},{"id":"omni:osum-echat","slug":"osum-echat","name":"OSUM-EChat","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Uses speech understanding to produce empathetic text and voice responses.","architecture":"OSUM speech understanding connected to expressive speech generation","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["OSUM-EChat"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-08-13","dateKind":"paper","links":{"code":"https://github.com/ASLP-lab/OSUM","weights":"https://huggingface.co/ASLP-lab/OSUM-EChat","paper":"https://arxiv.org/abs/2508.09600","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#osum-echat"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/ASLP-lab/OSUM/blob/main/LICENSE.txt","weightsUrl":"https://huggingface.co/ASLP-lab/OSUM-EChat/blob/974b3d37cc774acd00c097c07961d23674a35186/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"OSUM-EChat: Enhancing End-to-End Empathetic Spoken Chatbot via Understanding-Driven Spoken Dialogue","url":"https://arxiv.org/abs/2508.09600","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/ASLP-lab/OSUM","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ASLP-lab/OSUM-EChat — checkpoint and usage","url":"https://huggingface.co/ASLP-lab/OSUM-EChat","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/ASLP-lab/OSUM/blob/main/LICENSE.txt","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/ASLP-lab/OSUM-EChat/blob/974b3d37cc774acd00c097c07961d23674a35186/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--osum-echat.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.09600","originUrl":"https://arxiv.org/html/2508.09600v2/OSUM-Chat-Structure.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/osum-echat.png","sha256":"8da3179309d457b4d1ddf3299543ff0796bdef812daa929bd6f9cc9118da076c"}},{"id":"omni:parabridge","slug":"parabridge","name":"ParaBridge","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Adapts replies to vocal emotion and other nonverbal cues in a spoken request.","architecture":"On-policy self-distillation of a Qwen3-Omni reasoning model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["ParaBridge"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-06-09","dateKind":"paper","links":{"code":"https://github.com/AmphionTeam/ParaBridge","weights":"https://huggingface.co/YuxiangW/ParaBridge","paper":"https://arxiv.org/abs/2606.10581","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#parabridge"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/AmphionTeam/ParaBridge/blob/main/LICENSE","weightsUrl":"https://huggingface.co/YuxiangW/ParaBridge/blob/0f3bb834e4bcc3840a7b75905982f56205f4e0f5/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"ParaBridge: Bridging Paralinguistic Perception and Dialogue Behavior in Speech Language Models","url":"https://arxiv.org/abs/2606.10581","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/AmphionTeam/ParaBridge","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"YuxiangW/ParaBridge — checkpoint and usage","url":"https://huggingface.co/YuxiangW/ParaBridge","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/AmphionTeam/ParaBridge/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/YuxiangW/ParaBridge/blob/0f3bb834e4bcc3840a7b75905982f56205f4e0f5/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--parabridge.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2606.10581","originUrl":"https://arxiv.org/html/2606.10581v2/training.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/parabridge.png","sha256":"f6718ffe7ed067f12972eecb2a5e8b3a2ca47abfc022fb48fb2998e90c3a0c2e"}},{"id":"omni:parakeet-qwen3","slug":"parakeet-qwen3","name":"Parakeet-TDT-v2 + Qwen3-8B","collection":"omni","category":"omni:pipeline","categoryName":"Cascaded systems","summary":"Transcribes speech with Parakeet and sends the transcript to Qwen3 for a text response.","architecture":"Cascaded ASR and text language model","notes":"VoiceBench lists this ASR + LLM composition, but a matching evaluation wrapper was not located. The Code link provides the Parakeet implementation; Qwen3 uses its standard Transformers implementation. This is a composition of public components, not a separate trained checkpoint.","variants":[],"inputs":["S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/NVIDIA-NeMo/Speech","weights":"https://huggingface.co/Qwen/Qwen3-8B","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/pipelines.md#parakeet-qwen3"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/NVIDIA-NeMo/Speech/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Qwen/Qwen3-8B/blob/b968826d9c46dd6066d109eabc6255188de91218/README.md","notes":"","components":[{"role":"ASR","checkpoint":"nvidia/parakeet-tdt-0.6b-v2","weights_url":"https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2","weights_license":"CC-BY-4.0","weights_license_url":"https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md","weights_revision":"ae9ad07059c7c739ffaf932226a8fe64ae2620b0"}]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://github.com/MatthewCYM/VoiceBench","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/Qwen/Qwen3-8B","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/Qwen/Qwen3-8B/blob/b968826d9c46dd6066d109eabc6255188de91218/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2/blob/ae9ad07059c7c739ffaf932226a8fe64ae2620b0/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Cascade component implementation and license","url":"https://github.com/NVIDIA-NeMo/Speech","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Cascade component implementation and license","url":"https://github.com/NVIDIA-NeMo/Speech/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Cascade component implementation and license","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/qwen3","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--parakeet-qwen3.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/Qwen/Qwen3-8B","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/parakeet-qwen3.svg","sha256":"b43db24a7ede67f916cf00592506f8ff6804fb0dc3d1ec766b21116a2a71009d"}},{"id":"omni:personaplex","slug":"personaplex","name":"PersonaPlex","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Holds full-duplex conversations with a role set by text and a voice set by an audio prompt.","architecture":"Moshi-based speech model conditioned on role and voice prompts","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["personaplex-7b-v1"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":"2026-01-14","dateKind":"paper","links":{"code":"https://github.com/NVIDIA/personaplex","weights":"https://huggingface.co/nvidia/personaplex-7b-v1","paper":"https://arxiv.org/abs/2602.06053","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#personaplex"},"license":{"status":"restricted","code":"MIT","weights":"NVIDIA Open Model License","codeUrl":"https://github.com/NVIDIA/personaplex/blob/main/LICENSE-MIT","weightsUrl":"https://huggingface.co/nvidia/personaplex-7b-v1/blob/fdaf4090a61cb315c138a1faee287ffd6c716309/README.md","notes":"Custom NVIDIA weight terms; access requires accepting the model agreement.","components":[]},"sources":[{"kind":"paper","title":"PersonaPlex: Voice and Role Control for Full Duplex Conversational Speech Models","url":"https://arxiv.org/abs/2602.06053","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/NVIDIA/personaplex","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"nvidia/personaplex-7b-v1 — checkpoint and usage","url":"https://huggingface.co/nvidia/personaplex-7b-v1","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/NVIDIA/personaplex/blob/main/LICENSE-MIT","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/nvidia/personaplex-7b-v1/blob/fdaf4090a61cb315c138a1faee287ffd6c716309/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--personaplex.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2602.06053","originUrl":"https://arxiv.org/html/2602.06053v1/personaplex_release_arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/personaplex.png","sha256":"93129a86a0c774ce119776b1368a2a48e36aa1fefecfbbbea7f26095574d6103"}},{"id":"omni:phi-4-multimodal","slug":"phi-4-multimodal","name":"Phi-4-multimodal","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Reads images, transcribes or translates speech, and answers multimodal questions in text.","architecture":"Modality-specific adapters on a compact language backbone","notes":"The multimodal release accepts audio and vision; Phi-4-mini is a separate text model.","variants":["Phi-4-multimodal-instruct"],"inputs":["T","I","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-03-03","dateKind":"paper","links":{"code":"https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/93f923e1a7727d1c4f446756212d9d3e8fcc5d81/modeling_phi4mm.py","weights":"https://huggingface.co/microsoft/Phi-4-multimodal-instruct","paper":"https://arxiv.org/abs/2503.01743","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#phi-4-multimodal"},"license":{"status":"open","code":"Apache-2.0","weights":"MIT","codeUrl":"https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/93f923e1a7727d1c4f446756212d9d3e8fcc5d81/modeling_phi4mm.py","weightsUrl":"https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/93f923e1a7727d1c4f446756212d9d3e8fcc5d81/LICENSE","notes":"","components":[]},"sources":[{"kind":"paper","title":"Phi-4-Mini Technical Report: Compact yet Powerful Multimodal Language Models via Mixture-of-LoRAs","url":"https://arxiv.org/abs/2503.01743","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"documentation","title":"Phi-4-multimodal — released implementation","url":"https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/93f923e1a7727d1c4f446756212d9d3e8fcc5d81/modeling_phi4mm.py","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"model-card","title":"microsoft/Phi-4-multimodal-instruct — official model card","url":"https://huggingface.co/microsoft/Phi-4-multimodal-instruct","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Phi-4-multimodal — checkpoint license evidence","url":"https://huggingface.co/microsoft/Phi-4-multimodal-instruct/blob/93f923e1a7727d1c4f446756212d9d3e8fcc5d81/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--phi-4-multimodal.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2503.01743","originUrl":"https://arxiv.org/html/2503.01743v2/phi-o.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/phi-4-multimodal.png","sha256":"b725a9511524a3f14654e58f9a97a2db350d370d05e280c8f06f73a0b53e7390"}},{"id":"omni:qwen-audio","slug":"qwen-audio","name":"Qwen-Audio","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Transcribes, translates and describes speech, sounds and music through natural-language responses.","architecture":"Multitask audio encoder connected to Qwen","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Qwen-Audio-Chat"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2023-11-14","dateKind":"paper","links":{"code":"https://github.com/QwenLM/Qwen-Audio","weights":"https://huggingface.co/Qwen/Qwen-Audio-Chat","paper":"https://arxiv.org/abs/2311.07919","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#qwen-audio"},"license":{"status":"restricted","code":"Tongyi Qianwen license","weights":"Tongyi Qianwen license","codeUrl":"https://github.com/QwenLM/Qwen-Audio/blob/main/LICENSE","weightsUrl":"https://github.com/QwenLM/Qwen-Audio/blob/main/LICENSE","notes":"Custom Qwen terms include a large-user-base commercial permission requirement.","components":[]},"sources":[{"kind":"paper","title":"Qwen-Audio: Advancing Universal Audio Understanding via Unified Large-Scale Audio-Language Models","url":"https://arxiv.org/abs/2311.07919","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/QwenLM/Qwen-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Qwen/Qwen-Audio-Chat — checkpoint and usage","url":"https://huggingface.co/Qwen/Qwen-Audio-Chat","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/QwenLM/Qwen-Audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--qwen-audio.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2311.07919","originUrl":"https://arxiv.org/html/2311.07919v2/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/qwen-audio.png","sha256":"89f9fb79891e5bd2d9f86fcc649a994cfa75c60a662a5a9e1c28d9add7cef376"}},{"id":"omni:qwen2-audio","slug":"qwen2-audio","name":"Qwen2-Audio","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers spoken instructions and analyzes speech, environmental sounds and music in text.","architecture":"Whisper-large-v3 encoder and Qwen language decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Qwen2-Audio-7B-Instruct"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2024-07-15","dateKind":"paper","links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/qwen2_audio","weights":"https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct","paper":"https://arxiv.org/abs/2407.10759","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#qwen2-audio"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct/blob/0a095220c30b7b31434169c3086508ef3ea5bf0a/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Qwen2-Audio Technical Report","url":"https://arxiv.org/abs/2407.10759","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/qwen2_audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Qwen/Qwen2-Audio-7B-Instruct — checkpoint and usage","url":"https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct/blob/0a095220c30b7b31434169c3086508ef3ea5bf0a/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--qwen2-audio.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2407.10759","originUrl":"https://arxiv.org/html/2407.10759v1/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/qwen2-audio.png","sha256":"e24601ba7e265380c379f9f11ba96e13021e017fb65324e9c9b218f537c1ad99"}},{"id":"omni:qwen2-5-omni","slug":"qwen2-5-omni","name":"Qwen2.5-Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands images, video and sound and streams coordinated text and speech replies.","architecture":"Thinker-Talker architecture with aligned audiovisual positions","notes":"Processes audiovisual inputs and produces streaming speech with text.","variants":["Qwen2.5-Omni-7B"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-03-26","dateKind":"paper","links":{"code":"https://github.com/QwenLM/Qwen2.5-Omni","weights":"https://huggingface.co/Qwen/Qwen2.5-Omni-7B","paper":"https://arxiv.org/abs/2503.20215","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#qwen2-5-omni"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/QwenLM/Qwen2.5-Omni/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Qwen/Qwen2.5-Omni-7B/blob/ae9e1690543ffd5c0221dc27f79834d0294cba00/LICENSE","notes":"","components":[]},"sources":[{"kind":"paper","title":"Qwen2.5-Omni Technical Report","url":"https://arxiv.org/abs/2503.20215","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Qwen2.5-Omni — released implementation","url":"https://github.com/QwenLM/Qwen2.5-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Qwen/Qwen2.5-Omni-7B — official model card","url":"https://huggingface.co/Qwen/Qwen2.5-Omni-7B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Qwen2.5-Omni — code license evidence","url":"https://github.com/QwenLM/Qwen2.5-Omni/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Qwen2.5-Omni — checkpoint license evidence","url":"https://huggingface.co/Qwen/Qwen2.5-Omni-7B/blob/ae9e1690543ffd5c0221dc27f79834d0294cba00/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--qwen2-5-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2503.20215","originUrl":"https://arxiv.org/html/2503.20215v1/figures/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/qwen2-5-omni.png","sha256":"2d45a64b478bee273f677c527ee8acfbebfd2c69a2169a3d035afd90eb6c1889"}},{"id":"omni:qwen3-asr","slug":"qwen3-asr","name":"Qwen3-ASR","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Identifies languages and transcribes speech across languages and dialects.","architecture":"Speech-recognition adaptation of the Qwen3-Omni backbone","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Qwen3-ASR-1.7B"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-01-29","dateKind":"paper","links":{"code":"https://github.com/QwenLM/Qwen3-ASR","weights":"https://huggingface.co/Qwen/Qwen3-ASR-1.7B","paper":"https://arxiv.org/abs/2601.21337","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#qwen3-asr"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/QwenLM/Qwen3-ASR/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Qwen3-ASR Technical Report","url":"https://arxiv.org/abs/2601.21337","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/QwenLM/Qwen3-ASR","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Qwen/Qwen3-ASR-1.7B — checkpoint and usage","url":"https://huggingface.co/Qwen/Qwen3-ASR-1.7B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/QwenLM/Qwen3-ASR/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/Qwen/Qwen3-ASR-1.7B/blob/7278e1e70fe206f11671096ffdd38061171dd6e5/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--qwen3-asr.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2601.21337","originUrl":"https://arxiv.org/html/2601.21337v2/overview.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/qwen3-asr.png","sha256":"f936045afae8c3a7a2911fcbe9c179b3f1b576f569948cc25182fa342ae03ed5"}},{"id":"omni:qwen3-omni","slug":"qwen3-omni","name":"Qwen3-Omni Instruct","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Converses about text, images, video and audio and generates streaming speech.","architecture":"Mixture-of-experts Thinker and Talker","notes":"The verified Instruct checkpoint includes Thinker and Talker, supporting visual/audio understanding and speech output.","variants":["Qwen3-Omni-30B-A3B-Instruct"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-09-22","dateKind":"paper","links":{"code":"https://github.com/QwenLM/Qwen3-Omni","weights":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct","paper":"https://arxiv.org/abs/2509.17765","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#qwen3-omni"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/QwenLM/Qwen3-Omni/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/26291f793822fb6be9555850f06dfe95f2d7e695/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Qwen3-Omni Technical Report","url":"https://arxiv.org/abs/2509.17765","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Qwen3-Omni — released implementation","url":"https://github.com/QwenLM/Qwen3-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Qwen/Qwen3-Omni-30B-A3B-Instruct — official model card","url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Qwen3-Omni — code license evidence","url":"https://github.com/QwenLM/Qwen3-Omni/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Qwen3-Omni — checkpoint license evidence","url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/26291f793822fb6be9555850f06dfe95f2d7e695/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--qwen3-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2509.17765","originUrl":"https://arxiv.org/html/2509.17765v1/figures/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/qwen3-omni.png","sha256":"2fd6b7ba1cb6f2ebe75375bd22b1cdc807457a1114c95c05f894d76bb41edf2d"}},{"id":"omni:qwen3-omni-thinking","slug":"qwen3-omni-thinking","name":"Qwen3-Omni Thinking","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Uses explicit reasoning to answer questions about text, images, video and audio.","architecture":"Reasoning-tuned mixture-of-experts Thinker","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["Qwen3-Omni-30B-A3B-Thinking"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/QwenLM/Qwen3-Omni","weights":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Thinking","paper":"https://arxiv.org/abs/2509.17765","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#qwen3-omni-thinking"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/QwenLM/Qwen3-Omni/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Thinking/blob/2f443cfc4c54b14a815c0e2bb9a9d6cbcd9a748b/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Qwen3-Omni Technical Report","url":"https://arxiv.org/abs/2509.17765","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/QwenLM/Qwen3-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Qwen/Qwen3-Omni-30B-A3B-Thinking — checkpoint and usage","url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Thinking","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/QwenLM/Qwen3-Omni/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Thinking/blob/2f443cfc4c54b14a815c0e2bb9a9d6cbcd9a748b/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--qwen3-omni-thinking.webp","kind":"source","label":"Family figure — Figure 2","sourceUrl":"https://arxiv.org/abs/2509.17765","originUrl":"https://arxiv.org/html/2509.17765v1/figures/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/qwen3-omni.png","sha256":"2fd6b7ba1cb6f2ebe75375bd22b1cdc807457a1114c95c05f894d76bb41edf2d"}},{"id":"omni:r1-omni","slug":"r1-omni","name":"R1-Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Reasons about emotion using facial, vocal and contextual cues in videos.","architecture":"Reinforcement learning on a human-centric audiovisual model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["R1-Omni-0.5B"],"inputs":["T","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/HumanMLLM/R1-Omni","weights":"https://huggingface.co/StarJiaxing/R1-Omni-0.5B","paper":"https://arxiv.org/abs/2503.05379","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#r1-omni"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0","codeUrl":"https://github.com/HumanMLLM/R1-Omni/blob/main/README.md","weightsUrl":"https://huggingface.co/StarJiaxing/R1-Omni-0.5B/blob/b3774e27104d3f0b6cceb889f93d74f5a129e59b/README.md","notes":"Public code and weights were located, but a complete release license was not stated.","components":[]},"sources":[{"kind":"paper","title":"R1-Omni: Explainable Omni-Multimodal Emotion Recognition with Reinforcement Learning","url":"https://arxiv.org/abs/2503.05379","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/HumanMLLM/R1-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"StarJiaxing/R1-Omni-0.5B — checkpoint and usage","url":"https://huggingface.co/StarJiaxing/R1-Omni-0.5B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/HumanMLLM/R1-Omni/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/StarJiaxing/R1-Omni-0.5B/blob/b3774e27104d3f0b6cceb889f93d74f5a129e59b/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--r1-omni.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/StarJiaxing/R1-Omni-0.5B","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/r1-omni.svg","sha256":"67344d771107d8a658083e1ac54febd3519b17abdba8d64e54a7e9446b56f147"}},{"id":"omni:salmonn","slug":"salmonn","name":"SALMONN","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Transcribes speech, captions sound and music, and answers audio-grounded questions.","architecture":"Whisper and BEATs encoders fused through a window-level Q-Former","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["SALMONN-7B"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2023-10-20","dateKind":"paper","links":{"code":"https://github.com/bytedance/SALMONN","weights":"https://huggingface.co/tsinghua-ee/SALMONN-7B","paper":"https://arxiv.org/abs/2310.13289","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#salmonn"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Apache-2.0 + Llama 2 terms","codeUrl":"https://github.com/bytedance/SALMONN/blob/main/LICENSE","weightsUrl":"https://huggingface.co/tsinghua-ee/SALMONN-7B/blob/ed583f2c565c1110b3bf5772ed42d6744f3cc443/README.md","notes":"The Vicuna/Llama 2 language backbone retains its community license.","components":[]},"sources":[{"kind":"paper","title":"SALMONN: Towards Generic Hearing Abilities for Large Language Models","url":"https://arxiv.org/abs/2310.13289","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Llama-2-7b-hf/blob/main/LICENSE.txt","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/bytedance/SALMONN","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"tsinghua-ee/SALMONN-7B — checkpoint and usage","url":"https://huggingface.co/tsinghua-ee/SALMONN-7B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/bytedance/SALMONN/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/tsinghua-ee/SALMONN-7B/blob/ed583f2c565c1110b3bf5772ed42d6744f3cc443/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--salmonn.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2310.13289","originUrl":"https://arxiv.org/html/2310.13289v2/salmonn.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/salmonn.png","sha256":"73d323d98fed54a7c474968f9ab2ffeccac4f5f353649b1df84ebcf5432d08c7"}},{"id":"omni:salmonn-2","slug":"salmonn-2","name":"SALMONN-2","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Understands speech, sounds and music and supports context-aware transcription and audio questions.","architecture":"Self-supervised audio representations fused across language-model layers","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["SALMONN-2-8B"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://huggingface.co/marcoyang/SALMONN-2-8B/tree/1ee8613c2d4f718e1932d4d4dbb008e1c6dc7c13","weights":"https://huggingface.co/marcoyang/SALMONN-2-8B","paper":"https://arxiv.org/abs/2607.17079","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#salmonn-2"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://huggingface.co/marcoyang/SALMONN-2-8B/blob/1ee8613c2d4f718e1932d4d4dbb008e1c6dc7c13/LICENSE","weightsUrl":"https://huggingface.co/marcoyang/SALMONN-2-8B/blob/1ee8613c2d4f718e1932d4d4dbb008e1c6dc7c13/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"SALMONN-2: Advancing General-Purpose Hearing Abilities with Self-Supervised Representations","url":"https://arxiv.org/abs/2607.17079","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://huggingface.co/marcoyang/SALMONN-2-8B/tree/1ee8613c2d4f718e1932d4d4dbb008e1c6dc7c13","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"marcoyang/SALMONN-2-8B — checkpoint and usage","url":"https://huggingface.co/marcoyang/SALMONN-2-8B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://huggingface.co/marcoyang/SALMONN-2-8B/blob/1ee8613c2d4f718e1932d4d4dbb008e1c6dc7c13/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/marcoyang/SALMONN-2-8B/blob/1ee8613c2d4f718e1932d4d4dbb008e1c6dc7c13/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--salmonn-2.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2607.17079","originUrl":"https://arxiv.org/html/2607.17079v1/SALMONN2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/salmonn-2.png","sha256":"86946611d9b4f581296d9d7c474eeb410ff1e5e58a4a5e814c12d66b3dcb4ec6"}},{"id":"omni:slam-omni","slug":"slam-omni","name":"SLAM-Omni","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Conducts multi-turn voice conversations while allowing the output speaker's timbre to be changed.","architecture":"Grouped semantic speech tokens with speaker-conditioned vocoding","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["SLAM-Omni"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2024-12-20","dateKind":"paper","links":{"code":"https://github.com/X-LANCE/SLAM-LLM","weights":"https://huggingface.co/xcczach/SLAM-Omni","paper":"https://arxiv.org/abs/2412.15649","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#slam-omni"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/X-LANCE/SLAM-LLM/blob/main/LICENSE","weightsUrl":"https://huggingface.co/xcczach/SLAM-Omni/blob/1b9e53858d45b340d41d23f538790c2095e286fb/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"SLAM-Omni: Timbre-Controllable Voice Interaction System with Single-Stage Training","url":"https://arxiv.org/abs/2412.15649","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/X-LANCE/SLAM-LLM","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"xcczach/SLAM-Omni — checkpoint and usage","url":"https://huggingface.co/xcczach/SLAM-Omni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/X-LANCE/SLAM-LLM/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/xcczach/SLAM-Omni/blob/1b9e53858d45b340d41d23f538790c2095e286fb/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--slam-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2412.15649","originUrl":"https://arxiv.org/html/2412.15649v1/SLAM-Omni.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/slam-omni.png","sha256":"e059828944fa06156b9a21f20144578ebd86d763e48bac23e386bdcdaf83d607"}},{"id":"omni:soulx-duplug","slug":"soulx-duplug","name":"SoulX-Duplug","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Predicts when a voice assistant should keep listening, respond or handle an interruption.","architecture":"Lightweight speech-language turn-control model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["SoulX-Duplug-0.6B"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-03-16","dateKind":"paper","links":{"code":"https://github.com/Soul-AILab/SoulX-Duplug","weights":"https://huggingface.co/Soul-AILab/SoulX-Duplug-0.6B","paper":"https://arxiv.org/abs/2603.14877","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#soulx-duplug"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/Soul-AILab/SoulX-Duplug/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Soul-AILab/SoulX-Duplug-0.6B/blob/61701c4ab8193cc1ee2220d3848872ac6c720142/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"SoulX-Duplug: Plug-and-Play Streaming State Prediction Module for Realtime Full-Duplex Speech Conversation","url":"https://arxiv.org/abs/2603.14877","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/Soul-AILab/SoulX-Duplug","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"Soul-AILab/SoulX-Duplug-0.6B — checkpoint and usage","url":"https://huggingface.co/Soul-AILab/SoulX-Duplug-0.6B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/Soul-AILab/SoulX-Duplug/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/Soul-AILab/SoulX-Duplug-0.6B/blob/61701c4ab8193cc1ee2220d3848872ac6c720142/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--soulx-duplug.webp","kind":"source","label":"Figure 4","sourceUrl":"https://arxiv.org/abs/2603.14877","originUrl":"https://arxiv.org/html/2603.14877v1/duplex_model_3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/soulx-duplug.png","sha256":"cc77dd46f7f0901b844fdea265482303a90f950a678096055e9a20de2e5d3e80"}},{"id":"omni:speechgpt","slug":"speechgpt","name":"SpeechGPT","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Follows spoken instructions and generates speech using discrete speech units in a language model.","architecture":"Cross-modal instruction tuning of a Llama backbone","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["SpeechGPT-7B-cm"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2023-05-18","dateKind":"paper","links":{"code":"https://github.com/0nutation/SpeechGPT","weights":"https://huggingface.co/OpenMOSS-Team/SpeechGPT-7B-cm","paper":"https://arxiv.org/abs/2305.11000","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#speechgpt"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Not stated","codeUrl":"https://github.com/0nutation/SpeechGPT/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenMOSS-Team/SpeechGPT-7B-cm/blob/f6d3fc9cfd604e530c59905d1047463083b66cdc/README.md","notes":"The repository licenses code, but a release-specific weight license was not located.","components":[]},"sources":[{"kind":"paper","title":"SpeechGPT: Empowering Large Language Models with Intrinsic Cross-Modal Conversational Abilities","url":"https://arxiv.org/abs/2305.11000","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/0nutation/SpeechGPT","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"OpenMOSS-Team/SpeechGPT-7B-cm — checkpoint and usage","url":"https://huggingface.co/OpenMOSS-Team/SpeechGPT-7B-cm","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/0nutation/SpeechGPT/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/OpenMOSS-Team/SpeechGPT-7B-cm/blob/f6d3fc9cfd604e530c59905d1047463083b66cdc/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--speechgpt.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2305.11000","originUrl":"https://arxiv.org/html/2305.11000v2/SpeechGPT-main.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/speechgpt.png","sha256":"dc20c9795f3edf8247349b1935768439063f82642182f01c936f9c9d70dac2dc"}},{"id":"omni:speechgpt-2-0-preview","slug":"speechgpt-2-0-preview","name":"SpeechGPT 2.0 Preview","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Understands spoken requests and produces natural spoken responses through a speech-native language model.","architecture":"Language modeling with a dedicated semantic-acoustic speech codec","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["SpeechGPT-2.0-preview-7B"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/OpenMOSS/SpeechGPT-2.0-preview","weights":"https://huggingface.co/OpenMOSS-Team/SpeechGPT-2.0-preview-7B","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#speechgpt-2-0-preview"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Not stated","codeUrl":"https://github.com/OpenMOSS/SpeechGPT-2.0-preview/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenMOSS-Team/SpeechGPT-2.0-preview-7B/blob/bced655764e7aac2a9cbc15f93d392de46cd6af9/README.md","notes":"The repository licenses code, but a release-specific weight license was not located.","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/OpenMOSS/SpeechGPT-2.0-preview","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"OpenMOSS-Team/SpeechGPT-2.0-preview-7B — checkpoint and usage","url":"https://huggingface.co/OpenMOSS-Team/SpeechGPT-2.0-preview-7B","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/OpenMOSS/SpeechGPT-2.0-preview/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/OpenMOSS-Team/SpeechGPT-2.0-preview-7B/blob/bced655764e7aac2a9cbc15f93d392de46cd6af9/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--speechgpt-2-0-preview.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://github.com/OpenMOSS/SpeechGPT-2.0-preview","originUrl":"https://raw.githubusercontent.com/OpenMOSS/SpeechGPT-2.0-preview/main/asset/sg2_arch.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/speechgpt-2-0-preview.png","sha256":"72fb30c9da79eb491ed3563d1ebe2b71cdc07f21d4bf5cc74a27b343244550af"}},{"id":"omni:step-audio","slug":"step-audio","name":"Step-Audio","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Supports voice conversations with expressive speech, role-play and spoken tool requests.","architecture":"Joint speech-text language modeling with audio tokenization","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Step-Audio-Chat"],"inputs":["T","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-02-17","dateKind":"paper","links":{"code":"https://github.com/stepfun-ai/Step-Audio","weights":"https://huggingface.co/stepfun-ai/Step-Audio-Chat","paper":"https://arxiv.org/abs/2502.11946","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#step-audio"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/stepfun-ai/Step-Audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stepfun-ai/Step-Audio-Chat/blob/e0335e2510b20d8b10e0ac1e07b159ef86681f14/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Step-Audio: Unified Understanding and Generation in Intelligent Speech Interaction","url":"https://arxiv.org/abs/2502.11946","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/stepfun-ai/Step-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"stepfun-ai/Step-Audio-Chat — checkpoint and usage","url":"https://huggingface.co/stepfun-ai/Step-Audio-Chat","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/stepfun-ai/Step-Audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/stepfun-ai/Step-Audio-Chat/blob/e0335e2510b20d8b10e0ac1e07b159ef86681f14/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--step-audio.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2502.11946","originUrl":"https://arxiv.org/html/2502.11946v2/stepAudio-20250217-ff.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/step-audio.png","sha256":"76ac0f53f1ca2a4017aa177ca447465ac9a9801ebdadd0c7b5714e8618a6bc2c"}},{"id":"omni:step-audio-2","slug":"step-audio-2","name":"Step-Audio 2 Mini","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Understands audio and produces expressive spoken replies, with support for tool-assisted conversations.","architecture":"Latent audio encoder and interleaved text-speech decoder","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Step-Audio-2-mini"],"inputs":["T","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-07-22","dateKind":"paper","links":{"code":"https://github.com/stepfun-ai/Step-Audio2","weights":"https://huggingface.co/stepfun-ai/Step-Audio-2-mini","paper":"https://arxiv.org/abs/2507.16632","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#step-audio-2"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/stepfun-ai/Step-Audio2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stepfun-ai/Step-Audio-2-mini/blob/e36fdd5d71e0ea22f09dd94bbab9bfc544ca1e36/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Step-Audio 2 Technical Report","url":"https://arxiv.org/abs/2507.16632","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/stepfun-ai/Step-Audio2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"stepfun-ai/Step-Audio-2-mini — checkpoint and usage","url":"https://huggingface.co/stepfun-ai/Step-Audio-2-mini","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/stepfun-ai/Step-Audio2/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/stepfun-ai/Step-Audio-2-mini/blob/e36fdd5d71e0ea22f09dd94bbab9bfc544ca1e36/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--step-audio-2.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2507.16632","originUrl":"https://arxiv.org/html/2507.16632v3/architecture5.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/step-audio-2.png","sha256":"f443c1719a09085d98af90198c27a4cd32acc386f59b95829fa98f26d341cb56"}},{"id":"omni:step-audio-2-think","slug":"step-audio-2-think","name":"Step-Audio 2 Mini Think","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Reasons before answering spoken questions and complex audio-understanding tasks.","architecture":"Reasoning-tuned Step-Audio 2 Mini","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["Step-Audio-2-mini-Think"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/stepfun-ai/Step-Audio2","weights":"https://huggingface.co/stepfun-ai/Step-Audio-2-mini-Think","paper":"https://arxiv.org/abs/2507.16632","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#step-audio-2-think"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/stepfun-ai/Step-Audio2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stepfun-ai/Step-Audio-2-mini-Think/blob/729b52fd9f4d5075fe60197e0a710be7392680a1/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Step-Audio 2 Technical Report","url":"https://arxiv.org/abs/2507.16632","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/stepfun-ai/Step-Audio2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"stepfun-ai/Step-Audio-2-mini-Think — checkpoint and usage","url":"https://huggingface.co/stepfun-ai/Step-Audio-2-mini-Think","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/stepfun-ai/Step-Audio2/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/stepfun-ai/Step-Audio-2-mini-Think/blob/729b52fd9f4d5075fe60197e0a710be7392680a1/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--step-audio-2-think.webp","kind":"source","label":"Family figure — Figure 3","sourceUrl":"https://arxiv.org/abs/2507.16632","originUrl":"https://arxiv.org/html/2507.16632v3/architecture5.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/step-audio-2.png","sha256":"f443c1719a09085d98af90198c27a4cd32acc386f59b95829fa98f26d341cb56"}},{"id":"omni:step-audio-aqaa","slug":"step-audio-aqaa","name":"Step-Audio-AQAA","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Produces expressive spoken answers directly from audio questions.","architecture":"End-to-end interleaved text and audio token generation","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Step-Audio-AQAA"],"inputs":["T","A"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-06-10","dateKind":"paper","links":{"code":"https://github.com/stepfun-ai/Step-Audio","weights":"https://huggingface.co/stepfun-ai/Step-Audio-AQAA","paper":"https://arxiv.org/abs/2506.08967","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#step-audio-aqaa"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/stepfun-ai/Step-Audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stepfun-ai/Step-Audio-AQAA/blob/34a727e86410053580e40f3c5ffab4fd36a46dd6/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Step-Audio-AQAA: a Fully End-to-End Expressive Large Audio Language Model","url":"https://arxiv.org/abs/2506.08967","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/stepfun-ai/Step-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"stepfun-ai/Step-Audio-AQAA — checkpoint and usage","url":"https://huggingface.co/stepfun-ai/Step-Audio-AQAA","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/stepfun-ai/Step-Audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/stepfun-ai/Step-Audio-AQAA/blob/34a727e86410053580e40f3c5ffab4fd36a46dd6/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--step-audio-aqaa.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.08967","originUrl":"https://arxiv.org/html/2506.08967v2/AQAA_arch03.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/step-audio-aqaa.png","sha256":"8a574903728e54e752fff2e0a911806126568ba650cc1773a463de730b328b22"}},{"id":"omni:step-audio-r1","slug":"step-audio-r1","name":"Step-Audio-R1","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Uses reinforcement-learned reasoning to answer questions about speech, sound and music.","architecture":"Reasoning-optimized audio-language model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Step-Audio-R1"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-11-19","dateKind":"paper","links":{"code":"https://github.com/stepfun-ai/Step-Audio-R1","weights":"https://huggingface.co/stepfun-ai/Step-Audio-R1","paper":"https://arxiv.org/abs/2511.15848","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#step-audio-r1"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/stepfun-ai/Step-Audio-R1/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stepfun-ai/Step-Audio-R1/blob/60a56dbe86918b0a85b07ec29f2c8983025e7073/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Step-Audio-R1 Technical Report","url":"https://arxiv.org/abs/2511.15848","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/stepfun-ai/Step-Audio-R1","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"stepfun-ai/Step-Audio-R1 — checkpoint and usage","url":"https://huggingface.co/stepfun-ai/Step-Audio-R1","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/stepfun-ai/Step-Audio-R1/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/stepfun-ai/Step-Audio-R1/blob/60a56dbe86918b0a85b07ec29f2c8983025e7073/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--step-audio-r1.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2511.15848","originUrl":"https://arxiv.org/html/2511.15848v2/overview_final.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/step-audio-r1.png","sha256":"e3d9005b90f950855c297ac10bd67cecd2e559319f1317f4813062de8702044c"}},{"id":"omni:step-audio-r1-1","slug":"step-audio-r1-1","name":"Step-Audio-R1.1","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Extends audio reasoning into responsive spoken dialogue and voice interaction.","architecture":"Dual-brain reasoning and speech articulation with acoustic-grounded reasoning","notes":"The model card connects the Realtime variant to the Mind-Paced Speaking method shown in the figure; the published serving recipe should be consulted for the supported output mode.","variants":["Step-Audio-R1.1"],"inputs":["T","A"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/stepfun-ai/Step-Audio-R1","weights":"https://huggingface.co/stepfun-ai/Step-Audio-R1.1","paper":"https://arxiv.org/abs/2510.09592","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#step-audio-r1-1"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/stepfun-ai/Step-Audio-R1/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stepfun-ai/Step-Audio-R1.1/blob/8abb296c09123345b09de57dbc2830b6a12134b1/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/stepfun-ai/Step-Audio-R1","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"stepfun-ai/Step-Audio-R1.1 — checkpoint and usage","url":"https://huggingface.co/stepfun-ai/Step-Audio-R1.1","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/stepfun-ai/Step-Audio-R1/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/stepfun-ai/Step-Audio-R1.1/blob/8abb296c09123345b09de57dbc2830b6a12134b1/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"paper","title":"Mind-Paced Speaking: A Dual-Brain Approach to Real-Time Reasoning in Spoken Language Models","url":"https://arxiv.org/abs/2510.09592","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/omni--step-audio-r1-1.webp","kind":"source","label":"Related method — Mind-Paced Speaking, Figure 3","sourceUrl":"https://arxiv.org/abs/2510.09592","originUrl":"https://arxiv.org/html/2510.09592v2/structure-spkthksim-v8.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/step-audio-r1-1.png","sha256":"5d758907428a93a9e238495dc60ede7206b78d30c73a8d54788cacfef2a4b5b4"}},{"id":"omni:stream-omni","slug":"stream-omni","name":"Stream-Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Supports simultaneous text, image and spoken interaction while aligning text and speech outputs.","architecture":"Shared language backbone with aligned streaming speech and vision interfaces","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["stream-omni-8b"],"inputs":["T","I","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-06-16","dateKind":"paper","links":{"code":"https://github.com/ictnlp/Stream-Omni","weights":"https://huggingface.co/ICTNLP/stream-omni-8b","paper":"https://arxiv.org/abs/2506.13642","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#stream-omni"},"license":{"status":"restricted","code":"GPL-3.0","weights":"GPL-3.0 + Llama 3.1 terms","codeUrl":"https://github.com/ictnlp/Stream-Omni/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ICTNLP/stream-omni-8b/blob/cf6a7854ebe9d5de92b248d390719f02be672272/README.md","notes":"The included Llama 3.1 backbone retains its community license.","components":[]},"sources":[{"kind":"paper","title":"Stream-Omni: Simultaneous Multimodal Interactions with Large Language-Vision-Speech Model","url":"https://arxiv.org/abs/2506.13642","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/ictnlp/Stream-Omni","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"ICTNLP/stream-omni-8b — checkpoint and usage","url":"https://huggingface.co/ICTNLP/stream-omni-8b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/ictnlp/Stream-Omni/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/ICTNLP/stream-omni-8b/blob/cf6a7854ebe9d5de92b248d390719f02be672272/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--stream-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2506.13642","originUrl":"https://arxiv.org/html/2506.13642v2/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/stream-omni.png","sha256":"c5b7d002ad928dd4255a7a859b6cc043c1292486fd63100aeedd2bad4481a0d6"}},{"id":"omni:sympatheia","slug":"sympatheia","name":"Sympatheia","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Adjusts spoken replies to a speaker's changing emotional state.","architecture":"Continuous affect conditioning of a GLM-4-Voice backbone","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Sympatheia"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2026-05-30","dateKind":"paper","links":{"code":"https://github.com/susameddin/sympatheia","weights":"https://huggingface.co/susameddin/Sympatheia","paper":"https://arxiv.org/abs/2606.00851","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#sympatheia"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0 + GLM-4 terms","codeUrl":"https://github.com/susameddin/sympatheia/blob/master/README.md","weightsUrl":"https://huggingface.co/susameddin/Sympatheia/blob/6fbd7205be95147cd5dec4463500ffee4e2e428d/README.md","notes":"GLM-4-Voice base-model conditions also apply; repository code licensing is not stated.","components":[]},"sources":[{"kind":"paper","title":"Sympatheia: Emotionally Adaptive Voice Assistant with Continuous Affect Conditioning","url":"https://arxiv.org/abs/2606.00851","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/susameddin/sympatheia","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"susameddin/Sympatheia — checkpoint and usage","url":"https://huggingface.co/susameddin/Sympatheia","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/susameddin/sympatheia/blob/master/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/susameddin/Sympatheia/blob/6fbd7205be95147cd5dec4463500ffee4e2e428d/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--sympatheia.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2606.00851","originUrl":"https://arxiv.org/html/2606.00851v1/figures/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/sympatheia.png","sha256":"846b962b494804244180fad1407330903b51c23f11b795a8d2ac8c000f45dc08"}},{"id":"omni:turnguide","slug":"turnguide","name":"TurnGuide","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Maintains coherent spoken conversations while managing overlaps and turn changes.","architecture":"Dynamic turn-level text-speech interleaving","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["turnguide_loss_2_1"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/dreamtheater123/TurnGuide","weights":"https://huggingface.co/qqjz/turnguide_loss_2_1","paper":"https://arxiv.org/abs/2508.07375","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#turnguide"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Not stated","codeUrl":"https://github.com/dreamtheater123/TurnGuide/blob/main/LICENSE","weightsUrl":"https://huggingface.co/qqjz/turnguide_loss_2_1/blob/37b5b8517353513e9e36935fdc4f7a156c1e87ca/README.md","notes":"No release-specific weight license is stated; the GLM-4-Voice base has custom terms.","components":[]},"sources":[{"kind":"paper","title":"TurnGuide: Enhancing Meaningful Full Duplex Spoken Interactions via Dynamic Turn-Level Text-Speech Interleaving","url":"https://arxiv.org/abs/2508.07375","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/dreamtheater123/TurnGuide","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"qqjz/turnguide_loss_2_1 — checkpoint and usage","url":"https://huggingface.co/qqjz/turnguide_loss_2_1","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/dreamtheater123/TurnGuide/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/qqjz/turnguide_loss_2_1/blob/37b5b8517353513e9e36935fdc4f7a156c1e87ca/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--turnguide.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.07375","originUrl":"https://arxiv.org/html/2508.07375v3/methodology_overall.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/turnguide.png","sha256":"c888d6a3280b4d3d346270303566760cc4d654846dd33029c2b0e440165f4504"}},{"id":"omni:ultravox-0-4-1","slug":"ultravox-0-4-1","name":"Ultravox 0.4.1 Llama-3.1-8B","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Provides an earlier Ultravox speech-understanding checkpoint for text-based voice responses.","architecture":"Whisper encoder and multimodal adapter on Llama 3.1","notes":"The checkpoint produces text from speech. A separate TTS system is needed for a spoken assistant response. The image illustrates the shared model family; checkpoint-specific changes are described in the model card. The family figure shows a Llama configuration; use the linked checkpoint card for this release’s actual backbone and dimensions.","variants":["ultravox-v0_4_1-llama-3_1-8b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/fixie-ai/ultravox","weights":"https://huggingface.co/fixie-ai/ultravox-v0_4_1-llama-3_1-8b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ultravox-0-4-1"},"license":{"status":"restricted","code":"MIT","weights":"MIT + Llama community terms","codeUrl":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fixie-ai/ultravox-v0_4_1-llama-3_1-8b/blob/d1d4298148a6584e085c15111f3348ca0cd47a19/README.md","notes":"Ultravox code is MIT; the included Llama backbone retains its community license.","components":[]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/fixie-ai/ultravox","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"fixie-ai/ultravox-v0_4_1-llama-3_1-8b — checkpoint and usage","url":"https://huggingface.co/fixie-ai/ultravox-v0_4_1-llama-3_1-8b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/fixie-ai/ultravox-v0_4_1-llama-3_1-8b/blob/d1d4298148a6584e085c15111f3348ca0cd47a19/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ultravox-0-4-1.webp","kind":"source","label":"Family figure — Llama-based example","sourceUrl":"https://github.com/fixie-ai/ultravox","originUrl":"https://raw.githubusercontent.com/fixie-ai/ultravox/main/docs/assets/Ultravox%20Model%20Architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ultravox.png","sha256":"d00b92d335dc4fb352c5d4f72553771363305f0ae5d2b8389dd0cbb2eab8cd51"}},{"id":"omni:ultravox-0-5-8b","slug":"ultravox-0-5-8b","name":"Ultravox 0.5 Llama-3.1-8B","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Handles spoken questions and instruction following with text output.","architecture":"Whisper encoder and multimodal adapter on Llama 3.1","notes":"The checkpoint produces text from speech. A separate TTS system is needed for a spoken assistant response. The image illustrates the shared model family; checkpoint-specific changes are described in the model card. The family figure shows a Llama configuration; use the linked checkpoint card for this release’s actual backbone and dimensions.","variants":["ultravox-v0_5-llama-3_1-8b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/fixie-ai/ultravox","weights":"https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_1-8b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ultravox-0-5-8b"},"license":{"status":"restricted","code":"MIT","weights":"MIT + Llama community terms","codeUrl":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_1-8b/blob/94aa77f70ca548e669ea61f737e159b2fddbb7f7/README.md","notes":"Ultravox code is MIT; the included Llama backbone retains its community license.","components":[]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/fixie-ai/ultravox","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"fixie-ai/ultravox-v0_5-llama-3_1-8b — checkpoint and usage","url":"https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_1-8b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_1-8b/blob/94aa77f70ca548e669ea61f737e159b2fddbb7f7/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ultravox-0-5-8b.webp","kind":"source","label":"Family figure — Llama-based example","sourceUrl":"https://github.com/fixie-ai/ultravox","originUrl":"https://raw.githubusercontent.com/fixie-ai/ultravox/main/docs/assets/Ultravox%20Model%20Architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ultravox.png","sha256":"d00b92d335dc4fb352c5d4f72553771363305f0ae5d2b8389dd0cbb2eab8cd51"}},{"id":"omni:ultravox-0-5-1b","slug":"ultravox-0-5-1b","name":"Ultravox 0.5 Llama-3.2-1B","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Provides speech-to-text-answer interaction with a small 1B language backbone.","architecture":"Whisper encoder and multimodal adapter on Llama 3.2","notes":"The checkpoint produces text from speech. A separate TTS system is needed for a spoken assistant response. The image illustrates the shared model family; checkpoint-specific changes are described in the model card. The family figure shows a Llama configuration; use the linked checkpoint card for this release’s actual backbone and dimensions.","variants":["ultravox-v0_5-llama-3_2-1b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/fixie-ai/ultravox","weights":"https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_2-1b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ultravox-0-5-1b"},"license":{"status":"restricted","code":"MIT","weights":"MIT + Llama community terms","codeUrl":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_2-1b/blob/b95bec8ab291eeb04b5cd600dd473377f6b79026/README.md","notes":"Ultravox code is MIT; the included Llama backbone retains its community license.","components":[]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/fixie-ai/ultravox","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"fixie-ai/ultravox-v0_5-llama-3_2-1b — checkpoint and usage","url":"https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_2-1b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_2-1b/blob/b95bec8ab291eeb04b5cd600dd473377f6b79026/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ultravox-0-5-1b.webp","kind":"source","label":"Family figure — Llama-based example","sourceUrl":"https://github.com/fixie-ai/ultravox","originUrl":"https://raw.githubusercontent.com/fixie-ai/ultravox/main/docs/assets/Ultravox%20Model%20Architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ultravox.png","sha256":"d00b92d335dc4fb352c5d4f72553771363305f0ae5d2b8389dd0cbb2eab8cd51"}},{"id":"omni:ultravox-0-6-gemma3","slug":"ultravox-0-6-gemma3","name":"Ultravox 0.6 Gemma-3-27B","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers voice requests using Gemma 3's language capabilities.","architecture":"Whisper encoder and audio projection into Gemma 3","notes":"The checkpoint produces text from speech. A separate TTS system is needed for a spoken assistant response. The image illustrates the shared model family; checkpoint-specific changes are described in the model card. The family figure shows a Llama configuration; use the linked checkpoint card for this release’s actual backbone and dimensions.","variants":["ultravox-v0_6-gemma-3-27b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/fixie-ai/ultravox","weights":"https://huggingface.co/fixie-ai/ultravox-v0_6-gemma-3-27b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ultravox-0-6-gemma3"},"license":{"status":"restricted","code":"MIT","weights":"MIT + Gemma terms","codeUrl":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fixie-ai/ultravox-v0_6-gemma-3-27b/blob/17a79bc5441ef09916cce45aae525dc6a8b0c031/README.md","notes":"The included Gemma backbone retains its custom usage terms.","components":[]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://ai.google.dev/gemma/terms","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/fixie-ai/ultravox","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"fixie-ai/ultravox-v0_6-gemma-3-27b — checkpoint and usage","url":"https://huggingface.co/fixie-ai/ultravox-v0_6-gemma-3-27b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/fixie-ai/ultravox-v0_6-gemma-3-27b/blob/17a79bc5441ef09916cce45aae525dc6a8b0c031/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ultravox-0-6-gemma3.webp","kind":"source","label":"Family figure — Llama-based example","sourceUrl":"https://github.com/fixie-ai/ultravox","originUrl":"https://raw.githubusercontent.com/fixie-ai/ultravox/main/docs/assets/Ultravox%20Model%20Architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ultravox.png","sha256":"d00b92d335dc4fb352c5d4f72553771363305f0ae5d2b8389dd0cbb2eab8cd51"}},{"id":"omni:ultravox-0-6-8b","slug":"ultravox-0-6-8b","name":"Ultravox 0.6 Llama-3.1-8B","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Responds to spoken instructions in text using the 8B Ultravox 0.6 release.","architecture":"Whisper encoder and multimodal adapter on Llama 3.1","notes":"The checkpoint produces text from speech. A separate TTS system is needed for a spoken assistant response. The image illustrates the shared model family; checkpoint-specific changes are described in the model card. The family figure shows a Llama configuration; use the linked checkpoint card for this release’s actual backbone and dimensions.","variants":["ultravox-v0_6-llama-3_1-8b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/fixie-ai/ultravox","weights":"https://huggingface.co/fixie-ai/ultravox-v0_6-llama-3_1-8b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ultravox-0-6-8b"},"license":{"status":"restricted","code":"MIT","weights":"MIT + Llama community terms","codeUrl":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fixie-ai/ultravox-v0_6-llama-3_1-8b/blob/9db56c13737ad34e962d8be02d0df5075ec0a9da/README.md","notes":"Ultravox code is MIT; the included Llama backbone retains its community license.","components":[]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/fixie-ai/ultravox","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"fixie-ai/ultravox-v0_6-llama-3_1-8b — checkpoint and usage","url":"https://huggingface.co/fixie-ai/ultravox-v0_6-llama-3_1-8b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/fixie-ai/ultravox-v0_6-llama-3_1-8b/blob/9db56c13737ad34e962d8be02d0df5075ec0a9da/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ultravox-0-6-8b.webp","kind":"source","label":"Family figure — Llama-based example","sourceUrl":"https://github.com/fixie-ai/ultravox","originUrl":"https://raw.githubusercontent.com/fixie-ai/ultravox/main/docs/assets/Ultravox%20Model%20Architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ultravox.png","sha256":"d00b92d335dc4fb352c5d4f72553771363305f0ae5d2b8389dd0cbb2eab8cd51"}},{"id":"omni:ultravox-0-6-70b","slug":"ultravox-0-6-70b","name":"Ultravox 0.6 Llama-3.3-70B","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Turns spoken questions into text answers using a large Llama 3.3 backbone.","architecture":"Whisper encoder and multimodal adapter on Llama 3.3","notes":"The checkpoint produces text from speech. A separate TTS system is needed for a spoken assistant response. The image illustrates the shared model family; checkpoint-specific changes are described in the model card. The family figure shows a Llama configuration; use the linked checkpoint card for this release’s actual backbone and dimensions.","variants":["ultravox-v0_6-llama-3_3-70b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/fixie-ai/ultravox","weights":"https://huggingface.co/fixie-ai/ultravox-v0_6-llama-3_3-70b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ultravox-0-6-70b"},"license":{"status":"restricted","code":"MIT","weights":"MIT + Llama community terms","codeUrl":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fixie-ai/ultravox-v0_6-llama-3_3-70b/blob/48ce31171d03d4aaf4356d019c3262bc8fbae90a/README.md","notes":"Ultravox code is MIT; the included Llama backbone retains its community license.","components":[]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/fixie-ai/ultravox","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"fixie-ai/ultravox-v0_6-llama-3_3-70b — checkpoint and usage","url":"https://huggingface.co/fixie-ai/ultravox-v0_6-llama-3_3-70b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/fixie-ai/ultravox-v0_6-llama-3_3-70b/blob/48ce31171d03d4aaf4356d019c3262bc8fbae90a/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ultravox-0-6-70b.webp","kind":"source","label":"Family figure — Llama-based example","sourceUrl":"https://github.com/fixie-ai/ultravox","originUrl":"https://raw.githubusercontent.com/fixie-ai/ultravox/main/docs/assets/Ultravox%20Model%20Architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ultravox.png","sha256":"d00b92d335dc4fb352c5d4f72553771363305f0ae5d2b8389dd0cbb2eab8cd51"}},{"id":"omni:ultravox-0-6-qwen3","slug":"ultravox-0-6-qwen3","name":"Ultravox 0.6 Qwen3-32B","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Adds spoken-instruction understanding and text responses to Qwen3-32B.","architecture":"Whisper encoder and learned projection into Qwen3","notes":"The checkpoint produces text from speech. A separate TTS system is needed for a spoken assistant response. The image illustrates the shared model family; checkpoint-specific changes are described in the model card. The family figure shows a Llama configuration; use the linked checkpoint card for this release’s actual backbone and dimensions.","variants":["ultravox-v0_6-qwen-3-32b"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/fixie-ai/ultravox","weights":"https://huggingface.co/fixie-ai/ultravox-v0_6-qwen-3-32b","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ultravox-0-6-qwen3"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fixie-ai/ultravox-v0_6-qwen-3-32b/blob/783a02929a402cc425ddcf7638e27e09b961d744/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/fixie-ai/ultravox","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"fixie-ai/ultravox-v0_6-qwen-3-32b — checkpoint and usage","url":"https://huggingface.co/fixie-ai/ultravox-v0_6-qwen-3-32b","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/fixie-ai/ultravox-v0_6-qwen-3-32b/blob/783a02929a402cc425ddcf7638e27e09b961d744/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ultravox-0-6-qwen3.webp","kind":"source","label":"Family figure — Llama-based example","sourceUrl":"https://github.com/fixie-ai/ultravox","originUrl":"https://raw.githubusercontent.com/fixie-ai/ultravox/main/docs/assets/Ultravox%20Model%20Architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ultravox.png","sha256":"d00b92d335dc4fb352c5d4f72553771363305f0ae5d2b8389dd0cbb2eab8cd51"}},{"id":"omni:ultravox-0-7-glm-4-6","slug":"ultravox-0-7-glm-4-6","name":"Ultravox 0.7 GLM-4.6","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Answers spoken questions and follows voice instructions using a GLM-4.6 language backbone.","architecture":"Whisper audio encoder projected into a GLM language model","notes":"The checkpoint produces text from speech. A separate TTS system is needed for a spoken assistant response. The image illustrates the shared model family; checkpoint-specific changes are described in the model card. The family figure shows a Llama configuration; use the linked checkpoint card for this release’s actual backbone and dimensions.","variants":["ultravox-v0_7-glm-4_6"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/fixie-ai/ultravox","weights":"https://huggingface.co/fixie-ai/ultravox-v0_7-glm-4_6","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#ultravox-0-7-glm-4-6"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fixie-ai/ultravox-v0_7-glm-4_6/blob/a849a0957d3f67dadc25e4fbc2e1122d5daf8ea0/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/fixie-ai/ultravox","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"fixie-ai/ultravox-v0_7-glm-4_6 — checkpoint and usage","url":"https://huggingface.co/fixie-ai/ultravox-v0_7-glm-4_6","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/fixie-ai/ultravox/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/fixie-ai/ultravox-v0_7-glm-4_6/blob/a849a0957d3f67dadc25e4fbc2e1122d5daf8ea0/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--ultravox-0-7-glm-4-6.webp","kind":"source","label":"Family figure — Llama-based example","sourceUrl":"https://github.com/fixie-ai/ultravox","originUrl":"https://raw.githubusercontent.com/fixie-ai/ultravox/main/docs/assets/Ultravox%20Model%20Architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/ultravox.png","sha256":"d00b92d335dc4fb352c5d4f72553771363305f0ae5d2b8389dd0cbb2eab8cd51"}},{"id":"omni:uni-moe-2-0-omni","slug":"uni-moe-2-0-omni","name":"Uni-MoE 2.0 Omni","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands audiovisual requests and supports text, image and speech generation.","architecture":"Multimodal mixture-of-experts with perception and generation specialists","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Uni-MoE-2.0-Omni"],"inputs":["T","I","V","A"],"outputs":["T","I","A"],"interaction":"generation","specs":{},"date":"2025-11-16","dateKind":"paper","links":{"code":"https://github.com/HITsz-TMG/Uni-MoE/tree/master/Uni-MoE-2","weights":"https://huggingface.co/HIT-TMG/Uni-MoE-2.0-Omni","paper":"https://arxiv.org/abs/2511.12609","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#uni-moe-2-0-omni"},"license":{"status":"unclear","code":"Not stated","weights":"Apache-2.0","codeUrl":"https://github.com/HITsz-TMG/Uni-MoE/tree/master/Uni-MoE-2","weightsUrl":"https://huggingface.co/HIT-TMG/Uni-MoE-2.0-Omni/blob/e4bdc0346ccd025b4994f3b61fe19567f5fe80cb/README.md","notes":"Weights are Apache-2.0; no license for this implementation directory was located.","components":[]},"sources":[{"kind":"paper","title":"Uni-MoE-2.0-Omni: Scaling Language-Centric Omnimodal Large Model with Advanced MoE, Training and Data","url":"https://arxiv.org/abs/2511.12609","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/HITsz-TMG/Uni-MoE/tree/master/Uni-MoE-2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"HIT-TMG/Uni-MoE-2.0-Omni — checkpoint and usage","url":"https://huggingface.co/HIT-TMG/Uni-MoE-2.0-Omni","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/HIT-TMG/Uni-MoE-2.0-Omni/blob/e4bdc0346ccd025b4994f3b61fe19567f5fe80cb/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--uni-moe-2-0-omni.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2511.12609","originUrl":"https://arxiv.org/html/2511.12609v2/figures/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/uni-moe-2-0-omni.png","sha256":"407115466ab5ec1b007072ad4ce9ca8af667358ef11c92c4a64d47ff5e235b42"}},{"id":"omni:vibevoice-asr","slug":"vibevoice-asr","name":"VibeVoice-ASR","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Transcribes long recordings with speaker labels and timestamps in one pass.","architecture":"Unified long-form speech understanding","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["VibeVoice-ASR"],"inputs":["T","S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2026-01-26","dateKind":"paper","links":{"code":"https://github.com/microsoft/VibeVoice","weights":"https://huggingface.co/microsoft/VibeVoice-ASR","paper":"https://arxiv.org/abs/2601.18184","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#vibevoice-asr"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/microsoft/VibeVoice/blob/main/LICENSE","weightsUrl":"https://huggingface.co/microsoft/VibeVoice-ASR/blob/d0c9efdb8d614685062c04425d91e01b6f37d944/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"VIBEVOICE-ASR Technical Report","url":"https://arxiv.org/abs/2601.18184","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/microsoft/VibeVoice","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"microsoft/VibeVoice-ASR — checkpoint and usage","url":"https://huggingface.co/microsoft/VibeVoice-ASR","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/microsoft/VibeVoice/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/microsoft/VibeVoice-ASR/blob/d0c9efdb8d614685062c04425d91e01b6f37d944/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--vibevoice-asr.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2601.18184","originUrl":"https://arxiv.org/html/2601.18184v2/vibevoice-cropped.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/vibevoice-asr.png","sha256":"4bc22d3b439ce71d54c2aff42dcef6f55625568c6f1641c40a7dd8bef4c204a4"}},{"id":"omni:video-salmonn-2","slug":"video-salmonn-2","name":"video-SALMONN 2+ (7B)","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Captions videos and answers questions using both what is seen and heard.","architecture":"Captioning-enhanced audiovisual alignment","notes":"The selected release is the complete 7B checkpoint of the audio-visual 2+ generation. The paper and figure describe the broader caption-enhanced video-SALMONN 2 family.","variants":["video-SALMONN2_plus_7B_full"],"inputs":["T","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-06-18","dateKind":"paper","links":{"code":"https://github.com/bytedance/video-SALMONN-2","weights":"https://huggingface.co/tsinghua-ee/video-SALMONN2_plus_7B_full","paper":"https://arxiv.org/abs/2506.15220","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#video-salmonn-2"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/bytedance/video-SALMONN-2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/tsinghua-ee/video-SALMONN2_plus_7B_full/blob/aab28bd55258742f7dbaca319b2031f69b662f8f/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"video-SALMONN 2: Caption-Enhanced Audio-Visual Large Language Models","url":"https://arxiv.org/abs/2506.15220","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"video-SALMONN 2 — released implementation","url":"https://github.com/bytedance/video-SALMONN-2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"tsinghua-ee/video-SALMONN2_plus_7B_full — official model card","url":"https://huggingface.co/tsinghua-ee/video-SALMONN2_plus_7B_full","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"video-SALMONN 2 — code license evidence","url":"https://github.com/bytedance/video-SALMONN-2/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"video-SALMONN 2 — checkpoint license evidence","url":"https://huggingface.co/tsinghua-ee/video-SALMONN2_plus_7B_full/blob/aab28bd55258742f7dbaca319b2031f69b662f8f/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--video-salmonn-2.webp","kind":"source","label":"Family figure · Figure 1","sourceUrl":"https://arxiv.org/abs/2506.15220","originUrl":"https://arxiv.org/html/2506.15220v3/iclr26main.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/video-salmonn-2.png","sha256":"3516d9496b49a98f668f868cd9ff0162883dfb9e6dfab7feb67200cea668b145"}},{"id":"omni:videollama-2","slug":"videollama-2","name":"VideoLLaMA 2.1 AV","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Answers questions about videos using synchronized visual and audio information.","architecture":"Spatial-temporal visual connector and an audio branch","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["VideoLLaMA2.1-7B-AV"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/DAMO-NLP-SG/VideoLLaMA2","weights":"https://huggingface.co/DAMO-NLP-SG/VideoLLaMA2.1-7B-AV","paper":"https://arxiv.org/abs/2406.07476","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#videollama-2"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Research-only","codeUrl":"https://github.com/DAMO-NLP-SG/VideoLLaMA2/blob/main/LICENSE","weightsUrl":"https://github.com/DAMO-NLP-SG/VideoLLaMA2/blob/main/README.md","notes":"The repository restricts use to noncommercial research despite the checkpoint badge.","components":[]},"sources":[{"kind":"paper","title":"VideoLLaMA 2: Advancing Spatial-Temporal Modeling and Audio Understanding in Video-LLMs","url":"https://arxiv.org/abs/2406.07476","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/DAMO-NLP-SG/VideoLLaMA2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"DAMO-NLP-SG/VideoLLaMA2.1-7B-AV — checkpoint and usage","url":"https://huggingface.co/DAMO-NLP-SG/VideoLLaMA2.1-7B-AV","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/DAMO-NLP-SG/VideoLLaMA2/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://github.com/DAMO-NLP-SG/VideoLLaMA2/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--videollama-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2406.07476","originUrl":"https://arxiv.org/html/2406.07476v3/overall_pipeline_v3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/videollama-2.png","sha256":"238009006d0c7b23d5a431dc22a3cad9e9b05e66896fde10963007aacdb5165f"}},{"id":"omni:vita","slug":"vita","name":"VITA 1.0","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Understands images, video and speech in a multimodal assistant with interruptible interaction.","architecture":"Multimodal encoders on a language backbone with system-level interaction control","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["VITA"],"inputs":["T","I","V","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2024-08-09","dateKind":"paper","links":{"code":"https://github.com/VITA-MLLM/VITA","weights":"https://huggingface.co/VITA-MLLM/VITA","paper":"https://arxiv.org/abs/2408.05211","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#vita"},"license":{"status":"restricted","code":"Tencent research license","weights":"Tencent research license","codeUrl":"https://github.com/VITA-MLLM/VITA/blob/main/License.txt","weightsUrl":"https://github.com/VITA-MLLM/VITA/blob/main/License.txt","notes":"Academic/research use only; commercial and production use are excluded.","components":[]},"sources":[{"kind":"paper","title":"VITA: Towards Open-Source Interactive Omni Multimodal LLM","url":"https://arxiv.org/abs/2408.05211","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/VITA-MLLM/VITA","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"VITA-MLLM/VITA — checkpoint and usage","url":"https://huggingface.co/VITA-MLLM/VITA","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/VITA-MLLM/VITA/blob/main/License.txt","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--vita.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2408.05211","originUrl":"https://arxiv.org/html/2408.05211v3/teaser.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/vita.png","sha256":"ab5b8dea16b8ff80a31ab7e04c123dd0403fb0a556407bec98d7d2769fe6b463"}},{"id":"omni:vita-1-5","slug":"vita-1-5","name":"VITA 1.5","collection":"omni","category":"omni:omni","categoryName":"Omni & audio-visual","summary":"Provides visual and spoken interaction with streaming speech responses.","architecture":"Vision-speech encoders and language model with streaming speech output","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["VITA-1.5"],"inputs":["T","I","V","A"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-01-03","dateKind":"paper","links":{"code":"https://github.com/VITA-MLLM/VITA","weights":"https://huggingface.co/VITA-MLLM/VITA-1.5","paper":"https://arxiv.org/abs/2501.01957","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/omni.md#vita-1-5"},"license":{"status":"restricted","code":"Tencent research license","weights":"Tencent research license","codeUrl":"https://github.com/VITA-MLLM/VITA/blob/main/License.txt","weightsUrl":"https://github.com/VITA-MLLM/VITA/blob/main/License.txt","notes":"Academic/research use only; commercial and production use are excluded.","components":[]},"sources":[{"kind":"paper","title":"VITA-1.5: Towards GPT-4o Level Real-Time Vision and Speech Interaction","url":"https://arxiv.org/abs/2501.01957","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/VITA-MLLM/VITA","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"VITA-MLLM/VITA-1.5 — checkpoint and usage","url":"https://huggingface.co/VITA-MLLM/VITA-1.5","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/VITA-MLLM/VITA/blob/main/License.txt","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--vita-1-5.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2501.01957","originUrl":"https://arxiv.org/html/2501.01957v4/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/vita-1-5.png","sha256":"88db60ea81e97aa4bbf8f86ef06b907cc224c5a3866112725637988e2230afca"}},{"id":"omni:vita-audio","slug":"vita-audio","name":"VITA-Audio","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Streams spoken responses quickly by generating multiple audio tokens per language-model step.","architecture":"Lightweight multiple cross-modal token prediction modules","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["VITA-Audio-Boost"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":"2025-05-06","dateKind":"paper","links":{"code":"https://github.com/VITA-MLLM/VITA-Audio","weights":"https://huggingface.co/VITA-MLLM/VITA-Audio-Boost","paper":"https://arxiv.org/abs/2505.03739","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#vita-audio"},"license":{"status":"restricted","code":"Tencent research license","weights":"Tencent research license","codeUrl":"https://github.com/VITA-MLLM/VITA-Audio/blob/main/LICENSE","weightsUrl":"https://github.com/VITA-MLLM/VITA-Audio/blob/main/LICENSE","notes":"Academic/research use only; commercial and production use are excluded.","components":[]},"sources":[{"kind":"paper","title":"VITA-Audio: Fast Interleaved Cross-Modal Token Generation for Efficient Large Speech-Language Model","url":"https://arxiv.org/abs/2505.03739","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/VITA-MLLM/VITA-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"VITA-MLLM/VITA-Audio-Boost — checkpoint and usage","url":"https://huggingface.co/VITA-MLLM/VITA-Audio-Boost","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/VITA-MLLM/VITA-Audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--vita-audio.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2505.03739","originUrl":"https://arxiv.org/html/2505.03739v2/VITA_audio_framework_v1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/vita-audio.png","sha256":"391d5aa4f1e48898788a178a6ee9214dab56abae691ef7f1c93bab4530ba1ea7"}},{"id":"omni:vita-audio-plus","slug":"vita-audio-plus","name":"VITA-Audio Plus","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Provides the later VITA-Audio release with accelerated interleaved speech and text generation.","architecture":"Extended multiple cross-modal token prediction architecture","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["VITA-Audio-Plus-Boost"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/VITA-MLLM/VITA-Audio","weights":"https://huggingface.co/VITA-MLLM/VITA-Audio-Plus-Boost","paper":"https://arxiv.org/abs/2505.03739","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#vita-audio-plus"},"license":{"status":"restricted","code":"Tencent research license","weights":"Tencent research license","codeUrl":"https://github.com/VITA-MLLM/VITA-Audio/blob/main/LICENSE","weightsUrl":"https://github.com/VITA-MLLM/VITA-Audio/blob/main/LICENSE","notes":"Academic/research use only; commercial and production use are excluded.","components":[]},"sources":[{"kind":"paper","title":"VITA-Audio: Fast Interleaved Cross-Modal Token Generation for Efficient Large Speech-Language Model","url":"https://arxiv.org/abs/2505.03739","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/VITA-MLLM/VITA-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"VITA-MLLM/VITA-Audio-Plus-Boost — checkpoint and usage","url":"https://huggingface.co/VITA-MLLM/VITA-Audio-Plus-Boost","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/VITA-MLLM/VITA-Audio/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--vita-audio-plus.webp","kind":"source","label":"Family figure — Figure 2","sourceUrl":"https://arxiv.org/abs/2505.03739","originUrl":"https://arxiv.org/html/2505.03739v2/VITA_audio_framework_v1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/vita-audio.png","sha256":"391d5aa4f1e48898788a178a6ee9214dab56abae691ef7f1c93bab4530ba1ea7"}},{"id":"omni:voila-autonomous","slug":"voila-autonomous","name":"Voila Autonomous","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Supports autonomous voice interaction with simultaneous listening and speaking.","architecture":"Full-duplex extension of the hierarchical Voila model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["Voila-autonomous-preview"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"full-duplex","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/maitrix-org/Voila","weights":"https://huggingface.co/maitrix-org/Voila-autonomous-preview","paper":"https://arxiv.org/abs/2505.02707","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#voila-autonomous"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/maitrix-org/Voila/blob/main/LICENSE","weightsUrl":"https://huggingface.co/maitrix-org/Voila-autonomous-preview/blob/be9ffe8da05094a17178a11ba19332c463e29406/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Voila: Voice-Language Foundation Models for Real-Time Autonomous Interaction and Voice Role-Play","url":"https://arxiv.org/abs/2505.02707","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/maitrix-org/Voila","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"maitrix-org/Voila-autonomous-preview — checkpoint and usage","url":"https://huggingface.co/maitrix-org/Voila-autonomous-preview","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/maitrix-org/Voila/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/maitrix-org/Voila-autonomous-preview/blob/be9ffe8da05094a17178a11ba19332c463e29406/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--voila-autonomous.webp","kind":"source","label":"Family figure — Figure 2","sourceUrl":"https://arxiv.org/abs/2505.02707","originUrl":"https://arxiv.org/html/2505.02707v1/voila-archs-joint.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/voila.png","sha256":"6cd14d6bbb6b29c206e1b833fb1bfb743c76e22b47aed8fbe0a4c138a3b47063"}},{"id":"omni:voila","slug":"voila","name":"Voila Chat","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Holds voice conversations with speaker identity and speaking style controlled by prompts.","architecture":"Hierarchical voice-language backbone and acoustic Transformer","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Voila-chat"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2025-05-05","dateKind":"paper","links":{"code":"https://github.com/maitrix-org/Voila","weights":"https://huggingface.co/maitrix-org/Voila-chat","paper":"https://arxiv.org/abs/2505.02707","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#voila"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/maitrix-org/Voila/blob/main/LICENSE","weightsUrl":"https://huggingface.co/maitrix-org/Voila-chat/blob/ad4ee33219a02808841b575cbdbd439e0100848c/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Voila: Voice-Language Foundation Models for Real-Time Autonomous Interaction and Voice Role-Play","url":"https://arxiv.org/abs/2505.02707","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/maitrix-org/Voila","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"maitrix-org/Voila-chat — checkpoint and usage","url":"https://huggingface.co/maitrix-org/Voila-chat","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/maitrix-org/Voila/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/maitrix-org/Voila-chat/blob/ad4ee33219a02808841b575cbdbd439e0100848c/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--voila.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2505.02707","originUrl":"https://arxiv.org/html/2505.02707v1/voila-archs-joint.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/voila.png","sha256":"6cd14d6bbb6b29c206e1b833fb1bfb743c76e22b47aed8fbe0a4c138a3b47063"}},{"id":"omni:voxmind","slug":"voxmind","name":"VoxMind","collection":"omni","category":"omni:speech","categoryName":"Speech dialogue","summary":"Handles spoken requests that require reasoning, planning and external tool use.","architecture":"Think-before-speak dialogue model with dynamic tool management","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["VoxMind"],"inputs":["T","S"],"outputs":["T","S"],"interaction":"generation","specs":{},"date":"2026-04-17","dateKind":"paper","links":{"code":"https://github.com/MM-Speech/VoxMind","weights":"https://huggingface.co/leungtianle/VoxMind","paper":"https://arxiv.org/abs/2604.15710","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/speech.md#voxmind"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/MM-Speech/VoxMind/blob/main/README.md","weightsUrl":"https://huggingface.co/leungtianle/VoxMind/blob/0d5d1df306ff78da40058f5aee2c74a4a33dda20/README.md","notes":"Public code and weights were located, but a complete release license was not stated.","components":[]},"sources":[{"kind":"paper","title":"VoxMind: An End-to-End Agentic Spoken Dialogue System","url":"https://arxiv.org/abs/2604.15710","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/MM-Speech/VoxMind","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"leungtianle/VoxMind — checkpoint and usage","url":"https://huggingface.co/leungtianle/VoxMind","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/MM-Speech/VoxMind/blob/main/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/leungtianle/VoxMind/blob/0d5d1df306ff78da40058f5aee2c74a4a33dda20/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--voxmind.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2604.15710","originUrl":"https://arxiv.org/html/2604.15710v1/Architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/voxmind.png","sha256":"f5f65762d602a4779a7e3fd3d40d4ea785aed15f9bca5588bde959b37e288b4a"}},{"id":"omni:voxtral-mini","slug":"voxtral-mini","name":"Voxtral Mini","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Provides transcription and audio question answering in a smaller 3B configuration.","architecture":"Compact Voxtral audio-language architecture","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements. The image illustrates the shared model family; checkpoint-specific changes are described in the model card.","variants":["Voxtral-Mini-3B-2507"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/mistralai/mistral-inference","weights":"https://huggingface.co/mistralai/Voxtral-Mini-3B-2507","paper":"https://arxiv.org/abs/2507.13264","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#voxtral-mini"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/mistralai/mistral-inference/blob/main/LICENSE","weightsUrl":"https://huggingface.co/mistralai/Voxtral-Mini-3B-2507/blob/3060fe34b35ba5d44202ce9ff3c097642914f8f3/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Voxtral","url":"https://arxiv.org/abs/2507.13264","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/mistralai/mistral-inference","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"mistralai/Voxtral-Mini-3B-2507 — checkpoint and usage","url":"https://huggingface.co/mistralai/Voxtral-Mini-3B-2507","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/mistralai/mistral-inference/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/mistralai/Voxtral-Mini-3B-2507/blob/3060fe34b35ba5d44202ce9ff3c097642914f8f3/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--voxtral-mini.webp","kind":"source","label":"Family figure — Figure 1","sourceUrl":"https://arxiv.org/abs/2507.13264","originUrl":"https://arxiv.org/html/2507.13264v1/images/voxtral-arch.002.jpeg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/voxtral.png","sha256":"796c9731ac7bbc57217e7877c6275fd49ebf439290b5ec8cbcb6112a319aec4c"}},{"id":"omni:voxtral-realtime","slug":"voxtral-realtime","name":"Voxtral Realtime","collection":"omni","category":"omni:asr","categoryName":"Speech recognition","summary":"Transcribes live audio incrementally for low-latency applications.","architecture":"Streaming audio encoding and causal language decoding","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Voxtral-Mini-4B-Realtime-2602"],"inputs":["S"],"outputs":["T"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/voxtral_realtime","weights":"https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/asr.md#voxtral-realtime"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/huggingface/transformers/blob/main/LICENSE","weightsUrl":"https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602/blob/2769294da9567371363522aac9bbcfdd19447add/README.md","notes":"","components":[]},"sources":[{"kind":"repository","title":"Published model implementation","url":"https://github.com/huggingface/transformers/tree/main/src/transformers/models/voxtral_realtime","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"mistralai/Voxtral-Mini-4B-Realtime-2602 — checkpoint and usage","url":"https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/huggingface/transformers/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602/blob/2769294da9567371363522aac9bbcfdd19447add/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--voxtral-realtime.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/mistralai/Voxtral-Mini-4B-Realtime-2602","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/voxtral-realtime.svg","sha256":"c7e7352d110a87db9257fe431ac8d9c55b24521eda2cdd69bb6715beff64599f"}},{"id":"omni:voxtral","slug":"voxtral","name":"Voxtral Small","collection":"omni","category":"omni:audio","categoryName":"Audio understanding","summary":"Transcribes and translates speech and answers questions about long audio recordings.","architecture":"Audio encoder connected to a Mistral language model","notes":"Capabilities refer to the linked checkpoint; consult its model card for task prompts and deployment requirements.","variants":["Voxtral-Small-24B-2507"],"inputs":["T","A"],"outputs":["T"],"interaction":"text-output","specs":{},"date":"2025-07-17","dateKind":"paper","links":{"code":"https://github.com/mistralai/mistral-inference","weights":"https://huggingface.co/mistralai/Voxtral-Small-24B-2507","paper":"https://arxiv.org/abs/2507.13264","catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/audio.md#voxtral"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/mistralai/mistral-inference/blob/main/LICENSE","weightsUrl":"https://huggingface.co/mistralai/Voxtral-Small-24B-2507/blob/da5b42409f279fdd92febee0511a6c32828569c1/README.md","notes":"","components":[]},"sources":[{"kind":"paper","title":"Voxtral","url":"https://arxiv.org/abs/2507.13264","reviewed_on":"2026-09-13","review_level":"paper-sections"},{"kind":"repository","title":"Published model implementation","url":"https://github.com/mistralai/mistral-inference","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"model-card","title":"mistralai/Voxtral-Small-24B-2507 — checkpoint and usage","url":"https://huggingface.co/mistralai/Voxtral-Small-24B-2507","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"documentation","title":"Code license evidence","url":"https://github.com/mistralai/mistral-inference/blob/main/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Weight license evidence","url":"https://huggingface.co/mistralai/Voxtral-Small-24B-2507/blob/da5b42409f279fdd92febee0511a6c32828569c1/README.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--voxtral.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.13264","originUrl":"https://arxiv.org/html/2507.13264v1/images/voxtral-arch.002.jpeg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/voxtral.png","sha256":"796c9731ac7bbc57217e7877c6275fd49ebf439290b5ec8cbcb6112a319aec4c"}},{"id":"omni:whisper-v3-llama31","slug":"whisper-v3-llama31","name":"Whisper-v3-large + Llama-3.1-8B","collection":"omni","category":"omni:pipeline","categoryName":"Cascaded systems","summary":"Uses Whisper to transcribe a spoken request and Llama 3.1 to answer it.","architecture":"Cascaded Whisper ASR and Llama text generation","notes":"A benchmark composition of separately released models, not a separately trained speech-language checkpoint. The text LLM receives the ASR transcript.","variants":[],"inputs":["S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/src/models/naive.py","weights":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/pipelines.md#whisper-v3-llama31"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Llama community license","codeUrl":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/LICENSE","weightsUrl":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/0e9e39f249a16976918f6564b8830bc894c89659/README.md","notes":"The text backbone uses the Llama community license.","components":[{"role":"ASR","checkpoint":"openai/whisper-large-v3","weights_url":"https://huggingface.co/openai/whisper-large-v3","weights_license":"Apache-2.0","weights_license_url":"https://huggingface.co/openai/whisper-large-v3/blob/06f233fe06e710322aca913c1bc4249a0d71fce1/README.md","weights_revision":"06f233fe06e710322aca913c1bc4249a0d71fce1"}]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/src/models/naive.py","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/0e9e39f249a16976918f6564b8830bc894c89659/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/openai/whisper-large-v3","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/openai/whisper-large-v3/blob/06f233fe06e710322aca913c1bc4249a0d71fce1/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--whisper-v3-llama31.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/whisper-v3-llama31.svg","sha256":"a000f1c18d5da3f85d0e0aba2d45e133f6e94570b279981891477e242d61f681"}},{"id":"omni:whisper-turbo-llama31","slug":"whisper-turbo-llama31","name":"Whisper-v3-turbo + Llama-3.1-8B","collection":"omni","category":"omni:pipeline","categoryName":"Cascaded systems","summary":"Pairs faster Whisper transcription with Llama 3.1 text responses.","architecture":"Cascaded Whisper Turbo ASR and Llama text generation","notes":"A benchmark composition of separately released models, not a separately trained speech-language checkpoint. The text LLM receives the ASR transcript.","variants":[],"inputs":["S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/src/models/naive3.py","weights":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/pipelines.md#whisper-turbo-llama31"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Llama community license","codeUrl":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/LICENSE","weightsUrl":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/0e9e39f249a16976918f6564b8830bc894c89659/README.md","notes":"The text backbone uses the Llama community license.","components":[{"role":"ASR","checkpoint":"openai/whisper-large-v3-turbo","weights_url":"https://huggingface.co/openai/whisper-large-v3-turbo","weights_license":"MIT","weights_license_url":"https://huggingface.co/openai/whisper-large-v3-turbo/blob/41f01f3fe87f28c78e2fbf8b568835947dd65ed9/README.md","weights_revision":"41f01f3fe87f28c78e2fbf8b568835947dd65ed9"}]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/src/models/naive3.py","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/0e9e39f249a16976918f6564b8830bc894c89659/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/openai/whisper-large-v3-turbo","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/openai/whisper-large-v3-turbo/blob/41f01f3fe87f28c78e2fbf8b568835947dd65ed9/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--whisper-turbo-llama31.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/whisper-turbo-llama31.svg","sha256":"c4514355886abaed9d6f073815800b08a7bb58446bbbbd16e366b37b9d745ce1"}},{"id":"omni:whisper-turbo-llama32","slug":"whisper-turbo-llama32","name":"Whisper-v3-turbo + Llama-3.2-3B","collection":"omni","category":"omni:pipeline","categoryName":"Cascaded systems","summary":"Combines Whisper Turbo with a smaller Llama 3.2 model for spoken question answering.","architecture":"Cascaded Whisper Turbo ASR and compact Llama text generation","notes":"A benchmark composition of separately released models, not a separately trained speech-language checkpoint. The text LLM receives the ASR transcript.","variants":[],"inputs":["S"],"outputs":["T"],"interaction":"text-output","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/src/models/naive4.py","weights":"https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct","paper":null,"catalog":"https://github.com/kadirnar/awesome-omni-architectures/blob/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/models/pipelines.md#whisper-turbo-llama32"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Llama community license","codeUrl":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/LICENSE","weightsUrl":"https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct/blob/0cb88a4f764b7a12671c53f0838cd831a0843b95/README.md","notes":"The text backbone uses the Llama community license.","components":[{"role":"ASR","checkpoint":"openai/whisper-large-v3-turbo","weights_url":"https://huggingface.co/openai/whisper-large-v3-turbo","weights_license":"MIT","weights_license_url":"https://huggingface.co/openai/whisper-large-v3-turbo/blob/41f01f3fe87f28c78e2fbf8b568835947dd65ed9/README.md","weights_revision":"41f01f3fe87f28c78e2fbf8b568835947dd65ed9"}]},"sources":[{"kind":"documentation","title":"Release documentation and license evidence","url":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/src/models/naive4.py","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct/blob/0cb88a4f764b7a12671c53f0838cd831a0843b95/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/openai/whisper-large-v3-turbo","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://huggingface.co/openai/whisper-large-v3-turbo/blob/41f01f3fe87f28c78e2fbf8b568835947dd65ed9/README.md","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Release documentation and license evidence","url":"https://github.com/MatthewCYM/VoiceBench/blob/6992cf4fc51d0426c52c4805b5002e0aae49118a/LICENSE","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/omni--whisper-turbo-llama32.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-omni-architectures/8c5bae28e8e41f36e8db8ecca97e94b7c133a0fc/assets/architectures/whisper-turbo-llama32.svg","sha256":"0b2d1a88faab66ebf1bebdf1e5aa01666991babb6860791e509a7f223fdb7247"}},{"id":"tts:bark","slug":"bark","name":"Bark","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Text-conditioned speech, nonverbal sounds and some music generation.","architecture":"Hierarchical autoregressive audio tokens","notes":"Text-conditioned speech, nonverbal sounds and some music generation.","variants":["Bark","Bark Small"],"inputs":["T"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/suno-ai/bark","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#bark"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"suno-ai/bark — official repository","url":"https://github.com/suno-ai/bark","reviewed_on":"2026-09-12","review_level":"readme"}],"figure":{"path":"assets/figures/tts--bark.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/suno-ai/bark","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/bark.svg","sha256":"f561b96ccfe2993ef1e626f6afcc142ee1f7fd3281072a5607bad49d37725b0c"}},{"id":"tts:base-tts","slug":"base-tts","name":"BASE TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Models discrete speech codes with an incremental waveform decoder.","architecture":"Autoregressive speechcodes and convolutional decoder","notes":"Models discrete speech codes with an incremental waveform decoder.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2024-02-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2402.08093","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#base-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"BASE TTS: Lessons from building a billion-parameter Text-to-Speech model on 100K hours of data","url":"https://arxiv.org/abs/2402.08093","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--base-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2402.08093","originUrl":"https://arxiv.org/html/2402.08093v2/figures/gpt.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/base-tts.png","sha256":"f686dfa579a9f36372277c3d0201dca536ae0a3c84efd2b62f863fd2adc8e552"}},{"id":"tts:chatterbox","slug":"chatterbox","name":"Chatterbox","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Chatterbox synthesizes speech from text and a voice reference, with controls for expressive delivery. Its multilingual models focus on cross-language voice consistency, while Turbo and Nano use a smaller backbone and a single-step acoustic decoder. These variants serve different narration, conversational playback and local-device requirements; their capabilities are not interchangeable.","architecture":"Speech-token LM and acoustic decoder","notes":"Family of reference-conditioned speech synthesizers; capabilities vary by release.","variants":["Original","Multilingual V2","Multilingual V3","Turbo","Nano","Single Language Pack"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/resemble-ai/chatterbox","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#chatterbox"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"resemble-ai/chatterbox — official repository","url":"https://github.com/resemble-ai/chatterbox","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--chatterbox.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/resemble-ai/chatterbox","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/chatterbox.svg","sha256":"9dafd0fd97c5e5ac0f655337ef70215b57329a61ae5079e201c4d24f28e9c079"}},{"id":"tts:chattts","slug":"chattts","name":"ChatTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Synthesizes conversational delivery from supplied text; does not itself answer a user's questions.","architecture":"Autoregressive speech-token generation","notes":"Synthesizes conversational delivery from supplied text; does not itself answer a user's questions.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/2noise/ChatTTS","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#chattts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"2noise/ChatTTS — official repository","url":"https://github.com/2noise/ChatTTS","reviewed_on":"2026-09-12","review_level":"readme"}],"figure":{"path":"assets/figures/tts--chattts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/2noise/ChatTTS","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/chattts.svg","sha256":"6bf3df2420c6571ce500a77b75b14640f064baa4e6a0eaa941f8a6dfd8526279"}},{"id":"tts:clam-tts","slug":"clam-tts","name":"CLaM-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Predicts multiple codec tokens jointly to reduce generation cost.","architecture":"Probabilistic residual quantization and multi-token LM","notes":"Predicts multiple codec tokens jointly to reduce generation cost.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-04-03","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2404.02781","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#clam-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CLaM-TTS: Improving Neural Codec Language Model for Zero-Shot Text-to-Speech","url":"https://arxiv.org/abs/2404.02781","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--clam-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2404.02781","originUrl":"https://arxiv.org/html/2404.02781v1/Fig1_overview_R4.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/clam-tts.png","sha256":"171c285f64d3ed0e16a7327eda80ca52d684b1632fa45ca60dcdb4f3f3b6d39c"}},{"id":"tts:cosyvoice","slug":"cosyvoice","name":"CosyVoice","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Text-conditioned multilingual synthesis and voice conditioning.","architecture":"Supervised semantic tokens and flow decoder","notes":"Text-conditioned multilingual synthesis and voice conditioning.","variants":["300M","SFT","Instruct"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-07-07","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2407.05407","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#cosyvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CosyVoice: A Scalable Multilingual Zero-shot Text-to-speech Synthesizer based on Supervised Semantic Tokens","url":"https://arxiv.org/abs/2407.05407","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--cosyvoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2407.05407","originUrl":"https://arxiv.org/html/2407.05407v2/overview2.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cosyvoice.png","sha256":"363c28dc7d28629fb9f2da47a843fd48e40de8bc7c7bc1a90398325703f5bd0f"}},{"id":"tts:cosyvoice-2","slug":"cosyvoice-2","name":"CosyVoice 2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Supports streaming text input and audio output.","architecture":"Text/speech LM and chunk-aware flow matching","notes":"Supports streaming text input and audio output.","variants":["0.5B"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2024-12-13","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2412.10117","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#cosyvoice-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CosyVoice 2: Scalable Streaming Speech Synthesis with Large Language Models","url":"https://arxiv.org/abs/2412.10117","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--cosyvoice-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2412.10117","originUrl":"https://arxiv.org/html/2412.10117v3/CosyVoice2-overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cosyvoice-2.png","sha256":"963637ac577d0e2739d370598af5babea34f54ba3d38736f6b40e7b4e6db30fe"}},{"id":"tts:cosyvoice-3","slug":"cosyvoice-3","name":"CosyVoice 3","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"CosyVoice 3 extends streaming, reference-conditioned synthesis with a tokenizer trained on several speech-understanding tasks and a reward model for post-training. It targets reliable pronunciation, speaker identity and prosody across languages, dialects and less controlled text. The research scaling experiments and the downloadable Fun-CosyVoice3 checkpoint represent different model configurations.","architecture":"Scaled speech LM and post-training","notes":"Extends coverage and control of the CosyVoice streaming synthesis line.","variants":["Fun-CosyVoice3-0.5B"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-05-23","dateKind":"paper","links":{"code":"https://github.com/QwenAudio/CosyVoice","weights":null,"paper":"https://arxiv.org/abs/2505.17589","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#cosyvoice-3"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CosyVoice 3: Towards In-the-wild Speech Generation via Scaling-up and Post-training","url":"https://arxiv.org/abs/2505.17589","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"CosyVoice — official implementation","url":"https://github.com/QwenAudio/CosyVoice","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"CosyVoice 3 author demo","url":"https://funaudiollm.github.io/cosyvoice3","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--cosyvoice-3.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2505.17589","originUrl":"https://arxiv.org/html/2505.17589v2/overview.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cosyvoice-3.png","sha256":"a287801e5e05858bc157665b3f0b053557c8cc93e7f4564c023b447115e18b87"}},{"id":"tts:csm","slug":"csm","name":"CSM","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"CSM uses the text and audio of preceding speaker turns to shape the delivery of the next utterance. A Llama backbone predicts speech representations, and a smaller decoder completes Mimi audio codes. It is a contextual speech renderer: an application supplies the words to say, including any responses written by a separate language model.","architecture":"Llama backbone and Mimi-code decoder","notes":"Conversational Speech Model renders speech from text and audio context; the public checkpoint is not a complete voice assistant.","variants":["CSM-1B"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/sesameailabs/csm","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#csm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"sesameailabs/csm — official repository","url":"https://github.com/sesameailabs/csm","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--csm.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/sesameailabs/csm","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/csm.svg","sha256":"e5e1a07ea65370d91e74e639b2c40cc2025ea8c4a56bf79fa3cd7e37f1e8e598"}},{"id":"tts:dia","slug":"dia","name":"Dia","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Dia turns a speaker-tagged transcript into conversational audio, including supported nonverbal events such as laughter and coughing. Reference audio and its transcript can establish speaker identity and delivery. Its English checkpoint is useful for scripted exchanges and dialogue narration, with the conversation content supplied by the user rather than generated by the model.","architecture":"Dialogue-conditioned codec generation","notes":"Renders a speaker-tagged script into multi-speaker speech with optional audio context.","variants":["1.6B"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/nari-labs/dia","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#dia"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"nari-labs/dia — official repository","url":"https://github.com/nari-labs/dia","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--dia.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/nari-labs/dia","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dia.svg","sha256":"2f8209991c02ac4d8a6f32b2043a5f3cb308a0ab1d3bb7e07ff2c9ce7e9cae68"}},{"id":"tts:dia2","slug":"dia2","name":"Dia2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Dia2 begins synthesizing before the complete script is available, allowing an application to feed words incrementally. Audio prefixes provide speaker and conversational context, while a streaming codec path reconstructs the output. The released 1B and 2B checkpoints focus on English dialogue; prefix conditioning helps keep voices consistent between generations.","architecture":"Streaming dialogue TTS","notes":"Starts synthesis from partial text and supports audio prefix conditioning.","variants":["1B","2B"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/nari-labs/dia2","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#dia2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"nari-labs/dia2 — official repository","url":"https://github.com/nari-labs/dia2","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--dia2.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/nari-labs/dia2","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dia2.svg","sha256":"b59735e1d318cc684acdc0c4ade41dc62bfc7234e3860020a2588cbd64777fac"}},{"id":"tts:ella-v","slug":"ella-v","name":"ELLA-V","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Interleaves phonemes and acoustic codes to stabilize autoregressive synthesis.","architecture":"Alignment-guided token reordering","notes":"Interleaves phonemes and acoustic codes to stabilize autoregressive synthesis.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-01-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2401.07333","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#ella-v"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ELLA-V: Stable Neural Codec Language Modeling with Alignment-guided Sequence Reordering","url":"https://arxiv.org/abs/2401.07333","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--ella-v.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2401.07333","originUrl":"https://arxiv.org/html/2401.07333v1/overall4.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ella-v.png","sha256":"a87d48fb2751cfaa0897006d05b1dba9229be9288527be4fea6ee7ba72a62ded"}},{"id":"tts:fireredtts","slug":"fireredtts","name":"FireRedTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Foundation TTS system for expressive and reference-conditioned speech.","architecture":"Text-to-semantic LM and speech decoder","notes":"Foundation TTS system for expressive and reference-conditioned speech.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-09-05","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2409.03283","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#fireredtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FireRedTTS: A Foundation Text-To-Speech Framework for Industry-Level Generative Speech Applications","url":"https://arxiv.org/abs/2409.03283","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--fireredtts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2409.03283","originUrl":"https://arxiv.org/html/2409.03283v2/image/fireredtts.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fireredtts.png","sha256":"c3029824ebd963713a8ac247acf1ef2cada22e97066e623e86cdf0f8973df102"}},{"id":"tts:fireredtts-2","slug":"fireredtts-2","name":"FireRedTTS-2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"FireRedTTS-2 models chronological sequences of speaker-labeled text and speech using a large Transformer plus a smaller codebook decoder. A low-rate streaming tokenizer reduces the number of audio steps. It targets long conversations and podcasts where speaker changes, turn-specific delivery and continuity across utterances matter.","architecture":"Low-rate streaming codec and dialogue LM","notes":"Generates context-aware multi-speaker speech from dialogue text.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-09-02","dateKind":"paper","links":{"code":"https://github.com/FireRedTeam/FireRedTTS2","weights":null,"paper":"https://arxiv.org/abs/2509.02020","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#fireredtts-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FireRedTTS-2: Towards Long Conversational Speech Generation for Podcast and Chatbot","url":"https://arxiv.org/abs/2509.02020","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"FireRedTTS-2 author repository","url":"https://github.com/FireRedTeam/FireRedTTS2","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--fireredtts-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.02020","originUrl":"https://arxiv.org/html/2509.02020v2/image/tts_model_framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fireredtts-2.png","sha256":"e87b4985379a791a35d3ce64b3a3988d70cfe19b55062ccaad04727cac4178c5"}},{"id":"tts:openaudio-s1","slug":"openaudio-s1","name":"Fish Audio S1 / OpenAudio S1","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"The S1 family combines multilingual voice conditioning with explicit markers for emotion, tone and nonverbal sounds. Its full model and distilled S1-mini offer different deployment sizes, with reinforcement learning used to refine generation. It is suited to expressive narration and character dialogue, although access and capabilities depend on the selected release.","architecture":"Dual-autoregressive speech language model","notes":"Expressive TTS family also known as OpenAudio S1; the model card distinguishes the full S1 model and distilled S1-mini.","variants":["S1","S1-mini"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#openaudio-s1"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"model-card","title":"fishaudio/s1-mini · Hugging Face","url":"https://huggingface.co/fishaudio/s1-mini","reviewed_on":"2026-09-13","review_level":"model-card"}],"figure":{"path":"assets/figures/tts--openaudio-s1.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/fishaudio/s1-mini","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/openaudio-s1.svg","sha256":"1cef2d03a0da1bb71deff6befe7d99b90f460ffd154dc9ef72054afa3d0bf9e2"}},{"id":"tts:fish-audio-s2","slug":"fish-audio-s2","name":"Fish Audio S2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Fish Audio S2 extends the Fish speech-model line with natural-language delivery instructions and multi-speaker, multi-turn synthesis. Its training pipeline uses speech descriptions, quality assessment and reward modeling to improve controllability. The released inference stack supports streaming, making the model relevant to both scripted audio production and incremental spoken responses.","architecture":"Dual-autoregressive speech modeling","notes":"Instruction-controlled multi-speaker, multi-turn speech synthesis.","variants":["S2","S2-Pro"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-09","dateKind":"paper","links":{"code":"https://github.com/fishaudio/fish-speech","weights":null,"paper":"https://arxiv.org/abs/2603.08823","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#fish-audio-s2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Fish Audio S2 Technical Report","url":"https://arxiv.org/abs/2603.08823","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"fishaudio/fish-speech — official repository","url":"https://github.com/fishaudio/fish-speech","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--fish-audio-s2.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2603.08823","originUrl":"https://arxiv.org/html/2603.08823v2/figures/fish_audio_s2_arch_v16.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fish-audio-s2.png","sha256":"98254426d8f9a4d9e6e80b56b6a086ee12bacacd1c7de40717ba9584baf5b532"}},{"id":"tts:fish-speech","slug":"fish-speech","name":"Fish Speech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Separates semantic sequence modeling from grouped acoustic-token prediction.","architecture":"Dual-autoregressive slow/fast transformers","notes":"Separates semantic sequence modeling from grouped acoustic-token prediction.","variants":["Fish Speech release family"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-11-02","dateKind":"paper","links":{"code":"https://github.com/fishaudio/fish-speech","weights":null,"paper":"https://arxiv.org/abs/2411.01156","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#fish-speech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Fish-Speech: Leveraging Large Language Models for Advanced Multilingual Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2411.01156","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"fishaudio/fish-speech — official repository","url":"https://github.com/fishaudio/fish-speech","reviewed_on":"2026-09-12","review_level":"readme"}],"figure":{"path":"assets/figures/tts--fish-speech.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2411.01156","originUrl":"https://arxiv.org/html/2411.01156v2/DualAR2_0.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fish-speech.png","sha256":"d19ffcc8f5b0d6389972342976dfc8d816845f03636d6c82b8c55c5fac923add"}},{"id":"tts:gemini-2-5-tts","slug":"gemini-2-5-tts","name":"Gemini 2.5 TTS","collection":"tts","category":"tts:api","categoryName":"Commercial TTS","summary":"Gemini 2.5 TTS converts supplied text into speech with prompt-based control of accent, pace, style and emotion. The Flash and Pro interfaces support single-speaker narration and two-speaker scripts with separately assigned voices. These are dedicated speech-generation endpoints; their internal acoustic architecture is not fully disclosed in the public documentation.","architecture":"Gemini-derived text-to-speech","notes":"Specialized speech-generation endpoints; separate from the Live API.","variants":["2.5 Flash Preview TTS","2.5 Pro Preview TTS"],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/api.md#gemini-2-5-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"documentation","title":"Gemini 2.5 Flash text-to-speech | Gemini API | Google AI for Developers","url":"https://ai.google.dev/gemini-api/docs/models/gemini-2.5-flash-preview-tts","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Gemini 2.5 Pro text-to-speech | Gemini API | Google AI for Developers","url":"https://ai.google.dev/gemini-api/docs/models/gemini-2.5-pro-preview-tts","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Speech generation (text-to-speech) | Gemini API","url":"https://ai.google.dev/gemini-api/docs/speech-generation","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/tts--gemini-2-5-tts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://ai.google.dev/gemini-api/docs/models/gemini-2.5-flash-preview-tts","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/gemini-2-5-tts.svg","sha256":"2c623d9800dac2a6a96cbd6c59597f26a8b814642441e22649a17efd4423c39c"}},{"id":"tts:gemini-3-1-flash-tts","slug":"gemini-3-1-flash-tts","name":"Gemini 3.1 Flash TTS","collection":"tts","category":"tts:api","categoryName":"Commercial TTS","summary":"Gemini 3.1 Flash TTS adds expressive audio tags to prompt-steered speech generation, giving authors more local control over narration and delivery. It targets natural, responsive multilingual synthesis through a managed API. The public preview documentation describes the interface and controls, without enough architectural detail to reconstruct the underlying speech generator.","architecture":"Gemini-derived text-to-speech","notes":"Text-conditioned speech generation with controllable delivery.","variants":["gemini-3.1-flash-tts-preview"],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/api.md#gemini-3-1-flash-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"documentation","title":"Gemini 3.1 Flash TTS (text-to-speech) preview | Gemini API | Google AI for Developers","url":"https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Speech generation (text-to-speech) | Gemini API","url":"https://ai.google.dev/gemini-api/docs/speech-generation","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/tts--gemini-3-1-flash-tts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-tts-preview","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/gemini-3-1-flash-tts.svg","sha256":"b5f28cd0b00b5d72bb299bf3d2b40c476572a74f80fbbdaabd74ba3ca858b454"}},{"id":"tts:glm-tts","slug":"glm-tts","name":"GLM-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"GLM-TTS first predicts speech tokens autoregressively, then converts them into audio with a diffusion decoder. Pitch-aware tokenization and multi-reward reinforcement learning target pronunciation, speaker similarity and expression. Hybrid phoneme/text input and LoRA voice adaptation provide controls for applications that need repeatable pronunciation and customized voices.","architecture":"Autoregressive token model and diffusion decoder","notes":"Post-training improves controllable pronunciation, speaker similarity and prosody.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-12-16","dateKind":"paper","links":{"code":"https://github.com/zai-org/GLM-TTS","weights":null,"paper":"https://arxiv.org/abs/2512.14291","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#glm-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"GLM-TTS Technical Report","url":"https://arxiv.org/abs/2512.14291","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"zai-org/GLM-TTS — official repository","url":"https://github.com/zai-org/GLM-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--glm-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2512.14291","originUrl":"https://arxiv.org/html/2512.14291v1/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/glm-tts.png","sha256":"d7b9b14725bc499325ac615eb8ac414a1ec4c39d97b41ec7aeacc3971d342f4a"}},{"id":"tts:gpt-4o-mini-tts","slug":"gpt-4o-mini-tts","name":"GPT-4o Mini TTS","collection":"tts","category":"tts:api","categoryName":"Commercial TTS","summary":"GPT-4o Mini TTS combines the text to be spoken with instructions that steer accent, speed, tone and emotional delivery. The Speech API can stream audio before the full result is complete and supports several output formats. It provides a managed synthesis component for narration and voice applications; public documentation does not disclose the complete acoustic architecture.","architecture":"GPT-derived speech synthesizer","notes":"Generates spoken audio from text and delivery instructions.","variants":["gpt-4o-mini-tts"],"inputs":["T"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/api.md#gpt-4o-mini-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"documentation","title":"GPT-4o Mini TTS","url":"https://developers.openai.com/api/docs/models/gpt-4o-mini-tts","reviewed_on":"2026-09-13","review_level":"documentation"},{"kind":"documentation","title":"Text to speech | OpenAI API","url":"https://developers.openai.com/api/docs/guides/text-to-speech","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/tts--gpt-4o-mini-tts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://developers.openai.com/api/docs/models/gpt-4o-mini-tts","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/gpt-4o-mini-tts.svg","sha256":"70077006acb69db2760c986c3264f5fc8da9837a3bed980ddbe06756648bee67"}},{"id":"tts:gpt-sovits","slug":"gpt-sovits","name":"GPT-SoVITS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"GPT-SoVITS couples text-to-semantic token prediction with a reference-conditioned speech decoder. Its 2025 V3, V4 and V2 Pro releases extend a workflow that supports both zero-shot synthesis and voice adaptation from a small training set. The surrounding WebUI helps prepare data, while recognition and source-separation utilities remain separate components.","architecture":"Text-to-semantic GPT and speech decoder","notes":"Few-shot and zero-shot synthesis using reference audio; the WebUI's separate ASR tools are not part of its language model.","variants":["v1","v2","v3","v4","v2 Pro","v2 ProPlus"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/RVC-Boss/GPT-SoVITS","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#gpt-sovits"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"RVC-Boss/GPT-SoVITS — official repository","url":"https://github.com/RVC-Boss/GPT-SoVITS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"documentation","title":"GPT-SoVITS changelog","url":"https://github.com/RVC-Boss/GPT-SoVITS/blob/main/docs/en/Changelog_EN.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/tts--gpt-sovits.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/RVC-Boss/GPT-SoVITS","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/gpt-sovits.svg","sha256":"736557caf05ab881fc4c2a33f16304f3137fc47e9a5157a70c361c38da1fdd20"}},{"id":"tts:higgs-audio-v2","slug":"higgs-audio-v2","name":"Higgs Audio v2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Higgs Audio v2 combines interleaved text/audio modeling with a unified speech tokenizer and DualFFN layers for acoustic prediction. Reference clips and scene context influence voices, while text context shapes prosody across narration and multi-speaker scripts. The generation checkpoint covers synthesis; the separate understanding branch in the family diagram is not another output mode of this checkpoint.","architecture":"Text/audio language model with audio heads","notes":"Text and acoustic context condition expressive speech generation.","variants":["3B Base"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#higgs-audio-v2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"model-card","title":"bosonai/higgs-tts-2-3b-base · Hugging Face","url":"https://huggingface.co/bosonai/higgs-audio-v2-generation-3B-base","reviewed_on":"2026-09-13","review_level":"model-card"}],"figure":{"path":"assets/figures/tts--higgs-audio-v2.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://huggingface.co/bosonai/higgs-audio-v2-generation-3B-base","originUrl":"https://huggingface.co/bosonai/higgs-audio-v2-generation-3B-base/resolve/main/higgs_audio_v2_architecture_combined.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/higgs-audio-v2.png","sha256":"e51631c9b3c3852c7000ad60ce0910208cc166d67bc80f78f842875955ccf2f9"}},{"id":"tts:higgs-audio-v2-5","slug":"higgs-audio-v2-5","name":"Higgs Audio v2.5","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Higgs Audio v2.5, now documented as Higgs TTS 2.5, reduces the autoregressive audio Transformer to 1B parameters. GRPO-based alignment and a curated voice dataset refine pronunciation, cloning and expressive control tags. It targets production narration and conversational speech with lower computational requirements than the preceding 3B generation model.","architecture":"Autoregressive audio Transformer with GRPO alignment","notes":"Successor synthesis release in the Higgs Audio line.","variants":["v2.5"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-01-09","dateKind":"announcement","links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#higgs-audio-v2-5"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"announcement","title":"Boson AI Launches Higgs TTS 2.5 Voice Model – Boson AI","url":"https://www.boson.ai/blog/higgs-audio-v2.5","reviewed_on":"2026-09-13","review_level":"announcement"}],"figure":{"path":"assets/figures/tts--higgs-audio-v2-5.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://www.boson.ai/blog/higgs-audio-v2.5","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/higgs-audio-v2-5.svg","sha256":"9b4791037b212e4f7ff772cf34699c1e6d5cc40ca22e05874a1b9f610ba0fe67"}},{"id":"tts:higgs-audio-v3-tts","slug":"higgs-audio-v3-tts","name":"Higgs Audio v3 TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Higgs TTS 3 uses an autoregressive decoder over interleaved text and eight speech codebooks, with a delay pattern and fused input/output projections. Reference audio establishes a voice, while inline tokens control emotion, style, pauses and sound effects. The 4B release targets multilingual conversational speech and expressive response rendering.","architecture":"Autoregressive Transformer with delayed audio codebooks","notes":"Standalone successor with voice conditioning and text-controlled delivery.","variants":["4B","higgs-tts-3-4b"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#higgs-audio-v3-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"model-card","title":"bosonai/higgs-tts-3-4b · Hugging Face","url":"https://huggingface.co/bosonai/higgs-tts-3-4b","reviewed_on":"2026-09-13","review_level":"model-card"}],"figure":{"path":"assets/figures/tts--higgs-audio-v3-tts.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://huggingface.co/bosonai/higgs-tts-3-4b","originUrl":"https://huggingface.co/bosonai/higgs-tts-3-4b/resolve/main/assets/model_architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/higgs-audio-v3-tts.png","sha256":"fc95ec283388b684c02d3e0d9c7cfc8d939d62ee83d5918bf303cbef65f85faf"}},{"id":"tts:hume-octave-tts","slug":"hume-octave-tts","name":"Hume Octave TTS","collection":"tts","category":"tts:api","categoryName":"Commercial TTS","summary":"Octave uses text context and acting instructions to adjust pronunciation, emphasis, tempo and emotional delivery. Its API supports voice creation from descriptions, voice cloning and continuation across longer passages. Octave 1 and the Octave 2 preview have different feature coverage; the public interface is documented more fully than the internal speech-model architecture.","architecture":"LLM-based expressive speech synthesis","notes":"Commercial TTS interface with voice design and reference conditioning; EVI 4-mini combines it with an external language model.","variants":["Octave 1","Octave 2"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/api.md#hume-octave-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"documentation","title":"Text-to-Speech (TTS) | Hume API","url":"https://dev.hume.ai/docs/text-to-speech-tts/overview","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/tts--hume-octave-tts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://dev.hume.ai/docs/text-to-speech-tts/overview","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/hume-octave-tts.svg","sha256":"d64235a631248102128f4913bfa6034ec307a8a5375e4edf69d3dbb9c745c74c"}},{"id":"tts:indextts","slug":"indextts","name":"IndexTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"IndexTTS adapts the XTTS/Tortoise approach with a Conformer reference encoder and a BigVGAN2 speech decoder. Hybrid character/pinyin input gives explicit control over difficult Chinese pronunciations. Its central use case is zero-shot voice cloning with predictable text rendering, including content that benefits from pronunciation correction.","architecture":"Autoregressive acoustic generation","notes":"Extends the XTTS/Tortoise approach with improved pronunciation control.","variants":["IndexTTS","IndexTTS-1.5"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-08","dateKind":"paper","links":{"code":"https://github.com/index-tts/index-tts","weights":null,"paper":"https://arxiv.org/abs/2502.05512","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#indextts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"IndexTTS: An Industrial-Level Controllable and Efficient Zero-Shot Text-To-Speech System","url":"https://arxiv.org/abs/2502.05512","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"IndexTTS author repository","url":"https://github.com/index-tts/index-tts","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"IndexTTS author demo","url":"https://index-tts.github.io","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--indextts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.05512","originUrl":"https://arxiv.org/html/2502.05512v1/indexTTS3.drawio.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/indextts.png","sha256":"4ee4e4b584a519062f948431b28cd32b6f805972e96aef51c1591e75cf2f7d75"}},{"id":"tts:indextts-2-5","slug":"indextts-2-5","name":"IndexTTS 2.5","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"IndexTTS 2.5 shortens semantic sequences with a lower-rate codec and replaces the acoustic module's backbone with a Zipformer design. Multilingual training strategies and reinforcement learning extend pronunciation and emotion transfer across languages. It retains reference-based voice conditioning while reducing the cost of semantic and acoustic generation.","architecture":"Text-to-semantic LM and non-AR acoustic decoder","notes":"Extends IndexTTS2 with multilingual coverage and efficiency improvements.","variants":["2.5"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-07","dateKind":"paper","links":{"code":"https://github.com/index-tts/index-tts","weights":null,"paper":"https://arxiv.org/abs/2601.03888","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#indextts-2-5"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"IndexTTS 2.5 Technical Report","url":"https://arxiv.org/abs/2601.03888","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"IndexTTS 2.5 author repository","url":"https://github.com/index-tts/index-tts","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--indextts-2-5.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.03888","originUrl":"https://arxiv.org/html/2601.03888v5/tree1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/indextts-2-5.png","sha256":"2981b521c5023d1f62451ccb4eb225369f338d42f6a714ce0a8e4600db4b028f"}},{"id":"tts:indextts2","slug":"indextts2","name":"IndexTTS2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"IndexTTS2 separates speaker identity from emotional style so that different references can control timbre and delivery. Its autoregressive formulation also supports explicit output-token budgeting for duration control, alongside unconstrained generation. These mechanisms are intended for expressive speech and timing-sensitive work such as dubbing; availability of controls should be checked in the chosen implementation.","architecture":"Duration-controlled autoregressive synthesis","notes":"Separates emotion and timbre control and adds explicit duration conditioning.","variants":["2"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-23","dateKind":"paper","links":{"code":"https://github.com/index-tts/index-tts","weights":null,"paper":"https://arxiv.org/abs/2506.21619","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#indextts2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"IndexTTS2: A Breakthrough in Emotionally Expressive and Duration-Controlled Auto-Regressive Zero-Shot Text-to-Speech","url":"https://arxiv.org/abs/2506.21619","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"IndexTTS2 author repository","url":"https://github.com/index-tts/index-tts","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--indextts2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.21619","originUrl":"https://arxiv.org/html/2506.21619v2/indextts2-drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/indextts2.png","sha256":"5f26f337c2bc67d5ef9a420d37bf5c694f82c9953a30eaed58c01259c46b02f9"}},{"id":"tts:kyutai-tts-dsm","slug":"kyutai-tts-dsm","name":"Kyutai TTS (DSM)","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Kyutai TTS treats text and speech as aligned streams separated by a controlled delay. A decoder-only language model can therefore emit audio as text arrives, instead of waiting for a complete utterance. This formulation supports incremental synthesis for voice interfaces and long streams, with speaker conditioning supplied through the TTS implementation.","architecture":"Delayed Streams Modeling","notes":"Aligns text and audio streams with controlled delays for incremental synthesis.","variants":["Kyutai TTS"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-09-10","dateKind":"paper","links":{"code":"https://github.com/kyutai-labs/delayed-streams-modeling","weights":null,"paper":"https://arxiv.org/abs/2509.08753","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#kyutai-tts-dsm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Streaming Sequence-to-Sequence Learning with Delayed Streams Modeling","url":"https://arxiv.org/abs/2509.08753","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"kyutai-labs/delayed-streams-modeling — official repository","url":"https://github.com/kyutai-labs/delayed-streams-modeling","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--kyutai-tts-dsm.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.08753","originUrl":"https://arxiv.org/html/2509.08753v2/Figure_time_delay-3_cropped.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/kyutai-tts-dsm.png","sha256":"0cc8203482da1dad8a6fd2bb1fa6389eb44ae22f8f3c01187f11e71425e3bd15"}},{"id":"tts:llasa","slug":"llasa","name":"Llasa","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Llasa maps text and an optional speech prompt to a single stream of codec tokens using a Llama-style Transformer. The simple representation makes standard language-model scaling and sampling techniques applicable to synthesis. Its research also explores speech-model verifiers that select samples for content accuracy, voice consistency or emotional expression.","architecture":"Single-codebook codec and Llama","notes":"Uses a single Transformer for language-model-based speech synthesis.","variants":["1B","3B","8B"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-06","dateKind":"paper","links":{"code":"https://github.com/zhenye234/LLaSA_training","weights":null,"paper":"https://arxiv.org/abs/2502.04128","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#llasa"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Llasa: Scaling Train-Time and Inference-Time Compute for Llama-based Speech Synthesis","url":"https://arxiv.org/abs/2502.04128","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Llasa author repository","url":"https://github.com/zhenye234/LLaSA_training","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"Llasa author demo","url":"https://llasatts.github.io/llasatts/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--llasa.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2502.04128","originUrl":"https://arxiv.org/html/2502.04128v2/comparison_plot.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/llasa.png","sha256":"6f2a51ce8a6ed9ea9a69bd0cae7f50134d3ccc7be4afc041fc7731f8729be044"}},{"id":"tts:llasa-plus","slug":"llasa-plus","name":"Llasa+","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Llasa+ adds multi-token prediction modules to a frozen Llasa backbone and checks their proposals with that backbone. A causal codec decoder turns accepted tokens into streaming audio. The resulting design addresses autoregressive latency while retaining the original speech model, making it relevant to systems that need incremental playback without retraining an entire backbone.","architecture":"Llasa with multi-token prediction","notes":"Adds acceleration modules and streaming synthesis to the Llasa backbone.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-08-08","dateKind":"paper","links":{"code":"https://github.com/ASLP-lab/LLaSA_Plus","weights":null,"paper":"https://arxiv.org/abs/2508.06262","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#llasa-plus"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Llasa+: Free Lunch for Accelerated and Streaming Llama-Based Speech Synthesis","url":"https://arxiv.org/abs/2508.06262","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Llasa+ author repository","url":"https://github.com/ASLP-lab/LLaSA_Plus","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--llasa-plus.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2508.06262","originUrl":"https://arxiv.org/html/2508.06262v1/llasa+.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/llasa-plus.png","sha256":"fe5d9ee0f5299fc8a4281c7aa12a2102a520ff04e44bd9d653b8e03585cdd99e"}},{"id":"tts:maskgct","slug":"maskgct","name":"MaskGCT","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Non-autoregressive token language modeling without explicit text-speech alignment.","architecture":"Masked generative codec transformers","notes":"Non-autoregressive token language modeling without explicit text-speech alignment.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-09-01","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2409.00750","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#maskgct"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MaskGCT: Zero-Shot Text-to-Speech with Masked Generative Codec Transformer","url":"https://arxiv.org/abs/2409.00750","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--maskgct.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2409.00750","originUrl":"https://arxiv.org/html/2409.00750v3/x1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/maskgct.png","sha256":"73c061adedcaae12328f458cb508daf7b13ce248ff4e1e10cb45d183c0089c26"}},{"id":"tts:mega-tts","slug":"mega-tts","name":"Mega-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Uses separate inductive biases for content, timbre, prosody and waveform generation.","architecture":"Disentangled speech factors and prosody LM","notes":"Uses separate inductive biases for content, timbre, prosody and waveform generation.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-06-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2306.03509","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#mega-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Mega-TTS: Zero-Shot Text-to-Speech at Scale with Intrinsic Inductive Bias","url":"https://arxiv.org/abs/2306.03509","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mega-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2306.03509","originUrl":"https://arxiv.org/html/2306.03509v1/MegaTTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mega-tts.png","sha256":"260119ae6b244dd7ecf6161e0ae3f3b9bd4153045f6e071f5dad62e7c8f119fc"}},{"id":"tts:mega-tts-2","slug":"mega-tts-2","name":"Mega-TTS 2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Improves prompt utilization and transfer of speaking style.","architecture":"Prosody language model and multi-sentence prompting","notes":"Improves prompt utilization and transfer of speaking style.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-07-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2307.07218","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#mega-tts-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Mega-TTS 2: Boosting Prompting Mechanisms for Zero-Shot Speech Synthesis","url":"https://arxiv.org/abs/2307.07218","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mega-tts-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2307.07218","originUrl":"https://arxiv.org/html/2307.07218v4/Main_Arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mega-tts-2.png","sha256":"a0b11e0e5d9c5bc7feb47c3e73f81b06670be1d832ee92775215670d40d2fe15"}},{"id":"tts:minimax-speech","slug":"minimax-speech","name":"MiniMax-Speech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MiniMax-Speech extracts speaker characteristics directly from reference audio without requiring its transcript, then generates speech with an autoregressive Transformer and Flow-VAE. The research emphasizes multilingual zero-shot cloning and expressive delivery. Additional adaptation mechanisms support emotion control, description-based voice creation and more specialized voice cloning without replacing the base model.","architecture":"Autoregressive Transformer and Flow-VAE","notes":"Learnable speaker conditioning supports expressive multilingual TTS.","variants":["Research model family"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.07916","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#minimax-speech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MiniMax-Speech: Intrinsic Zero-Shot Text-to-Speech with a Learnable Speaker Encoder","url":"https://arxiv.org/abs/2505.07916","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--minimax-speech.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.07916","originUrl":"https://arxiv.org/html/2505.07916v1/overview.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/minimax-speech.png","sha256":"bc7c95db53c1fb81cefca8a05bbc75e282a5543deddb6191a60fcb4cd931bcf4"}},{"id":"tts:moss-tts","slug":"moss-tts","name":"MOSS-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MOSS-TTS offers two generators over a shared discrete audio representation: a delay-pattern model and a model with a frame-local Transformer. They balance long-context control against efficient codebook prediction and speaker preservation. The family supports reference-conditioned synthesis, pronunciation and duration controls, with later checkpoints extending language handling and explicit pauses.","architecture":"Delay-pattern LM and local Transformer alternatives","notes":"Family includes structurally distinct generators over MOSS audio tokens.","variants":["Delay 8B v1/v1.5","Local Transformer 1.7B/4B v1.5"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-18","dateKind":"paper","links":{"code":"https://github.com/OpenMOSS/MOSS-TTS","weights":null,"paper":"https://arxiv.org/abs/2603.18090","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#moss-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MOSS-TTS Technical Report","url":"https://arxiv.org/abs/2603.18090","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"OpenMOSS/MOSS-TTS — official repository","url":"https://github.com/OpenMOSS/MOSS-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--moss-tts.webp","kind":"source","label":"Figure 2, PDF p. 8","sourceUrl":"https://arxiv.org/abs/2603.18090","originUrl":"https://arxiv.org/pdf/2603.18090","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/moss-tts.png","sha256":"477a6a8892fc7d33eac25586c41018e9f274769a62b3a45c28153b5544a7e769"}},{"id":"tts:moss-tts-nano","slug":"moss-tts-nano","name":"MOSS-TTS-Nano","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MOSS-TTS-Nano packages multilingual voice cloning into a roughly 100M-parameter speech generator with a compact audio tokenizer. Streaming output and an ONNX inference path make it relevant to CPU-based readers and local applications. Its published performance depends on the runtime and hardware, and the tokenizer is a separate part of the deployment footprint.","architecture":"Compact speech language model","notes":"Lightweight member of the MOSS-TTS family.","variants":["MOSS-TTS-Nano"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/OpenMOSS/MOSS-TTS","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#moss-tts-nano"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"OpenMOSS/MOSS-TTS — official repository","url":"https://github.com/OpenMOSS/MOSS-TTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"repository","title":"MOSS-TTS-Nano — official implementation","url":"https://github.com/OpenMOSS/MOSS-TTS-Nano","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--moss-tts-nano.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://github.com/OpenMOSS/MOSS-TTS","originUrl":"https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/arch_moss_tts_nano.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/moss-tts-nano.png","sha256":"e15e4330ed8df7f144437359115c229cf422411e94dbbd6f0928bf1f909955c7"}},{"id":"tts:moss-tts-realtime","slug":"moss-tts-realtime","name":"MOSS-TTS-Realtime","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MOSS-TTS-Realtime uses a Qwen3-derived backbone for linguistic context and a smaller local Transformer to predict audio codebooks. Text and speech tokens are handled at different levels so the system can accept text and emit audio incrementally. It targets low-latency spoken responses while preserving context across the generated utterance.","architecture":"Streaming autoregressive speech generator","notes":"The real-time TTS specialization of the MOSS speech-generation family.","variants":["MOSS-TTS-Realtime"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/OpenMOSS/MOSS-TTS","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#moss-tts-realtime"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"OpenMOSS/MOSS-TTS — official repository","url":"https://github.com/OpenMOSS/MOSS-TTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"documentation","title":"MOSS-TTS-Realtime — model and inference guide","url":"https://github.com/OpenMOSS/MOSS-TTS/blob/main/moss_tts_realtime/README.md","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--moss-tts-realtime.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/OpenMOSS/MOSS-TTS","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/moss-tts-realtime.svg","sha256":"328678c1361861db37e6565517372db416649b2f1f6af3695b5fea101d746e23"}},{"id":"tts:moss-ttsd","slug":"moss-ttsd","name":"MOSS-TTSD","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MOSS-TTSD turns a dialogue script with explicit speaker tags into a continuous multi-party recording. Long-context modeling helps maintain speaker identity, turn assignment and acoustic continuity, while short references can define voices. It is designed for podcasts, commentary and other scripted conversations; the source paper evaluates dialogue-specific consistency as well as intelligibility.","architecture":"Long-context dialogue speech LM","notes":"Turns speaker-tagged scripts into multi-party spoken dialogue.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-20","dateKind":"paper","links":{"code":"https://github.com/OpenMOSS/MOSS-TTSD","weights":null,"paper":"https://arxiv.org/abs/2603.19739","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#moss-ttsd"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MOSS-TTSD: Text to Spoken Dialogue Generation","url":"https://arxiv.org/abs/2603.19739","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MOSS-TTSD author repository","url":"https://github.com/OpenMOSS/MOSS-TTSD","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--moss-ttsd.webp","kind":"source","label":"Figure 2, PDF p. 4","sourceUrl":"https://arxiv.org/abs/2603.19739","originUrl":"https://arxiv.org/pdf/2603.19739","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/moss-ttsd.png","sha256":"a97fe22a97397a076f5f8f555759df197ef1e3d61093dd97366441e0a0715bf0"}},{"id":"tts:moss-voicegenerator","slug":"moss-voicegenerator","name":"MOSS-VoiceGenerator","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MOSS-VoiceGenerator creates a speaking voice from a natural-language description rather than requiring an example speaker recording. Training on expressive cinematic speech exposes it to varied delivery and acoustic conditions. The model is intended for character design, storytelling and role-based narration where the desired voice must be specified in words.","architecture":"Instruction-conditioned voice generation","notes":"Creates speaking voices from natural-language descriptions.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-30","dateKind":"paper","links":{"code":"https://github.com/OpenMOSS/MOSS-TTS","weights":null,"paper":"https://arxiv.org/abs/2603.28086","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#moss-voicegenerator"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MOSS-VoiceGenerator: Create Realistic Voices with Natural Language Descriptions","url":"https://arxiv.org/abs/2603.28086","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MOSS-VoiceGenerator author repository","url":"https://github.com/OpenMOSS/MOSS-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--moss-voicegenerator.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.28086","originUrl":"https://arxiv.org/html/2603.28086v1/its-modelarchv2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/moss-voicegenerator.png","sha256":"7c49aba12e120fe165769c654bd2ca0cdd3021c894116876f3ff296da2c86e41"}},{"id":"tts:omnivoice","slug":"omnivoice","name":"OmniVoice","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"OmniVoice predicts multiple acoustic codebooks directly from text using a masked, non-autoregressive diffusion language model. Random masking across codebooks and initialization from a pretrained language model support multilingual generation without a separate text-to-semantic stage. It focuses on broad-language zero-shot synthesis and voice conditioning, rather than visual or general-purpose omni interaction.","architecture":"Discrete diffusion language model","notes":"Omnilingual TTS over multiple acoustic codebooks; its name does not imply vision input.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-01","dateKind":"paper","links":{"code":"https://github.com/k2-fsa/OmniVoice","weights":null,"paper":"https://arxiv.org/abs/2604.00688","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#omnivoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"OmniVoice: Towards Omnilingual Zero-Shot Text-to-Speech with Diffusion Language Models","url":"https://arxiv.org/abs/2604.00688","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"k2-fsa/OmniVoice — official repository","url":"https://github.com/k2-fsa/OmniVoice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--omnivoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2604.00688","originUrl":"https://arxiv.org/html/2604.00688v3/architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/omnivoice.png","sha256":"7b3ab671bac467d736b0893aa6dddb715e8f0bf3dadb16d68f573eebfad59940"}},{"id":"tts:orpheus-tts","slug":"orpheus-tts","name":"Orpheus TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Orpheus TTS repurposes a Llama-family language model to generate speech codec tokens from text. Emotion tags and speaker conditioning guide expressive delivery, while the project supplies inference and adaptation workflows. English releases and multilingual previews have different coverage, so a checkpoint's documented capabilities matter when selecting it for narration or a voice application.","architecture":"Llama-based codec language model","notes":"Expressive synthesis with textual emotion cues and voice conditioning.","variants":["3B English","multilingual preview"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/canopyai/Orpheus-TTS","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#orpheus-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"canopyai/Orpheus-TTS — official repository","url":"https://github.com/canopyai/Orpheus-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--orpheus-tts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/canopyai/Orpheus-TTS","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/orpheus-tts.svg","sha256":"c675ed8bf1043ae45c7512dd9356eab84f08ee1ce0990b0bf522b34d5c914601"}},{"id":"tts:outetts","slug":"outetts","name":"OuteTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"OuteTTS represents speech in a form that can be generated by a decoder-only language model and reconstructed by an audio decoder. A speaker reference guides vocal identity, style and accent. Its 1.0 line supports standard LLM serving backends, but generation settings and repetition handling need to follow the matching model implementation.","architecture":"Decoder-only speech-token LM","notes":"TTS models usable through standard LLM inference backends.","variants":["0.1","0.2","0.3","1.0"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/edwko/OuteTTS","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#outetts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"edwko/OuteTTS — official repository","url":"https://github.com/edwko/OuteTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--outetts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/edwko/OuteTTS","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/outetts.svg","sha256":"768d47b8a723474ae70a756c8f82579efac3d03c51420305e9ea330b725a2cba"}},{"id":"tts:parler-tts","slug":"parler-tts","name":"Parler-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Uses text descriptions of the speaker and delivery to guide synthesis.","architecture":"Description-conditioned codec language model","notes":"Uses text descriptions of the speaker and delivery to guide synthesis.","variants":["Mini","Large","Mini Expresso"],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/huggingface/parler-tts","weights":null,"paper":"https://arxiv.org/abs/2402.01912","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#parler-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"huggingface/parler-tts — official repository","url":"https://github.com/huggingface/parler-tts","reviewed_on":"2026-09-12","review_level":"readme"},{"kind":"paper","title":"Natural language guidance of high-fidelity text-to-speech with synthetic annotations","url":"https://arxiv.org/abs/2402.01912","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--parler-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2402.01912","originUrl":"https://arxiv.org/html/2402.01912v1/architecture_v3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/parler-tts.png","sha256":"a9ca5174fd0f4005725d816a75bb34979e47df032634287175bda0b75162a54f"}},{"id":"tts:pocket-tts","slug":"pocket-tts","name":"Pocket TTS","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"Pocket TTS uses continuous autoregressive speech modeling with a flow-based output mechanism, avoiding long sequences of discrete acoustic codebooks. It combines a small language-model backbone with streaming audio reconstruction and reusable voice conditioning. The project targets CPU-based speech synthesis; language-specific models and runtime choices affect its speed and voice behavior.","architecture":"Continuous autoregressive FlowLM","notes":"Small streaming synthesizer derived from continuous audio language modeling.","variants":["100M"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/kyutai-labs/pocket-tts","weights":null,"paper":"https://arxiv.org/abs/2509.06926","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#pocket-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"kyutai-labs/pocket-tts — official repository","url":"https://github.com/kyutai-labs/pocket-tts","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"paper","title":"Continuous Audio Language Models","url":"https://arxiv.org/abs/2509.06926","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--pocket-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.06926","originUrl":"https://arxiv.org/html/2509.06926v3/fig1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/pocket-tts.png","sha256":"8e2c55b3a8ea4b9a9082e8302675862edb46580aed7e8e9406e6432a47613420"}},{"id":"tts:qwen3-tts","slug":"qwen3-tts","name":"Qwen3-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Qwen3-TTS combines a dual-track speech language model with tokenizers designed for compact streaming audio. The released 12Hz line separates Base voice cloning, CustomVoice preset-speaker control and VoiceDesign creation from descriptions. These variants support different conditioning interfaces, allowing applications to choose between reproducing a reference voice and directing a new voice through text.","architecture":"Dual-track speech language model","notes":"Family offers voice cloning, preset voices and description-driven voice creation; input conditioning depends on variant.","variants":["0.6B","1.7B","Base","CustomVoice","VoiceDesign","12Hz/25Hz tokenizers"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-01-22","dateKind":"paper","links":{"code":"https://github.com/QwenLM/Qwen3-TTS","weights":null,"paper":"https://arxiv.org/abs/2601.15621","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#qwen3-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Qwen3-TTS Technical Report","url":"https://arxiv.org/abs/2601.15621","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"QwenLM/Qwen3-TTS — official repository","url":"https://github.com/QwenLM/Qwen3-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--qwen3-tts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2601.15621","originUrl":"https://arxiv.org/html/2601.15621v1/figures/Qwen3TTS_1104.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/qwen3-tts.png","sha256":"d7372e583e5fb7b11d63dcaae39415ca3b1137e50da3be5ce297b539f3bdc046"}},{"id":"tts:rall-e","slug":"rall-e","name":"RALL-E","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Predicts pitch and duration as intermediate conditions for speech synthesis.","architecture":"Prosody-guided codec language modeling","notes":"Predicts pitch and duration as intermediate conditions for speech synthesis.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-04-04","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2404.03204","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#rall-e"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"RALL-E: Robust Codec Language Modeling with Chain-of-Thought Prompting for Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2404.03204","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--rall-e.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2404.03204","originUrl":"https://arxiv.org/html/2404.03204v3/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/rall-e.png","sha256":"0edcc5b685b545d66ecfeb245819c8843dd775c07b2e3cff02f7ab981aaeea29"}},{"id":"tts:seed-tts","slug":"seed-tts","name":"Seed-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Research family with controllable synthesis and speaker prompting.","architecture":"Autoregressive speech foundation model","notes":"Research family with controllable synthesis and speaker prompting.","variants":["Autoregressive Seed-TTS"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-06-04","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2406.02430","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#seed-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Seed-TTS: A Family of High-Quality Versatile Speech Generation Models","url":"https://arxiv.org/abs/2406.02430","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--seed-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2406.02430","originUrl":"https://arxiv.org/html/2406.02430v1/seedtts-cpd-p1.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/seed-tts.png","sha256":"d7c392ae3b2b0e074b2cc05ea33ef29b3a7d19f755541877e303623f6461524f"}},{"id":"tts:spark-tts","slug":"spark-tts","name":"Spark-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Spark-TTS uses BiCodec to separate changing linguistic content from global speaker attributes, then predicts these tokens with a Qwen2.5 backbone. That separation supports both reference-based cloning and direct control of attributes such as speaking rate and pitch. It is useful for controllable speech generation where a reference recording alone is insufficient to specify the desired delivery.","architecture":"Qwen2.5 and single-stream BiCodec","notes":"Separates semantic tokens from global speaker tokens for controllable synthesis.","variants":["0.5B"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-03-03","dateKind":"paper","links":{"code":"https://github.com/SparkAudio/Spark-TTS","weights":null,"paper":"https://arxiv.org/abs/2503.01710","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#spark-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Spark-TTS: An Efficient LLM-Based Text-to-Speech Model with Single-Stream Decoupled Speech Tokens","url":"https://arxiv.org/abs/2503.01710","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"SparkAudio/Spark-TTS — official repository","url":"https://github.com/SparkAudio/Spark-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--spark-tts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2503.01710","originUrl":"https://arxiv.org/html/2503.01710v1/spark-tts-lm.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/spark-tts.png","sha256":"9977a45e812e2eaf324ac18d7d267666eaf67b61c8cb21168e0f14547f22f518"}},{"id":"tts:spear-tts","slug":"spear-tts","name":"SPEAR-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Separates reading and speaking to reduce paired-data requirements.","architecture":"Text-to-semantic and semantic-to-acoustic LMs","notes":"Separates reading and speaking to reduce paired-data requirements.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-02-07","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2302.03540","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#spear-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Speak, Read and Prompt: High-Fidelity Text-to-Speech with Minimal Supervision","url":"https://arxiv.org/abs/2302.03540","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--spear-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2302.03540","originUrl":"https://arxiv.org/html/2302.03540v1/arch_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/spear-tts.png","sha256":"997242768c2005e2aacc2532fe9be6d4b20e8689d86ee8e198e76e3e7e57fac9"}},{"id":"tts:speecht5","slug":"speecht5","name":"SpeechT5","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"This card covers the TTS path of the shared speech/text encoder-decoder. ASR and voice conversion are other tasks of the broader model.","architecture":"Shared encoder–decoder with modality interfaces","notes":"This card covers the TTS path of the shared speech/text encoder-decoder. ASR and voice conversion are other tasks of the broader model.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-10-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2110.07205","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#speecht5"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing","url":"https://arxiv.org/abs/2110.07205","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--speecht5.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2110.07205","originUrl":"https://arxiv.org/html/2110.07205v3/architecture-cb-v2.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/speecht5.png","sha256":"dc042614f45c54916b058c063e48110cc448320a456b9d94fc22e90e4adab8bf"}},{"id":"tts:speechx","slug":"speechx","name":"SpeechX","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This card covers the text-to-speech path. The broader codec language model also supports speech editing, enhancement and other transformations.","architecture":"Prompted neural codec language model","notes":"This card covers the text-to-speech path. The broader codec language model also supports speech editing, enhancement and other transformations.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-08-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2308.06873","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#speechx"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SpeechX: Neural Codec Language Model as a Versatile Speech Transformer","url":"https://arxiv.org/abs/2308.06873","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--speechx.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2308.06873","originUrl":"https://arxiv.org/html/2308.06873v2/overview-v5.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/speechx.png","sha256":"4c1213ce9852f5c7f3a64fd35c694d1db4a5350e5f576b7a685a8ddb46379686"}},{"id":"tts:step-audio-tts","slug":"step-audio-tts","name":"Step-Audio-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Step-Audio-TTS is the compact synthesis component produced through the broader Step-Audio speech-data and distillation pipeline. Text and voice conditioning drive speech-token generation, while the released workflow exposes delivery controls. The TTS checkpoint is used to render supplied content; reasoning, tool use and dialogue management belong to other components of the system.","architecture":"Speech-token language model","notes":"Dedicated synthesizer distributed with Step-Audio; separate from Step-Audio-Chat.","variants":["3B"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-17","dateKind":"paper","links":{"code":"https://github.com/stepfun-ai/Step-Audio","weights":null,"paper":"https://arxiv.org/abs/2502.11946","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#step-audio-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Step-Audio: Unified Understanding and Generation in Intelligent Speech Interaction","url":"https://arxiv.org/abs/2502.11946","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Step-Audio — official implementation","url":"https://github.com/stepfun-ai/Step-Audio","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--step-audio-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2502.11946","originUrl":"https://arxiv.org/html/2502.11946v2/stepAudio-20250217-ff.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/step-audio-tts.png","sha256":"76ac0f53f1ca2a4017aa177ca447465ac9a9801ebdadd0c7b5714e8618a6bc2c"}},{"id":"tts:tortoise-tts","slug":"tortoise-tts","name":"Tortoise TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Reference-conditioned synthesis from a scaled generative speech pipeline.","architecture":"Autoregressive transformer and diffusion decoder","notes":"Reference-conditioned synthesis from a scaled generative speech pipeline.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-05-12","dateKind":"paper","links":{"code":"https://github.com/neonbjb/tortoise-tts","weights":null,"paper":"https://arxiv.org/abs/2305.07243","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#tortoise-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Better speech synthesis through scaling","url":"https://arxiv.org/abs/2305.07243","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"neonbjb/tortoise-tts — official repository","url":"https://github.com/neonbjb/tortoise-tts","reviewed_on":"2026-09-12","review_level":"readme"}],"figure":{"path":"assets/figures/tts--tortoise-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2305.07243","originUrl":"https://arxiv.org/html/2305.07243v2/figures/tortoise_diagram.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tortoise-tts.png","sha256":"21cb63756c4183888d5a9679eab475ab33f576e3ab1b6f11558d5a1986838ac4"}},{"id":"tts:vall-e","slug":"vall-e","name":"VALL-E","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Treats synthesis as conditional prediction of neural codec tokens.","architecture":"Autoregressive and non-AR codec LMs","notes":"Treats synthesis as conditional prediction of neural codec tokens.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-01-05","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2301.02111","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#vall-e"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Neural Codec Language Models are Zero-Shot Text to Speech Synthesizers","url":"https://arxiv.org/abs/2301.02111","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vall-e.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2301.02111","originUrl":"https://arxiv.org/html/2301.02111v1/prompt.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vall-e.png","sha256":"eb52cc2cf99467aff0125af9acfb35ea07d53bce1a5575496ee846672dfdc5d3"}},{"id":"tts:vall-e-2","slug":"vall-e-2","name":"VALL-E 2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Improves robustness of zero-shot speech synthesis.","architecture":"Grouped codec modeling and repetition-aware sampling","notes":"Improves robustness of zero-shot speech synthesis.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-06-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2406.05370","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#vall-e-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VALL-E 2: Neural Codec Language Models are Human Parity Zero-Shot Text to Speech Synthesizers","url":"https://arxiv.org/abs/2406.05370","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vall-e-2.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2406.05370","originUrl":"https://arxiv.org/html/2406.05370v2/inference.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vall-e-2.png","sha256":"6f4e377a089fce336fbb2679625b49a58ed859f740d965d81b5e58f47b202b67"}},{"id":"tts:vall-e-x","slug":"vall-e-x","name":"VALL-E X","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Uses reference speech and target text for cross-lingual synthesis and translation.","architecture":"Cross-lingual neural codec language model","notes":"Uses reference speech and target text for cross-lingual synthesis and translation.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-03-07","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2303.03926","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#vall-e-x"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Speak Foreign Languages with Your Own Voice: Cross-Lingual Neural Codec Language Modeling","url":"https://arxiv.org/abs/2303.03926","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vall-e-x.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2303.03926","originUrl":"https://arxiv.org/html/2303.03926v1/vallex_framework.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vall-e-x.png","sha256":"0d54fb04eae3afcba38e42c28ed985ae8860ab662b32628cbd39310de844c51d"}},{"id":"tts:vall-t","slug":"vall-t","name":"VALL-T","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Adds monotonic generation constraints to prompt-based speech synthesis.","architecture":"Decoder-only generative transducer","notes":"Adds monotonic generation constraints to prompt-based speech synthesis.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-01-25","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2401.14321","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#vall-t"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VALL-T: Decoder-Only Generative Transducer for Robust and Decoding-Controllable Text-to-Speech","url":"https://arxiv.org/abs/2401.14321","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vall-t.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2401.14321","originUrl":"https://arxiv.org/html/2401.14321v5/main_figure.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vall-t.png","sha256":"782eb0c580382a4060e71a472b618e698301a7f6dc3a5d2a2dc87ac838411c5c"}},{"id":"tts:vibevoice","slug":"vibevoice","name":"VibeVoice","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"VibeVoice autoregressively generates continuous speech latents with a diffusion head, using compressed acoustic and semantic representations to extend context. It targets long, multi-speaker recordings in which voices and delivery should remain coherent across a script. The original research studies recordings lasting up to 90 minutes with four speakers; those limits do not describe every family variant.","architecture":"Next-token diffusion over continuous speech latents","notes":"Synthesizes long scripted conversations; ASR is a separate model in the same project.","variants":["1.5B","7B"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.19205","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#vibevoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VibeVoice Technical Report","url":"https://arxiv.org/abs/2508.19205","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vibevoice.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.19205","originUrl":"https://arxiv.org/html/2508.19205v1/vv_fig1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vibevoice.png","sha256":"a552e51e3e9b43ad4af826d6a77016f69e68e287fa9ae634f908196e028d8e0a"}},{"id":"tts:vibevoice-realtime","slug":"vibevoice-realtime","name":"VibeVoice-Realtime","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"VibeVoice-Realtime interleaves incoming text chunks with diffusion-based acoustic generation so playback can begin while more text is arriving. The smaller 0.5B design keeps the acoustic tokenizer and removes the semantic tokenizer used by the longer-form model. It focuses on single-speaker streaming, with preset voice embeddings and experimental language coverage documented separately.","architecture":"Streaming next-token diffusion","notes":"Incremental text-to-speech member of the VibeVoice family.","variants":["0.5B"],"inputs":["T"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/microsoft/VibeVoice","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#vibevoice-realtime"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"microsoft/VibeVoice — official repository","url":"https://github.com/microsoft/VibeVoice","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"documentation","title":"VibeVoice-Realtime-0.5B Model Card","url":"https://github.com/microsoft/VibeVoice/blob/main/docs/vibevoice-realtime-0.5b.md","reviewed_on":"2026-09-13","review_level":"model-card"}],"figure":{"path":"assets/figures/tts--vibevoice-realtime.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://github.com/microsoft/VibeVoice","originUrl":"https://raw.githubusercontent.com/microsoft/VibeVoice/main/Figures/VibeVoice_Realtime.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vibevoice-realtime.png","sha256":"2706fbc554cd8ed0b3ad3f22c4f19e30c1d9f5d1070cfd5ad2eac4fe834b8abf"}},{"id":"tts:voicecraft","slug":"voicecraft","name":"VoiceCraft","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Supports both speech editing and reference-conditioned TTS.","architecture":"Token-infilling codec language model","notes":"Supports both speech editing and reference-conditioned TTS.","variants":["330M","830M"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-03-25","dateKind":"paper","links":{"code":"https://github.com/jasonppy/VoiceCraft","weights":null,"paper":"https://arxiv.org/abs/2403.16973","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#voicecraft"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoiceCraft: Zero-Shot Speech Editing and Text-to-Speech in the Wild","url":"https://arxiv.org/abs/2403.16973","reviewed_on":"2026-09-12","review_level":"abstract"},{"kind":"repository","title":"jasonppy/VoiceCraft — official repository","url":"https://github.com/jasonppy/VoiceCraft","reviewed_on":"2026-09-12","review_level":"readme"}],"figure":{"path":"assets/figures/tts--voicecraft.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2403.16973","originUrl":"https://arxiv.org/html/2403.16973v3/archi.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voicecraft.png","sha256":"9f79b261204d0dff28f9aa7d0159897f978c934ad1de78fa3ff6cd5dceb36d5f"}},{"id":"tts:voxcpm","slug":"voxcpm","name":"VoxCPM","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"VoxCPM divides synthesis into semantic/prosodic planning and residual acoustic modeling, then uses a local diffusion decoder to produce continuous speech latents. These components are trained together without an external discrete speech tokenizer. Reference conditioning supports voice cloning, while text context helps determine the rhythm and expressive shape of the generated utterance.","architecture":"Hierarchical semantic/acoustic LM and diffusion","notes":"Tokenizer-free speech generation with voice conditioning.","variants":["VoxCPM","VoxCPM1.5"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-09-29","dateKind":"paper","links":{"code":"https://github.com/OpenBMB/VoxCPM","weights":null,"paper":"https://arxiv.org/abs/2509.24650","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#voxcpm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning","url":"https://arxiv.org/abs/2509.24650","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"VoxCPM author repository","url":"https://github.com/OpenBMB/VoxCPM","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"VoxCPM author demo","url":"https://openbmb.github.io/VoxCPM-demopage/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--voxcpm.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.24650","originUrl":"https://arxiv.org/html/2509.24650v1/images/draft_1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voxcpm.png","sha256":"d1fa96d6fec42133cf6ea73593161ae92ee9142c3db9fa5c0f100875c755051f"}},{"id":"tts:voxcpm2","slug":"voxcpm2","name":"VoxCPM2","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"VoxCPM2 expands the same hierarchical approach to multilingual synthesis, voice design and style-controlled cloning within one backbone. An asymmetric AudioVAE encodes lower-rate audio and reconstructs higher-rate output, while a unified input sequence expresses the different generation tasks. It serves applications that need both reference-based voices and description-driven control from one model family.","architecture":"Hierarchical diffusion-autoregressive model","notes":"Extends VoxCPM with multilingual generation and description-based voice control.","variants":["2B"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-06-05","dateKind":"paper","links":{"code":"https://github.com/OpenBMB/VoxCPM","weights":null,"paper":"https://arxiv.org/abs/2606.06928","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#voxcpm2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoxCPM2 Technical Report","url":"https://arxiv.org/abs/2606.06928","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"OpenBMB/VoxCPM — official repository","url":"https://github.com/OpenBMB/VoxCPM","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"VoxCPM2 author demo","url":"https://openbmb.github.io/voxcpm2-demopage/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--voxcpm2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.06928","originUrl":"https://arxiv.org/html/2606.06928v1/VoxCPM2-fig.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voxcpm2.png","sha256":"1ccc8b67bdda21356d00900117db694ba68f7a13969359615f00141277f5dcfd"}},{"id":"tts:xtts","slug":"xtts","name":"XTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Multilingual reference-conditioned TTS derived from Tortoise.","architecture":"Autoregressive acoustic codes and decoder","notes":"Multilingual reference-conditioned TTS derived from Tortoise.","variants":["XTTS","XTTS-v2"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2024-06-07","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2406.04904","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#xtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"XTTS: a Massively Multilingual Zero-Shot Text-to-Speech Model","url":"https://arxiv.org/abs/2406.04904","reviewed_on":"2026-09-12","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--xtts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2406.04904","originUrl":"https://arxiv.org/html/2406.04904v1/Images/XTTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/xtts.png","sha256":"7a6de71919db10f9a5950041154fceec721124b964b3a6b5eefe42616ea83659"}},{"id":"tts:zonos","slug":"zonos","name":"Zonos","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Zonos phonemizes text and predicts DAC codec tokens with either a Transformer or a hybrid backbone. Speaker embeddings and audio prefixes condition the voice, alongside explicit controls for rate, pitch variation and emotion. Its v0.1 variants provide alternative architectures for expressive multilingual synthesis and reference-based cloning.","architecture":"Transformer or hybrid backbone with codec prediction","notes":"Speech synthesis conditioned on text, speaker references and delivery attributes.","variants":["v0.1 Transformer","v0.1 Hybrid"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/Zyphra/Zonos","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#zonos"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"Zyphra/Zonos — official repository","url":"https://github.com/Zyphra/Zonos","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--zonos.webp","kind":"source","label":"Official architecture diagram","sourceUrl":"https://github.com/Zyphra/Zonos","originUrl":"https://raw.githubusercontent.com/Zyphra/Zonos/main/assets/ArchitectureDiagram.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/zonos.png","sha256":"44b53a55f6eee58cc07a1ecd4d86575549f8957bab2f905f8db0f3c1edb71e6a"}},{"id":"tts:tacotron","slug":"tacotron","name":"Tacotron","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Maps characters to mel and linear spectrograms, then reconstructs a waveform with Griffin-Lim.","architecture":"Attention-based recurrent spectrogram synthesis","notes":"Maps characters to mel and linear spectrograms, then reconstructs a waveform with Griffin-Lim.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2017-03-29","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1703.10135","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#tacotron"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Tacotron: Towards End-to-End Speech Synthesis","url":"https://arxiv.org/abs/1703.10135","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--tacotron.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/1703.10135","originUrl":"https://arxiv.org/html/1703.10135v2/model.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tacotron.png","sha256":"fb0a0159f7e008aa0f421c5bb9382a0b04f1d804e3cde942748591597fb78a98"}},{"id":"tts:tacotron-2","slug":"tacotron-2","name":"Tacotron 2","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Predicts mel spectrograms from characters before waveform synthesis with a separately trained neural vocoder.","architecture":"Recurrent attention model and WaveNet vocoder","notes":"Predicts mel spectrograms from characters before waveform synthesis with a separately trained neural vocoder.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2017-12-16","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1712.05884","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#tacotron-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Natural TTS Synthesis by Conditioning WaveNet on Mel Spectrogram Predictions","url":"https://arxiv.org/abs/1712.05884","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--tacotron-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/1712.05884","originUrl":"https://arxiv.org/html/1712.05884v2/TTSArchitecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tacotron-2.png","sha256":"16d3aa6ac6b7a3422c196c5cf5f9d6ebc47aff0e2e05b50c2ab664bce4da49db"}},{"id":"tts:deep-voice","slug":"deep-voice","name":"Deep Voice","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Combines grapheme-to-phoneme, duration and pitch models with a waveform generator. Segmentation is used during training.","architecture":"Neural TTS pipeline with autoregressive WaveNet synthesis","notes":"Combines grapheme-to-phoneme, duration and pitch models with a waveform generator. Segmentation is used during training.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2017-02-25","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1702.07825","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#deep-voice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Deep Voice: Real-time Neural Text-to-Speech","url":"https://arxiv.org/abs/1702.07825","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--deep-voice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/1702.07825","originUrl":"https://arxiv.org/html/1702.07825v2/system-diagram.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/deep-voice.png","sha256":"673d8e4ee360582d47626d3dc74f20548db977ef90631f9a23dbeded47cbc870"}},{"id":"tts:deep-voice-2","slug":"deep-voice-2","name":"Deep Voice 2","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Extends Deep Voice with speaker embeddings and improved components. The paper also evaluates a separate multi-speaker Tacotron system.","architecture":"Speaker-conditioned neural pipeline and WaveNet synthesis","notes":"Extends Deep Voice with speaker embeddings and improved components. The paper also evaluates a separate multi-speaker Tacotron system.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2017-05-24","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1705.08947","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#deep-voice-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Deep Voice 2: Multi-Speaker Neural Text-to-Speech","url":"https://arxiv.org/abs/1705.08947","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--deep-voice-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/1705.08947","originUrl":"https://arxiv.org/html/1705.08947v2/system_model.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/deep-voice-2.png","sha256":"254b52eda702eed8d34be845fc3513b769d88afeb5029bdc263f9f4642520a3d"}},{"id":"tts:deep-voice-3","slug":"deep-voice-3","name":"Deep Voice 3","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Autoregressively predicts acoustic features; a converter prepares the representation required by the selected waveform synthesizer.","architecture":"Convolutional attention encoder-decoder and converter","notes":"Autoregressively predicts acoustic features; a converter prepares the representation required by the selected waveform synthesizer.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2017-10-20","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1710.07654","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#deep-voice-3"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Deep Voice 3: Scaling Text-to-Speech with Convolutional Sequence Learning","url":"https://arxiv.org/abs/1710.07654","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--deep-voice-3.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/1710.07654","originUrl":"https://arxiv.org/html/1710.07654v3/spectranet_concise.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/deep-voice-3.png","sha256":"902b62ee0efebd6b413733788c4efd588a7db314b3c37ba2333631d6c0445427"}},{"id":"tts:transformer-tts","slug":"transformer-tts","name":"Transformer TTS","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Replaces recurrent layers with multi-head attention in the text-to-mel model. Parallel training does not make its speech decoding non-autoregressive.","architecture":"Autoregressive Transformer and WaveNet vocoder","notes":"Replaces recurrent layers with multi-head attention in the text-to-mel model. Parallel training does not make its speech decoding non-autoregressive.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2018-09-19","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1809.08895","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#transformer-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Neural Speech Synthesis with Transformer Network","url":"https://arxiv.org/abs/1809.08895","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--transformer-tts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/1809.08895","originUrl":"https://arxiv.org/html/1809.08895v3/Transtron.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/transformer-tts.png","sha256":"04cf4b9cc513ab8822f777dcaf47ab2432a938bafc3b7bdea9d26358df947cdb"}},{"id":"tts:fastspeech","slug":"fastspeech","name":"FastSpeech","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Generates mel frames in parallel using durations learned from an autoregressive teacher, followed by a vocoder.","architecture":"Feed-forward Transformer and duration-based length regulator","notes":"Generates mel frames in parallel using durations learned from an autoregressive teacher, followed by a vocoder.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2019-05-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1905.09263","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#fastspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FastSpeech: Fast, Robust and Controllable Text to Speech","url":"https://arxiv.org/abs/1905.09263","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--fastspeech.webp","kind":"source","label":"Figure 1, PDF p. 4","sourceUrl":"https://arxiv.org/abs/1905.09263","originUrl":"https://arxiv.org/pdf/1905.09263","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fastspeech.png","sha256":"91c3248f676b028920036467c3ab3fbd84ba8ef829aa891b19eaf4af4f535f91"}},{"id":"tts:fastspeech-2","slug":"fastspeech-2","name":"FastSpeech 2","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Learns from ground-truth speech without teacher distillation. FastSpeech 2s adds a parallel waveform decoder.","architecture":"Parallel Transformer with duration, pitch and energy prediction","notes":"Learns from ground-truth speech without teacher distillation. FastSpeech 2s adds a parallel waveform decoder.","variants":["FastSpeech 2s"],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2020-06-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2006.04558","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#fastspeech-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FastSpeech 2: Fast and High-Quality End-to-End Text to Speech","url":"https://arxiv.org/abs/2006.04558","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--fastspeech-2.webp","kind":"source","label":"Figure 1, PDF p. 3","sourceUrl":"https://arxiv.org/abs/2006.04558","originUrl":"https://arxiv.org/pdf/2006.04558","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fastspeech-2.png","sha256":"825a663693b22c3b83ee522c8711f83ea994a3a4c66309dbc6ea647ac01e8066"}},{"id":"tts:fastpitch","slug":"fastpitch","name":"FastPitch","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Predicts token durations and pitch before mel-spectrogram generation; pitch predictions can be edited to control delivery.","architecture":"Parallel Transformer with explicit pitch prediction","notes":"Predicts token durations and pitch before mel-spectrogram generation; pitch predictions can be edited to control delivery.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2020-06-11","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2006.06873","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#fastpitch"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FastPitch: Parallel Text-to-speech with Pitch Prediction","url":"https://arxiv.org/abs/2006.06873","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--fastpitch.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2006.06873","originUrl":"https://arxiv.org/html/2006.06873v2/model.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fastpitch.png","sha256":"f55d6131dc63e2420dcee0871091eccf82bc5390f450b70831ba4f64696ed9d1"}},{"id":"tts:parallel-tacotron","slug":"parallel-tacotron","name":"Parallel Tacotron","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Uses lightweight convolutions and residual variation modeling to generate mel spectrograms, followed by a neural vocoder.","architecture":"Parallel acoustic model with a variational residual encoder","notes":"Uses lightweight convolutions and residual variation modeling to generate mel spectrograms, followed by a neural vocoder.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2020-10-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2010.11439","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#parallel-tacotron"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Parallel Tacotron: Non-Autoregressive and Controllable TTS","url":"https://arxiv.org/abs/2010.11439","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--parallel-tacotron.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2010.11439","originUrl":"https://arxiv.org/html/2010.11439v1/PTacoBlock.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/parallel-tacotron.png","sha256":"689446ef45b2d2eef7bc51577a65a03d356cd31921242968d707988d06a245c8"}},{"id":"tts:parallel-tacotron-2","slug":"parallel-tacotron-2","name":"Parallel Tacotron 2","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Learns alignment and token durations through differentiable upsampling and a Soft-DTW reconstruction objective.","architecture":"Parallel synthesis with differentiable duration modeling","notes":"Learns alignment and token durations through differentiable upsampling and a Soft-DTW reconstruction objective.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-03-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2103.14574","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#parallel-tacotron-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Parallel Tacotron 2: A Non-Autoregressive Neural TTS Model with Differentiable Duration Modeling","url":"https://arxiv.org/abs/2103.14574","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--parallel-tacotron-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2103.14574","originUrl":"https://arxiv.org/html/2103.14574v7/MoreSimplifiedPTacoBlock.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/parallel-tacotron-2.png","sha256":"43a40ae5cc6dbf291c2f48bceedcbc330bcb5f13e74d64ce8fd93aa3f662e13b"}},{"id":"tts:speedyspeech","slug":"speedyspeech","name":"SpeedySpeech","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"A teacher provides durations for the parallel student; a MelGAN vocoder converts predicted spectrograms to audio.","architecture":"Residual convolutional acoustic model with duration expansion","notes":"A teacher provides durations for the parallel student; a MelGAN vocoder converts predicted spectrograms to audio.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2020-08-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2008.03802","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#speedyspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SpeedySpeech: Efficient Neural Speech Synthesis","url":"https://arxiv.org/abs/2008.03802","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--speedyspeech.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2008.03802","originUrl":"https://arxiv.org/html/2008.03802v1/speedy_speech.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/speedyspeech.png","sha256":"547787d1688c9927976d7b129e00868eba11807f8aec9ef6305a51b84c48ac4c"}},{"id":"tts:lightspeech","slug":"lightspeech","name":"LightSpeech","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Searches lightweight convolutional and attention components for compact acoustic modeling. This entry covers the Luo et al. model.","architecture":"FastSpeech-derived architecture found by neural architecture search","notes":"Searches lightweight convolutional and attention components for compact acoustic modeling. This entry covers the Luo et al. model.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-02-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2102.04040","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#lightspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"LightSpeech: Lightweight and Fast Text to Speech with Neural Architecture Search","url":"https://arxiv.org/abs/2102.04040","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--lightspeech.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://arxiv.org/abs/2102.04040","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/lightspeech.svg","sha256":"96d47a9cb6802013c1b10ff5b4f3b872bf4f77ca28f767da3bf2030764c2d7a4"}},{"id":"tts:glow-tts","slug":"glow-tts","name":"Glow-TTS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"Learns text-to-speech alignment without an external aligner and generates mel spectrograms in parallel before vocoding.","architecture":"Normalizing-flow acoustic model and monotonic alignment search","notes":"Learns text-to-speech alignment without an external aligner and generates mel spectrograms in parallel before vocoding.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2020-05-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2005.11129","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#glow-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Glow-TTS: A Generative Flow for Text-to-Speech via Monotonic Alignment Search","url":"https://arxiv.org/abs/2005.11129","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--glow-tts.webp","kind":"source","label":"Figure 1, PDF p. 3","sourceUrl":"https://arxiv.org/abs/2005.11129","originUrl":"https://arxiv.org/pdf/2005.11129","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/glow-tts.png","sha256":"86851cb9d3bc5f4884ca41a79b827dca64914c74b30aca0ea53989ce81f24a40"}},{"id":"tts:vits","slug":"vits","name":"VITS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"Jointly trains text-to-waveform synthesis with a stochastic duration predictor. Reference-audio cloning is not an intrinsic capability of the original model.","architecture":"Conditional VAE, normalizing flows and adversarial waveform decoding","notes":"Jointly trains text-to-waveform synthesis with a stochastic duration predictor. Reference-audio cloning is not an intrinsic capability of the original model.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-06-11","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2106.06103","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#vits"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech","url":"https://arxiv.org/abs/2106.06103","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vits.webp","kind":"source","label":"Figure 1, PDF p. 3","sourceUrl":"https://arxiv.org/abs/2106.06103","originUrl":"https://arxiv.org/pdf/2106.06103","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vits.png","sha256":"7742958606343abc1a39da628ab3c39f18de73738d1229c2e25bb409929b130d"}},{"id":"tts:vits-2","slug":"vits-2","name":"VITS2","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"Improves duration prediction, flow layers and speaker conditioning while reducing dependence on phoneme conversion.","architecture":"VITS with adversarial duration training and revised flows","notes":"Improves duration prediction, flow layers and speaker conditioning while reducing dependence on phoneme conversion.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-07-31","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2307.16430","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#vits-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VITS2: Improving Quality and Efficiency of Single-Stage Text-to-Speech with Adversarial Learning and Architecture Design","url":"https://arxiv.org/abs/2307.16430","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vits-2.webp","kind":"source","label":"Figure 1, PDF p. 2","sourceUrl":"https://arxiv.org/abs/2307.16430","originUrl":"https://arxiv.org/pdf/2307.16430","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vits-2.png","sha256":"05544ac2416678fcc668e0e214355dd485fe11f2b032983715ea137337d96ca1"}},{"id":"tts:yourtts","slug":"yourtts","name":"YourTTS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"Extends VITS for zero-shot multi-speaker synthesis; the broader model also supports voice conversion and speaker adaptation.","architecture":"Multilingual VITS with speaker-reference conditioning","notes":"Extends VITS for zero-shot multi-speaker synthesis; the broader model also supports voice conversion and speaker adaptation.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-12-04","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2112.02418","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#yourtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"YourTTS: Towards Zero-Shot Multi-Speaker TTS and Zero-Shot Voice Conversion for everyone","url":"https://arxiv.org/abs/2112.02418","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--yourtts.webp","kind":"source","label":"Figure 1, PDF p. 3","sourceUrl":"https://arxiv.org/abs/2112.02418","originUrl":"https://arxiv.org/pdf/2112.02418","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/yourtts.png","sha256":"92503d329b06df7fec26a2c4faeea92cb6d6edb14d7036d4a5f7f9f1b5db8bfe"}},{"id":"tts:flowtron","slug":"flowtron","name":"Flowtron","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"An invertible acoustic model supports sampling and style transfer. A separate vocoder produces the waveform.","architecture":"Autoregressive normalizing flows over mel spectrograms","notes":"An invertible acoustic model supports sampling and style transfer. A separate vocoder produces the waveform.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2020-05-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2005.05957","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#flowtron"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Flowtron: an Autoregressive Flow-based Generative Network for Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2005.05957","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--flowtron.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2005.05957","originUrl":"https://arxiv.org/html/2005.05957v3/images/flowtron.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/flowtron.png","sha256":"06ed3872bcdc1ac2ab2e43baa96eda84f70ddf28852ad7128a5032fa5c8d9ef9"}},{"id":"tts:styletts","slug":"styletts","name":"StyleTTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Extracts speaking style from reference speech and predicts duration, pitch and energy for acoustic generation.","architecture":"Style-conditioned parallel synthesis with a transferable aligner","notes":"Extracts speaking style from reference speech and predicts duration, pitch and energy for acoustic generation.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2022-05-30","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2205.15439","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#styletts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"StyleTTS: A Style-Based Generative Model for Natural and Diverse Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2205.15439","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--styletts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2205.15439","originUrl":"https://arxiv.org/html/2205.15439v2/Figure1_new_1.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/styletts.png","sha256":"bab31fec1536eaff9abcef38a1d98e6a75063e7b9e7ab27e15de13efdb73d5f1"}},{"id":"tts:grad-tts","slug":"grad-tts","name":"Grad-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Transforms text-conditioned noise into a mel spectrogram; the number of diffusion steps controls the inference tradeoff.","architecture":"Score-based diffusion decoder and monotonic alignment search","notes":"Transforms text-conditioned noise into a mel spectrogram; the number of diffusion steps controls the inference tradeoff.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-05-13","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2105.06337","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#grad-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Grad-TTS: A Diffusion Probabilistic Model for Text-to-Speech","url":"https://arxiv.org/abs/2105.06337","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--grad-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2105.06337","originUrl":"https://arxiv.org/html/2105.06337v2/grad-tts-new.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/grad-tts.png","sha256":"944739fc185e8dfbb14b239e11d50636e22f25652063dd46933477f06a0eab84"}},{"id":"tts:diff-tts","slug":"diff-tts","name":"Diff-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Uses a non-autoregressive diffusion process to generate mel spectrograms and accelerated sampling before vocoding.","architecture":"Text-conditioned denoising diffusion acoustic model","notes":"Uses a non-autoregressive diffusion process to generate mel spectrograms and accelerated sampling before vocoding.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-04-03","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2104.01409","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#diff-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Diff-TTS: A Denoising Diffusion Model for Text-to-Speech","url":"https://arxiv.org/abs/2104.01409","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--diff-tts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2104.01409","originUrl":"https://arxiv.org/html/2104.01409v1/final_model.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/diff-tts.png","sha256":"478428727a9d636fdd4569e7c6138b5d44ea80d530e2a3056a526799104403c5"}},{"id":"tts:naturalspeech","slug":"naturalspeech","name":"NaturalSpeech","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"Combines phoneme pretraining, a differentiable durator, bidirectional prior/posterior modeling and a memory-equipped decoder.","architecture":"Text-to-waveform VAE with enhanced prior and duration modeling","notes":"Combines phoneme pretraining, a differentiable durator, bidirectional prior/posterior modeling and a memory-equipped decoder.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2022-05-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2205.04421","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#naturalspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"NaturalSpeech: End-to-End Text to Speech Synthesis with Human-Level Quality","url":"https://arxiv.org/abs/2205.04421","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--naturalspeech.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2205.04421","originUrl":"https://arxiv.org/html/2205.04421v2/x1.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/naturalspeech.png","sha256":"f67b99a4f7e06531d9b15962024f4edb60024589497996f3e12a2679f28f933b"}},{"id":"tts:naturalspeech-2","slug":"naturalspeech-2","name":"NaturalSpeech 2","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Speech prompts condition duration, pitch and latent generation for zero-shot synthesis. Singing is an additional task in the paper.","architecture":"Latent diffusion over neural-codec representations","notes":"Speech prompts condition duration, pitch and latent generation for zero-shot synthesis. Singing is an additional task in the paper.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-04-18","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2304.09116","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#naturalspeech-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"NaturalSpeech 2: Latent Diffusion Models are Natural and Zero-Shot Speech and Singing Synthesizers","url":"https://arxiv.org/abs/2304.09116","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--naturalspeech-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2304.09116","originUrl":"https://arxiv.org/html/2304.09116v3/x1.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/naturalspeech-2.png","sha256":"a27ae0e50785fb4c10813b3860ff8f787cb10f3706fb77c1e595fe869b4ab19b"}},{"id":"tts:naturalspeech-3","slug":"naturalspeech-3","name":"NaturalSpeech 3","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Separates content, prosody, timbre and acoustic detail, then generates the corresponding speech representations with diffusion models.","architecture":"Factorized speech codec and attribute-wise diffusion","notes":"Separates content, prosody, timbre and acoustic detail, then generates the corresponding speech representations with diffusion models.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-03-05","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2403.03100","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#naturalspeech-3"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"NaturalSpeech 3: Zero-Shot Speech Synthesis with Factorized Codec and Diffusion Models","url":"https://arxiv.org/abs/2403.03100","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--naturalspeech-3.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2403.03100","originUrl":"https://arxiv.org/html/2403.03100v3/fig-ns3-gen-new.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/naturalspeech-3.png","sha256":"0970a23f96ca83cdc7cfe3c0adf3612fda5a6fbece77838b62eb96e287087349"}},{"id":"tts:styletts-2","slug":"styletts-2","name":"StyleTTS 2","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Samples style conditioned on text; multi-speaker synthesis can additionally use a voice reference. The speech model acts as a discriminator during training.","architecture":"Style diffusion and adversarial training with speech-model discriminators","notes":"Samples style conditioned on text; multi-speaker synthesis can additionally use a voice reference. The speech model acts as a discriminator during training.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-06-13","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2306.07691","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#styletts-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"StyleTTS 2: Towards Human-Level Text-to-Speech through Style Diffusion and Adversarial Training with Large Speech Language Models","url":"https://arxiv.org/abs/2306.07691","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--styletts-2.webp","kind":"source","label":"Figure 1, PDF p. 4","sourceUrl":"https://arxiv.org/abs/2306.07691","originUrl":"https://arxiv.org/pdf/2306.07691","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/styletts-2.png","sha256":"85daeeeba6af286bb1f6bd96f302510de2cb295b462a8584eca458410f79aaa5"}},{"id":"tts:voicebox","slug":"voicebox","name":"Voicebox","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Uses text and surrounding audio for zero-shot synthesis. Editing, denoising and style transfer are other applications of the model.","architecture":"Non-autoregressive flow matching for speech infilling","notes":"Uses text and surrounding audio for zero-shot synthesis. Editing, denoising and style transfer are other applications of the model.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-06-23","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2306.15687","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#voicebox"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Voicebox: Text-Guided Multilingual Universal Speech Generation at Scale","url":"https://arxiv.org/abs/2306.15687","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--voicebox.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2306.15687","originUrl":"https://arxiv.org/html/2306.15687v2/voicebox_model_diagram.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voicebox.png","sha256":"0eed6448eb134d9572cb4c07f376364be2ee8a28ae09769b4a8218257f7290a1"}},{"id":"tts:matcha-tts","slug":"matcha-tts","name":"Matcha-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"An ODE decoder generates mel spectrograms from aligned text representations, followed by a neural vocoder.","architecture":"Conditional flow matching with an encoder-decoder acoustic model","notes":"An ODE decoder generates mel spectrograms from aligned text representations, followed by a neural vocoder.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-09-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2309.03199","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#matcha-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Matcha-TTS: A fast TTS architecture with conditional flow matching","url":"https://arxiv.org/abs/2309.03199","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--matcha-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2309.03199","originUrl":"https://arxiv.org/html/2309.03199v2/architectural_overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/matcha-tts.png","sha256":"269d95efc9b0edbc7430f0d860f826a3913081ab49b26c78af304974df261c9e"}},{"id":"tts:f5-tts","slug":"f5-tts","name":"F5-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"F5-TTS learns text-guided speech infilling with flow matching, refining character representations before a Transformer predicts the speech trajectory. A reference clip supplies voice context, and Sway Sampling controls how inference steps are distributed. The 2025 v1 Base release refines training and inference within the same general architecture for zero-shot speech synthesis.","architecture":"Flow-matching Transformer with ConvNeXt text refinement","notes":"Pads character sequences to speech length and learns text-guided audio infilling; Sway Sampling adjusts inference steps.","variants":["F5-TTS v1 Base"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-10-09","dateKind":"paper","links":{"code":"https://github.com/SWivid/F5-TTS","weights":null,"paper":"https://arxiv.org/abs/2410.06885","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#f5-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"F5-TTS: A Fairytaler that Fakes Fluent and Faithful Speech with Flow Matching","url":"https://arxiv.org/abs/2410.06885","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"F5-TTS — official implementation","url":"https://github.com/SWivid/F5-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--f5-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2410.06885","originUrl":"https://arxiv.org/html/2410.06885v3/F5-TTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/f5-tts.png","sha256":"8abcae1a304cf4b781ee4d905f48e1531ffca785f8242d7ac24ba1c09cdc22e0"}},{"id":"tts:e2-tts","slug":"e2-tts","name":"E2 TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Learns speech infilling without a phoneme aligner or explicit duration predictor. The paper describes alternate input representations.","architecture":"Flow matching with filler-token text conditioning","notes":"Learns speech infilling without a phoneme aligner or explicit duration predictor. The paper describes alternate input representations.","variants":["E2 TTS X1","E2 TTS X2"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-06-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2406.18009","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#e2-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"E2 TTS: Embarrassingly Easy Fully Non-Autoregressive Zero-Shot TTS","url":"https://arxiv.org/abs/2406.18009","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--e2-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2406.18009","originUrl":"https://arxiv.org/html/2406.18009v2/e2tts_main_figure_c.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/e2-tts.png","sha256":"5969e3bc6bbd5201bce5071d50d51578ce6abbe817dfb3ef2624e2e180e319c7"}},{"id":"tts:ditto-tts","slug":"ditto-tts","name":"DiTTo-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Uses text conditioning and a total-length predictor instead of phoneme-level durations for zero-shot speech synthesis.","architecture":"Latent diffusion Transformer with speech-length prediction","notes":"Uses text conditioning and a total-length predictor instead of phoneme-level durations for zero-shot speech synthesis.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2024-06-17","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2406.11427","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#ditto-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DiTTo-TTS: Diffusion Transformers for Scalable Text-to-Speech without Domain-Specific Factors","url":"https://arxiv.org/abs/2406.11427","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--ditto-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2406.11427","originUrl":"https://arxiv.org/html/2406.11427v2/thisis9.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ditto-tts.png","sha256":"06bb0b9f5609c27a7829e57f2fef8799d561ca974f1afaec4de86ee2cf382e60"}},{"id":"tts:mega-tts-3","slug":"mega-tts-3","name":"MegaTTS 3","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"MegaTTS 3 guides a latent diffusion Transformer with sparse text-speech alignment boundaries, leaving the model room to learn finer timing. Classifier-free guidance controls accent strength, while piecewise rectified flow reduces sampling work. The design targets robust zero-shot voice synthesis with more flexible alignment than a fully fixed duration sequence.","architecture":"Sparse-alignment latent diffusion Transformer","notes":"Sparse boundaries guide text-speech alignment; piecewise rectified flow accelerates synthesis and guidance controls accent strength.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-26","dateKind":"paper","links":{"code":"https://github.com/bytedance/MegaTTS3","weights":null,"paper":"https://arxiv.org/abs/2502.18924","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#mega-tts-3"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MegaTTS 3: Sparse Alignment Enhanced Latent Diffusion Transformer for Zero-Shot Speech Synthesis","url":"https://arxiv.org/abs/2502.18924","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MegaTTS 3 author repository","url":"https://github.com/bytedance/MegaTTS3","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--mega-tts-3.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.18924","originUrl":"https://arxiv.org/html/2502.18924v4/Main_Arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mega-tts-3.png","sha256":"329aa026a97936fd8c0aedb554bafe14dd7eb65017ce10789259262320aac2fa"}},{"id":"tts:portaspeech","slug":"portaspeech","name":"PortaSpeech","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"Combines word-level and phoneme-level alignment with a lightweight VAE; the post-net restores acoustic detail before vocoding.","architecture":"Variational acoustic model and flow-based post-net","notes":"Combines word-level and phoneme-level alignment with a lightweight VAE; the post-net restores acoustic detail before vocoding.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-09-30","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2109.15166","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#portaspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"PortaSpeech: Portable and High-Quality Generative Text-to-Speech","url":"https://arxiv.org/abs/2109.15166","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--portaspeech.webp","kind":"source","label":"Figure 1, PDF p. 4","sourceUrl":"https://arxiv.org/abs/2109.15166","originUrl":"https://arxiv.org/pdf/2109.15166","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/portaspeech.png","sha256":"cd2280e6e0d3cd1204ad2c6fd1f1a4850387a7f1c5c80c4e0d131220a17b4d41"}},{"id":"tts:delightfultts","slug":"delightfultts","name":"DelightfulTTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Predicts acoustic features and variation factors, then uses HiFiNet to synthesize higher-sample-rate audio.","architecture":"Conformer acoustic model with explicit and implicit prosody","notes":"Predicts acoustic features and variation factors, then uses HiFiNet to synthesize higher-sample-rate audio.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2021-10-25","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2110.12612","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#delightfultts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DelightfulTTS: The Microsoft Speech Synthesis System for Blizzard Challenge 2021","url":"https://arxiv.org/abs/2110.12612","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--delightfultts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2110.12612","originUrl":"https://arxiv.org/html/2110.12612v2/v16.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/delightfultts.png","sha256":"39e78c47d3758f609b6554393259c3d7944ab900a1e9a99facd96df834966b21"}},{"id":"tts:jets","slug":"jets","name":"JETS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Trains the acoustic model and waveform generator together, avoiding a separately supplied text-speech alignment.","architecture":"Joint FastSpeech 2 and HiFi-GAN with learned alignment","notes":"Trains the acoustic model and waveform generator together, avoiding a separately supplied text-speech alignment.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2022-03-31","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2203.16852","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#jets"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"JETS: Jointly Training FastSpeech2 and HiFi-GAN for End to End Text to Speech","url":"https://arxiv.org/abs/2203.16852","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--jets.webp","kind":"source","label":"Figure 1, PDF p. 2","sourceUrl":"https://arxiv.org/abs/2203.16852","originUrl":"https://arxiv.org/pdf/2203.16852","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/jets.png","sha256":"5e65ba50a3438ade5594a4a9fc28f436f8bc0fff0a16c4922fec3064c3cfb831"}},{"id":"tts:mellotron","slug":"mellotron","name":"Mellotron","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Reference audio can provide prosody. The broader model also accepts musical-score controls and supports singing synthesis.","architecture":"Tacotron 2 with global style tokens, pitch and rhythm conditioning","notes":"Reference audio can provide prosody. The broader model also accepts musical-score controls and supports singing synthesis.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2019-10-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1910.11997","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#mellotron"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Mellotron: Multispeaker expressive voice synthesis by conditioning on rhythm, pitch and global style tokens","url":"https://arxiv.org/abs/1910.11997","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mellotron.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://arxiv.org/abs/1910.11997","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mellotron.svg","sha256":"c0794d9ed62ae104f62f16dddadb5c6b9f15a89bbe0536d3bbd4b19382f90fb3"}},{"id":"tts:gst-tacotron","slug":"gst-tacotron","name":"GST-Tacotron","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Unsupervised style embeddings control delivery; synthesis can use reference audio or manually selected token weights.","architecture":"Tacotron with a reference encoder and global style tokens","notes":"Unsupervised style embeddings control delivery; synthesis can use reference audio or manually selected token weights.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2018-03-23","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/1803.09017","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#gst-tacotron"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Style Tokens: Unsupervised Style Modeling, Control and Transfer in End-to-End Speech Synthesis","url":"https://arxiv.org/abs/1803.09017","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--gst-tacotron.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/1803.09017","originUrl":"https://arxiv.org/html/1803.09017v1/st_token_small.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/gst-tacotron.png","sha256":"a33b44021e260343e2830ef575ac1b45b82cb5690508079a0fa2484775d64384"}},{"id":"tts:wave-tacotron","slug":"wave-tacotron","name":"Wave-Tacotron","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"Models long-range dependencies across waveform blocks autoregressively and samples each block with a conditional flow.","architecture":"Autoregressive waveform blocks with a normalizing-flow decoder","notes":"Models long-range dependencies across waveform blocks autoregressively and samples each block with a conditional flow.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2020-11-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2011.03568","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#wave-tacotron"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Wave-Tacotron: Spectrogram-free end-to-end text-to-speech synthesis","url":"https://arxiv.org/abs/2011.03568","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--wave-tacotron.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2011.03568","originUrl":"https://arxiv.org/html/2011.03568v2/2020-09-10-waveform_tacotron_system_diagram-v6-noR.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/wave-tacotron.png","sha256":"4df9b259f0ca7cdf3cbda7d840117c2c3ce82a301edee7bf9381e64bb37d234a"}},{"id":"tts:prompttts","slug":"prompttts","name":"PromptTTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Accepts a spoken script plus a natural-language description of the desired voice and delivery.","architecture":"Style and content text encoders with a speech decoder","notes":"Accepts a spoken script plus a natural-language description of the desired voice and delivery.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2022-11-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2211.12171","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#prompttts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"PromptTTS: Controllable Text-to-Speech with Text Descriptions","url":"https://arxiv.org/abs/2211.12171","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--prompttts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2211.12171","originUrl":"https://arxiv.org/html/2211.12171v1/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/prompttts.png","sha256":"0e930fad2904100ebc04a81955e868905950a3224fd2e1be36d7c930fa809a95"}},{"id":"tts:prompttts-2","slug":"prompttts-2","name":"PromptTTS 2","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Samples voice variability not captured by the text description. Reference speech is used in training; the prompt-generation LLM is a separate data-construction component.","architecture":"Prompt-conditioned TTS with a diffusion variation network","notes":"Samples voice variability not captured by the text description. Reference speech is used in training; the prompt-generation LLM is a separate data-construction component.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2023-09-05","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2309.02285","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#prompttts-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"PromptTTS 2: Describing and Generating Voices with Text Prompt","url":"https://arxiv.org/abs/2309.02285","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--prompttts-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2309.02285","originUrl":"https://arxiv.org/html/2309.02285v2/figs/model.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/prompttts-2.png","sha256":"8675e9628674ea2ce8584cf35b63281b4b763406ea831e40572e2556eb87de01"}},{"id":"tts:kokoro","slug":"kokoro","name":"Kokoro","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"Kokoro's 2025 v1.0 release uses a compact StyleTTS 2-derived decoder with an iSTFTNet waveform generator. The released model relies on preset voice representations and omits style diffusion and a reference encoder. It is suited to lightweight narration and application speech where a small synthesis model and ready-made voices are useful.","architecture":"StyleTTS 2-derived decoder with iSTFTNet","notes":"The released 82M model omits style diffusion and a reference encoder; preset voice representations condition synthesis.","variants":["Kokoro-82M"],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#kokoro"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"model-card","title":"Kokoro-82M model card","url":"https://huggingface.co/hexgrad/Kokoro-82M","reviewed_on":"2026-09-13","review_level":"model-card"}],"figure":{"path":"assets/figures/tts--kokoro.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/hexgrad/Kokoro-82M","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/kokoro.svg","sha256":"aad79a6b8aa76da822af0086e7b2340a995b1237253f209e2eaefcf5d1bcaf9d"}},{"id":"tts:kitten-tts","slug":"kitten-tts","name":"KittenTTS","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"KittenTTS provides small ONNX speech models with built-in voices and adjustable playback speed. The Mini, Micro and Nano releases offer different size and inference tradeoffs for CPU-oriented applications. The official README documents usage more fully than internal acoustic design, so the catalog presents it as a compact synthesis family without asserting an undisclosed architecture.","architecture":"Compact neural speech synthesis with ONNX inference","notes":"The official release provides CPU-oriented models and preset voices. Its README does not fully disclose the internal acoustic architecture.","variants":["Mini 0.8","Micro 0.8","Nano 0.8","Nano 0.8 int8"],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/KittenML/KittenTTS","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#kitten-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"KittenTTS official repository","url":"https://github.com/KittenML/KittenTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--kitten-tts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/KittenML/KittenTTS","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/kitten-tts.svg","sha256":"d9da471b2d6f0e017dc8ffa12bcfcb4ae42b95f491f8494a98f7eed64ebb4185"}},{"id":"tts:melo-tts","slug":"melo-tts","name":"MeloTTS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"The implementation builds on VITS, VITS2 and Bert-VITS2. Language and speaker selections depend on the released checkpoint.","architecture":"VITS-family multilingual speech synthesis","notes":"The implementation builds on VITS, VITS2 and Bert-VITS2. Language and speaker selections depend on the released checkpoint.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/myshell-ai/MeloTTS","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#melo-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"MeloTTS official repository","url":"https://github.com/myshell-ai/MeloTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--melo-tts.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/myshell-ai/MeloTTS","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/melo-tts.svg","sha256":"1a504982709a44e3eceaa5d577412e915989fc59ad8d92ff40c18e0837ebc3c9"}},{"id":"tts:piper","slug":"piper","name":"Piper (VITS voices)","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"This entry covers the trained voice family distributed through Piper. The runtime and individual voice packs are not counted as separate architectures.","architecture":"VITS voice models exported for local inference","notes":"This entry covers the trained voice family distributed through Piper. The runtime and individual voice packs are not counted as separate architectures.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/OHF-Voice/piper1-gpl","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#piper"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"Piper official repository","url":"https://github.com/OHF-Voice/piper1-gpl","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"documentation","title":"Piper voice training guide","url":"https://github.com/rhasspy/piper/blob/master/TRAINING.md","reviewed_on":"2026-09-13","review_level":"documentation"}],"figure":{"path":"assets/figures/tts--piper.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/OHF-Voice/piper1-gpl","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/piper.svg","sha256":"d2871c869003780f1559d23341c79bdae1f3a20e92229bbb80aa82c75c0e8452"}},{"id":"tts:neutts-air","slug":"neutts-air","name":"NeuTTS Air","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"NeuTTS Air pairs a phoneme-conditioned language model with NeuCodec to synthesize a reference voice locally. Quantized GGUF backbones support incremental generation through the documented streaming backend. It targets embedded and desktop voice applications, with the codec's compute and memory requirements considered alongside those of the language-model backbone.","architecture":"Compact language-model backbone and single-codebook NeuCodec","notes":"Phoneme-conditioned voice cloning with quantized local inference; streaming is documented for the GGUF backend.","variants":["PyTorch","Q4 GGUF","Q8 GGUF"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/neuphonic/neutts","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#neutts-air"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"NeuTTS official repository","url":"https://github.com/neuphonic/neutts","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--neutts-air.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/neuphonic/neutts","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/neutts-air.svg","sha256":"5597097b46b146ef683bd91ff35623232a8284892d3bee936707b347465f2016"}},{"id":"tts:neutts-nano","slug":"neutts-nano","name":"NeuTTS Nano","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"NeuTTS Nano reduces the speech-model backbone while retaining phoneme conditioning, reference-based cloning and NeuCodec reconstruction. Its English, German, French and Spanish models share a design but use separate language-specific checkpoints. Quantized variants support compact local deployments, and streaming depends on the selected inference backend.","architecture":"Smaller speech language model with NeuCodec decoding","notes":"A smaller phoneme-input family; language-specific checkpoints share an architecture. Streaming is documented for GGUF inference.","variants":["English","German","French","Spanish","Q4 / Q8 GGUF"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/neuphonic/neutts","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#neutts-nano"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"NeuTTS official repository","url":"https://github.com/neuphonic/neutts","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--neutts-nano.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/neuphonic/neutts","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/neutts-nano.svg","sha256":"bdcf094182d3a98a83b8111992bd15e4679ab8966691aba40a47cf8fb16879fd"}},{"id":"tts:neutts-2e","slug":"neutts-2e","name":"NeuTTS-2E","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"NeuTTS-2E accepts text directly and adds explicit emotional delivery to the NeuTTS language-model-and-codec pipeline. The released configuration supplies four fixed speaker presets instead of arbitrary reference-based cloning. It is intended for compact expressive speech applications, with streaming available through the documented GGUF inference path.","architecture":"Text-input speech language model with emotion control","notes":"Uses fixed speaker presets and emotion controls. The official release documents streaming through GGUF inference.","variants":["PyTorch","Q4 GGUF","Q8 GGUF"],"inputs":["T"],"outputs":["S"],"interaction":"streaming","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/neuphonic/neutts","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#neutts-2e"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"NeuTTS official repository","url":"https://github.com/neuphonic/neutts","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--neutts-2e.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/neuphonic/neutts","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/neutts-2e.svg","sha256":"eae0d70ac773ff14fefa57b0027eeb164a19e35a10c1f5a025d639b20ad06e08"}},{"id":"tts:whisperspeech","slug":"whisperspeech","name":"WhisperSpeech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Builds semantic units from Whisper and predicts acoustic tokens for waveform decoding. Voice references condition synthesis; this is separate from Whisper ASR.","architecture":"Text-to-semantic and semantic-to-acoustic token models","notes":"Builds semantic units from Whisper and predicts acoustic tokens for waveform decoding. Voice references condition synthesis; this is separate from Whisper ASR.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/WhisperSpeech/WhisperSpeech","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#whisperspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"WhisperSpeech official repository","url":"https://github.com/WhisperSpeech/WhisperSpeech","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--whisperspeech.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/WhisperSpeech/WhisperSpeech","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/whisperspeech.svg","sha256":"139612a66dd13205eb2d7907e4ea5b9526b5a4ca1a2e34ad7eecab44489fb897"}},{"id":"tts:emotivoice","slug":"emotivoice","name":"EmotiVoice","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Released English and Chinese synthesizers accept text prompts describing emotion and delivery. Voice adaptation is a separate training workflow.","architecture":"PromptTTS-derived style and content conditioning","notes":"Released English and Chinese synthesizers accept text prompts describing emotion and delivery. Voice adaptation is a separate training workflow.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":"https://github.com/netease-youdao/EmotiVoice","weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#emotivoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"repository","title":"EmotiVoice official repository","url":"https://github.com/netease-youdao/EmotiVoice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--emotivoice.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://github.com/netease-youdao/EmotiVoice","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emotivoice.svg","sha256":"fa2916e84b808aff9d1984239889af824bc09c5d1bfd0159b48e08753b58b10f"}},{"id":"tts:supertonic","slug":"supertonic","name":"Supertonic","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"The SupertonicTTS research system compresses speech into continuous latents and predicts them from character-level text with flow matching. ConvNeXt blocks, temporal compression and a separate duration predictor keep synthesis compact. The later Supertonic ONNX release exposes preset voice-style assets; its packaged configurations should not be equated with the paper's 44M-parameter research model.","architecture":"Compact latent flow matching with a speech autoencoder and duration predictor","notes":"The paper supports reference-conditioned synthesis, while the original open-weight ONNX release provides preset English voices. The source date is the research paper's first submission, not the ONNX release date.","variants":["SupertonicTTS research model","Supertonic ONNX release"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-03-29","dateKind":"paper","links":{"code":"https://github.com/supertone-inc/supertonic","weights":null,"paper":"https://arxiv.org/abs/2503.23108","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#supertonic"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"model-card","title":"Supertonic model card","url":"https://huggingface.co/Supertone/supertonic","reviewed_on":"2026-09-13","review_level":"model-card"},{"kind":"paper","title":"SupertonicTTS: Towards Highly Efficient and Streamlined Text-to-Speech System","url":"https://arxiv.org/abs/2503.23108","reviewed_on":"2026-09-13","review_level":"paper"},{"kind":"repository","title":"Supertonic author repository","url":"https://github.com/supertone-inc/supertonic","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"Supertonic author demo","url":"https://supertonictts.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--supertonic.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2503.23108","originUrl":"https://arxiv.org/html/2503.23108v3/SupertonicTTS-overview-cropped.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/supertonic.png","sha256":"ac16f21319411f5bdf8c276374ff3542143da699112d1b1934194740212593ab"}},{"id":"tts:supertonic-2","slug":"supertonic-2","name":"Supertonic 2","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"Supertonic 2 extends the local ONNX synthesis line to five languages while retaining voice-style conditioning and a compact model. It provides a practical path to multilingual narration on devices that can run the supplied inference stack. Creating a new voice-style asset is a separate workflow from generating speech with an existing asset.","architecture":"Multilingual compact speech synthesis with ONNX inference","notes":"Extends the local preset-voice release to English, Korean, Spanish, Portuguese and French. Voice-style assets condition synthesis.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#supertonic-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"model-card","title":"Supertonic 2 model card","url":"https://huggingface.co/Supertone/supertonic-2","reviewed_on":"2026-09-13","review_level":"model-card"}],"figure":{"path":"assets/figures/tts--supertonic-2.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/Supertone/supertonic-2","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/supertonic-2.svg","sha256":"5517b22f37cfd355f96a6fdd419156ff21792ee9c18ab6aaf64fe586c3adcef3"}},{"id":"tts:supertonic-3","slug":"supertonic-3","name":"Supertonic 3","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"Supertonic 3 expands language coverage and adds expression tags while keeping local ONNX inference and preset voice styles. The release targets more reliable reading across short and long text, with controls for events such as breaths or laughter. Custom voice-style creation is offered through a separate service; downloaded styles can then condition local synthesis.","architecture":"Compact multilingual synthesis with expression controls","notes":"The open-weight package includes preset voices and expression tags across 31 languages. Creating custom voice-style assets is a separate service workflow.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":null,"dateKind":null,"links":{"code":null,"weights":null,"paper":null,"catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#supertonic-3"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"model-card","title":"Supertonic 3 model card","url":"https://huggingface.co/Supertone/supertonic-3","reviewed_on":"2026-09-13","review_level":"model-card"}],"figure":{"path":"assets/figures/tts--supertonic-3.svg","kind":"editorial","label":"Input/output diagram","sourceUrl":"https://huggingface.co/Supertone/supertonic-3","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/supertonic-3.svg","sha256":"b83d3629098794b4ce13a62f5b0b755c5fc61667d3b24f8696e33dc57a2174ab"}},{"id":"tts:cde-styletts","slug":"cde-styletts","name":"CDE-StyleTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"CDE-StyleTTS lets acoustic states evolve continuously along a phoneme sequence whose timing comes from durations. Sampling this trajectory supplies the acoustic decoder with timing-sensitive representations, providing a way to transfer changing expressive style instead of merely repeating phoneme embeddings.","architecture":"Duration-aware neural controlled differential equations","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-09-10","dateKind":"paper","links":{"code":"https://github.com/Mattias421/CDE_StyleTTS","weights":null,"paper":"https://arxiv.org/abs/2609.11725","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#cde-styletts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Continuous-Time Acoustic Modelling with Neural Controlled Differential Equations","url":"https://arxiv.org/abs/2609.11725","reviewed_on":"2026-09-13","review_level":"paper"},{"kind":"repository","title":"CDE-StyleTTS author repository","url":"https://github.com/Mattias421/CDE_StyleTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--cde-styletts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2609.11725","originUrl":"https://arxiv.org/html/2609.11725v1/cde_styletts-Page-4.drawio-5.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cde-styletts.png","sha256":"c0a0a9c8ac357efbaeaf8e2ebe3501d21f4554a0ff313e0911baf743b46829a0"}},{"id":"tts:auk","slug":"auk","name":"AuK","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"AuK combines language-model conditioning, an audio VAE and successive multimodal and single-stream diffusion blocks. One model handles reference-based speech synthesis and instruction-guided editing; its distilled AuK-Flash variant reduces the number of generation steps.","architecture":"Instruction-conditioned hybrid rectified-flow Transformer","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":["AuK","AuK-Flash"],"inputs":["T","S"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":"2026-09-08","dateKind":"paper","links":{"code":"https://github.com/Tencent-Hunyuan/AuK","weights":null,"paper":"https://arxiv.org/abs/2609.08936","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#auk"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"AuK Technical Report: An Open-Source Foundational Model for Speech Generation and Editing","url":"https://arxiv.org/abs/2609.08936","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"AuK author repository","url":"https://github.com/Tencent-Hunyuan/AuK","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--auk.webp","kind":"source","label":"Figure 4","sourceUrl":"https://arxiv.org/abs/2609.08936","originUrl":"https://arxiv.org/html/2609.08936v1/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/auk.png","sha256":"d7a1dabcb3fe4b325a4e7e3081d4d6d27f3fd5c5b8b0f84c576734d30170bceb"}},{"id":"tts:tontaubev1","slug":"tontaubev1","name":"TontaubeV1","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"TontaubeV1 assigns a large Transformer to semantic speech tokens and smaller Transformers to successive acoustic refinements. Character-level text, synchronized position markers and a causal waveform reconstruction path support long recordings and streaming voice cloning, primarily in English and German.","architecture":"Hierarchical codec prediction with bounded streaming context","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-09-08","dateKind":"paper","links":{"code":"https://github.com/craitech/tontaube","weights":null,"paper":"https://arxiv.org/abs/2609.08703","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#tontaubev1"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"TontaubeV1: Streaming Text-to-Speech with Hierarchical Codec Modeling and Bounded Context","url":"https://arxiv.org/abs/2609.08703","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"TontaubeV1 author repository","url":"https://github.com/craitech/tontaube","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--tontaubev1.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2609.08703","originUrl":"https://arxiv.org/html/2609.08703v1/gmos_results_figure.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tontaubev1.png","sha256":"e9ea4ff733186278f37fe4fff39bdde6e38ff35629620263dd63e062d3d78436"}},{"id":"tts:kaburi-tts","slug":"kaburi-tts","name":"KABURI-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"KABURI-TTS renders each participant on a separate audio channel from time-aligned phonemes and speaker activity. Supplying the timing layout explicitly lets the system synthesize overlapping speech, backchannels and interruptions for two-speaker conversations.","architecture":"Phoneme-raster and activity-conditioned two-channel synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-09-07","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2609.07200","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#kaburi-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"KABURI-TTS: Phoneme-Keyed Activity-conditioned Bi-channel Utterance Rendering for Interaction","url":"https://arxiv.org/abs/2609.07200","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--kaburi-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2609.07200","originUrl":"https://arxiv.org/html/2609.07200v1/acoustic_model_architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/kaburi-tts.png","sha256":"40a647c0d0e394b4369fe9684b6716687d188bc743870338eed42e9388ffabf8"}},{"id":"tts:fireredaudio","slug":"fireredaudio","name":"FireRedAudio","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"FireRedAudio uses different acoustic encoders for understanding audio and conditioning speech generation. Its shared language model drives a flow-matching decoder over continuous RedAE latents, supporting voice cloning, instruction-controlled synthesis and speech editing within the broader audio model.","architecture":"Shared language model with separate continuous speech pathways","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":"2026-08-25","dateKind":"paper","links":{"code":"https://github.com/FireRedTeam/FireRedAudio","weights":null,"paper":"https://arxiv.org/abs/2608.24168","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#fireredaudio"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FireRedAudio: A General-Purpose Audio Language Model with Decoupled Continuous Representations for Understanding and Generation","url":"https://arxiv.org/abs/2608.24168","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"FireRedAudio author repository","url":"https://github.com/FireRedTeam/FireRedAudio","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--fireredaudio.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2608.24168","originUrl":"https://arxiv.org/html/2608.24168v2/image/FireRedAudio_model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fireredaudio.png","sha256":"7c41a5674aa2426df37d55badf84bcd80bb625f269d08e3a768b88aa65dd06e4"}},{"id":"tts:emotra-tts","slug":"emotra-tts","name":"EmoTra-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"EmoTra-TTS introduces frame-level valence, arousal and dominance controls into both prosodic planning and acoustic decoding. Synthetic transition examples teach the model to move between emotions within an utterance while keeping its wording and speaker identity consistent.","architecture":"Language model and flow decoder with time-varying emotion conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-24","dateKind":"paper","links":{"code":"https://github.com/Liu-Tianchi/EmoTra-TTS","weights":null,"paper":"https://arxiv.org/abs/2608.23791","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#emotra-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"EmoTra-TTS: Smooth Intra-Utterance Emotion Transitions for Speech Synthesis","url":"https://arxiv.org/abs/2608.23791","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"EmoTra-TTS author repository","url":"https://github.com/Liu-Tianchi/EmoTra-TTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"EmoTra-TTS author demo","url":"https://liu-tianchi.github.io/EmoTra_DemoPage/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--emotra-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2608.23791","originUrl":"https://arxiv.org/html/2608.23791v1/EmoTrav2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emotra-tts.png","sha256":"184daadcae42f8e9108ac8b0f494c430140b2e2aa11c48cb1046dae80a099c70"}},{"id":"tts:x2streaming-tts","slug":"x2streaming-tts","name":"X2Streaming-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"X2Streaming-TTS consumes text as it arrives and retains speech-generation state across successive segments. A frontend delays ambiguous text until its pronunciation can be decided, while the talker and waveform decoder produce incremental audio with bounded context.","architecture":"Causal streaming codec TTS with speech-state inheritance","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-08-19","dateKind":"paper","links":{"code":"https://github.com/X-Square-Robot/X2Streaming-TTS","weights":null,"paper":"https://arxiv.org/abs/2608.18661","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#x2streaming-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"X2Streaming-TTS: Causal Token-Level Text-to-Speech from Streaming Text with Speech-State Inheritance","url":"https://arxiv.org/abs/2608.18661","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"X2Streaming-TTS author repository","url":"https://github.com/X-Square-Robot/X2Streaming-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--x2streaming-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2608.18661","originUrl":"https://arxiv.org/html/2608.18661v1/figures/fig1_pipeline.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/x2streaming-tts.png","sha256":"567d560fa97987b8e4996a27b486d14d6fa9e82a39f93b929423a4664c3e62f7"}},{"id":"tts:fireredtts3","slug":"fireredtts3","name":"FireRedTTS3","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"FireRedTTS3 uses a semantically supervised audio autoencoder to make continuous speech representations easier to predict. The Base variant provides multilingual reference cloning; the Instruct variant adds natural-language voice design and editing of spoken content or acoustic attributes.","architecture":"Semantic-enriched continuous representations with autoregressive generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":["Base","Instruct"],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-18","dateKind":"paper","links":{"code":"https://github.com/FireRedTeam/FireRedTTS3","weights":null,"paper":"https://arxiv.org/abs/2608.17492","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#fireredtts3"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FireRedTTS3: Unified Speech Generation and Editing with Semantically Enriched Speech Representations","url":"https://arxiv.org/abs/2608.17492","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"FireRedTTS3 author repository","url":"https://github.com/FireRedTeam/FireRedTTS3","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--fireredtts3.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2608.17492","originUrl":"https://arxiv.org/html/2608.17492v2/image/fireredtts3_arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fireredtts3.png","sha256":"88aa4c4097edc1bbe36df7d34b12efb8126500e413bdc3f4597682b19dafc705"}},{"id":"tts:voicechat-tts","slug":"voicechat-tts","name":"VoiceChat-TTS","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"VoiceChat-TTS generates speech directly from an incoming language-model text stream. Control tokens allow interruptions and silence, and generation can resume without discarding the attention cache, making the synthesizer suitable for interactive agent playback.","architecture":"Continuous autoregressive synthesis driven by streaming text tokens","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-13","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2608.13831","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#voicechat-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoiceChat-TTS: A Low-Latency Continuous Speech Synthesis Model for Interactive Agents","url":"https://arxiv.org/abs/2608.13831","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--voicechat-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2608.13831","originUrl":"https://arxiv.org/html/2608.13831v1/Images/VoiceChat-TTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voicechat-tts.png","sha256":"5001f8151811b9603af8dbbe738cd46ad0ba9edb72ea707cc947bee677235c87"}},{"id":"tts:voicedesigner","slug":"voicedesigner","name":"VoiceDesigner","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"VoiceDesigner conditions speech generation on descriptions of a desired voice and supports modifying an existing voice. A shared diffusion model learns these tasks from natural and augmented voice examples, including voices with unusual or fictional characteristics.","architecture":"Unified diffusion Transformer for voice generation and editing","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2608.13613","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#voicedesigner"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoiceDesigner: Text-to-Voice Generation and Editing via Unified Diffusion Modeling and Data Augmentation","url":"https://arxiv.org/abs/2608.13613","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"VoiceDesigner author demo","url":"https://voicedesigner-demo.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--voicedesigner.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2608.13613","originUrl":"https://arxiv.org/html/2608.13613v1/figs/dreamvoice_framework_v2.drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voicedesigner.png","sha256":"67c7476b157706820b5be0556266f621215ffad52f239e5d046c2cdf81f582f3"}},{"id":"tts:midashenglm-gen","slug":"midashenglm-gen","name":"MiDashengLM-Gen","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"MiDashengLM-Gen trains a language model together with a conditional flow-matching output head to generate variable-length audio. Its text conditioning supports scenes containing intelligible speech alongside music or other sounds, with generation performed over continuous audio representations.","architecture":"Language-model-driven autoregressive flow matching for mixed audio","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":"2026-08-12","dateKind":"paper","links":{"code":"https://github.com/xiaomi-research/midashenglm-gen","weights":null,"paper":"https://arxiv.org/abs/2608.11804","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#midashenglm-gen"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MiDashengLM-Gen: Unified Audio Scene Generation via LLM-Driven Autoregressive Flow Matching","url":"https://arxiv.org/abs/2608.11804","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MiDashengLM-Gen author repository","url":"https://github.com/xiaomi-research/midashenglm-gen","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"MiDashengLM-Gen author demo","url":"https://xingws.github.io/midashenglm-gen-demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--midashenglm-gen.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2608.11804","originUrl":"https://arxiv.org/html/2608.11804v1/train.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/midashenglm-gen.png","sha256":"6c53d0b02892b97f6a4617b5691c67f71bffd825bd68fce38ad84c0ac932e5d7"}},{"id":"tts:phoenix-tts","slug":"phoenix-tts","name":"Phoenix TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Phoenix TTS aligns its speech tokenizer with the downstream flow-matching acoustic decoder during training. An autoregressive language model predicts the resulting discrete representation, supporting text-to-speech and voice conversion without separating token design from acoustic reconstruction.","architecture":"Jointly learned speech tokens with an autoregressive LM and flow decoder","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2608.11737","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#phoenix-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Phoenix TTS: High-Fidelity Synthesis and Voice Conversion via Flow-Matching-Driven Speech Tokenization","url":"https://arxiv.org/abs/2608.11737","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--phoenix-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2608.11737","originUrl":"https://arxiv.org/html/2608.11737v1/PhoenixTTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/phoenix-tts.png","sha256":"fc1dfe10ff4ccf57fd599e7862171a4a646a13f72340002f1f98b9022ef2498b"}},{"id":"tts:confucius4-tts","slug":"confucius4-tts","name":"Confucius4-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Confucius4-TTS extracts voice characteristics from self-supervised speech features without requiring a transcript of the reference recording. A language model predicts semantic tokens and a flow decoder generates mel-spectrograms, enabling voice cloning within and across fourteen languages.","architecture":"Speaker-conditioned text-to-semantic LM and acoustic flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-12","dateKind":"paper","links":{"code":"https://github.com/netease-youdao/Confucius4-TTS","weights":null,"paper":"https://arxiv.org/abs/2608.11650","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#confucius4-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Confucius4-TTS: Transcript-Free Cross-Lingual Zero-Shot TTS with a Learnable Speaker Encoder","url":"https://arxiv.org/abs/2608.11650","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Confucius4-TTS author repository","url":"https://github.com/netease-youdao/Confucius4-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--confucius4-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2608.11650","originUrl":"https://arxiv.org/html/2608.11650v1/img/confucius4-tts-model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/confucius4-tts.png","sha256":"a493531bde54ab046f8032d42ba068616669514fc42fca8a724875123741a384"}},{"id":"tts:luna-tts","slug":"luna-tts","name":"Luna-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Luna-TTS adapts an autoregressive text backbone into a speech diffusion language model. Its parallel and Realtime variants share a tokenizer and training lineage; Realtime predicts successive codec blocks while denoising each block in parallel for incremental audio delivery.","architecture":"Discrete diffusion language modeling with parallel and block-causal variants","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":["Luna-TTS (parallel)","Luna-TTS-Realtime"],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-08-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2608.11593","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#luna-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Luna-TTS Family Technical Report","url":"https://arxiv.org/abs/2608.11593","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"Luna-TTS author demo","url":"https://vuilabs-ai.github.io/luna-tts","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--luna-tts.webp","kind":"source","label":"Figure 1 (paper page 4)","sourceUrl":"https://arxiv.org/abs/2608.11593","originUrl":"https://arxiv.org/pdf/2608.11593v1","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/luna-tts.png","sha256":"f7957f46a7e9173abfc4fd88f563fc5e83b077bb1ebbf7134666b9c4796e9b9d"}},{"id":"tts:cookvoice","slug":"cookvoice","name":"CookVoice","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"CookVoice aligns textual content, style and prosodic controls to acoustic frames before speech generation. The same compact model supports spoken and sung voices, reference imitation and editing, allowing individual voice attributes to be controlled within a shared synthesis pipeline.","architecture":"Factorized content, prosody and style with frame-aligned flow synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2608.11590","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#cookvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CookVoice: Unified Framework for Style Controllable Multi-Modal Human Voice Generation","url":"https://arxiv.org/abs/2608.11590","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"CookVoice author demo","url":"https://haoweilou.github.io/CookVoice/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--cookvoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2608.11590","originUrl":"https://arxiv.org/html/2608.11590v2/cookvoice.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cookvoice.png","sha256":"3b0e755a76638884b8ce2bbc3b73aeec2d0877be8e12c45880a5f3710230b9a2"}},{"id":"tts:cutetts","slug":"cutetts","name":"CuteTTS","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"CuteTTS combines a causal audio VAE, an autoregressive patch model and an explicitly speaker-conditioned flow head. Generating patches rather than individual frames reduces sequential work; a distilled variant targets faster streaming while preserving reference-voice synthesis.","architecture":"Patch-autoregressive continuous latents with a flow-matching head","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-08-09","dateKind":"paper","links":{"code":"https://github.com/OPPO-Mente-Lab/CuteTTS","weights":null,"paper":"https://arxiv.org/abs/2608.08638","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#cutetts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CuteTTS: Efficient and High-Quality Speech Synthesis via Autoregressive Modeling of Continuous Latents","url":"https://arxiv.org/abs/2608.08638","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"CuteTTS author repository","url":"https://github.com/OPPO-Mente-Lab/CuteTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--cutetts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2608.08638","originUrl":"https://arxiv.org/html/2608.08638v2/cute_backbone.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cutetts.png","sha256":"373d682b738ec57a4f1bc78c0136c05c70d0f50c262f718a2b3586761652ed80"}},{"id":"tts:ctrlspeech","slug":"ctrlspeech","name":"CtrlSpeech","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"CtrlSpeech adds local pitch, loudness and duration conditioning to a patch-autoregressive diffusion synthesizer. A separate global speaker condition preserves the reference voice while users modify the delivery of individual words or phonemes.","architecture":"DiTAR synthesis with global voice and phoneme-level prosody controls","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-08","dateKind":"paper","links":{"code":"https://github.com/zszheng147/ctrlspeech","weights":null,"paper":"https://arxiv.org/abs/2608.08362","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#ctrlspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CtrlSpeech: Coarse-to-Fine Control for Expressive Speech Synthesis","url":"https://arxiv.org/abs/2608.08362","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"CtrlSpeech author repository","url":"https://github.com/zszheng147/ctrlspeech","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--ctrlspeech.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2608.08362","originUrl":"https://arxiv.org/html/2608.08362v1/architecture1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ctrlspeech.png","sha256":"446474c286a44a21d639a96dda7acb6ef89f10959d2d05eace8126ffc7ff415d"}},{"id":"tts:sembridge","slug":"sembridge","name":"SemBridge","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"SemBridge uses discrete semantic targets during training to organize both acoustic latents and language-model hidden states. The resulting continuous generator supports zero-shot speech synthesis with stronger content alignment, without needing to generate the auxiliary semantic tokens during inference.","architecture":"Semantic-anchored continuous autoregression with an aligned acoustic VAE","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-08-07","dateKind":"paper","links":{"code":"https://github.com/ASLP-lab/SemBridge","weights":null,"paper":"https://arxiv.org/abs/2608.07462","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#sembridge"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SemBridge: Semantic Token Anchoring for Continuous-Latent Autoregressive Speech Generation","url":"https://arxiv.org/abs/2608.07462","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"SemBridge author repository","url":"https://github.com/ASLP-lab/SemBridge","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--sembridge.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2608.07462","originUrl":"https://arxiv.org/html/2608.07462v1/sembridge_v7.drawio.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/sembridge.png","sha256":"2bf517df40716ff0523f8160a131deb31aaa4dc51a896f6095afb43a37e3f952"}},{"id":"tts:daien-tts","slug":"daien-tts","name":"DAIEN-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"DAIEN-TTS separates a reference recording into speech and environmental components, then conditions acoustic generation on them independently. Its extended formulation additionally models reverberation and uses separate guidance controls for speech, noise and room acoustics, enabling voice cloning into a chosen environment.","architecture":"Disentangled speech and environment prompts in an F5-TTS flow model","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","A"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-18","dateKind":"paper","links":{"code":"https://github.com/yxlu-0102/DAIEN-TTS","weights":null,"paper":"https://arxiv.org/abs/2608.03011","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#daien-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Towards Real-world Environment-aware Zero-shot Text-to-speech Synthesis via Disentangled Audio Infilling","url":"https://arxiv.org/abs/2608.03011","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"DAIEN-TTS author demo","url":"https://yxlu-0102.github.io/DAIEN-TTS/journal","reviewed_on":"2026-09-13","review_level":"project"},{"kind":"paper","title":"DAIEN-TTS: Disentangled Audio Infilling for Environment-Aware Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2509.14684","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DAIEN-TTS author repository","url":"https://github.com/yxlu-0102/DAIEN-TTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"DAIEN-TTS author demo","url":"https://yxlu-0102.github.io/DAIEN-TTS","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--daien-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2608.03011","originUrl":"https://arxiv.org/html/2608.03011v1/model_train.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/daien-tts.png","sha256":"c7498501220999de79f449dfb8131ef12532781e9056b3a8b0379fca8f480428"}},{"id":"tts:mp-eld","slug":"mp-eld","name":"MP-ELD","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"MP-ELD predicts low-rate continuous speech tokens through several information paths with separate local encoders. A flow decoder combines their predictions, while the accompanying Locodec representation is designed to limit accumulated errors during long speech generation.","architecture":"Multi-path continuous autoregression with residual flow guidance","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-31","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.29363","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#mp-eld"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Stable Autoregressive Speech Generation with Low-Frame-Rate High-Dimensional Continuous Tokens","url":"https://arxiv.org/abs/2607.29363","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mp-eld.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2607.29363","originUrl":"https://arxiv.org/html/2607.29363v1/x2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mp-eld.png","sha256":"aa447d9170af052e894be6ae68c66ea48a39fe6544d7f2131f34dba80cf391aa"}},{"id":"tts:face-adapted-styletts-2","slug":"face-adapted-styletts-2","name":"Face-adapted StyleTTS 2","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"This model maps facial features into the style space of StyleTTS 2 through a lightweight learned adapter. It synthesizes text in a plausible face-conditioned voice without an audio reference; the paper evaluates transfer to unseen identities and another synthesis language.","architecture":"Face-to-style adapter over a frozen style-diffusion synthesizer","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-29","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.26742","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#face-adapted-styletts-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Zero-Shot Face-to-Speech Synthesis via Latent Space Adaptation of a Style-Diffusion TTS Model","url":"https://arxiv.org/abs/2607.26742","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--face-adapted-styletts-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2607.26742","originUrl":"https://arxiv.org/html/2607.26742v1/architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/face-adapted-styletts-2.png","sha256":"096050c4f75bfc4f5df185158efe38001e69243c4f5b3e71aba1de5a0594a589"}},{"id":"tts:facialtalker","slug":"facialtalker","name":"FacialTalker","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"FacialTalker quantizes facial action information and combines it with text and speech history in a conversational synthesizer. Joint preference training over visual and speech tokens helps the generated delivery reflect the interlocutor's facial expression and dialogue context.","architecture":"Facial-expression tokens and conversational speech language modeling","notes":"The author repository currently lists dataset and processing releases as TODO; a synthesis implementation is not documented in its README.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-27","dateKind":"paper","links":{"code":"https://github.com/walker-hyf/FacialTalker","weights":null,"paper":"https://arxiv.org/abs/2607.24430","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#facialtalker"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Let Me Look at You: Advanced Facial Expression Modeling for Conversational Speech Synthesis","url":"https://arxiv.org/abs/2607.24430","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"FacialTalker author repository","url":"https://github.com/walker-hyf/FacialTalker","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--facialtalker.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2607.24430","originUrl":"https://arxiv.org/html/2607.24430v1/2026-MM-Model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/facialtalker.png","sha256":"49a5bd57b66d361af63c0094ce3b50c2fb8145e7796a4ee68494e8fc5d2a378d"}},{"id":"tts:qwen-audio-3-0-tts","slug":"qwen-audio-3-0-tts","name":"Qwen-Audio-3.0-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Qwen-Audio-3.0-TTS combines compact semantic speech tokens with progressively trained language and acoustic models. Natural-language instructions and inline tags control delivery, while multilingual reference conditioning supports voice cloning and longer speech generation under varied recording conditions.","architecture":"Low-rate speech LM with a flow-matching acoustic decoder","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-27","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.23938","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#qwen-audio-3-0-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Qwen-Audio-3.0-TTS: Freely Controllable and Highly Robust Speech Synthesis with Multi-Stage Training Paradigm","url":"https://arxiv.org/abs/2607.23938","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"Qwen-Audio-3.0-TTS author demo","url":"https://qwenaudio.github.io/FunAudioLLM.github.io/qwen-audio-3.0-tts/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--qwen-audio-3-0-tts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2607.23938","originUrl":"https://arxiv.org/html/2607.23938v1/qwen_audio_3_tts_architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/qwen-audio-3-0-tts.png","sha256":"f0ff0e77ef178457151ca1492269f4f0b635c64623dc4f71b258195c3f4f0326"}},{"id":"tts:stellartts","slug":"stellartts","name":"StellarTTS","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"StellarTTS encodes phoneme timing sparsely and uses a lightweight masked Transformer to generate speech tokens in parallel. A semantic-aware codec supports waveform reconstruction, while explicit timing representations provide control over pronunciation, duration and prosody.","architecture":"Sparse temporal conditioning and masked codec-token generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.19859","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#stellartts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"StellarTTS: Sparse Temporal Embedding for Low-Latency and Robust Speech Synthesis","url":"https://arxiv.org/abs/2607.19859","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"StellarTTS author demo","url":"https://stellartts.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--stellartts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2607.19859","originUrl":"https://arxiv.org/html/2607.19859v1/illustration3.6.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/stellartts.png","sha256":"df687a676748e3ec4e7a1fd08f097ced25573ec28f0f0d7a030a6a8f6db6ac39"}},{"id":"tts:ttsyoruba","slug":"ttsyoruba","name":"TTSYoruba","collection":"tts","category":"tts:concatenative","categoryName":"Concatenative","summary":"TTSYoruba converts tone-marked Yoruba text into speech by selecting and joining recorded diphones. Handwritten phonological rules resolve nasal forms and contextual tone contours, providing a language-specific synthesis system rather than a neural acoustic generator.","architecture":"Tone-aware rules and concatenative diphone synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-17","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.18317","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/concatenative.md#ttsyoruba"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"A Situational Speech Synthesizer for Yoruba: System Design, Phonological Rule Architecture, and Orthographic Extensions for Contour","url":"https://arxiv.org/abs/2607.18317","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--ttsyoruba.svg","kind":"editorial","label":"Editorial input/output diagram","sourceUrl":"https://arxiv.org/abs/2607.18317","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ttsyoruba.svg","sha256":"bc6577c4cc26d493c6a92a9b48c2803bbf84d8370c370117a0dd0b5cdc85f1ef"}},{"id":"tts:auemochat","slug":"auemochat","name":"AuEmoChat","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"AuEmoChat learns a discrete emotion representation from speech and compresses dialogue history while retaining emotionally relevant information. Its language model predicts emotion and speech tokens, which a context-conditioned flow decoder renders into expressive conversational audio.","architecture":"Emotion-token language modeling with contextual flow rendering","notes":"The linked author repository is a placeholder that says code will be made public after publication.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-17","dateKind":"paper","links":{"code":"https://github.com/AI-S2-Lab/AuEmoChat","weights":null,"paper":"https://arxiv.org/abs/2607.15755","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#auemochat"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"AuEmoChat: Authentic Emotion Understanding and Rendering for Conversational Speech Synthesis","url":"https://arxiv.org/abs/2607.15755","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"AuEmoChat author repository","url":"https://github.com/AI-S2-Lab/AuEmoChat","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--auemochat.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2607.15755","originUrl":"https://arxiv.org/html/2607.15755v2/Figs/auemochat-c.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/auemochat.png","sha256":"b616942465d8f3fb4c62715e68a2b1f721cd4639a8bdf76496725dfe0b41316f"}},{"id":"tts:autosift","slug":"autosift","name":"AutoSIFT","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"AutoSIFT divides a reference voice's style into attribute-specific components and a residual representation. Text instructions replace selected attributes while unmentioned characteristics remain conditioned on the reference, allowing partial style editing during speech synthesis.","architecture":"Style disentanglement and selective infilling over flow-matching TTS","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.12706","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#autosift"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"AutoSIFT: Automatic Style Sifting for Controllable Speech Generation with Arbitrary Style Infilling","url":"https://arxiv.org/abs/2607.12706","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--autosift.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2607.12706","originUrl":"https://arxiv.org/html/2607.12706v2/autosift_cropped.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/autosift.png","sha256":"d14374f0dce70d356a13fab508e8c1182a3d3502cc34624da791d69563a6d017"}},{"id":"tts:freyatts","slug":"freyatts","name":"FreyaTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"FreyaTTS is a Turkish-focused non-autoregressive synthesizer that maps character sequences to continuous audio latents. A frozen waveform autoencoder reconstructs the speech, while duration prediction and voice-focused post-training support efficient conversational playback without phoneme or discrete speech tokenization.","architecture":"Character-conditioned flow matching over continuous AudioVAE2 latents","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.09530","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#freyatts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FreyaTTS: A Compact Tokenizer-Free Flow-Matching Transformer for Turkish-First Speech Synthesis","url":"https://arxiv.org/abs/2607.09530","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--freyatts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2607.09530","originUrl":"https://arxiv.org/html/2607.09530v2/freyatts_architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/freyatts.png","sha256":"3ffcf9f1f0ab7bc65bd9bd99212ef9564d01d8abe4404a5bec450458c8c7970c"}},{"id":"tts:regenvoice","slug":"regenvoice","name":"ReGenVoice","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"ReGenVoice applies the ReGen representation-and-waveform modeling approach to text-to-speech. Multiple levels of generated conditioning help reconstruct detailed waveforms from compressed latents, linking efficient acoustic representation with reference-conditioned speech synthesis.","architecture":"Latent diffusion TTS with hierarchical multi-prompt waveform generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.09134","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#regenvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ReGen: Hierarchical Multi-Prompt Representation Generation for Efficient Waveform Diffusion Models","url":"https://arxiv.org/abs/2607.09134","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"ReGenVoice author demo","url":"https://regenvoice.github.io/demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--regenvoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2607.09134","originUrl":"https://arxiv.org/html/2607.09134v1/fig1_final.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/regenvoice.png","sha256":"360275c369d1f80162d9b636413f58a2fd2a28451d17bf8b60948e702d44e109"}},{"id":"tts:wordvoice","slug":"wordvoice","name":"WordVoice","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"WordVoice associates controllable acoustic attributes with individual words before rendering their speech. Users can accept the language model's predicted delivery or override selected local attributes, enabling detailed expressive adjustments within a single utterance.","architecture":"Explicit word-level acoustic planning with bound speech tokens","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-07","dateKind":"paper","links":{"code":"https://github.com/XXH333/WordVoice-main","weights":null,"paper":"https://arxiv.org/abs/2607.06461","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#wordvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"WordVoice: Explicit and Decoupled Multi-Dimensional Word-Level Control for LLM-Based TTS","url":"https://arxiv.org/abs/2607.06461","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"WordVoice author repository","url":"https://github.com/XXH333/WordVoice-main","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"WordVoice author demo","url":"https://xxh333.github.io/wordvoice-demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--wordvoice.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2607.06461","originUrl":"https://arxiv.org/html/2607.06461v1/Figures/wordvoice.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/wordvoice.png","sha256":"897a278b217f62918cdee6440881df0af142a50b8199befb79b377d9ef9f8a15"}},{"id":"tts:delta-tts","slug":"delta-tts","name":"DELTA-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DELTA-TTS converts a pretrained speech language model to parallel discrete diffusion using lightweight adaptation. Local convolution and confidence-based decoding help retain acoustic structure while the model fills speech-token positions in an order determined by prediction confidence.","architecture":"Autoregressive-to-diffusion adaptation with local acoustic convolution","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-05","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.04140","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#delta-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DELTA-TTS: Adapting Autoregressive Model into Diffusion Language Model for Text-to-Speech","url":"https://arxiv.org/abs/2607.04140","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--delta-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2607.04140","originUrl":"https://arxiv.org/html/2607.04140v2/fig1_architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/delta-tts.png","sha256":"8ae0f96e2b45477dd836cac48a844af72d7978b44c6b9eaca7e564f233cca612"}},{"id":"tts:graft","slug":"graft","name":"GRAFT","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"GRAFT attaches codec tokens from a spoken word example to that word's location in the text prompt. Separate target-speaker conditioning allows the pronunciation hint to come from another voice while the synthesized sentence retains the desired speaker.","architecture":"Word-local reference-code conditioning for pronunciation control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-07-02","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2607.02633","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#graft"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"GRAFT: Grafted Reference Audio for Fine-grained Pronunciation in Zero-shot Text-to-Speech","url":"https://arxiv.org/abs/2607.02633","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--graft.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2607.02633","originUrl":"https://arxiv.org/html/2607.02633v1/assets/teaser.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/graft.png","sha256":"c9daf072b22843b9c8f9c6932b6b55bd0c65031a3615ffeda00cbf11f35e632f"}},{"id":"tts:oscillatts","slug":"oscillatts","name":"OscillaTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"OscillaTTS changes the periodic nonlinearities used in a style-diffusion synthesis backbone. Adjustable oscillatory modulation is designed to capture rapid pitch and amplitude changes while a linear bypass stabilizes the acoustic signal, targeting sharper expressive prosody.","architecture":"StyleTTS 2 with adaptive oscillatory decoder nonlinearities","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-24","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.25424","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#oscillatts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Adaptive Oscillatory Inductive Bias for Modeling Sharp Prosodic Dynamics in Diffusion-Based TTS","url":"https://arxiv.org/abs/2606.25424","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--oscillatts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.25424","originUrl":"https://arxiv.org/html/2606.25424v1/OscillaTTS-new-1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/oscillatts.png","sha256":"8bb6425ba38d8ddfed65d4336d6ef7827456b087aef5fcbd392832ecd7aa4b50"}},{"id":"tts:crossaccent-tts","slug":"crossaccent-tts","name":"CrossAccent-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"CrossAccent-TTS separates speaker identity from accent-related information in a speech synthesis model. Weighted language embeddings control the accent subspace, allowing gradual accent changes and cross-language synthesis while retaining the reference speaker's timbre.","architecture":"Disentangled speaker and accent conditioning with intensity control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-24","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.25403","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#crossaccent-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CrossAccent-TTS: Cross-Lingual Accent-Intensity Controllable Text-to-Speech via Disentangled Speaker and Accent Representations","url":"https://arxiv.org/abs/2606.25403","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--crossaccent-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.25403","originUrl":"https://arxiv.org/html/2606.25403v1/Picture1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/crossaccent-tts.png","sha256":"af7f402d8427892069934ab244f10e8a39c0e1f48c8f44be6c1f998a0d288a59"}},{"id":"tts:sarashina2-2-tts","slug":"sarashina2-2-tts","name":"Sarashina2.2-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Sarashina2.2-TTS emphasizes reliable Japanese pronunciation, including characters with several possible readings. Its semantic language model and acoustic flow decoder use reference speech for voice conditioning, with balanced multilingual training to reduce dependence on the reference language.","architecture":"Semantic speech LM followed by flow matching and vocoding","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-24","dateKind":"paper","links":{"code":"https://github.com/sbintuitions/sarashina2.2-tts","weights":null,"paper":"https://arxiv.org/abs/2606.25369","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#sarashina2-2-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Sarashina2.2-TTS: Tackling Kanji Polyphony in Japanese Speech Generation via Data Scaling and Targeted Data Synthesis","url":"https://arxiv.org/abs/2606.25369","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Sarashina2.2-TTS author repository","url":"https://github.com/sbintuitions/sarashina2.2-tts","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--sarashina2-2-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.25369","originUrl":"https://arxiv.org/html/2606.25369v1/img/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/sarashina2-2-tts.png","sha256":"b74fbc0fc7949a5c31ca860596ef8d32e49ebc60016eafbc03248753dac7187b"}},{"id":"tts:lombard-matcha-tts","slug":"lombard-matcha-tts","name":"Lombard Matcha-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"This Matcha-TTS extension learns vocal effort and articulation from automatically derived labels. It provides continuous controls over speech clarity and loudness-related effort, together with word-level emphasis, to synthesize the clearer delivery used in noisy listening conditions.","architecture":"Flow matching with separate vocal-effort and articulation controls","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.23176","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#lombard-matcha-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Synthesizing the Lombard Effect: Multi-Level Control of Speech Clarity and Vocal Effort in TTS","url":"https://arxiv.org/abs/2606.23176","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--lombard-matcha-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.23176","originUrl":"https://arxiv.org/html/2606.23176v1/matcha_tts_arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/lombard-matcha-tts.png","sha256":"3e0d87bc2735520ccdcb5259fdf5a29aa0be7bb1eb025a15df3a5b494f054f20"}},{"id":"tts:bagpiper-tts","slug":"bagpiper-tts","name":"Bagpiper-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Bagpiper-TTS converts a natural-language request into a detailed speech plan containing words and delivery information. The generator follows that plan for tasks ranging from ordinary narration to multi-speaker rendering, role-play and singing.","architecture":"Rich-caption planning followed by universal speech generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.22811","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#bagpiper-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Bagpiper-TTS: Natural Language Guided Universal Speech Synthesis","url":"https://arxiv.org/abs/2606.22811","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"Bagpiper-TTS author demo","url":"https://bagpipertts.github.io/bagpiper_tts_demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--bagpiper-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.22811","originUrl":"https://arxiv.org/html/2606.22811v1/cot_demo.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/bagpiper-tts.png","sha256":"7701546ccab38d804261a145ead03083665a903fc5c0b5e6ddf75a4e94687a7d"}},{"id":"tts:s5-tts","slug":"s5-tts","name":"S5-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"S5-TTS adapts T5-TTS for word-by-word synthesis using limited future text. Lookahead-aware masks, convolutional auxiliary attention and distillation let the model start speaking before the complete sentence is available while retaining voice conditioning and alignment.","architecture":"Streaming encoder-decoder TTS with monotonic alignment and lookahead masks","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-06-20","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.21882","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#s5-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Streaming T5-based Text-to-Speech Synthesis with Limited Lookahead","url":"https://arxiv.org/abs/2606.21882","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--s5-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.21882","originUrl":"https://arxiv.org/html/2606.21882v1/architecture_figure.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/s5-tts.png","sha256":"8482d0c7c3042ce400e27cc85be83e84a5227a52325137c1f2b616cd5a6f51e3"}},{"id":"tts:disspeech","slug":"disspeech","name":"DisSpeech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DisSpeech maps Mandarin text and marked stuttering events to semantic speech tokens without temporal autoregression. Pitch and energy modeling guide acoustic reconstruction, enabling controlled repetitions and other disfluencies for speech synthesis and recognition-data augmentation.","architecture":"Masked semantic-token prediction with explicit stuttering labels","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-19","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.21457","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#disspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DisSpeech: Low-Resource Controllable Mandarin Stuttered Speech Synthesis for ASR Augmentation","url":"https://arxiv.org/abs/2606.21457","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--disspeech.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2606.21457","originUrl":"https://arxiv.org/html/2606.21457v1/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/disspeech.png","sha256":"aa27520ca2e35c700cbf51b7ea90a4c567cf557dee4f24fa1995d5571217c991"}},{"id":"tts:rtfree-f5","slug":"rtfree-f5","name":"RTFree-F5","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"RTFree-F5 replaces the transcript normally associated with an F5-TTS reference recording with projected speech features. A lightweight adapter reuses the pretrained generator, enabling transcript-free voice conditioning, including references whose pronunciation makes transcription unreliable.","architecture":"Self-supervised reference features adapted to F5-TTS text conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-18","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.20266","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#rtfree-f5"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Transcript-Free Flow-Matching Text-to-Speech via Speech Feature Conditioning","url":"https://arxiv.org/abs/2606.20266","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--rtfree-f5.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.20266","originUrl":"https://arxiv.org/html/2606.20266v1/rtfree-f5_revised.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/rtfree-f5.png","sha256":"fc842fa4041b85c1f875979d59e03d82d8532ed1ae64ac6a63730f75f71534ef"}},{"id":"tts:finecombo-tts","slug":"finecombo-tts","name":"FineCombo-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"FineCombo-TTS interprets style descriptions relative to a supplied speech reference. A flow-based variance predictor models how acoustic attributes should change, enabling precise relative edits without requiring an explicit independent embedding for every voice attribute.","architecture":"Text-guided speech-variance flow prediction with unified reference features","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-17","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.19209","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#finecombo-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FineCombo-TTS: Collaborative and Precise Controllable Speech Synthesis Using Text Descriptions and Reference Speech","url":"https://arxiv.org/abs/2606.19209","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"FineCombo-TTS author demo","url":"https://thuhcsi.github.io/interspeech2026-FineCombo-TTS","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--finecombo-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2606.19209","originUrl":"https://arxiv.org/html/2606.19209v2/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/finecombo-tts.png","sha256":"87adf8fcca2a0d2a94e8d6253e836d28139f8b3a18e6d1e0da856f0a5247c418"}},{"id":"tts:magpietts-lf","slug":"magpietts-lf","name":"MagpieTTS-LF","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MagpieTTS-LF extends MagpieTTS at inference time by retaining acoustic and textual context across sentence boundaries. Soft alignment priors and history-aware encoding support coherent longer narration without retraining the synthesizer on long recordings.","architecture":"Stateful long-form codec synthesis with soft alignment priors","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-16","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.18485","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#magpietts-lf"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MagpieTTS-LF: Inference-Time Long-Form Speech Generation Without Training on Long-Form data","url":"https://arxiv.org/abs/2606.18485","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--magpietts-lf.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.18485","originUrl":"https://arxiv.org/html/2606.18485v1/longformdiagramwide.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/magpietts-lf.png","sha256":"865905cf9f05642ed9c23b4d3f0e73a195d4bbcc758d82bf493acd326051cc6d"}},{"id":"tts:joycent","slug":"joycent","name":"Joycent","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Joycent separates accent information from speaker identity using an adversarially trained accent encoder. It injects accent and speaker features at different text-encoder layers, supporting accent-conditioned synthesis without requiring a separate accented phoneme prediction stage.","architecture":"Accent-disentangled encoding with layer-specific flow conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-15","dateKind":"paper","links":{"code":"https://github.com/oshindow/Joycent-code","weights":null,"paper":"https://arxiv.org/abs/2606.16417","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#joycent"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Joycent: Multi-Accent TTS via Disentangled Accent Modeling and Layer-Specific Conditioning","url":"https://arxiv.org/abs/2606.16417","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Joycent author repository","url":"https://github.com/oshindow/Joycent-code","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"Joycent author demo","url":"https://oshindow.github.io/joycent/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--joycent.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.16417","originUrl":"https://arxiv.org/html/2606.16417v4/overall.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/joycent.png","sha256":"de0c013c8679ba56a5af17112d46db56d6a946ac74ca19d48e7babffcce9f67c"}},{"id":"tts:mel-llm-tts","slug":"mel-llm-tts","name":"Mel-LLM (TTS)","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"The synthesis experiment in Mel-LLM extends a language model to predict mel-based acoustic information directly. Its next-token VAE decoder demonstrates a text-to-speech path within an otherwise understanding-focused model; the paper presents this as a proof of concept with quality limitations.","architecture":"Direct mel-patch language modeling with a next-token VAE decoder","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.10231","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#mel-llm-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"LLM can Read Spectrogram: Encoder-free Speech-Language Modeling","url":"https://arxiv.org/abs/2606.10231","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mel-llm-tts.webp","kind":"source","label":"Fig. 1 (paper page 2)","sourceUrl":"https://arxiv.org/abs/2606.10231","originUrl":"https://arxiv.org/pdf/2606.10231v3","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mel-llm-tts.png","sha256":"0c44d69f8ce77d9e5899e62396bf81c4754f804c73273820299f805e45454b57"}},{"id":"tts:nushu-pitchvits","slug":"nushu-pitchvits","name":"Nüshu-PitchVITS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"Nüshu-PitchVITS uses pitch annotations from Nüshu's writing system to guide acoustic generation under very limited data. A frame-level pitch predictor conditions the VITS waveform path, allowing syllable recordings and linguistic tone knowledge to support sentence synthesis.","architecture":"VITS with explicit pitch supervision from Nüshu tone notation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.09295","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#nushu-pitchvits"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"NüshuVoice: Reviving the Voice of Endangered Nüshu with Pitch-Aware Text-to-Speech","url":"https://arxiv.org/abs/2606.09295","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--nushu-pitchvits.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2606.09295","originUrl":"https://arxiv.org/html/2606.09295v1/train_infer.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/nushu-pitchvits.png","sha256":"424694bed3e0efbb69e4f5231454d720924e81767730c9597fa730ab7942ea6a"}},{"id":"tts:end-to-end-discrete-token-tts","slug":"end-to-end-discrete-token-tts","name":"End-to-end discrete-token TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This system trains the discrete speech representation together with the language and acoustic models that consume it. Reconstruction and recognition feedback also update token prediction, reducing mismatches between separately trained components in reference-conditioned speech synthesis.","architecture":"Joint optimization of speech tokenizer, LM, flow decoder and reward model","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.09234","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#end-to-end-discrete-token-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"End-to-End Training for Discrete Token LLM based TTS System","url":"https://arxiv.org/abs/2606.09234","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--end-to-end-discrete-token-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.09234","originUrl":"https://arxiv.org/html/2606.09234v1/framework.drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/end-to-end-discrete-token-tts.png","sha256":"bb3d2be25d678507e63622c542ad885f5fca836cd00ab569c22803cc80d7141a"}},{"id":"tts:flashtts","slug":"flashtts","name":"FlashTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"FlashTTS processes incoming text and speech context on staggered tracks so synthesis can begin before sentence completion. Multi-token prediction accelerates the language model, while a distilled acoustic decoder reduces waveform-generation delay.","architecture":"Lagged streaming tracks with multi-token prediction and mean-flow decoding","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-06-08","dateKind":"paper","links":{"code":"https://github.com/ASLP-lab/FlashTTS","weights":null,"paper":"https://arxiv.org/abs/2606.09141","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#flashtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FlashTTS: Fast Streaming TTS with MTP Acceleration and X-pred Mean Flow Distillation","url":"https://arxiv.org/abs/2606.09141","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"FlashTTS author repository","url":"https://github.com/ASLP-lab/FlashTTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"FlashTTS author demo","url":"https://aslp-lab.github.io/flashtts_demo","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--flashtts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.09141","originUrl":"https://arxiv.org/html/2606.09141v2/flashtts-v4.drawio.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/flashtts.png","sha256":"63964eb4a39f906ba6b8d30fd01ec77002a5c09222fb21462dc37af2972f386e"}},{"id":"tts:holidubber","slug":"holidubber","name":"HoliDubber","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"HoliDubber conditions audio generation on video and a text prompt describing speech and sound effects. A causal model plans successive latent patches and a local diffusion Transformer generates their detail, supporting synchronized dubbing within complex acoustic scenes.","architecture":"Patch-autoregressive audiovisual fusion with local diffusion decoding","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.09098","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#holidubber"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"HoliDubber: Holistic Video Dubbing for Complex Acoustic Scenes via Text-Guided Audio Synthesis","url":"https://arxiv.org/abs/2606.09098","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"HoliDubber author demo","url":"https://holidubber.github.io","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--holidubber.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2606.09098","originUrl":"https://arxiv.org/html/2606.09098v2/holidubber_arc.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/holidubber.png","sha256":"c131b743b0b40bbc9e3f36bdcdd8f64270f4caa01dc1c3174fb6509b6d57c0ff"}},{"id":"tts:barewave","slug":"barewave","name":"BareWave","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"BareWave generates speech in waveform space using a single inference path. Representation alignment, staged noise scheduling and perceptual objectives guide training, replacing the separate acoustic-feature and waveform-reconstruction stages common in other TTS systems.","architecture":"Direct waveform flow matching without an intermediate acoustic decoder","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.09048","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#barewave"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"BareWave: Waveform-Native Flow-Matching Text-to-Speech","url":"https://arxiv.org/abs/2606.09048","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"BareWave author demo","url":"https://barewave.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--barewave.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2606.09048","originUrl":"https://arxiv.org/html/2606.09048v1/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/barewave.png","sha256":"5ea5ec4b9ebdf612bc5a9fb8ba56867038e726a6ac5556c68df40817b88a53c8"}},{"id":"tts:tldr-tts","slug":"tldr-tts","name":"TLDR (TTS)","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"TLDR groups consecutive speech tokens into learned patches and adapts a pretrained speech language model to predict the shorter sequence. A speaker-conditioned extractor reconstructs the detailed codec tokens, reducing sequential decoding and attention-cache use.","architecture":"Compressed codec patches with speaker-conditioned token reconstruction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.09019","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#tldr-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"TLDR: Compressing Audio Tokens for Efficient Autoregressive Text-to-Speech","url":"https://arxiv.org/abs/2606.09019","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--tldr-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.09019","originUrl":"https://arxiv.org/html/2606.09019v1/overview.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tldr-tts.png","sha256":"0602c08de17344e2bbd7afb2cdbb1c7a92a9b1f560446db4e8c98796c075ff9d"}},{"id":"tts:dots-tts","slug":"dots-tts","name":"dots.tts","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"dots.tts predicts continuous acoustic representations from multilingual text and voice references. Its flow-matching output head supports both audio streaming and streaming text input; guidance-aware distillation reduces the work needed to generate each audio packet.","architecture":"Continuous autoregressive speech modeling with a distilled flow head","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-06-05","dateKind":"paper","links":{"code":"https://github.com/studio-dots-ai/dots.tts","weights":null,"paper":"https://arxiv.org/abs/2606.07080","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#dots-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"dots.tts Technical Report","url":"https://arxiv.org/abs/2606.07080","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"dots.tts author repository","url":"https://github.com/studio-dots-ai/dots.tts","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"dots.tts author demo","url":"https://studio-dots-ai.github.io/dots.tts-demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--dots-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2606.07080","originUrl":"https://arxiv.org/html/2606.07080v2/figure1_2.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dots-tts.png","sha256":"bd449e8c57480f24a8586dabf4a86ce080e10516b44befaffe394af46a8d838a"}},{"id":"tts:univoice-speech-and-singing","slug":"univoice-speech-and-singing","name":"UniVoice (speech and singing)","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"This UniVoice model separates linguistic content, melody and speaker characteristics before conditioning a shared diffusion Transformer. It supports speech and singing in one generator, with input encoders chosen for each type of condition.","architecture":"Shared flow-matching Transformer with content, melody and timbre encoders","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-04","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2606.05852","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#univoice-speech-and-singing"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniVoice: A Unified Model for Speech and Singing Voice Generation","url":"https://arxiv.org/abs/2606.05852","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--univoice-speech-and-singing.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2606.05852","originUrl":"https://arxiv.org/html/2606.05852v1/architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/univoice-speech-and-singing.png","sha256":"b37bb1a7c225e78b4b9f4c2e2f94bc86a67bdf6cbe7f2a5be3d02e84ff645408"}},{"id":"tts:wavtts","slug":"wavtts","name":"WavTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"WavTTS operates directly on raw waveform patches instead of mel-spectrograms or codec tokens. Multiscale spectral supervision guides acoustic quality during training, supporting reference-based speech generation within a direct text-to-waveform architecture.","architecture":"Waveform-patch diffusion Transformer trained with flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-02","dateKind":"paper","links":{"code":"https://github.com/cwx-worst-one/WavTTS","weights":null,"paper":"https://arxiv.org/abs/2606.03455","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#wavtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"WavTTS: Towards High-Quality Zero-Shot TTS via Direct Raw Waveform Modeling","url":"https://arxiv.org/abs/2606.03455","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"WavTTS author repository","url":"https://github.com/cwx-worst-one/WavTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--wavtts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2606.03455","originUrl":"https://arxiv.org/html/2606.03455v1/figures/WavTTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/wavtts.png","sha256":"9f332b5923de8a8be5828d83a4e0bbd7aa2f935ce2dda3b43cf20ffcbec1a85d"}},{"id":"tts:univocal","slug":"univocal","name":"UniVocal","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"UniVocal learns when text should be spoken or sung and can move between these modes within one sequence. Its language model predicts refined pitch and semantic tokens before acoustic reconstruction, with optional descriptions and voice references controlling the performance.","architecture":"Joint semantic and pitch-token planning for speech-singing synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-06-01","dateKind":"paper","links":{"code":"https://github.com/FunAudioLLM/FunResearch","weights":null,"paper":"https://arxiv.org/abs/2606.01677","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#univocal"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniVocal: Unified Speech-Singing Code-Switching Synthesis","url":"https://arxiv.org/abs/2606.01677","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"UniVocal author repository","url":"https://github.com/FunAudioLLM/FunResearch","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"UniVocal author demo","url":"https://project-univocal-demo.github.io/demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--univocal.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2606.01677","originUrl":"https://arxiv.org/html/2606.01677v1/univocal.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/univocal.png","sha256":"fcbd6bb1a3c9ca33f08d92b3f9a21546db6d11d42d31204ef0b1b262cfb0d6a3"}},{"id":"tts:unison","slug":"unison","name":"UNISON","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"UNISON uses one diffusion backbone for speech generation, general audio generation and editing. Hidden states from multiple language-model layers guide the acoustic model, while task masks and optional source latents specify the requested generation or modification.","architecture":"Latent flow matching with layer-wise language-model conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","A"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":"2026-05-29","dateKind":"paper","links":{"code":"https://github.com/lizhaoqing/UNISON","weights":null,"paper":"https://arxiv.org/abs/2605.31530","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#unison"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UNISON: A Unified Sound Generation and Editing Framework via Deep LLM Fusion","url":"https://arxiv.org/abs/2605.31530","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"UNISON author repository","url":"https://github.com/lizhaoqing/UNISON","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"UNISON author demo","url":"https://lizhaoqing.github.io/UNISON-demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--unison.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2605.31530","originUrl":"https://arxiv.org/html/2605.31530v2/fig2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/unison.png","sha256":"ea30e3fd8e35b55a0b9f8088ac0dc536b8bd79a8e08ba71793cf99bf1c550c1b"}},{"id":"tts:swanvoice","slug":"swanvoice","name":"SwanVoice","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"SwanVoice generates monologues or dialogues with up to four speakers using raw text, voice references and speaker-turn conditions. Pause markers and optional pronunciation substitutions provide textual control, while staged dialogue training supports longer expressive speech.","architecture":"Speaker-turn-conditioned flow matching over continuous speech latents","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-29","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2605.30993","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#swanvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SwanVoice: Expressive Long-Form Zero-Shot Speech Synthesis for Both Monologue and Dialogue","url":"https://arxiv.org/abs/2605.30993","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"SwanVoice author demo","url":"https://swanaigc.github.io/#/swanvoice","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--swanvoice.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2605.30993","originUrl":"https://arxiv.org/html/2605.30993v1/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/swanvoice.png","sha256":"08ac0bcd8b9a662378104d012d528b06bd93b3f4bb22d63eb2c729067d73c480"}},{"id":"tts:immersivetts","slug":"immersivetts","name":"ImmersiveTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"ImmersiveTTS jointly models spoken content and its surrounding acoustic scene in a multimodal diffusion Transformer. Speech and general-audio representations provide complementary training signals, helping generated wording remain intelligible within the requested environmental context.","architecture":"Dual-stream multimodal diffusion for speech and acoustic environment","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","A"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":"2026-05-29","dateKind":"paper","links":{"code":"https://github.com/immersivetts/ImmersiveTTS","weights":null,"paper":"https://arxiv.org/abs/2605.30965","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#immersivetts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ImmersiveTTS: Environment-Aware Text-to-Speech with Multimodal Diffusion Transformer and Domain-Specific Representation Alignment","url":"https://arxiv.org/abs/2605.30965","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"ImmersiveTTS author repository","url":"https://github.com/immersivetts/ImmersiveTTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"ImmersiveTTS author demo","url":"https://jjunak-yun.github.io/ImmersiveTTS","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--immersivetts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.30965","originUrl":"https://arxiv.org/html/2605.30965v1/camera_ready_main_architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/immersivetts.png","sha256":"5b288fbb42327e9466e30254dd9060903fd708bcef91863f39d38c8112f6da95"}},{"id":"tts:chatterbox-flash","slug":"chatterbox-flash","name":"Chatterbox-Flash","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Chatterbox-Flash generates speech-token blocks in parallel while keeping block-by-block streaming. Calibration against common-token priors and confidence-based stopping improve its discrete diffusion decoding after adaptation from a pretrained autoregressive synthesizer.","architecture":"Prior-calibrated block diffusion adapted from an autoregressive TTS model","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-05-29","dateKind":"paper","links":{"code":"https://github.com/resemble-ai/chatterbox-flash","weights":null,"paper":"https://arxiv.org/abs/2605.30748","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#chatterbox-flash"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Chatterbox-Flash: Prior-Calibrated Block Diffusion for Streaming Zero-Shot TTS","url":"https://arxiv.org/abs/2605.30748","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Chatterbox-Flash author repository","url":"https://github.com/resemble-ai/chatterbox-flash","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--chatterbox-flash.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2605.30748","originUrl":"https://arxiv.org/html/2605.30748v3/figures/attn_mask.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/chatterbox-flash.png","sha256":"78a1231ca333174600ad144cda197dddb11f2d53736d282eb010fcbabfdc66bf"}},{"id":"tts:holitok-tts","slug":"holitok-tts","name":"HoliTok (TTS)","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"HoliTok combines linguistic and acoustic information in a continuous representation designed for both understanding and generation. Its downstream autoregressive model and diffusion decoder demonstrate a text-to-speech path using the same latents employed for recognition.","architecture":"Holistic continuous speech latents with shared autoregressive-diffusion modeling","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-28","dateKind":"paper","links":{"code":"https://github.com/bovod-sjtu/HoliTok","weights":null,"paper":"https://arxiv.org/abs/2605.29948","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#holitok-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"HoliTok: A Continuous Holistic Tokenization with Robust Dual Capabilities of Speech Generation and Understanding","url":"https://arxiv.org/abs/2605.29948","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"HoliTok (TTS) author repository","url":"https://github.com/bovod-sjtu/HoliTok","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--holitok-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.29948","originUrl":"https://arxiv.org/html/2605.29948v3/holitok.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/holitok-tts.png","sha256":"7882dee756220d835c416a0646b4578815464ee7dec9442e129d46d38b680c19"}},{"id":"tts:meld","slug":"meld","name":"MELD","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MELD learns discrete latent variables from mel-spectrograms jointly with its speech language model. This shared optimization supports zero-shot synthesis and recognition while addressing omissions and excessive silence associated with less coordinated acoustic representations.","architecture":"Jointly trained discrete mel latents and autoregressive speech modeling","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-28","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2605.29859","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#meld"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MELD: Mel-Spectrogram-Based Speech Language Modeling with Discrete Latent Variables","url":"https://arxiv.org/abs/2605.29859","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--meld.webp","kind":"source","label":"Figures 1–2 (paper page 2; panels assembled)","sourceUrl":"https://arxiv.org/abs/2605.29859","originUrl":"https://arxiv.org/pdf/2605.29859v2","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/meld.png","sha256":"2d102e308dc932a4bf080900728c50da1ecdfe0a8f09368426d011f711e3e283"}},{"id":"tts:pilottts","slug":"pilottts","name":"PilotTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"PilotTTS uses paired recordings and Q-Former conditioning to separate a speaker's identity from delivery style. The model supports reference cloning, emotional and nonverbal expression, and Chinese dialect synthesis within a shared autoregressive pipeline.","architecture":"Compact autoregression with Q-Former speaker and style conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-26","dateKind":"paper","links":{"code":"https://github.com/AMAPVOICE/PilotTTS","weights":null,"paper":"https://arxiv.org/abs/2605.27258","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#pilottts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"PilotTTS: A Disciplined Modular Recipe for Competitive Speech Synthesis","url":"https://arxiv.org/abs/2605.27258","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"PilotTTS author repository","url":"https://github.com/AMAPVOICE/PilotTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--pilottts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2605.27258","originUrl":"https://arxiv.org/html/2605.27258v2/figure/PilotTTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/pilottts.png","sha256":"3211672701b15d14f88f25ba92bc42477b58843d269c0c9a0c6b38dac4790e25"}},{"id":"tts:cosyedit2","slug":"cosyedit2","name":"CosyEdit2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"CosyEdit2 adapts a text-speech language model and acoustic decoder for consistent speech editing, then refines them with editing-specific rewards. The paper also evaluates the resulting improvement in zero-shot text-to-speech, connecting local editing consistency with reference-based synthesis.","architecture":"CosyVoice speech generation with editing-oriented post-training","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-25","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2605.25930","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#cosyedit2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CosyEdit2: Speech-Editing-Oriented Reinforcement Learning Unlocks Better Zero-Shot TTS","url":"https://arxiv.org/abs/2605.25930","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"CosyEdit2 author demo","url":"https://cjy1018.github.io/CosyEdit2","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--cosyedit2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.25930","originUrl":"https://arxiv.org/html/2605.25930v2/CosyEdit2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cosyedit2.png","sha256":"fc1dbd778efb4e6492329bcb23053e177ee904d589f56c201cf79b75ff193ca0"}},{"id":"tts:fc-tts","slug":"fc-tts","name":"FC-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"FC-TTS conditions generation on two recordings, one supplying delivery style and the other speaker identity. Specialized representation processing and auxiliary training objectives aim to prevent either condition from leaking unwanted attributes into the synthesized voice.","architecture":"Separate reference representations for speech style and timbre","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-23","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2605.24618","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#fc-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FC-TTS: Style and Timbre Control in Zero-Shot Text-to-Speech with Disentangled Speech Representations","url":"https://arxiv.org/abs/2605.24618","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--fc-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.24618","originUrl":"https://arxiv.org/html/2605.24618v1/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fc-tts.png","sha256":"e073ef6cabf047e6d13c8bcda3dcfdb5be6d0c588236cccc7f3a3f8a8b4a2e57"}},{"id":"tts:stepaudio-2-5-tts","slug":"stepaudio-2-5-tts","name":"StepAudio 2.5 TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"The TTS mode of StepAudio 2.5 uses a shared speech-language foundation with synthesis-specific decoding and preference training. Rich contextual supervision and feedback target controllable expression; this card describes its speech-rendering path within the broader system.","architecture":"Shared audio-language backbone with a dedicated expressive synthesis mode","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2605.23463","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#stepaudio-2-5-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"StepAudio 2.5 Technical Report","url":"https://arxiv.org/abs/2605.23463","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--stepaudio-2-5-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.23463","originUrl":"https://arxiv.org/html/2605.23463v1/figures/stepaudio25-unified-foundation-architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/stepaudio-2-5-tts.png","sha256":"3743e6c99200f4e364bebeb51bf51b7ba43ab7f95ab270559b4db34a4b7a4bc9"}},{"id":"tts:raon-opentts","slug":"raon-opentts","name":"Raon-OpenTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Raon-OpenTTS is a family of reference-conditioned diffusion synthesizers trained on a large, documented English speech collection. The release pairs its models with data processing and evaluation resources, allowing robustness across varied acoustic conditions to be examined alongside clean-speech quality.","architecture":"Diffusion Transformer synthesis trained on curated open speech","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-20","dateKind":"paper","links":{"code":"https://github.com/krafton-ai/RAON-OpenTTS","weights":null,"paper":"https://arxiv.org/abs/2605.20830","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#raon-opentts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Raon-OpenTTS: Open Models and Data for Robust Text-to-Speech","url":"https://arxiv.org/abs/2605.20830","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Raon-OpenTTS author repository","url":"https://github.com/krafton-ai/RAON-OpenTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--raon-opentts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.20830","originUrl":"https://arxiv.org/html/2605.20830v2/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/raon-opentts.png","sha256":"d322e7a83353d9daf0f49d43c69202f23be244fde5b55121faa48e8b5fcfbffd"}},{"id":"tts:agentsteertts","slug":"agentsteertts","name":"AgentSteerTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"AgentSteerTTS separates identity and emotional-prosodic representations, then grounds compound instructions in retrieved acoustic examples. A controller combines these conditions and uses feedback to refine expressive speech while preserving the intended speaker.","architecture":"Disentangled voice controls with retrieval and feedback-guided synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2605.17583","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#agentsteertts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"AgentSteerTTS: A Multi-Agent Closed-Loop Framework for Composite-Instruction Text-to-Speech","url":"https://arxiv.org/abs/2605.17583","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--agentsteertts.webp","kind":"source","label":"Figure 4","sourceUrl":"https://arxiv.org/abs/2605.17583","originUrl":"https://arxiv.org/html/2605.17583v1/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/agentsteertts.png","sha256":"17d48cb2e4b5363d18c8040b06e9cf4bc41c8cb2c280409c95c469d7f81d8374"}},{"id":"tts:semavoice","slug":"semavoice","name":"SemaVoice","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"SemaVoice organizes its audio VAE latents using guidance from speech foundation-model representations. A continuous autoregressive backbone and patch-level diffusion head then synthesize reference-conditioned speech with greater emphasis on linguistic coherence.","architecture":"Semantic-aligned continuous autoregression with next-patch diffusion","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-16","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2605.16964","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#semavoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SemaVoice: Semantic-Aware Continuous Autoregressive Speech Synthesis","url":"https://arxiv.org/abs/2605.16964","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--semavoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.16964","originUrl":"https://arxiv.org/html/2605.16964v1/semavoice.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/semavoice.png","sha256":"7e50bc762bf75528a959efda5378fb5c882b84ee94ff5ca9585de5197dc64aa8"}},{"id":"tts:gibbstts","slug":"gibbstts","name":"GibbsTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"GibbsTTS generates discrete speech tokens through a continuous-time jump process. Metric-aware transition scheduling and a finite-step correction improve how token states evolve, supporting zero-shot voice synthesis with discrete flow matching.","architecture":"Metric-induced discrete flow matching with corrected transition scheduling","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-10","dateKind":"paper","links":{"code":"https://github.com/ydqmkkx/GibbsTTS","weights":null,"paper":"https://arxiv.org/abs/2605.09386","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#gibbstts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Kinetic-Optimal Scheduling with Moment Correction for Metric-Induced Discrete Flow Matching in Zero-Shot Text-to-Speech","url":"https://arxiv.org/abs/2605.09386","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"GibbsTTS author repository","url":"https://github.com/ydqmkkx/GibbsTTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"GibbsTTS author demo","url":"https://ydqmkkx.github.io/GibbsTTSProject","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--gibbstts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.09386","originUrl":"https://arxiv.org/html/2605.09386v1/architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/gibbstts.png","sha256":"d0e283158223cd0f155edb647e3282513886dcf3695dd8d555add6ee7273b660"}},{"id":"tts:x-voice","slug":"x-voice","name":"X-Voice","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"X-Voice extends a flow-matching synthesizer with language information at both textual and acoustic-generation stages. Speaker-consistent training references and independently scheduled guidance support transcript-free cross-language cloning across thirty languages.","architecture":"Language-conditioned F5-TTS with multilingual reference alignment","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-07","dateKind":"paper","links":{"code":"https://github.com/sunnyxrxrx/X-Voice","weights":null,"paper":"https://arxiv.org/abs/2605.05611","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#x-voice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"X-Voice: Enabling Everyone to Speak 30 Languages via Zero-Shot Cross-Lingual Voice Cloning","url":"https://arxiv.org/abs/2605.05611","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"X-Voice author repository","url":"https://github.com/sunnyxrxrx/X-Voice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--x-voice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2605.05611","originUrl":"https://arxiv.org/html/2605.05611v2/overview.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/x-voice.png","sha256":"36b5d1556de6811c9f8c195fc4162f8443f441d65085d41cb9096abc73143f88"}},{"id":"tts:tibetan-tts","slug":"tibetan-tts","name":"Tibetan-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Tibetan-TTS adapts a large speech generator through language-specific text representation, tokenizer changes and cross-language training. Its data preparation and quality enhancement target scarce Tibetan recordings and the differences between written forms and spoken pronunciation.","architecture":"Tibetan-adapted large speech synthesis model","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-05-04","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2605.02496","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#tibetan-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Tibetan-TTS:Low-Resource Tibetan Speech Synthesis with Large Model Adaptation","url":"https://arxiv.org/abs/2605.02496","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--tibetan-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2605.02496","originUrl":"https://arxiv.org/html/2605.02496v1/pngs/tibetan-tts_model_1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tibetan-tts.png","sha256":"271f642e6b551a083f6f4c405705d2ac6630ef2c40ed2bb1e665d6d65c47dd4f"}},{"id":"tts:jaitts","slug":"jaitts","name":"JaiTTS","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"JaiTTS continually trains a VoxCPM-derived synthesizer on Thai-centered speech data. Its semantic planning, residual acoustic modeling and local diffusion decoding retain reference-based voice cloning while targeting fluent Thai pronunciation and delivery.","architecture":"Thai-adapted VoxCPM continuous autoregressive synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-30","dateKind":"paper","links":{"code":"https://github.com/JTS-AI-Team/JaiTTS","weights":null,"paper":"https://arxiv.org/abs/2604.27607","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#jaitts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"JaiTTS: A Thai Voice Cloning Model","url":"https://arxiv.org/abs/2604.27607","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"JaiTTS author repository","url":"https://github.com/JTS-AI-Team/JaiTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--jaitts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2604.27607","originUrl":"https://arxiv.org/html/2604.27607v2/voxcpm_architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/jaitts.png","sha256":"8ade136ad8485893195272e2519bf688aee7e563e3d87d08a6f6f7089b4ba923"}},{"id":"tts:unisonate","slug":"unisonate","name":"UniSonate","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"UniSonate uses a shared diffusion model to generate speech, music and sound effects from textual instructions. Phonemes structure spoken or sung content, while learned tokens organize unstructured effects and provide a common timing interface.","architecture":"Instruction-and-content-conditioned multimodal flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-24","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2604.22209","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#unisonate"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniSonate: A Unified Model for Speech, Music, and Sound Effect Generation with Text Instructions","url":"https://arxiv.org/abs/2604.22209","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"UniSonate author demo","url":"https://qiangchunyu.github.io/UniSonate/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--unisonate.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2604.22209","originUrl":"https://arxiv.org/html/2604.22209v1/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/unisonate.png","sha256":"4b15ac48ded8ebd1b3f4a2f52c2d4009596c9b1ee6ea442f4b90ca7635b66b26"}},{"id":"tts:magic-tts","slug":"magic-tts","name":"MAGIC-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MAGIC-TTS exposes timing controls for selected speech tokens and pauses. Training includes incomplete control signals so the model can follow local edits where provided and infer natural timing elsewhere, supporting precise pacing without requiring every segment to be specified.","architecture":"Token-level duration conditioning with explicit local pause control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-23","dateKind":"paper","links":{"code":"https://github.com/yongaifadian1/MAGIC-TTS","weights":null,"paper":"https://arxiv.org/abs/2604.21164","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#magic-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MAGIC-TTS: Fine-Grained Controllable Speech Synthesis with Explicit Local Duration and Pause Control","url":"https://arxiv.org/abs/2604.21164","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MAGIC-TTS author repository","url":"https://github.com/yongaifadian1/MAGIC-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--magic-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2604.21164","originUrl":"https://arxiv.org/html/2604.21164v2/method_overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/magic-tts.png","sha256":"52c1bf3af3bba7362a75f2c4a2ead08c4889182a82fc63062b6a303f75672342"}},{"id":"tts:chain-of-details-tts","slug":"chain-of-details-tts","name":"Chain-of-Details TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Chain-of-Details TTS progressively predicts speech at increasing temporal resolutions using a shared decoder. The coarsest stage provides an implicit phonetic plan, while subsequent stages recover timing detail without a separate phoneme-duration predictor.","architecture":"Shared-decoder temporal coarse-to-fine speech-token generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-21","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2604.19330","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#chain-of-details-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Text-To-Speech with Chain-of-Details: modeling temporal dynamics in speech generation","url":"https://arxiv.org/abs/2604.19330","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--chain-of-details-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2604.19330","originUrl":"https://arxiv.org/html/2604.19330v2/masked_audio_token_modeling.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/chain-of-details-tts.png","sha256":"ab0c9eba73399b0dbbfd24f4e3be4b36056b36db1e4922e35c503d36790118bf"}},{"id":"tts:atrie","slug":"atrie","name":"ATRIE","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"ATRIE converts character descriptions into separate voice-identity and dynamic prosody conditions. A compact adapter learns from a larger language-model teacher and modulates a GPT-SoVITS-based synthesizer, supporting expressive persona-driven speech generation.","architecture":"Persona-conditioned dual-track timbre and prosody synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-21","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2604.19055","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#atrie"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ATRIE: Adaptive Tuning for Robust Inference and Emotion in Persona-Driven Speech Synthesis","url":"https://arxiv.org/abs/2604.19055","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--atrie.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2604.19055","originUrl":"https://arxiv.org/html/2604.19055v2/Main_structure.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/atrie.png","sha256":"6f0b7bc9b65ed872059c2febf4fcb8aa12cf24e8d0b7d764204d8749decc1ed1"}},{"id":"tts:block-wise-mimi-tts","slug":"block-wise-mimi-tts","name":"Block-wise Mimi TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This streaming system replaces continuous acoustic regression with direct prediction of Mimi codec layers. A modified FastSpeech 2 backbone supplies aligned features and a depth-wise decoder fills residual codebooks, producing successive speech blocks without temporal autoregression.","architecture":"FastSpeech 2 conditioning with depth-wise Mimi codec prediction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-04-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2604.12438","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#block-wise-mimi-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"An Ultra-Low Latency, End-to-End Streaming Speech Synthesis Architecture via Block-Wise Generation and Depth-Wise Codec Decoding","url":"https://arxiv.org/abs/2604.12438","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--block-wise-mimi-tts.webp","kind":"source","label":"Figure 1 (paper page 10)","sourceUrl":"https://arxiv.org/abs/2604.12438","originUrl":"https://arxiv.org/pdf/2604.12438v1","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/block-wise-mimi-tts.png","sha256":"32b7023c52af8406cb320eb95c131a1a1b576a833fb89753023b9a50b1c7a7bd"}},{"id":"tts:cosyncdit","slug":"cosyncdit","name":"CoSyncDiT","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"CoSyncDiT guides flow-based speech synthesis through acoustic-style adaptation, visual calibration and timed context alignment. These stages connect the supplied transcript and scene information to expressive, synchronized movie dubbing.","architecture":"Visual and temporal conditioning in a dubbing diffusion Transformer","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-14","dateKind":"paper","links":{"code":"https://github.com/GalaxyCong/CoSyncDiT","weights":null,"paper":"https://arxiv.org/abs/2604.12292","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#cosyncdit"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CoSyncDiT: Cognitive Synchronous Diffusion Transformer for Movie Dubbing","url":"https://arxiv.org/abs/2604.12292","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"CoSyncDiT author repository","url":"https://github.com/GalaxyCong/CoSyncDiT","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--cosyncdit.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2604.12292","originUrl":"https://arxiv.org/html/2604.12292v1/ECCV_Method.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cosyncdit.png","sha256":"232f4c8872f202e89b60ebf81700307596aaa1d5be4a31f58c70f904c7c13d15"}},{"id":"tts:saslm","slug":"saslm","name":"SASLM","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"SASLM derives expressive intent from its own evolving semantic states through an information bottleneck. Acoustic feedback aligns generated speech with that intent, reducing the need for externally supplied emotion labels in context-sensitive speech rendering.","architecture":"Speech language modeling with latent intent and acoustic feedback","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-13","dateKind":"paper","links":{"code":"https://github.com/wangkevin02/SASLM","weights":null,"paper":"https://arxiv.org/abs/2604.11424","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#saslm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Bridging What the Model Thinks and How It Speaks: Expressive Speech Generation via Self-Aware Intent-Realization Alignment","url":"https://arxiv.org/abs/2604.11424","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"SASLM author repository","url":"https://github.com/wangkevin02/SASLM","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"SASLM author demo","url":"https://wangkevin02.github.io/SASLM/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--saslm.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2604.11424","originUrl":"https://arxiv.org/html/2604.11424v2/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/saslm.png","sha256":"f844ed3ff26dfac8721b9d0028f402ffbb6775a10d6c37ad044fd16fb567c318"}},{"id":"tts:ps-tts","slug":"ps-tts","name":"PS-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"PS-TTS uses vowel-based alignment to coordinate the timing and phonetic structure of dubbed speech. Its PS-Comet variant also considers semantic preservation when choosing translated text, connecting translation choices with a TTS rendering stage.","architecture":"Phonetic alignment and translation-conditioned dubbing synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2604.09111","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#ps-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"PS-TTS: Phonetic Synchronization in Text-to-Speech for Achieving Natural Automated Dubbing","url":"https://arxiv.org/abs/2604.09111","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--ps-tts.webp","kind":"source","label":"Fig. 1 (paper page 3)","sourceUrl":"https://arxiv.org/abs/2604.09111","originUrl":"https://arxiv.org/pdf/2604.09111v4","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ps-tts.png","sha256":"9d8ec81eadfb8a1fe64f9633c1c1cfccc029f4414697948963528fa2ea39102e"}},{"id":"tts:captalk","slug":"captalk","name":"CapTalk","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"CapTalk designs voices from descriptions for individual utterances and multi-speaker dialogue. Hierarchical conditioning separates stable speaker identity from changing turn-level delivery, while explicit planning tokens control dynamic expressive attributes.","architecture":"Caption-conditioned autoregression with hierarchical variational voice control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-04-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2604.08363","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#captalk"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CapTalk: Unified Voice Design for Single-Utterance and Dialogue Speech Generation","url":"https://arxiv.org/abs/2604.08363","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--captalk.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2604.08363","originUrl":"https://arxiv.org/html/2604.08363v1/captalk_model.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/captalk.png","sha256":"c11f32825557ab7cd62c41607e5897e135c5dea1fccf07860625cd9e7b8344c7"}},{"id":"tts:mambavoicecloning","slug":"mambavoicecloning","name":"MambaVoiceCloning","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"MambaVoiceCloning uses state-space modules to encode phonemes, learn their timing and condition expressive synthesis. A training-only alignment teacher supplies timing supervision, while the generation path targets efficient long sequences and limited-lookahead speech streaming.","architecture":"Bidirectional Mamba text and timing encoders with expressive diffusion control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-03-31","dateKind":"paper","links":{"code":"https://github.com/sahilkumar15/MVC","weights":null,"paper":"https://arxiv.org/abs/2604.00292","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#mambavoicecloning"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MambaVoiceCloning: Efficient and Expressive Text-to-Speech via State-Space Modeling and Diffusion Control","url":"https://arxiv.org/abs/2604.00292","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MambaVoiceCloning author repository","url":"https://github.com/sahilkumar15/MVC","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--mambavoicecloning.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2604.00292","originUrl":"https://arxiv.org/html/2604.00292v1/figures/MVC_1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mambavoicecloning.png","sha256":"95e67f853223a0183ab238dddef69cf642024db84f206aff4ec331619180b81e"}},{"id":"tts:longcat-audiodit","slug":"longcat-audiodit","name":"LongCat-AudioDiT","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"LongCat-AudioDiT maps text and reference speech into continuous waveform latents using a diffusion model. A jointly considered waveform autoencoder and adapted inference guidance address the interaction between acoustic reconstruction and zero-shot synthesis quality.","architecture":"Non-autoregressive diffusion directly in waveform-VAE latent space","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-31","dateKind":"paper","links":{"code":"https://github.com/meituan-longcat/LongCat-AudioDiT","weights":null,"paper":"https://arxiv.org/abs/2603.29339","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#longcat-audiodit"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"LongCat-AudioDiT: High-Fidelity Diffusion Text-to-Speech in the Waveform Latent Space","url":"https://arxiv.org/abs/2603.29339","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"LongCat-AudioDiT author repository","url":"https://github.com/meituan-longcat/LongCat-AudioDiT","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--longcat-audiodit.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.29339","originUrl":"https://arxiv.org/html/2603.29339v1/diagram_cropped.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/longcat-audiodit.png","sha256":"cc0b1ee9589ff955751e55eec56b57b32ab49bd7faf12942210b86af822d3bad"}},{"id":"tts:llada-tts","slug":"llada-tts","name":"LLaDA-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"LLaDA-TTS adapts a speech language model to fill masked token sequences in parallel. Its bidirectional generation also supports inserting, replacing or deleting spoken words, combining reference-based TTS and speech editing through the same model.","architecture":"Masked diffusion replacing autoregressive speech-token prediction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-27","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2603.26364","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#llada-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"LLaDA-TTS: Unifying Speech Synthesis and Zero-Shot Editing via Masked Diffusion Modeling","url":"https://arxiv.org/abs/2603.26364","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--llada-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.26364","originUrl":"https://arxiv.org/html/2603.26364v1/architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/llada-tts.png","sha256":"96e8165d07ec10492177b8a338b4389fba15ec098939aec48f4511a883c8efd8"}},{"id":"tts:voxtral-tts","slug":"voxtral-tts","name":"Voxtral TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Voxtral TTS conditions multilingual speech on a short voice recording. Its hybrid decoder predicts semantic tokens sequentially and acoustic information through flow matching, supporting expressive reference cloning across languages.","architecture":"Autoregressive semantic speech tokens with acoustic flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2603.25551","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#voxtral-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Voxtral TTS","url":"https://arxiv.org/abs/2603.25551","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--voxtral-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2603.25551","originUrl":"https://arxiv.org/html/2603.25551v2/voxtral-tts.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voxtral-tts.png","sha256":"73d8f2783bfa8179223545351fd0ad1c7ab4b5ca5844353eca5e3e91d46a64f6"}},{"id":"tts:selftts","slug":"selftts","name":"SelfTTS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"SelfTTS learns separate representations of a speaker's identity and emotional delivery using contrastive and adversarial objectives. It then improves synthesis through self-generated training examples, allowing emotion transfer to speakers originally recorded with neutral expression.","architecture":"Disentangled speaker and emotion embeddings with self-refined synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-23","dateKind":"paper","links":{"code":"https://github.com/AI-Unicamp/SelfTTS","weights":null,"paper":"https://arxiv.org/abs/2603.22252","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#selftts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SelfTTS: cross-speaker style transfer through explicit embedding disentanglement and self-refinement using self-augmentation","url":"https://arxiv.org/abs/2603.22252","reviewed_on":"2026-09-13","review_level":"paper"},{"kind":"repository","title":"SelfTTS author repository","url":"https://github.com/AI-Unicamp/SelfTTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"SelfTTS author demo","url":"https://ai-unicamp.github.io/publications/tts/selftts/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--selftts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.22252","originUrl":"https://arxiv.org/html/2603.22252v1/figs/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/selftts.png","sha256":"c11481cf1f65787aa715cccc4553ace210acc8d34ddff8fe78844a4c2b4bc482"}},{"id":"tts:borderless-long-speech-synthesis","slug":"borderless-long-speech-synthesis","name":"Borderless Long Speech Synthesis","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"This system organizes speech instructions at global, sentence and token levels to guide extended recordings. A continuous-token backbone uses explicit planning and condition dropout to combine voice design, multi-speaker rendering and changing acoustic or emotional context.","architecture":"Continuous-token synthesis with hierarchical textual control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-20","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2603.19798","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#borderless-long-speech-synthesis"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Borderless Long Speech Synthesis","url":"https://arxiv.org/abs/2603.19798","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--borderless-long-speech-synthesis.svg","kind":"editorial","label":"Editorial input/output diagram","sourceUrl":"https://arxiv.org/abs/2603.19798","originUrl":null,"originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/borderless-long-speech-synthesis.svg","sha256":"a953f08e7d5b2a32653a2327020095ce60df74152f05f87a862b6093f26b04e2"}},{"id":"tts:cast-tts","slug":"cast-tts","name":"CAST-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"CAST-TTS maps a voice description or a reference recording into a common timbre-conditioning interface. Cross-attention delivers that information to the synthesizer, enabling voice design and reference-based cloning within the same generation framework.","architecture":"Unified textual and acoustic timbre conditioning through cross-attention","notes":"The author-linked GitHub repository is currently empty; an inference implementation was not available for review.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-17","dateKind":"paper","links":{"code":"https://github.com/HiRookie9/CAST-TTS","weights":null,"paper":"https://arxiv.org/abs/2603.16280","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#cast-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CAST-TTS: A Simple Cross-Attention Framework for Unified Timbre Control in TTS","url":"https://arxiv.org/abs/2603.16280","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"CAST-TTS author repository","url":"https://github.com/HiRookie9/CAST-TTS","reviewed_on":"2026-09-13","review_level":"project"},{"kind":"project","title":"CAST-TTS author demo","url":"https://HiRookie9.github.io/CAST-TTS-Page","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--cast-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.16280","originUrl":"https://arxiv.org/html/2603.16280v1/model_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cast-tts.png","sha256":"8e3534a833b8e9164c94992903c7ef026a20abe229db14047dd132b74598d523"}},{"id":"tts:cosywhisper-whispsynth","slug":"cosywhisper-whispsynth","name":"CosyWhisper (WhispSynth)","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"The WhispSynth generation pipeline combines a CosyVoice synthesizer with pitch-free digital signal processing to produce whispered speech. It supports multilingual whisper generation while avoiding the voiced pitch patterns of ordinary speech synthesis.","architecture":"CosyVoice-based synthesis with pitch-free DDSP reconstruction","notes":"The paper links the CosyWhisper repository, but no README or packaged implementation was available for review.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-16","dateKind":"paper","links":{"code":"https://github.com/tan90xx/cosywhisper","weights":null,"paper":"https://arxiv.org/abs/2603.14853","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#cosywhisper-whispsynth"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"WhispSynth: Scaling Multilingual Whisper Corpus through Real Data Curation and A Novel Pitch-free Generative Framework","url":"https://arxiv.org/abs/2603.14853","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"CosyWhisper (WhispSynth) author repository","url":"https://github.com/tan90xx/cosywhisper","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--cosywhisper-whispsynth.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2603.14853","originUrl":"https://arxiv.org/html/2603.14853v1/test3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cosywhisper-whispsynth.png","sha256":"d062c7bed398dbb6921e1c1fbf111ffdec4fb1a36d8269b782785890ecc83346"}},{"id":"tts:affectron","slug":"affectron","name":"Affectron","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Affectron extends a verbal-speech backbone to place nonverbal vocalizations in emotionally and contextually appropriate positions. Augmented training examples and structural masking enable expressive utterances containing events such as laughter while preserving the spoken content.","architecture":"VoiceCraft adaptation with structured nonverbal-vocalization conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-15","dateKind":"paper","links":{"code":"https://github.com/Choddeok/Affectron","weights":null,"paper":"https://arxiv.org/abs/2603.14432","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#affectron"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Affectron: Emotional Speech Synthesis with Affective and Contextually Aligned Nonverbal Vocalizations","url":"https://arxiv.org/abs/2603.14432","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Affectron author repository","url":"https://github.com/Choddeok/Affectron","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"Affectron author demo","url":"https://choddeok.github.io/Affectron/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--affectron.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2603.14432","originUrl":"https://arxiv.org/html/2603.14432v2/250106_main_cropped.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/affectron.png","sha256":"59b4bdb421b6b50a7982ba8a5dab0dc9fb3cba667057806d8350d8c95525832d"}},{"id":"tts:diflowdubber","slug":"diflowdubber","name":"DiFlowDubber","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DiFlowDubber first learns linguistic content and separate prosodic-acoustic tokens through a discrete-flow TTS model. A subsequent video-dubbing stage aligns those representations to visual timing, connecting voice generation with synchronized lip movements.","architecture":"Factorized discrete-flow synthesis with audiovisual synchronization","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-15","dateKind":"paper","links":{"code":"https://github.com/Fsoft-AIC/DiFlowDubber","weights":null,"paper":"https://arxiv.org/abs/2603.14267","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#diflowdubber"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DiFlowDubber: Discrete Flow Matching for Automated Video Dubbing via Cross-Modal Alignment and Synchronization","url":"https://arxiv.org/abs/2603.14267","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DiFlowDubber author repository","url":"https://github.com/Fsoft-AIC/DiFlowDubber","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"DiFlowDubber author demo","url":"https://nngocson2002.github.io/projects/diflowdubber","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--diflowdubber.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2603.14267","originUrl":"https://arxiv.org/html/2603.14267v4/components.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/diflowdubber.png","sha256":"abfb659593b527fbf54031f1d7864eedfe73ab9af0da406db09bc745474cab93"}},{"id":"tts:voxtream2","slug":"voxtream2","name":"VoXtream2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"VoXtream2 consumes text incrementally while allowing the speaking rate to change during the utterance. Guidance over conditioning signals and a distribution-matching mechanism for duration states support reference cloning with responsive, locally adjustable pacing.","architecture":"Full-stream autoregressive speech with dynamic duration-state control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-03-13","dateKind":"paper","links":{"code":"https://github.com/herimor/voxtream","weights":null,"paper":"https://arxiv.org/abs/2603.13518","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#voxtream2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoXtream2: Full-stream TTS with dynamic speaking rate control","url":"https://arxiv.org/abs/2603.13518","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"VoXtream2 author repository","url":"https://github.com/herimor/voxtream","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"VoXtream2 author demo","url":"https://herimor.github.io/voxtream2/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--voxtream2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.13518","originUrl":"https://arxiv.org/html/2603.13518v1/architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voxtream2.png","sha256":"3f58c9711e97d08565b25e510abe686e5085bfb1cbe2d7c727847f7a88fa9c31"}},{"id":"tts:mamtra","slug":"mamtra","name":"MamTra","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"MamTra mixes state-space and attention layers to retain global context while reducing the cost of long sequences. Knowledge transfer from a pretrained Transformer initializes the hybrid synthesizer, combining efficient local processing with expressive acoustic modeling.","architecture":"Interleaved Mamba and Transformer speech-synthesis backbone","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-12","dateKind":"paper","links":{"code":"https://github.com/kaistmm/mamtra_inference","weights":null,"paper":"https://arxiv.org/abs/2603.12342","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#mamtra"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MamTra: A Hybrid Mamba-Transformer Backbone for Speech Synthesis","url":"https://arxiv.org/abs/2603.12342","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MamTra author repository","url":"https://github.com/kaistmm/mamtra_inference","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"MamTra author demo","url":"https://mamtratts.github.io","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--mamtra.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.12342","originUrl":"https://arxiv.org/html/2603.12342v2/hybrid_distillation.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mamtra.png","sha256":"3eef8dd968c1ed1b4df241fe5c0f8514c7c7bc389ebbd6ec10299775f30e6d86"}},{"id":"tts:causal-prosody-fastspeech-2","slug":"causal-prosody-fastspeech-2","name":"Causal-prosody FastSpeech 2","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"This FastSpeech 2 extension explicitly models emotion alongside duration, pitch and energy. Counterfactual training separates emotional changes from linguistic content, allowing users to modify prosody while preserving the intended words.","architecture":"Emotion-conditioned variance adaptation with counterfactual prosody training","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2603.11683","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#causal-prosody-fastspeech-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Causal Prosody Mediation for Text-to-Speech:Counterfactual Training of Duration, Pitch, and Energy in FastSpeech2","url":"https://arxiv.org/abs/2603.11683","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--causal-prosody-fastspeech-2.webp","kind":"source","label":"Figure 1 (paper page 3)","sourceUrl":"https://arxiv.org/abs/2603.11683","originUrl":"https://arxiv.org/pdf/2603.11683v1","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/causal-prosody-fastspeech-2.png","sha256":"cb82cd04be917e46ec4a1c140d05cb7e72e1f19eed064295a91d2678c9c6c6c9"}},{"id":"tts:uscf-conditioned-tts","slug":"uscf-conditioned-tts","name":"USCF-conditioned TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"USCF-TTS uses a low-rank transformation to suppress speaker timbre in its speech-content representation. A separate voice prompt conditions reconstruction, demonstrating how a simple invertible factorization can support speaker-controlled synthesis.","architecture":"Text-conditioned synthesis over linearly factorized speech content","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-09","dateKind":"paper","links":{"code":"https://github.com/HSTEHSTEHSTE/uscf","weights":null,"paper":"https://arxiv.org/abs/2603.08977","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#uscf-conditioned-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Universal Speech Content Factorization","url":"https://arxiv.org/abs/2603.08977","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"USCF-conditioned TTS author repository","url":"https://github.com/HSTEHSTEHSTE/uscf","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--uscf-conditioned-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.08977","originUrl":"https://arxiv.org/html/2603.08977v2/USCF_VC.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/uscf-conditioned-tts.png","sha256":"f1f80d36ee8de8465538c4d1a8fb0e3b3b94a8221dea28cdc7298d3e5a91120a"}},{"id":"tts:bolbosh","slug":"bolbosh","name":"Bolbosh","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Bolbosh adapts Matcha-TTS to Kashmiri with language-aware text processing and cross-language training. Its design targets the pronunciation and script challenges of a low-resource language while retaining efficient non-autoregressive acoustic generation.","architecture":"Script-aware Kashmiri adaptation of Matcha-TTS flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-08","dateKind":"paper","links":{"code":"https://github.com/gaash-lab/Bolbosh","weights":null,"paper":"https://arxiv.org/abs/2603.07513","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#bolbosh"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Bolbosh: Script-Aware Flow Matching for Kashmiri Text-to-Speech","url":"https://arxiv.org/abs/2603.07513","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Bolbosh author repository","url":"https://github.com/gaash-lab/Bolbosh","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--bolbosh.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.07513","originUrl":"https://arxiv.org/html/2603.07513v1/spectrogram_fig_comparison.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/bolbosh.png","sha256":"e66a1d2bc1336f99ed870f633798ded750e64b8f9b1c124a88d70f3b9bf504f8"}},{"id":"tts:dars","slug":"dars","name":"DARS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"DARS separately models pathological timing and acoustic style to synthesize dysarthric speech. A multistage rhythm predictor and conditioned flow model generate targeted examples for improving recognition under limited real-world speech data.","architecture":"Dysarthria-conditioned Matcha-TTS with a specialized rhythm predictor","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-03-02","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2603.01369","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#dars"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DARS: Dysarthria-Aware Rhythm-Style Synthesis for ASR Enhancement","url":"https://arxiv.org/abs/2603.01369","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--dars.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2603.01369","originUrl":"https://arxiv.org/html/2603.01369v1/framework_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dars.png","sha256":"b5670730f5748dbcfd0ff73d4faa117a3423b34a2ba42c34b4fda6347fe95dcc"}},{"id":"tts:tada","slug":"tada","name":"TADA","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"TADA aligns text tokens one-to-one with continuous acoustic units. A language model with a flow-matching head predicts these synchronized representations, reducing the ambiguity of text-speech alignment during reference-conditioned synthesis.","architecture":"Text-acoustic dual alignment with continuous autoregressive flow generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-02-26","dateKind":"paper","links":{"code":"https://github.com/HumeAI/tada","weights":null,"paper":"https://arxiv.org/abs/2602.23068","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#tada"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"TADA: A Generative Framework for Speech Modeling via Text-Acoustic Dual Alignment","url":"https://arxiv.org/abs/2602.23068","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"TADA author repository","url":"https://github.com/HumeAI/tada","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--tada.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2602.23068","originUrl":"https://arxiv.org/html/2602.23068v1/imgs/encdec.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tada.png","sha256":"36a2e9ce80e3d2654bc1e7ee4adcf4867878c0651cc7e9b44cd83b7f0e2ba3cd"}},{"id":"tts:ctc-tts","slug":"ctc-tts","name":"CTC-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"CTC-TTS uses automatically derived alignment and two-word interleaving to train an incremental speech language model. Its length-concatenated and feature-stacked variants make different trade-offs between speech quality and generation latency.","architecture":"CTC-aligned text-speech interleaving for dual-streaming synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2026-02-23","dateKind":"paper","links":{"code":"https://github.com/thu-spmi/CTC-TTS","weights":null,"paper":"https://arxiv.org/abs/2602.19574","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#ctc-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CTC-TTS: LLM-based dual-streaming text-to-speech with CTC alignment","url":"https://arxiv.org/abs/2602.19574","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"CTC-TTS author repository","url":"https://github.com/thu-spmi/CTC-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--ctc-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2602.19574","originUrl":"https://arxiv.org/html/2602.19574v2/ctc_tts.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ctc-tts.png","sha256":"bb65ed00c04555f03307ba173cce421438c21d60826cd0859301121275dd4abd"}},{"id":"tts:unitaf","slug":"unitaf","name":"UniTAF","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"UniTAF connects a text-to-speech model with a facial-animation model through internal representations. The joint architecture explores consistent emotional expression across generated audio and facial movement, focusing on the feasibility of feature sharing rather than claiming a production-quality avatar system. The authors withdrew the paper in March 2026 because some reported results require verification.","architecture":"Joint TTS and facial-animation modeling with shared intermediate features","notes":"The authors withdrew the paper on 2026-03-03, citing results that require further verification. The figure is taken from the original v1 submission, not the withdrawn v2 notice.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-02-17","dateKind":"paper","links":{"code":"https://github.com/GoldenFishes/UniTAF","weights":null,"paper":"https://arxiv.org/abs/2602.15651","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#unitaf"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniTAF: A Modular Framework for Joint Text-to-Speech and Audio-to-Face Modeling [withdrawn; see card notes]","url":"https://arxiv.org/abs/2602.15651","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"UniTAF author repository","url":"https://github.com/GoldenFishes/UniTAF","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--unitaf.webp","kind":"source","label":"Figure 2 (paper page 5)","sourceUrl":"https://arxiv.org/abs/2602.15651","originUrl":"https://arxiv.org/pdf/2602.15651v1","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/unitaf.png","sha256":"fb44136c06a59759cdc80e006834ff073af1baee3f04796867c3254ea9433e54"}},{"id":"tts:protodisent-tts","slug":"protodisent-tts","name":"ProtoDisent-TTS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"ProtoDisent-TTS learns a codebook of healthy and dysarthric articulation patterns separately from speaker identity. Adversarial constraints reduce pathological information in the speaker representation, enabling controlled synthesis of articulation characteristics in a target voice.","architecture":"Prototype-based separation of speaker timbre and dysarthric articulation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-02-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2602.08696","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#protodisent-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Prototype-Based Disentanglement for Controllable Dysarthric Speech Synthesis","url":"https://arxiv.org/abs/2602.08696","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"ProtoDisent-TTS author demo","url":"https://mors20.github.io/ProtoDisent-TTS","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--protodisent-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2602.08696","originUrl":"https://arxiv.org/html/2602.08696v2/Fig1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/protodisent-tts.png","sha256":"eede01b8dac94d388196dff7c4b8f5c7caf75a36ff5ea9f2bfbb685118e18e3a"}},{"id":"tts:archi-tts","slug":"archi-tts","name":"ARCHI-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"ARCHI-TTS uses a dedicated semantic alignment module to coordinate text and reference acoustic features. Reusing encoder features across denoising steps reduces repeated computation while the flow model generates the target speech.","architecture":"Flow-matching synthesis with a self-supervised semantic aligner","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-02-05","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2602.05207","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#archi-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ARCHI-TTS: A flow-matching-based Text-to-Speech Model with Self-supervised Semantic Aligner and Accelerated Inference","url":"https://arxiv.org/abs/2602.05207","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"ARCHI-TTS author demo","url":"https://archimickey.github.io/architts","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--archi-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2602.05207","originUrl":"https://arxiv.org/html/2602.05207v1/assets/arch23.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/archi-tts.png","sha256":"b572f2faf3159327f47eabf0aa0227e52d310515f624c73cb044ee2d671176aa"}},{"id":"tts:pfluxtts","slug":"pfluxtts","name":"PFluxTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"PFluxTTS combines two acoustic-generation paths by fusing their predicted vector fields at inference. Sequential reference embeddings support transcript-free cross-language voice cloning, and a super-resolution vocoder reconstructs high-rate output audio.","architecture":"Fusion of duration-guided and alignment-free flow decoders","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-02-04","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2602.04160","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#pfluxtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"PFluxTTS: Hybrid Flow-Matching TTS with Robust Cross-Lingual Voice Cloning and Inference-Time Model Fusion","url":"https://arxiv.org/abs/2602.04160","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"PFluxTTS author demo","url":"https://braskai.github.io/pfluxtts/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--pfluxtts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2602.04160","originUrl":"https://arxiv.org/html/2602.04160v2/raskttslast_v4.drawio.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/pfluxtts.png","sha256":"5365aac9bcc3f2e5c6df5f23b8db375d9181627a5aabdc73fb2805e56b29e40b"}},{"id":"tts:vividvoice","slug":"vividvoice","name":"VividVoice","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"VividVoice maps visual scene information to voice and environmental acoustic conditions using separate memory representations. Cross-modal supervision aligns those conditions with speech generation, allowing the synthesized voice and its surroundings to fit a supplied scene.","architecture":"Scene-aware visual conditioning with disentangled acoustic memories","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","I","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-02-01","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2602.02591","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#vividvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VividVoice: A Unified Framework for Scene-Aware Visually-Driven Speech Synthesis","url":"https://arxiv.org/abs/2602.02591","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"VividVoice author demo","url":"https://chengyuann.github.io/VividVoice/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--vividvoice.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2602.02591","originUrl":"https://arxiv.org/html/2602.02591v1/net1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vividvoice.png","sha256":"60f1d3a0d51c81c34776dd34e4b517c6554cb54f2e7b1ca1c82d79a6cee1c402"}},{"id":"tts:emoshift","slug":"emoshift","name":"EmoShift","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"EmoShift adds a lightweight layer that learns emotion-dependent changes to a TTS model's hidden representation. The controls adjust expressive delivery and emotion intensity while leaving most of the underlying synthesis backbone intact.","architecture":"Learned emotion steering in a speech model's output embeddings","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-30","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2601.22873","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#emoshift"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"EmoShift: Lightweight Activation Steering for Enhanced Emotion-Aware Speech Synthesis","url":"https://arxiv.org/abs/2601.22873","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--emoshift.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2601.22873","originUrl":"https://arxiv.org/html/2601.22873v1/emo-steer.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emoshift.png","sha256":"a30124c16acf281a6f4c250b2d9954cf51e5abaf9a9aa75bceccc44be8fb5eb6"}},{"id":"tts:deepasmr","slug":"deepasmr","name":"DeepASMR","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DeepASMR separates ASMR delivery from the reference speaker's identity using discrete speech representations. A language model predicts content and style, while a flow-based acoustic decoder transfers the requested performance into the target voice.","architecture":"Content-style speech LM with timbre-conditioned flow reconstruction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2601.15596","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#deepasmr"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DeepASMR: LLM-Based Zero-Shot ASMR Speech Generation for Anyone of Any Voice","url":"https://arxiv.org/abs/2601.15596","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--deepasmr.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2601.15596","originUrl":"https://arxiv.org/html/2601.15596v1/20260115pipelinev3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/deepasmr.png","sha256":"eebf51a241e51458f7c210d6172156677a4db8354f4284cf1f8709730c9cc105"}},{"id":"tts:habibi","slug":"habibi","name":"Habibi","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Habibi trains an Arabic synthesizer progressively from standard language to regional dialects using curated public speech. It targets zero-shot voice cloning across dialects and reading without mandatory diacritic marks.","architecture":"Arabic dialect synthesis through curriculum-trained flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-20","dateKind":"paper","links":{"code":"https://github.com/SWivid/Habibi-TTS","weights":null,"paper":"https://arxiv.org/abs/2601.13802","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#habibi"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Habibi: Laying the Open-Source Foundation of Unified-Dialectal Arabic Speech Synthesis","url":"https://arxiv.org/abs/2601.13802","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Habibi author repository","url":"https://github.com/SWivid/Habibi-TTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"Habibi author demo","url":"https://SWivid.github.io/Habibi/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--habibi.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.13802","originUrl":"https://arxiv.org/html/2601.13802v2/main_small.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/habibi.png","sha256":"87f8c77691d3b5232b304d82a5f98b9b1c2d7e18e44a63fa16d77410999bf7e9"}},{"id":"tts:speechedit","slug":"speechedit","name":"SpeechEdit","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"SpeechEdit combines text, instruction tokens and reference audio in a shared codec-language-model sequence. Paired examples that differ in selected attributes teach localized changes while retaining other aspects of the reference voice and delivery.","architecture":"Instruction-conditioned codec LM for selective speech attributes","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-18","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2601.12480","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#speechedit"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"A Unified Neural Codec Language Model for Selective Editable Text to Speech Generation","url":"https://arxiv.org/abs/2601.12480","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"SpeechEdit author demo","url":"https://speech-editing.github.io/speech-editing/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--speechedit.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.12480","originUrl":"https://arxiv.org/html/2601.12480v1/structure.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/speechedit.png","sha256":"d5bb10ab0c960599606aa68d0902576d257cc725d10dabda401aba2d4d6bf56c"}},{"id":"tts:wenetspeech-wu-tts","slug":"wenetspeech-wu-tts","name":"WenetSpeech-Wu TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"The WenetSpeech-Wu release includes TTS models trained on a curated Wu Chinese speech collection. Its synthesis paths cover ordinary and instruction-conditioned generation, alongside separate recognition and translation resources described in the same paper.","architecture":"Wu-dialect speech synthesis and instruction-conditioned model releases","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-16","dateKind":"paper","links":{"code":"https://github.com/ASLP-lab/WenetSpeech-Wu-Repo","weights":null,"paper":"https://arxiv.org/abs/2601.11027","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#wenetspeech-wu-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"WenetSpeech-Wu: Datasets, Benchmarks, and Models for a Unified Chinese Wu Dialect Speech Processing Ecosystem","url":"https://arxiv.org/abs/2601.11027","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"WenetSpeech-Wu TTS author repository","url":"https://github.com/ASLP-lab/WenetSpeech-Wu-Repo","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--wenetspeech-wu-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.11027","originUrl":"https://arxiv.org/html/2601.11027v1/overall_v3_roma.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/wenetspeech-wu-tts.png","sha256":"adeafa11353752b95800f8d23da1c55761ae028a697be461b3c373cb78da39f9"}},{"id":"tts:gpa","slug":"gpa","name":"GPA","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"General-Purpose Audio uses one autoregressive backbone to predict discrete speech tokens across synthesis, recognition and conversion tasks. Its TTS path combines target text with voice context, with shared multitask training and scalable inference.","architecture":"Unified autoregressive codec-language modeling across speech tasks","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-15","dateKind":"paper","links":{"code":"https://github.com/AutoArk/GPA","weights":null,"paper":"https://arxiv.org/abs/2601.10770","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#gpa"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Unifying Speech Recognition, Synthesis and Conversion with Autoregressive Transformers","url":"https://arxiv.org/abs/2601.10770","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"GPA author repository","url":"https://github.com/AutoArk/GPA","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--gpa.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.10770","originUrl":"https://arxiv.org/html/2601.10770v1/figures/GPA.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/gpa.png","sha256":"97770c47576ba0dc49d02dd0335e81bd8159824d0858fd279c3a31174bb4efff"}},{"id":"tts:voicesculptor","slug":"voicesculptor","name":"VoiceSculptor","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"VoiceSculptor first creates a voice matching a natural-language description, then renders an audio prompt for a cloning model. This two-part pipeline supports designing a new timbre and consistently using it to synthesize further text.","architecture":"Instruction-based voice design followed by reference cloning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-15","dateKind":"paper","links":{"code":"https://github.com/ASLP-lab/VoiceSculptor","weights":null,"paper":"https://arxiv.org/abs/2601.10629","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#voicesculptor"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoiceSculptor: Your Voice, Designed By You","url":"https://arxiv.org/abs/2601.10629","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"VoiceSculptor author repository","url":"https://github.com/ASLP-lab/VoiceSculptor","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"VoiceSculptor author demo","url":"https://hujingbin1.github.io/VoiceSculptor-Demo","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--voicesculptor.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.10629","originUrl":"https://arxiv.org/html/2601.10629v2/figures/llasa3b.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voicesculptor.png","sha256":"1fc7869b3cb85ce4ecf73529eaf922ecbf383faa04d4d9afce89eecb1810488f"}},{"id":"tts:flexivoice","slug":"flexivoice","name":"FlexiVoice","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"FlexiVoice accepts an optional style instruction and an optional reference voice alongside the target text. Progressive preference training teaches the model to follow both conditions while reducing unwanted coupling between wording, speaker identity and delivery.","architecture":"Speech language modeling with independently trained style and timbre control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2601.04656","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#flexivoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FlexiVoice: Enabling Flexible Style Control in Zero-Shot TTS with Natural Language Instructions","url":"https://arxiv.org/abs/2601.04656","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"FlexiVoice author demo","url":"https://flexi-voice.github.io","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--flexivoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.04656","originUrl":"https://arxiv.org/html/2601.04656v1/main.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/flexivoice.png","sha256":"ea31c35b35d47704726f8437a28039a407533fc70abdd1ec52282a6a2f2816f0"}},{"id":"tts:restyle-tts","slug":"restyle-tts","name":"ReStyle-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"ReStyle-TTS changes vocal attributes relative to a reference recording. Independent text and reference guidance, composable style adapters and timbre-consistency optimization allow continuous expressive edits while limiting changes to the speaker's identity.","architecture":"Decoupled guidance and orthogonal style adapters for relative control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-07","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2601.03632","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#restyle-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ReStyle-TTS: Relative and Continuous Style Control for Zero-Shot Speech Synthesis","url":"https://arxiv.org/abs/2601.03632","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--restyle-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.03632","originUrl":"https://arxiv.org/html/2601.03632v2/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/restyle-tts.png","sha256":"ad1ee48494459b9139355d1e470305e8badbf4e48b48754ec69d79a979b7740f"}},{"id":"tts:ted-tts","slug":"ted-tts","name":"TED-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"TED-TTS modifies conditioning and decoding in a pretrained zero-shot synthesizer to control different parts of an utterance. Segment-specific emotion masks and duration steering support local changes while coordinating transitions and the overall stopping point.","architecture":"Segment-aware inference control for emotion and duration","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-06","dateKind":"paper","links":{"code":"https://github.com/Simon-leong/TED-TTS","weights":null,"paper":"https://arxiv.org/abs/2601.03170","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#ted-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"TED-TTS: Training-Free Intra-Utterance Emotion and Duration Control for Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2601.03170","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"TED-TTS author repository","url":"https://github.com/Simon-leong/TED-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--ted-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.03170","originUrl":"https://arxiv.org/html/2601.03170v2/images/Intro.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ted-tts.png","sha256":"639395578387a70cd1b901b082f4e5e806c4053c73d301d259f3400ae8b9a859"}},{"id":"tts:vclip","slug":"vclip","name":"Vclip","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"Vclip learns an association between facial appearance and voice representations. Given a portrait, it retrieves and combines plausible speaker candidates for a downstream TTS model; the generated voice is an inferred match rather than a reconstruction of an unheard recording.","architecture":"CLIP-based face-to-voice retrieval with speaker-distribution modeling","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2601.02753","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#vclip"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Vclip: Face-based Speaker Generation by Face-voice Association Learning","url":"https://arxiv.org/abs/2601.02753","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vclip.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.02753","originUrl":"https://arxiv.org/html/2601.02753v1/fig-overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vclip.png","sha256":"2b425ba314204c8f81d1331c1943fb562cfc561346936e736877b5278c437ab5"}},{"id":"tts:ov-instructtts","slug":"ov-instructtts","name":"OV-InstructTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"OV-InstructTTS interprets voice and delivery descriptions beyond a fixed inventory of style labels. Its reasoning-based conditioning connects broader textual requests with expressive speech generation, supported by a dedicated instruction-speech dataset.","architecture":"Reasoning-guided speech synthesis from open-vocabulary style instructions","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-04","dateKind":"paper","links":{"code":"https://github.com/y-ren16/OV-InstructTTS","weights":null,"paper":"https://arxiv.org/abs/2601.01459","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#ov-instructtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"OV-InstructTTS: Towards Open-Vocabulary Instruct Text-to-Speech","url":"https://arxiv.org/abs/2601.01459","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"OV-InstructTTS author repository","url":"https://github.com/y-ren16/OV-InstructTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--ov-instructtts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2601.01459","originUrl":"https://arxiv.org/html/2601.01459v1/OV-InstructTTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ov-instructtts.png","sha256":"b616f25bc005ccd23fa39f85e1fddc687a62a64bb0068665070a7737966d187f"}},{"id":"tts:depflow","slug":"depflow","name":"DepFlow","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"DepFlow separates a depression-related acoustic representation from speaker identity and spoken content. The representation conditions a flow-matching synthesizer, enabling controlled synthetic speech for investigating acoustic cues and data augmentation.","architecture":"Disentangled condition encoding with FiLM-modulated flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2026-01-01","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2601.00303","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#depflow"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DepFlow: Disentangled Speech Generation to Mitigate Semantic Bias in Depression Detection","url":"https://arxiv.org/abs/2601.00303","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--depflow.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2601.00303","originUrl":"https://arxiv.org/html/2601.00303v1/Training.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/depflow.png","sha256":"fb20d670ea0cbb47ad9ecc8764f628c92de72bafbf7f234e839c57a319d5b911"}},{"id":"tts:manchutts","slug":"manchutts","name":"ManchuTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"ManchuTTS builds multilevel text representations suited to Manchu and feeds them into a convolutional diffusion Transformer. Its non-autoregressive generator and augmented training data target speech synthesis where naturally recorded material is scarce.","architecture":"Hierarchical linguistic features with convolutional flow-matching synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-12-27","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2512.22491","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#manchutts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ManchuTTS: Towards High-Quality Manchu Speech Synthesis via Flow Matching and Hierarchical Text Representation","url":"https://arxiv.org/abs/2512.22491","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--manchutts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2512.22491","originUrl":"https://arxiv.org/html/2512.22491v1/Model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/manchutts.png","sha256":"7aa32c9add35522adee5f7ee7e0031f1d737860cd90887cc61a5024579fff902"}},{"id":"tts:joyvoice","slug":"joyvoice","name":"JoyVoice","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"JoyVoice conditions long-form speech on speaker-labeled text and shared conversational context. Autoregressive hidden states feed an acoustic diffusion decoder, allowing multiple speakers, changing expression and more flexible turn boundaries within one synthesis system.","architecture":"End-to-end autoregressive planning with dynamic-chunk diffusion decoding","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-12-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2512.19090","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#joyvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"JoyVoice: Long-Context Conditioning for Anthropomorphic Multi-Speaker Conversational Synthesis","url":"https://arxiv.org/abs/2512.19090","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"JoyVoice author demo","url":"https://jea-speech.github.io/JoyVoice","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--joyvoice.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2512.19090","originUrl":"https://arxiv.org/html/2512.19090v1/Joyvoice.drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/joyvoice.png","sha256":"c8aadafdb586e99796608a52cdb1d9d922f4ea676b37a7b787190fd9a39a3ca1"}},{"id":"tts:disco-speech","slug":"disco-speech","name":"DisCo-Speech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DisCo-Speech learns a codec that separates content, delivery and speaker identity. A language model predicts combined content-prosody tokens while the decoder receives a separate timbre representation, enabling reference cloning with independently controllable vocal attributes.","architecture":"Disentangled content-prosody tokens with a separate timbre condition","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-12-15","dateKind":"paper","links":{"code":"https://github.com/disco-speech/DisCo-Speech-main","weights":null,"paper":"https://arxiv.org/abs/2512.13251","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#disco-speech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DisCo-Speech: Controllable Zero-Shot Speech Generation with A Disentangled Speech Codec","url":"https://arxiv.org/abs/2512.13251","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DisCo-Speech author repository","url":"https://github.com/disco-speech/DisCo-Speech-main","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"DisCo-Speech author demo","url":"https://disco-speech.github.io/DisCo-demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--disco-speech.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2512.13251","originUrl":"https://arxiv.org/html/2512.13251v3/img/llm.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/disco-speech.png","sha256":"676d3529ce2d190c25aef4b72cbf678fafe7d4e2c0c3fa6082d92759d6cfc518"}},{"id":"tts:dmp-tts","slug":"dmp-tts","name":"DMP-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"DMP-TTS maps style descriptions and reference recordings into a shared conditioning space. Chained guidance controls content, timbre and style separately, allowing detailed synthesis adjustments within a latent diffusion Transformer.","architecture":"Latent diffusion with aligned text-audio style embeddings and chained guidance","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-12-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2512.09504","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#dmp-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DMP-TTS: Disentangled multi-modal Prompting for Controllable Text-to-Speech with Chained Guidance","url":"https://arxiv.org/abs/2512.09504","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--dmp-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2512.09504","originUrl":"https://arxiv.org/html/2512.09504v1/Clap_TTS.drawio_2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dmp-tts.png","sha256":"f0379f6dc64a4d2cfa4fe2c5b505a20dbb6ba7856a76cd52d6ee24949c957448"}},{"id":"tts:m3-tts","slug":"m3-tts","name":"M3-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"M3-TTS uses joint text-audio diffusion layers to learn alignment without first stretching text into a guessed acoustic timeline. Additional single-stream layers refine acoustic details, producing reference-conditioned speech through a compressed mel representation.","architecture":"Multimodal diffusion alignment over mel-VAE latents","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-12-04","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2512.04720","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#m3-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"M3-TTS: Multi-modal DiT Alignment & Mel-latent for Zero-shot High-fidelity Speech Synthesis","url":"https://arxiv.org/abs/2512.04720","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--m3-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2512.04720","originUrl":"https://arxiv.org/html/2512.04720v1/M3-TTS-framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/m3-tts.png","sha256":"0490d5f721199cebc94d81855f0bf65249d566b2d354ace80a7c6e05c56145c3"}},{"id":"tts:vspeechlm","slug":"vspeechlm","name":"VSpeechLM","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"VSpeechLM aligns video timing with a text transcript before generating speech tokens. Its phoneme expansion and acoustic decoders support intelligible speech whose timing matches the visible speaker, targeting visual TTS and dubbing.","architecture":"Video-aligned phoneme expansion with speech-language-model decoding","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-11-27","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2511.22229","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#vspeechlm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VSpeechLM: A Visual Speech Language Model for Visual Text-to-Speech Task","url":"https://arxiv.org/abs/2511.22229","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vspeechlm.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2511.22229","originUrl":"https://arxiv.org/html/2511.22229v1/model_0809.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vspeechlm.png","sha256":"a4ad1587895f34d5fd06cc7fbbc324cdb4d5104bc90a29b6267dc5c88ac001d8"}},{"id":"tts:instructaudio","slug":"instructaudio","name":"InstructAudio","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"InstructAudio combines natural-language instructions with phonemes or lyrics in a common generation format. Joint and single-stream diffusion layers synthesize speech or music while controlling attributes such as voice, emotion, accent or musical character.","architecture":"Shared multimodal diffusion for instruction-controlled speech and music","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-11-23","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2511.18487","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#instructaudio"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"InstructAudio: Unified speech and music generation with natural language instruction","url":"https://arxiv.org/abs/2511.18487","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"InstructAudio author demo","url":"https://qiangchunyu.github.io/InstructAudio/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--instructaudio.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2511.18487","originUrl":"https://arxiv.org/html/2511.18487v1/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/instructaudio.png","sha256":"509d5e4d2f6bb5f11ae0ef8e129c68a18edfc4bcb9deaf805899a86e23206ec5"}},{"id":"tts:spotlight-tts","slug":"spotlight-tts","name":"Spotlight-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Spotlight-TTS extracts expressive reference information primarily from voiced speech and adjusts the resulting style direction before acoustic generation. The method targets more faithful expression transfer; the catalog combines its related paper records under one model.","architecture":"Voiced-region style extraction and adjusted style conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-27","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2511.14824","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#spotlight-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Voiced-Aware Style Extraction and Style Direction Adjustment for Expressive Text-to-Speech","url":"https://arxiv.org/abs/2511.14824","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"paper","title":"Spotlight-TTS: Spotlighting the Style via Voiced-Aware Style Extraction and Style Direction Adjustment for Expressive Text-to-Speech","url":"https://arxiv.org/abs/2505.20868","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--spotlight-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.20868","originUrl":"https://arxiv.org/html/2505.20868v2/Figure1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/spotlight-tts.png","sha256":"ddade00598cbfa284b153a1cb393872dac3dd89012e8efdb33e337c81299f670"}},{"id":"tts:authentic-dubber","slug":"authentic-dubber","name":"Authentic-Dubber","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Authentic-Dubber retrieves emotionally relevant audiovisual examples and progressively incorporates them into speech generation. Its director-actor formulation connects a scene's visual context and reference delivery to the target transcript for expressive movie dubbing.","architecture":"Reference-footage retrieval with graph-conditioned expressive dubbing","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-11-18","dateKind":"paper","links":{"code":"https://github.com/AI-S2-Lab/Authentic-Dubber","weights":null,"paper":"https://arxiv.org/abs/2511.14249","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#authentic-dubber"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Towards Authentic Movie Dubbing with Retrieve-Augmented Director-Actor Interaction Learning","url":"https://arxiv.org/abs/2511.14249","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Authentic-Dubber author repository","url":"https://github.com/AI-S2-Lab/Authentic-Dubber","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--authentic-dubber.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2511.14249","originUrl":"https://arxiv.org/html/2511.14249v1/figure_2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/authentic-dubber.png","sha256":"47445db5a021cb9e85038001a1d2d0309638eb95998be56f02db2dc8a8d95735"}},{"id":"tts:vocalnet-m2","slug":"vocalnet-m2","name":"VocalNet-M2","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"VocalNet-M2 directly predicts multiple codec streams from a speech-language backbone. Its talker combines multi-codebook representations and multi-token decoding to reduce latency, avoiding a separate flow-matching stage for acoustic reconstruction.","architecture":"Multi-codebook speech generation with multi-token prediction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-11-13","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2511.10232","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#vocalnet-m2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VocalNet-M2: Advancing Low-Latency Spoken Language Modeling via Integrated Multi-Codebook Tokenization and Multi-Token Prediction","url":"https://arxiv.org/abs/2511.10232","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--vocalnet-m2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2511.10232","originUrl":"https://arxiv.org/html/2511.10232v1/VocalNet-m2-full-v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vocalnet-m2.png","sha256":"b8fa6bb7f7ca37ef18e6904dbc46a1bfd674ccbb4b9b616d848edc26c01c83f0"}},{"id":"tts:step-audio-editx","slug":"step-audio-editx","name":"Step-Audio-EditX","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Step-Audio-EditX supports reference-based synthesis and repeated edits to emotion, speaking style or nonverbal delivery. Training on deliberately contrasting synthetic examples teaches the model to follow expressive changes without a separate attribute-embedding module.","architecture":"Speech language modeling for zero-shot TTS and iterative expressive editing","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-11-05","dateKind":"paper","links":{"code":"https://github.com/stepfun-ai/Step-Audio-EditX","weights":null,"paper":"https://arxiv.org/abs/2511.03601","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#step-audio-editx"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Step-Audio-EditX Technical Report","url":"https://arxiv.org/abs/2511.03601","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Step-Audio-EditX author repository","url":"https://github.com/stepfun-ai/Step-Audio-EditX","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--step-audio-editx.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2511.03601","originUrl":"https://arxiv.org/html/2511.03601v2/figures/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/step-audio-editx.png","sha256":"7461dbdcc7160f929bdea700b3ff4cb4c65e0f3a40190f3ec795a4cc5b330dd7"}},{"id":"tts:belle","slug":"belle","name":"BELLE","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"BELLE predicts both speech values and their uncertainty in a continuous autoregressive synthesizer. Multiple synthetic renditions of the same text provide training support for the variance estimate, enabling richer acoustic distributions without adding an iterative inference stage.","architecture":"Bayesian continuous speech modeling with learned predictive variance","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-28","dateKind":"paper","links":{"code":"https://github.com/OpenTSLab/BELLE","weights":null,"paper":"https://arxiv.org/abs/2510.24372","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#belle"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Bayesian Speech Synthesizers Can Learn from Multiple Teachers","url":"https://arxiv.org/abs/2510.24372","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"BELLE author repository","url":"https://github.com/OpenTSLab/BELLE","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"BELLE author demo","url":"https://belletts.github.io/Belle/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--belle.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2510.24372","originUrl":"https://arxiv.org/html/2510.24372v3/BELLE_v2.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/belle.png","sha256":"6e0855fc1796ebc7090d777730058197342d96cf752b81e432d54c9d81860bff"}},{"id":"tts:soulx-podcast","slug":"soulx-podcast","name":"SoulX-Podcast","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"SoulX-Podcast synthesizes conversational scripts with reference voices, dialect choices and nonverbal expression. Its long-form training targets consistent speaker identity and natural transitions across turns, while also supporting ordinary single-speaker TTS.","architecture":"Long-context multi-speaker speech language modeling","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-27","dateKind":"paper","links":{"code":"https://github.com/Soul-AILab/SoulX-Podcast","weights":null,"paper":"https://arxiv.org/abs/2510.23541","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#soulx-podcast"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SoulX-Podcast: Towards Realistic Long-form Podcasts with Dialectal and Paralinguistic Diversity","url":"https://arxiv.org/abs/2510.23541","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"SoulX-Podcast author repository","url":"https://github.com/Soul-AILab/SoulX-Podcast","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"SoulX-Podcast author demo","url":"https://soul-ailab.github.io/soulx-podcast/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--soulx-podcast.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2510.23541","originUrl":"https://arxiv.org/html/2510.23541v2/soulxpodcast_overview.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/soulx-podcast.png","sha256":"9a5d0c624d1ff31fac2c6faec6e7fe8a661b4ed9823deb9869738b82c1e42470"}},{"id":"tts:parastyletts","slug":"parastyletts","name":"ParaStyleTTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"ParaStyleTTS converts textual style prompts into separate controls for prosody and broader paralinguistic characteristics. The lightweight adaptation design targets expressive speech from descriptions while making the roles of the two conditioning levels explicit.","architecture":"Two-level adaptation separating prosodic and paralinguistic style","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-21","dateKind":"paper","links":{"code":"https://github.com/haoweilou/ParaStyleTTS","weights":null,"paper":"https://arxiv.org/abs/2510.18308","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#parastyletts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ParaStyleTTS: Toward Efficient and Robust Paralinguistic Style Control for Expressive Text-to-Speech Generation","url":"https://arxiv.org/abs/2510.18308","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"ParaStyleTTS author repository","url":"https://github.com/haoweilou/ParaStyleTTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"ParaStyleTTS author demo","url":"https://parastyletts.github.io/ParaStyleTTS_Demo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--parastyletts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2510.18308","originUrl":"https://arxiv.org/html/2510.18308v1/ParaStyleTTS_train.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/parastyletts.png","sha256":"aa39a8025287b1e0868c6fc32c6d234e46a5517af1139fc9dd9436f2bcb39dd8"}},{"id":"tts:continuous-token-diffusion-tts","slug":"continuous-token-diffusion-tts","name":"Continuous-token diffusion TTS","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"This model combines a language head that predicts boundaries with a diffusion head that generates continuous acoustic frames. Masked and staged training stabilize speaker-reference conditioning, providing a text-to-speech path within a multimodal language-model architecture.","architecture":"Dual language and diffusion heads for continuous speech generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.12995","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#continuous-token-diffusion-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Continuous-Token Diffusion for Speaker-Referenced TTS in Multimodal LLMs","url":"https://arxiv.org/abs/2510.12995","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--continuous-token-diffusion-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2510.12995","originUrl":"https://arxiv.org/html/2510.12995v2/training2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/continuous-token-diffusion-tts.png","sha256":"2144af4a948f943979ee8a9d368df6f1a3eb6237f73df757724a8741363d9bf0"}},{"id":"tts:distar","slug":"distar","name":"DiSTAR","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DiSTAR first drafts blocks of residual-quantized speech tokens with a language model. A masked diffusion decoder then fills acoustic detail within each block, combining temporal planning and parallel refinement entirely in discrete codec space.","architecture":"Autoregressive codec patches refined by masked discrete diffusion","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.12210","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#distar"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DiSTAR: Diffusion over a Scalable Token Autoregressive Representation for Speech Generation","url":"https://arxiv.org/abs/2510.12210","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--distar.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2510.12210","originUrl":"https://arxiv.org/html/2510.12210v2/overall1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/distar.png","sha256":"880d2038c6e72428fd9aba82e3bdc39f3264c655bfa4dc45fb96366cadb45ef0"}},{"id":"tts:bridgetts","slug":"bridgetts","name":"BridgeTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"BridgeTTS uses the BridgeCode dual representation to shorten the sequence predicted by its autoregressive language model. Bridging modules reconstruct more detailed continuous acoustic features from those sparse tokens, balancing generation speed with voice fidelity.","architecture":"Sparse discrete prediction with rich continuous acoustic reconstruction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-13","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.11646","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#bridgetts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"BridgeCode: A Dual Speech Representation Paradigm for Autoregressive Zero-Shot Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2510.11646","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--bridgetts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2510.11646","originUrl":"https://arxiv.org/html/2510.11646v1/1_2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/bridgetts.png","sha256":"4bec4547438073334e822c0533adea2b08bfa2e87140ae7307eaf2938f99f0a8"}},{"id":"tts:emm-tts","slug":"emm-tts","name":"EMM-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"EMM-TTS separates emotional content modeling from speaker-specific acoustic generation. Speaker-consistency objectives and adaptive normalization help transfer emotion across languages while retaining the reference speaker's timbre.","architecture":"Perturbed speech representations with separate emotion and speaker stages","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-13","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.11124","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#emm-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Perturbation Self-Supervised Representations for Cross-Lingual Emotion TTS: Stage-Wise Modeling of Emotion and Speaker","url":"https://arxiv.org/abs/2510.11124","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--emm-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2510.11124","originUrl":"https://arxiv.org/html/2510.11124v1/fig2_new.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emm-tts.png","sha256":"c540683428fdc2a0691848ad50e118215081209ad1c6a9af90e2a0d0130d210b"}},{"id":"tts:dialospeech","slug":"dialospeech","name":"DialoSpeech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DialoSpeech models two speakers on separate tracks and uses chunked flow matching for acoustic rendering. The design supports expressive scripted dialogue, including interactions whose timing is difficult to reproduce by joining independent utterances.","architecture":"Dual-track language modeling with chunked acoustic flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.08373","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#dialospeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DialoSpeech: Dual-Speaker Dialogue Generation with LLM and Flow Matching","url":"https://arxiv.org/abs/2510.08373","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"DialoSpeech author demo","url":"https://tiamojames.github.io/DialoSpeech","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--dialospeech.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2510.08373","originUrl":"https://arxiv.org/html/2510.08373v1/pic/diaspeech_over.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dialospeech.png","sha256":"5477b97c799e23fdb074fb2e4fac53d6024144ec08af51dea5b0a7daa873c713"}},{"id":"tts:intmeanflow","slug":"intmeanflow","name":"IntMeanFlow","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"IntMeanFlow distills a flow-based speech generator to predict integrated acoustic updates over larger intervals. A search for effective sampling steps further reduces decoding work, enabling reference-conditioned synthesis with fewer iterative refinements.","architecture":"Few-step synthesis through integral-velocity distillation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.07979","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#intmeanflow"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"IntMeanFlow: Few-step Speech Generation with Integral Velocity Distillation","url":"https://arxiv.org/abs/2510.07979","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"IntMeanFlow author demo","url":"https://vvwangvv.github.io/intmeanflow","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--intmeanflow.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2510.07979","originUrl":"https://arxiv.org/html/2510.07979v1/imf.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/intmeanflow.png","sha256":"47fc99ad68dc849dcad38d85c4fcbbbb52872d79920a3da643b88c44eee3cdae"}},{"id":"tts:ectspeech","slug":"ectspeech","name":"ECTSpeech","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"ECTSpeech gradually tightens consistency constraints on a pretrained diffusion synthesizer. The resulting generator maps a noise state to speech in one step, reducing repeated denoising while retaining the conditioning of the original TTS model.","architecture":"One-step speech synthesis through easy consistency tuning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-07","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.05984","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#ectspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ECTSpeech: Enhancing Efficient Speech Synthesis via Easy Consistency Tuning","url":"https://arxiv.org/abs/2510.05984","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--ectspeech.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2510.05984","originUrl":"https://arxiv.org/html/2510.05984v1/figure1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ectspeech.png","sha256":"d31ca46929659171afd27e7e52561f704355e5aa6cfc0255c4bc2a28966b0d04"}},{"id":"tts:mave","slug":"mave","name":"MAVE","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MAVE combines a state-space backbone with cross-attention to generate speech conditioned on text and acoustic context. It supports zero-shot voice synthesis and editing while reducing the attention-memory requirements of a comparable Transformer-based codec model.","architecture":"Cross-attentive Mamba autoregression for codec speech synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.04738","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#mave"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Speak, Edit, Repeat: High-Fidelity Voice Editing and Zero-Shot TTS with Cross-Attentive Mamba","url":"https://arxiv.org/abs/2510.04738","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mave.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2510.04738","originUrl":"https://arxiv.org/html/2510.04738v1/proposed_arch_cropped.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mave.png","sha256":"9fddc62694e390c1d6a8ccc218947c47b1db1f9f58d0d40427ccbe36a575c9e3"}},{"id":"tts:univoice-asr-and-tts","slug":"univoice-asr-and-tts","name":"UniVoice (ASR and TTS)","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"This UniVoice model shares an LLM framework between speech recognition and text-to-speech. Its synthesis path uses flow matching over continuous representations, providing zero-shot generation alongside an autoregressive recognition path.","architecture":"Unified language modeling and continuous acoustic flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-06","dateKind":"paper","links":{"code":"https://github.com/gwh22/UniVoice","weights":null,"paper":"https://arxiv.org/abs/2510.04593","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#univoice-asr-and-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniVoice: Unifying Autoregressive ASR and Flow-Matching based TTS with Large Language Models","url":"https://arxiv.org/abs/2510.04593","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"UniVoice (ASR and TTS) author repository","url":"https://github.com/gwh22/UniVoice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--univoice-asr-and-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2510.04593","originUrl":"https://arxiv.org/html/2510.04593v4/univoice_new5.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/univoice-asr-and-tts.png","sha256":"66f4a6a23098dc5dd83d836e45c8e0f0d0bfd707a5d042dcbce914fef7060cbe"}},{"id":"tts:flamed-tts","slug":"flamed-tts","name":"Flamed-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Flamed-TTS combines representations of different speech attributes in an attention-free generator. Its reformulated flow-matching process targets efficient zero-shot synthesis with flexible pacing and reduced sequential computation.","architecture":"Attention-free flow matching with discrete and continuous speech conditions","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-03","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2510.02848","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#flamed-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Flamed-TTS: Flow Matching Attention-Free Models for Efficient Generating and Dynamic Pacing Zero-shot Text-to-Speech","url":"https://arxiv.org/abs/2510.02848","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"Flamed-TTS author demo","url":"https://flamed-tts.github.io","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--flamed-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2510.02848","originUrl":"https://arxiv.org/html/2510.02848v1/flame-tts.drawio.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/flamed-tts.png","sha256":"9cdc00595cf9b3a2d9da43801af4da353e4493d68da514131b1473f5ec9c57cd"}},{"id":"tts:emotion-timbre-disentangled-tts","slug":"emotion-timbre-disentangled-tts","name":"Emotion-timbre disentangled TTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"This emotional synthesizer learns separate reference encoders for timbre and emotion. A mutual-information objective reduces their overlap, while phoneme-level emotion prediction carries changing expression into the generated acoustic sequence.","architecture":"Phoneme-level emotion prediction with mutual-information disentanglement","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-10-02","dateKind":"paper","links":{"code":"https://github.com/BaleYang/emotion-timbre-disentangled-tts-code","weights":null,"paper":"https://arxiv.org/abs/2510.01722","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#emotion-timbre-disentangled-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Emotional Text-To-Speech Based on Mutual-Information-Guided Emotion-Timbre Disentanglement","url":"https://arxiv.org/abs/2510.01722","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Emotion-timbre disentangled TTS author repository","url":"https://github.com/BaleYang/emotion-timbre-disentangled-tts-code","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"Emotion-timbre disentangled TTS author demo","url":"https://baleyang.github.io/emotion-timbre-disentangled-tts/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--emotion-timbre-disentangled-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2510.01722","originUrl":"https://arxiv.org/html/2510.01722v1/architecture_all.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emotion-timbre-disentangled-tts.png","sha256":"2489b50c5734091c02c10295d25c6e80b599e1ca7e394c9e7de353e474bd567f"}},{"id":"tts:batontts-batonvoice","slug":"batontts-batonvoice","name":"BatonTTS (BatonVoice)","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"BatonVoice interprets a user's expressive request and translates it into controls for its dedicated BatonTTS generator. Separating instruction interpretation from acoustic rendering enables more explicit feature control, including transfer to languages outside the control-training data.","architecture":"Feature-controlled speech synthesis driven by structured linguistic instructions","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-30","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.26514","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#batontts-batonvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"BatonVoice: An Operationalist Framework for Enhancing Controllable Speech Synthesis with Linguistic Intelligence from LLMs","url":"https://arxiv.org/abs/2509.26514","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--batontts-batonvoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.26514","originUrl":"https://arxiv.org/html/2509.26514v1/infer.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/batontts-batonvoice.png","sha256":"d884693c41982be1cbf457a7094ea2191d4eca24a880a2f4a2864632736c5879"}},{"id":"tts:histyle","slug":"histyle","name":"HiStyle","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"HiStyle predicts a voice's timbre first and finer delivery attributes afterward from textual descriptions. Contrastive text-audio alignment organizes these style representations before they condition a speech synthesizer.","architecture":"Hierarchical text-to-style embedding prediction for controllable TTS","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-30","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.25842","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#histyle"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"HiStyle: Hierarchical Style Embedding Predictor for Text-Prompt-Guided Controllable Speech Synthesis","url":"https://arxiv.org/abs/2509.25842","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--histyle.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2509.25842","originUrl":"https://arxiv.org/html/2509.25842v1/new.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/histyle.png","sha256":"f114890a5a43abc13993810a55f89be862b216902c0cdc409326481398bfedc8"}},{"id":"tts:wescon","slug":"wescon","name":"WeSCon","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"WeSCon creates training examples with local changes in emotion and speaking speed from a pretrained zero-shot TTS model. Transition smoothing and dynamic rate control help the adapted synthesizer deliver different parts of one sentence with distinct expression.","architecture":"Word-level emotion and rate control through self-trained synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-29","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.24629","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#wescon"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Word-Level Emotional Expression Control in Zero-Shot Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2509.24629","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"WeSCon author demo","url":"https://wangtianrui.github.io/wescon/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--wescon.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2509.24629","originUrl":"https://arxiv.org/html/2509.24629v2/overview2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/wescon.png","sha256":"7cecc95540a5cfe1381f4b3b7066c46ee0342c95b2ece79a4588a9fb749e212b"}},{"id":"tts:uniflow-audio","slug":"uniflow-audio","name":"UniFlow-Audio","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"UniFlow-Audio separates conditions that align to audio time from global descriptive information. Temporal fusion and cross-attention combine text, audio or visual cues in a shared generator that supports speech as well as broader audio tasks.","architecture":"Unified audio flow matching with temporal and non-temporal condition fusion","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","A","I","V"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":"2025-09-29","dateKind":"paper","links":{"code":"https://github.com/wsntxxn/UniFlow-Audio","weights":null,"paper":"https://arxiv.org/abs/2509.24391","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#uniflow-audio"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniFlow-Audio: Unified Flow Matching for Audio Generation from Omni-Modalities","url":"https://arxiv.org/abs/2509.24391","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"UniFlow-Audio author repository","url":"https://github.com/wsntxxn/UniFlow-Audio","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"UniFlow-Audio author demo","url":"https://wsntxxn.github.io/uniflow_audio","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--uniflow-audio.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2509.24391","originUrl":"https://arxiv.org/html/2509.24391v1/uniflow_audio-uncolored.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/uniflow-audio.png","sha256":"9860960aac96d2de0202dbc2578055a6443a20a02fe53cb0d44ca9554f34d3c6"}},{"id":"tts:cat-tts","slug":"cat-tts","name":"CaT-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"CaT-TTS separates textual understanding from acoustic generation in a two-Transformer architecture. During decoding, a masked parallel inference procedure guides speech-token predictions to reduce local errors in zero-shot voice synthesis.","architecture":"Dual-Transformer comprehension and speech rendering","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.22062","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#cat-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Comprehend and Talk: Text to Speech Synthesis via Dual Language Modeling","url":"https://arxiv.org/abs/2509.22062","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--cat-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2509.22062","originUrl":"https://arxiv.org/html/2509.22062v1/RQ_Transformer_Overview.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cat-tts.png","sha256":"c75b1ff7cc2395639a3134d2c4f32c095639bcaf31ce21c7100a55bbe85d20cd"}},{"id":"tts:frame-stacked-local-transformer-tts","slug":"frame-stacked-local-transformer-tts","name":"Frame-stacked local Transformer TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This architecture lets a global language model predict several speech frames at a time and delegates their codec entries to a smaller local Transformer. The paper compares sequential local decoding with iterative masked prediction, showing how frame stacking changes the trade-off between synthesis throughput and acoustic fidelity.","architecture":"Hierarchical multi-codebook generation with stacked frames","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-23","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.19592","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#frame-stacked-local-transformer-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Frame-Stacked Local Transformers For Efficient Multi-Codebook Speech Generation","url":"https://arxiv.org/abs/2509.19592","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--frame-stacked-local-transformer-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.19592","originUrl":"https://arxiv.org/html/2509.19592v2/lt-diagram.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/frame-stacked-local-transformer-tts.png","sha256":"c7161a84ed95c95140df71d90f00581ac0c05d6e0540f82c39734f11694be76f"}},{"id":"tts:hd-ppt","slug":"hd-ppt","name":"HD-PPT","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"HD-PPT learns speech codes that distinguish spoken content from instruction-related preferences. A language model predicts semantic information, expressive style and acoustic detail in sequence, improving the mapping from natural-language requests to controllable speech.","architecture":"Hierarchical prediction of content, style and acoustic tokens","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-23","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.19001","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#hd-ppt"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"HD-PPT: Hierarchical Decoding of Content- and Prompt-Preference Tokens for Instruction-based TTS","url":"https://arxiv.org/abs/2509.19001","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--hd-ppt.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.19001","originUrl":"https://arxiv.org/html/2509.19001v2/mot.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/hd-ppt.png","sha256":"7020705b0a458c5a73e4368396333a753bd2611ec0900911d8142a6dfc1fa3e3"}},{"id":"tts:tmd-tts","slug":"tmd-tts","name":"TMD-TTS (formerly FMSD-TTS)","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"TMD-TTS combines dialect labels with a routing network that models differences among three Tibetan dialects. The framework synthesizes parallel dialectal recordings from text, supporting both multi-dialect TTS and downstream speech-conversion research. The earlier FMSD-TTS preprint was withdrawn in favor of the restructured TMD-TTS paper.","architecture":"Dialect-conditioned synthesis with specialized dynamic routing","notes":"The FMSD-TTS submission was withdrawn on 2026-04-24 in favor of arXiv:2509.18060. Both paper links are retained to document the renamed family; the architecture image comes from TMD-TTS.","variants":["FMSD-TTS (withdrawn predecessor)","TMD-TTS (restructured paper)"],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-20","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.18060","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#tmd-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"TMD-TTS: A Unified Tibetan Multi-Dialect Text-to-Speech Framework for Ü-Tsang, Amdo and Kham Speech Dataset Generation","url":"https://arxiv.org/abs/2509.18060","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"paper","title":"FMSD-TTS: Few-shot Multi-Speaker Multi-Dialect Text-to-Speech Synthesis for Ü-Tsang, Amdo and Kham Speech Dataset Generation [withdrawn; see card notes]","url":"https://arxiv.org/abs/2505.14351","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--tmd-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.18060","originUrl":"https://arxiv.org/html/2509.18060v2/figures/tmd-tts.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tmd-tts.png","sha256":"6ca905d2894093aae97974fe1ccb141ba9f9d1c57c1c6a5d115170bd9cebcb39"}},{"id":"tts:audiobook-cc","slug":"audiobook-cc","name":"Audiobook-CC","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Audiobook-CC models context beyond individual sentences and separates style instructions from voice prompts. Distillation strengthens emotional expression, supporting multi-character narration with more consistent voices and performance across longer passages.","architecture":"Long-context speech generation with disentangled expressive conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.17516","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#audiobook-cc"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Audiobook-CC: Controllable Long-context Speech Generation for Multicast Audiobook","url":"https://arxiv.org/abs/2509.17516","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--audiobook-cc.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.17516","originUrl":"https://arxiv.org/html/2509.17516v1/pipeline_v7.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/audiobook-cc.png","sha256":"324f2a424ca35dd8d3dca6d43e94e24f99c6e79bb9002425ca6fe3f412b0833e"}},{"id":"tts:voxtream","slug":"voxtream","name":"VoXtream","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"VoXtream begins synthesis from the first available words using a streaming phoneme encoder. A temporal model predicts semantic and duration information, while a depth Transformer generates codec detail, enabling low-delay reference voice cloning.","architecture":"Incremental phoneme, temporal and depth Transformers","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-09-19","dateKind":"paper","links":{"code":"https://github.com/herimor/voxtream","weights":null,"paper":"https://arxiv.org/abs/2509.15969","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#voxtream"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoXtream: Full-Stream Text-to-Speech with Extremely Low Latency","url":"https://arxiv.org/abs/2509.15969","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"VoXtream author repository","url":"https://github.com/herimor/voxtream","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"VoXtream author demo","url":"https://herimor.github.io/voxtream","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--voxtream.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.15969","originUrl":"https://arxiv.org/html/2509.15969v2/img/architecture_v4.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voxtream.png","sha256":"1885cd5debbfd55fe6e1fa41a5082597ab3e8075a8a72c986368c6c37fe58e96"}},{"id":"tts:deepdubbing","slug":"deepdubbing","name":"DeepDubbing","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DeepDubbing assigns voices to characters and conditions speech rendering on the surrounding script. Its voice-design and instruction-synthesis stages support multi-participant audiobook production while maintaining character identity and context-sensitive expression.","architecture":"Text-to-timbre design with context-aware instruction TTS","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-19","dateKind":"paper","links":{"code":"https://github.com/TME-Lyra-Lab/DeepDubbing","weights":null,"paper":"https://arxiv.org/abs/2509.15845","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#deepdubbing"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Deep Dubbing: End-to-End Auto-Audiobook System with Text-to-Timbre and Context-Aware Instruct-TTS","url":"https://arxiv.org/abs/2509.15845","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DeepDubbing author repository","url":"https://github.com/TME-Lyra-Lab/DeepDubbing","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--deepdubbing.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.15845","originUrl":"https://arxiv.org/html/2509.15845v1/tsne_gender_by_age_subplots.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/deepdubbing.png","sha256":"3f96d71d250d7db6f4bdb1b64c8231797db29a806ec368467f799613c893594f"}},{"id":"tts:bvs","slug":"bvs","name":"BVS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Beyond Video-to-SFX predicts audio semantic tokens from visual information and phonetic cues, then refines them into acoustic tokens. The two-stage generator produces intelligible speech whose timing and environmental sound fit the supplied video.","architecture":"Video-guided semantic and acoustic speech generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","V"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":"2025-09-19","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.15492","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#bvs"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Beyond Video-to-SFX: Video to Audio Synthesis with Environmentally Aware Speech","url":"https://arxiv.org/abs/2509.15492","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--bvs.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.15492","originUrl":"https://arxiv.org/html/2509.15492v1/surface.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/bvs.png","sha256":"998748758e3885c93f066608dd53967ba986ad9d54af77a476de7792292a8828"}},{"id":"tts:mela-tts","slug":"mela-tts","name":"MELA-TTS","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"MELA-TTS predicts continuous mel-spectrogram frames from text and speaker conditions. A training-time alignment module connects the decoder to recognition-derived semantic features, helping the joint Transformer-diffusion model retain linguistic structure without discrete speech tokenization.","architecture":"Autoregressive mel generation with diffusion and semantic alignment","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-18","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.14784","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#mela-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MELA-TTS: Joint transformer-diffusion model with representation alignment for speech synthesis","url":"https://arxiv.org/abs/2509.14784","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mela-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.14784","originUrl":"https://arxiv.org/html/2509.14784v2/llm_diffusion.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mela-tts.png","sha256":"4617d146fb5d3b9851decd8025b5a9fb0fc1abee11bd6d8989443a0ed69617e3"}},{"id":"tts:cross-lingual-f5-tts","slug":"cross-lingual-f5-tts","name":"Cross-Lingual F5-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Cross-Lingual F5-TTS changes reference preparation and training so the generated text need not be paired with a reference transcript. Word-aligned acoustic prompts support cross-language voice cloning while reusing the flow-matching synthesis backbone.","architecture":"Transcript-independent reference conditioning in F5-TTS","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-18","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.14579","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#cross-lingual-f5-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Cross-Lingual F5-TTS: Towards Language-Agnostic Voice Cloning and Speech Synthesis","url":"https://arxiv.org/abs/2509.14579","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"Cross-Lingual F5-TTS author demo","url":"https://qingyuliu0521.github.io/Cross_lingual-F5-TTS/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--cross-lingual-f5-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.14579","originUrl":"https://arxiv.org/html/2509.14579v4/tts.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cross-lingual-f5-tts.png","sha256":"7939b95418f51e530a4bf2a699dd493472bc46ce5ee640dfeb9647b98b6d795c"}},{"id":"tts:diflow-tts","slug":"diflow-tts","name":"DiFlow-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DiFlow-TTS maps phonemes into linguistic content and generates separate prosody and acoustic token streams through discrete flow matching. Factorizing these responsibilities supports compact zero-shot synthesis with fewer sequential generation steps.","architecture":"Deterministic content mapping with factorized discrete-flow denoising","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-11","dateKind":"paper","links":{"code":"https://github.com/Fsoft-AIC/DiFlowTTS","weights":null,"paper":"https://arxiv.org/abs/2509.09631","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#diflow-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DiFlow-TTS: Compact and Low-Latency Zero-Shot Text-to-Speech with Discrete Flow Matching","url":"https://arxiv.org/abs/2509.09631","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DiFlow-TTS author repository","url":"https://github.com/Fsoft-AIC/DiFlowTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--diflow-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2509.09631","originUrl":"https://arxiv.org/html/2509.09631v5/components.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/diflow-tts.png","sha256":"32bccf182456d43972a9470c94cf17076798b14696c4e6ef7ff0c8a53719b096"}},{"id":"tts:progressive-face-conditioned-tts","slug":"progressive-face-conditioned-tts","name":"Progressive face-conditioned TTS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"This face-conditioned synthesizer combines local facial regions into progressively broader visual representations. Joint visual and acoustic attribute learning and multiple photographs of each training speaker align the face representation with voice characteristics, conditioning speech generation on text and a face image.","architecture":"Multi-granularity facial conditioning for speech synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.07376","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#progressive-face-conditioned-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Progressive Facial Granularity Aggregation with Bilateral Attribute-based Enhancement for Face-to-Speech Synthesis","url":"https://arxiv.org/abs/2509.07376","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--progressive-face-conditioned-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.07376","originUrl":"https://arxiv.org/html/2509.07376v3/architecture-v4-revised.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/progressive-face-conditioned-tts.png","sha256":"6b258c153c84e7f7b46b604abf4886d480e3cb01eabeb4791188ef776bd9c1db"}},{"id":"tts:mfcig-css","slug":"mfcig-css","name":"MFCIG-CSS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MFCIG-CSS represents dialogue history through separate graphs of meaning and vocal expression. Fine-grained multimodal interactions condition the speech synthesizer, helping each scripted response fit the surrounding conversation.","architecture":"Semantic and prosodic interaction graphs for conversational TTS","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-07","dateKind":"paper","links":{"code":"https://github.com/AI-S2-Lab/MFCIG-CSS","weights":null,"paper":"https://arxiv.org/abs/2509.06074","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#mfcig-css"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Multimodal Fine-grained Context Interaction Graph Modeling for Conversational Speech Synthesis","url":"https://arxiv.org/abs/2509.06074","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MFCIG-CSS author repository","url":"https://github.com/AI-S2-Lab/MFCIG-CSS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--mfcig-css.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.06074","originUrl":"https://arxiv.org/html/2509.06074v1/fine-intera-css2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mfcig-css.png","sha256":"f0e1cbec600eae05b312cb610bda9d0ed6dbb5e73750a93eac3311289305ae67"}},{"id":"tts:latinx","slug":"latinx","name":"LatinX","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"LatinX uses staged text-to-audio training, voice-cloning adaptation and automatic preference alignment. The resulting multilingual Transformer renders text in the source speaker's voice, supporting the synthesis stage of cross-language speech translation.","architecture":"Multilingual decoder-only codec TTS with preference alignment","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.05863","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#latinx"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"LatinX: Aligning a Multilingual TTS Model with Direct Preference Optimization","url":"https://arxiv.org/abs/2509.05863","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"LatinX author demo","url":"https://seu-usuario.github.io/latinx-demo","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--latinx.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2509.05863","originUrl":"https://arxiv.org/html/2509.05863v1/dpo_training.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/latinx.png","sha256":"26add3a56b05deed685f638412df04f9ab18fbb7672e7a3a2d355c99548e5c80"}},{"id":"tts:mixedg2p-t5","slug":"mixedg2p-t5","name":"MixedG2P-T5","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MixedG2P-T5 learns acoustic units from speech and uses a language-model synthesis path for text containing mixed scripts. It reduces dependence on manually designed grapheme-to-phoneme rules while retaining accent and intonation information in the speech representation.","architecture":"Speech-derived discrete units with T5-based mixed-script synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-09-01","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2509.01391","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#mixedg2p-t5"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MixedG2P-T5: G2P-free Speech Synthesis for Mixed-script texts using Speech Self-Supervised Learning and Language Model","url":"https://arxiv.org/abs/2509.01391","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mixedg2p-t5.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2509.01391","originUrl":"https://arxiv.org/html/2509.01391v1/images/ronbun_3types.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mixedg2p-t5.png","sha256":"38a62355143c633554936bbcec2fd3257ad44253617f6c8e8d5d86d30391e850"}},{"id":"tts:clear","slug":"clear","name":"CLEAR","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"CLEAR models speech directly in a continuous latent space, avoiding discrete codec-token prediction. Its zero-shot generator combines reference-voice conditioning with incremental audio production, targeting a balance between naturalness and response latency.","architecture":"Streaming autoregression over continuous acoustic latents","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.19098","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#clear"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CLEAR: Continuous Latent Autoregressive Modeling for High-quality and Low-latency Speech Synthesis","url":"https://arxiv.org/abs/2508.19098","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--clear.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2508.19098","originUrl":"https://arxiv.org/html/2508.19098v1/carm_jj2-2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/clear.png","sha256":"07094801fedf9218ec7bb5d2c6d6865eb0e40186436a474219f31a7bbd50468e"}},{"id":"tts:avlm-expressive-speech","slug":"avlm-expressive-speech","name":"AVLM (expressive speech)","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This audio-visual language model adds full-face information to an expressive speech backbone. Training on emotion and dialogue tasks connects facial cues with spoken delivery, enabling speech generation that uses visual as well as acoustic conversational context.","architecture":"Facial-feature fusion into an expressive speech language model","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-22","dateKind":"paper","links":{"code":"https://github.com/steventan0110/AVLM","weights":null,"paper":"https://arxiv.org/abs/2508.16188","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#avlm-expressive-speech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Seeing is Believing: Emotion-Aware Audio-Visual Language Modeling for Expressive Speech Generation","url":"https://arxiv.org/abs/2508.16188","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"AVLM (expressive speech) author repository","url":"https://github.com/steventan0110/AVLM","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--avlm-expressive-speech.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.16188","originUrl":"https://arxiv.org/html/2508.16188v2/modality_fusion.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/avlm-expressive-speech.png","sha256":"656f9893034eee9e6d1292114019df021a5f6649295a0db43da07c10507dab8d"}},{"id":"tts:cam-tts","slug":"cam-tts","name":"CAM-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"CAM-TTS retains global narrative information and retrieves local details through an updatable memory block. Prefix attention combines those memories with preceding context, guiding sentence-level synthesis across longer paragraphs.","architecture":"Context-aware memory for long-form speech language modeling","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-20","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.14713","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#cam-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Long-Context Speech Synthesis with Context-Aware Memory","url":"https://arxiv.org/abs/2508.14713","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--cam-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.14713","originUrl":"https://arxiv.org/html/2508.14713v1/figure2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/cam-tts.png","sha256":"2298bb6ff00e9fac05c79379b95ba5504408082e0498446d02830dff9d2660f8"}},{"id":"tts:fnh-tts","slug":"fnh-tts","name":"FNH-TTS","collection":"tts","category":"tts:flow-vae","categoryName":"Flows & variational TTS","summary":"FNH-TTS routes linguistic and speaker information through several duration experts to model varied timing patterns. It combines this predictor with changes to waveform generation, aiming for robust end-to-end speech synthesis across voices and prosodic conditions.","architecture":"VITS with a mixture-of-experts duration predictor","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-16","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.12001","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/flow-vae.md#fnh-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FNH-TTS: Mixture-of-Experts Duration Modeling for Robust Neural Speech Synthesis","url":"https://arxiv.org/abs/2508.12001","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--fnh-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2508.12001","originUrl":"https://arxiv.org/html/2508.12001v4/Overall.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fnh-tts.png","sha256":"cf4eaa0d2ddb920ec354d402ebbbff93384d216481a03d906aaa9fb5acd178ee"}},{"id":"tts:moe-tts","slug":"moe-tts","name":"MoE-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MoE-TTS augments a frozen text language model with speech-specific expert parameters. Retaining the original language knowledge helps the synthesizer interpret unfamiliar style descriptions while learning the acoustic generation task.","architecture":"Description-conditioned speech generation with modality-specific experts","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-15","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.11326","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#moe-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MoE-TTS: Enhancing Out-of-Domain Text Understanding for Description-based TTS via Mixture-of-Experts","url":"https://arxiv.org/abs/2508.11326","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--moe-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2508.11326","originUrl":"https://arxiv.org/html/2508.11326v1/moe-tts-framework.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/moe-tts.png","sha256":"5c85616a45c811328cf5ae94f40115e1f7e7d570a3b6cbbb46402d9db1e5e114"}},{"id":"tts:emosslsphere","slug":"emosslsphere","name":"EmoSSLSphere","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"EmoSSLSphere combines an emotion representation constrained to a sphere with discrete units derived from self-supervised speech features. The model synthesizes emotional speech across languages while organizing expressive controls independently of the textual content.","architecture":"Spherical emotion vectors and self-supervised discrete speech tokens","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-15","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.11273","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#emosslsphere"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"EmoSSLSphere: Multilingual Emotional Speech Synthesis with Spherical Vectors and Discrete Speech Tokens","url":"https://arxiv.org/abs/2508.11273","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--emosslsphere.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.11273","originUrl":"https://arxiv.org/html/2508.11273v2/training_inference.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emosslsphere.png","sha256":"ae2f5a9b5fb1eb69db8ec9e758e25f19cf42bad8e8ace44e4203d93e678d3d44"}},{"id":"tts:dualspeechlm","slug":"dualspeechlm","name":"DualSpeechLM","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DualSpeechLM uses understanding-oriented speech tokens as input and acoustic codec tokens for generation. Semantic supervision and staged conditioning coordinate the two representations, supporting speech synthesis within a unified understanding-and-generation architecture.","architecture":"Different input and output speech representations in one language model","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.08961","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#dualspeechlm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DualSpeechLM: Towards Unified Speech Understanding and Generation via Dual Speech Token Modeling with Large Language Models","url":"https://arxiv.org/abs/2508.08961","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--dualspeechlm.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2508.08961","originUrl":"https://arxiv.org/html/2508.08961v3/mllm.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dualspeechlm.png","sha256":"3b5e5c59a73c8a5347f2f0f556ed0d313edb857f7baae0dc9004f9c7241f23da"}},{"id":"tts:xemorag","slug":"xemorag","name":"XEmoRAG","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"XEmoRAG retrieves speech examples to guide emotion transfer from Chinese to Thai without parallel emotional recordings. Its language-model TTS path exposes intensity control while balancing the reference speaker's timbre with target-language pronunciation.","architecture":"Retrieved emotional references for cross-language speech synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.07302","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#xemorag"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"XEmoRAG: Cross-Lingual Emotion Transfer with Controllable Intensity Using Retrieval-Augmented Generation","url":"https://arxiv.org/abs/2508.07302","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--xemorag.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2508.07302","originUrl":"https://arxiv.org/html/2508.07302v2/overview-0605.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/xemorag.png","sha256":"7c7724dd10c20acbbc247fef6be00cd64b1aee4c4f05960c426944778150281d"}},{"id":"tts:meitei-mayek-tts","slug":"meitei-mayek-tts","name":"Meitei Mayek TTS","collection":"tts","category":"tts:autoregressive","categoryName":"Autoregressive acoustic","summary":"This Manipuri speech synthesizer maps Meitei Mayek writing to an ARPAbet-based phoneme representation before acoustic generation with Tacotron 2. A HiFi-GAN vocoder reconstructs the waveform. The paper develops a single-speaker system for a language with limited training resources and tonal pronunciation requirements.","architecture":"Tacotron 2 with a Meitei Mayek phoneme frontend and HiFi-GAN","notes":"The paper illustrates the Tacotron 2 reference architecture with a WaveNet block; its reported Meitei Mayek implementation uses HiFi-GAN for waveform reconstruction.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.06870","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/autoregressive.md#meitei-mayek-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Text to Speech System for Meitei Mayek Script","url":"https://arxiv.org/abs/2508.06870","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--meitei-mayek-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2508.06870","originUrl":"https://arxiv.org/html/2508.06870v1/architecture.PNG","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/meitei-mayek-tts.png","sha256":"56c2427e90de8c68f5f8d3c052993157a0e09ceb4e6aa79f97eb8032612ab52a"}},{"id":"tts:unitalker","slug":"unitalker","name":"UniTalker","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"UniTalker predicts emotional context before generating speech and face-motion tokens. A facial codec and synchronized decoding coordinate expression, wording and duration, allowing a scripted conversational response to be rendered as both voice and facial animation.","architecture":"Joint conversational speech and facial-token generation","notes":"The author-linked GitHub repository has no README available in the reviewed main or master branches.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-06","dateKind":"paper","links":{"code":"https://github.com/AI-S2-Lab/UniTalker","weights":null,"paper":"https://arxiv.org/abs/2508.04585","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#unitalker"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniTalker: Conversational Speech-Visual Synthesis","url":"https://arxiv.org/abs/2508.04585","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"UniTalker author repository","url":"https://github.com/AI-S2-Lab/UniTalker","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--unitalker.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.04585","originUrl":"https://arxiv.org/html/2508.04585v2/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/unitalker.png","sha256":"0585e593dc0c02b491aad35cbb6c68a9809bcd9194e1d5bddd792cf623cbc96c"}},{"id":"tts:nvspeech-tts","slug":"nvspeech-tts","name":"NVSpeech TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"The NVSpeech pipeline includes a TTS model that renders text with explicitly marked nonverbal events. Word-level annotations connect ordinary speech with vocalizations such as laughter, providing a shared representation for recognition and controllable audio generation.","architecture":"Inline paralinguistic-token conditioning for controllable synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.04195","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#nvspeech-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"NVSpeech: An Integrated and Scalable Pipeline for Human-Like Speech Modeling with Paralinguistic Vocalizations","url":"https://arxiv.org/abs/2508.04195","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"NVSpeech TTS author demo","url":"https://nvspeech170k.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--nvspeech-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2508.04195","originUrl":"https://arxiv.org/html/2508.04195v1/asr-pipeline.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/nvspeech-tts.png","sha256":"9608b4f95beba672597b0a7710e461fe7b9f9e535f8e7efb4a6dd66560fee69d"}},{"id":"tts:parallel-gpt","slug":"parallel-gpt","name":"Parallel GPT","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Parallel GPT divides speech generation between a general autoregressive predictor and a non-autoregressive detail model. The parallel refinement stage conditions on the initial tokens, balancing independence and interaction between semantic and acoustic information.","architecture":"Autoregressive general tokens with parallel coupled acoustic refinement","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.04141","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#parallel-gpt"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Parallel GPT: Harmonizing the Independence and Interdependence of Acoustic and Semantic Information for Zero-Shot Text-to-Speech","url":"https://arxiv.org/abs/2508.04141","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--parallel-gpt.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2508.04141","originUrl":"https://arxiv.org/html/2508.04141v2/parallel-gpt_3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/parallel-gpt.png","sha256":"a9c35aa062ccac31a7a768475e4f96939e0fe9119443548fcb69194a6a558a7d"}},{"id":"tts:emosteer-tts","slug":"emosteer-tts","name":"EmoSteer-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"EmoSteer-TTS extracts emotion-related directions from a pretrained synthesizer's internal activations. Applying those directions during inference changes, mixes or removes emotional expression without retraining; the paper evaluates the method across several different TTS backbones.","architecture":"Inference-time emotional activation steering for pretrained TTS","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-05","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2508.03543","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#emosteer-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"EmoSteer-TTS: Fine-Grained and Training-Free Emotion-Controllable Text-to-Speech via Activation Steering","url":"https://arxiv.org/abs/2508.03543","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--emosteer-tts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2508.03543","originUrl":"https://arxiv.org/html/2508.03543v3/figure-framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emosteer-tts.png","sha256":"b97133b19f8d0630be4370973eae5260d507b57b1eb4e736781b5b6c728bf558"}},{"id":"tts:marco-voice","slug":"marco-voice","name":"Marco-Voice","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Marco-Voice learns separate speaker and emotion representations using contrastive training. Rotating the emotional representation provides smooth expressive control while preserving the reference voice across different delivery styles.","architecture":"Contrastively disentangled speaker and emotion speech generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-08-04","dateKind":"paper","links":{"code":"https://github.com/AIDC-AI/Marco-Voice","weights":null,"paper":"https://arxiv.org/abs/2508.02038","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#marco-voice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Marco-Voice Technical Report","url":"https://arxiv.org/abs/2508.02038","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Marco-Voice author repository","url":"https://github.com/AIDC-AI/Marco-Voice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--marco-voice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2508.02038","originUrl":"https://arxiv.org/html/2508.02038v6/marco_vocie_fig_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/marco-voice.png","sha256":"6bdb98ecd5acb8d2c2c907379c802ff38eba208f13ae80249bd73640e4c1d31e"}},{"id":"tts:dragon-fm","slug":"dragon-fm","name":"Dragon-FM","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Dragon-FM predicts successive speech chunks autoregressively while refining the tokens inside each chunk with bidirectional flow matching. Compact acoustic codes and cross-chunk caching reduce generation overhead and support longer content such as podcasts.","architecture":"Chunk-autoregressive flow matching over compact codec representations","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-30","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2507.22746","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#dragon-fm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Next Tokens Denoising for Speech Synthesis","url":"https://arxiv.org/abs/2507.22746","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"Dragon-FM author demo","url":"https://cognitivespeech.github.io/dragon-fm","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--dragon-fm.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.22746","originUrl":"https://arxiv.org/html/2507.22746v2/model2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dragon-fm.png","sha256":"8a4692d88354ac4783fde6026fe77d1515c6a56150afe42073618fd01a69fc35"}},{"id":"tts:inworld-tts-1","slug":"inworld-tts-1","name":"Inworld TTS-1","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Inworld TTS-1 and TTS-1-Max are multilingual autoregressive synthesizers designed for low-latency speech output. Textual audio markup controls emotions and nonverbal vocalizations, with model variants offering different capacity and inference-cost trade-offs.","architecture":"Autoregressive Transformer speech synthesis with expressive markup","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-22","dateKind":"paper","links":{"code":"https://github.com/inworld-ai/tts","weights":null,"paper":"https://arxiv.org/abs/2507.21138","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#inworld-tts-1"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"TTS-1 Technical Report","url":"https://arxiv.org/abs/2507.21138","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Inworld TTS-1 author repository","url":"https://github.com/inworld-ai/tts","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--inworld-tts-1.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.21138","originUrl":"https://arxiv.org/html/2507.21138v1/graphics/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/inworld-tts-1.png","sha256":"06fcb6fff7c99fe28220905b5f47d7e5745edd99618d075c01577b5cfa147085"}},{"id":"tts:a2tts","slug":"a2tts","name":"A2TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"A2TTS extracts a voice embedding from a short recording and conditions a diffusion acoustic decoder on it. Reference-aware duration prediction improves timing consistency for multilingual synthesis in low-resource Indian languages.","architecture":"Speaker-conditioned diffusion with reference-aware duration attention","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-21","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2507.15272","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#a2tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"A2TTS: TTS for Low Resource Indian Languages","url":"https://arxiv.org/abs/2507.15272","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--a2tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.15272","originUrl":"https://arxiv.org/html/2507.15272v1/tts_architecture.drawio_4.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/a2tts.png","sha256":"abab277c698e496ff7fa55b0b1646fee0b6c0e4b5f2a4be5ce46268f98db8746"}},{"id":"tts:dmospeech-2","slug":"dmospeech-2","name":"DMOSpeech 2","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"DMOSpeech 2 adds reinforcement learning to duration prediction in an already metric-optimized synthesizer. Rewards derived from speaker similarity and transcription accuracy guide timing choices, linking prosodic planning with reference-voice and content objectives.","architecture":"Metric-optimized acoustic generation with a learned duration policy","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-20","dateKind":"paper","links":{"code":"https://github.com/yl4579/DMOSpeech2","weights":null,"paper":"https://arxiv.org/abs/2507.14988","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#dmospeech-2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DMOSpeech 2: Reinforcement Learning for Duration Prediction in Metric-Optimized Speech Synthesis","url":"https://arxiv.org/abs/2507.14988","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DMOSpeech 2 author repository","url":"https://github.com/yl4579/DMOSpeech2","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"DMOSpeech 2 author demo","url":"https://dmospeech2.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--dmospeech-2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.14988","originUrl":"https://arxiv.org/html/2507.14988v1/dmospeech2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dmospeech-2.png","sha256":"258b2a75b6f25a001ecebcd121b2c85937185f3a1516178d3cefc97c9b66f5f2"}},{"id":"tts:qtts","slug":"qtts","name":"QTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"QTTS predicts residual speech codes produced by its QDAC tokenizer. Hierarchical parallel and delayed multihead variants organize codebook dependencies differently, offering alternative balances between acoustic detail and sequential decoding cost.","architecture":"Hierarchical or delayed multi-codebook autoregressive synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-16","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2507.12197","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#qtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Quantize More, Lose Less: Autoregressive Generation from Residually Quantized Speech Representations","url":"https://arxiv.org/abs/2507.12197","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--qtts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2507.12197","originUrl":"https://arxiv.org/html/2507.12197v1/graph2.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/qtts.png","sha256":"7b62ede6c6c376909247ab5a956b26f17345026b9c4d66ef1715509bbec96a12"}},{"id":"tts:eme-tts","slug":"eme-tts","name":"EME-TTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"EME-TTS jointly models emotional delivery and local emphasis instead of treating them as independent effects. Automatically derived emphasis labels and variance-related features help users stress selected material while retaining a recognizable target emotion.","architecture":"Emotion-aware emphasis modeling with weakly supervised controls","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-16","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2507.12015","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#eme-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"EME-TTS: Unlocking the Emphasis and Emotion Link in Speech Synthesis","url":"https://arxiv.org/abs/2507.12015","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--eme-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.12015","originUrl":"https://arxiv.org/html/2507.12015v1/aaa.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/eme-tts.png","sha256":"130c1ce8809f457cd6df0de6b434220940d2cd3a311caf9170e0edd438f8b01e"}},{"id":"tts:dualdub","slug":"dualdub","name":"DualDub","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DualDub generates spoken dialogue and background sound together from video and textual conditions. A cross-modal aligner coordinates the two decoding heads, targeting temporally synchronized soundtracks rather than speech rendered in isolation.","architecture":"Multimodal alignment with separate speech and background-audio heads","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2507.10109","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#dualdub"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DualDub: Video-to-Soundtrack Generation via Joint Speech and Background Audio Synthesis","url":"https://arxiv.org/abs/2507.10109","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--dualdub.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2507.10109","originUrl":"https://arxiv.org/html/2507.10109v1/DualDub_0414.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dualdub.png","sha256":"d675dc5b3634c7e941cd2c2910ad42c73d08c3e2433b5ac3d3736b27ad190414"}},{"id":"tts:zipvoice-dialog","slug":"zipvoice-dialog","name":"ZipVoice-Dialog","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"ZipVoice-Dialog extends flow-based synthesis to scripted conversations between reference speakers. Curriculum training stabilizes text alignment, and speaker-turn embeddings control who speaks when; dedicated variants also generate separate stereo dialogue channels.","architecture":"Non-autoregressive dialogue flow matching with speaker-turn embeddings","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-12","dateKind":"paper","links":{"code":"https://github.com/k2-fsa/ZipVoice","weights":null,"paper":"https://arxiv.org/abs/2507.09318","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#zipvoice-dialog"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ZipVoice-Dialog: Non-Autoregressive Spoken Dialogue Generation with Flow Matching","url":"https://arxiv.org/abs/2507.09318","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"ZipVoice-Dialog author repository","url":"https://github.com/k2-fsa/ZipVoice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--zipvoice-dialog.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.09318","originUrl":"https://arxiv.org/html/2507.09318v2/training.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/zipvoice-dialog.png","sha256":"325763fc458ff2b2e3fd58d263968d5d643eae12c1c2362a42f0c9d6dfc7a470"}},{"id":"tts:tts-ctrlnet","slug":"tts-ctrlnet","name":"TTS-CtrlNet","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"TTS-CtrlNet adds a trainable control branch to a pretrained TTS model while preserving its original generation backbone. Time-varying emotion signals guide local delivery, combining explicit expressive control with the base model's voice-cloning capability.","architecture":"Trainable ControlNet conditioning over a frozen flow-matching synthesizer","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2507.04349","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#tts-ctrlnet"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"TTS-CtrlNet: Time varying emotion aligned text-to-speech generation with ControlNet","url":"https://arxiv.org/abs/2507.04349","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"TTS-CtrlNet author demo","url":"https://curryjung.github.io/ttsctrlnet_project_page","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--tts-ctrlnet.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2507.04349","originUrl":"https://arxiv.org/html/2507.04349v1/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tts-ctrlnet.png","sha256":"1bccef7b95fb1c196980bb2cbe635923cbf65862019ed098e0718a51a9402c0b"}},{"id":"tts:speechaccentllm","slug":"speechaccentllm","name":"SpeechAccentLLM","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"SpeechAccentLLM uses a content tokenizer trained with transcription alignment and jointly learns accent conversion and synthesis. A reconstruction refinement stage improves generated speech while separating accent-related changes from the target speaker's identity.","architecture":"Shared accent conversion and TTS over CTC-aligned speech codes","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-07-02","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2507.01348","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#speechaccentllm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SpeechAccentLLM: A Unified Framework for Foreign Accent Conversion and Text to Speech","url":"https://arxiv.org/abs/2507.01348","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--speechaccentllm.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2507.01348","originUrl":"https://arxiv.org/html/2507.01348v2/FACTTS-eps-converted-to.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/speechaccentllm.png","sha256":"ddc0fbb58deb9c43d8ba39d3bd1b1f3f1423c010bc7ba6e4af7af1bebf7742e3"}},{"id":"tts:tinywave","slug":"tinywave","name":"TinyWave","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"TinyWave is a family of compact models trained for mixed text-speech continuation and speech generation from phonetic or expressive tokens. Distillation transfers capabilities into smaller backbones, supporting controllable audio rendering alongside broader speech-language tasks.","architecture":"Distilled interleaved speech-text language modeling","notes":"The paper links the TinyWave repository, but no README was available in the reviewed main or master branches.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-30","dateKind":"paper","links":{"code":"https://github.com/mohammadmahdinoori/TinyWave","weights":null,"paper":"https://arxiv.org/abs/2506.23670","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#tinywave"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Efficient Interleaved Speech Modeling through Knowledge Distillation","url":"https://arxiv.org/abs/2506.23670","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"TinyWave author repository","url":"https://github.com/mohammadmahdinoori/TinyWave","reviewed_on":"2026-09-13","review_level":"project"},{"kind":"project","title":"TinyWave author demo","url":"https://mohammadmahdinoori.github.io/tinywave-landing/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--tinywave.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.23670","originUrl":"https://arxiv.org/html/2506.23670v2/distillation.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tinywave.png","sha256":"bcebd5fbebe94946719cd9fa3081dc92b13d599f0a0a1d15e198a28396803c30"}},{"id":"tts:jam-flow","slug":"jam-flow","name":"JAM-Flow","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"JAM-Flow couples audio and motion diffusion modules within a shared model. Text, voice references and optional motion conditions support synchronized speech and facial movement, with an infilling objective allowing several conditioning combinations.","architecture":"Joint audio and facial-motion generation through multimodal flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-30","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.23552","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#jam-flow"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"JAM-Flow: Joint Audio-Motion Synthesis with Flow Matching","url":"https://arxiv.org/abs/2506.23552","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--jam-flow.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2506.23552","originUrl":"https://arxiv.org/html/2506.23552v2/archi.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/jam-flow.png","sha256":"ba0efed115818ac9bd4ec0c57617b95554a1dd8b76f78e094c74c21deb517121"}},{"id":"tts:dcar","slug":"dcar","name":"DCAR","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DCAR changes the number of acoustic tokens predicted at each autoregressive step. Adapting the chunk size to the generation state shortens sequential processing while maintaining content alignment and reference-conditioned speech quality.","architecture":"Dynamic chunk-wise autoregressive speech-token prediction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-27","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.22023","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#dcar"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Robust and Efficient Autoregressive Speech Synthesis with Dynamic Chunk-wise Prediction Policy","url":"https://arxiv.org/abs/2506.22023","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--dcar.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.22023","originUrl":"https://arxiv.org/html/2506.22023v1/DCAR_fig1.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/dcar.png","sha256":"e1d84f1accf26a74d20c042589f277b485b52b6471c4eefe92ee31ddee66a033"}},{"id":"tts:multistage-multimodal-tts","slug":"multistage-multimodal-tts","name":"Multistage multimodal TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"This framework learns face and text conditioning in separate stages before using them for voice synthesis. Visual knowledge distillation and training across text-face and text-speech pairs reduce reliance on fully matched multimodal recordings.","architecture":"Three-stage face and text style alignment for controllable synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.20945","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#multistage-multimodal-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"A Multi-Stage Framework for Multimodal Controllable Speech Synthesis","url":"https://arxiv.org/abs/2506.20945","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--multistage-multimodal-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2506.20945","originUrl":"https://arxiv.org/html/2506.20945v1/f1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/multistage-multimodal-tts.png","sha256":"37d8e9e88d0bff89b54a0c6d765397f0d2dbf03fcddd84b63cf48de4236b42c7"}},{"id":"tts:opuslm","slug":"opuslm","name":"OpusLM","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"OpusLM extends text language models through speech-text pretraining on public data. Its interleaved representation supports speech recognition, text-conditioned synthesis and textual continuation within a transparent family of shared backbones.","architecture":"Decoder-only models with interleaved text and multi-stream speech codes","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-21","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.17611","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#opuslm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"OpusLM: A Family of Open Unified Speech Language Models","url":"https://arxiv.org/abs/2506.17611","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--opuslm.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.17611","originUrl":"https://arxiv.org/html/2506.17611v1/delay.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/opuslm.png","sha256":"4cb495c6d9eb6bc6010807af4dc2310cd1ad3b2a3a21b3aadb7b172294d86c39"}},{"id":"tts:rapflow-tts","slug":"rapflow-tts","name":"RapFlow-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"RapFlow-TTS regularizes the acoustic velocity field so longer generation steps remain consistent. Time-interval scheduling and adversarial objectives improve the resulting few-step synthesizer, reducing the iterations needed to render reference-conditioned speech.","architecture":"Few-step acoustic flow matching with velocity-consistency constraints","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-20","dateKind":"paper","links":{"code":"https://github.com/naver-ai/RapFlow-TTS","weights":null,"paper":"https://arxiv.org/abs/2506.16741","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#rapflow-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"RapFlow-TTS: Rapid and High-Fidelity Text-to-Speech with Improved Consistency Flow Matching","url":"https://arxiv.org/abs/2506.16741","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"RapFlow-TTS author repository","url":"https://github.com/naver-ai/RapFlow-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--rapflow-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.16741","originUrl":"https://arxiv.org/html/2506.16741v1/rap.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/rapflow-tts.png","sha256":"32e25210c049dc7d9671d65de77dc1b96b06a2eee47c32c142349f6978855984"}},{"id":"tts:v-cass","slug":"v-cass","name":"V-CASS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"V-CASS extracts paralinguistic cues from video with a vision-language model and passes them to an expressive speech generator. It renders supplied content in a style suited to the scene, supporting more informative audio descriptions of visual media.","architecture":"Visual-context interpretation conditioning expressive speech rendering","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-20","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.16716","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#v-cass"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"V-CASS: Vision-context-aware Expressive Speech Synthesis for Enhancing User Understanding of Videos","url":"https://arxiv.org/abs/2506.16716","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--v-cass.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2506.16716","originUrl":"https://arxiv.org/html/2506.16716v1/img/pipeline.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/v-cass.png","sha256":"4720d120f44d091729c84ee5b0f87cc6f21d2669958431111c75ce641fbeb289"}},{"id":"tts:parler-tts-hinglish-adaptation","slug":"parler-tts-hinglish-adaptation","name":"Parler-TTS Hinglish adaptation","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This Parler-TTS extension introduces language-specific phonetic alignment and emotion embeddings for Hindi and Indian English. Its conditioning targets code-switched utterances whose accent and emotional delivery change coherently across language boundaries.","architecture":"Accent-aware Parler-TTS with phoneme alignment and emotion conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-19","dateKind":"paper","links":{"code":"https://github.com/En1gma02/Parler-TTS-Hinglish-Accent-Emotions","weights":null,"paper":"https://arxiv.org/abs/2506.16310","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#parler-tts-hinglish-adaptation"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Optimizing Multilingual Text-To-Speech with Accents & Emotions","url":"https://arxiv.org/abs/2506.16310","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Parler-TTS Hinglish adaptation author repository","url":"https://github.com/En1gma02/Parler-TTS-Hinglish-Accent-Emotions","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--parler-tts-hinglish-adaptation.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2506.16310","originUrl":"https://arxiv.org/html/2506.16310v1/Media/Model/INF.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/parler-tts-hinglish-adaptation.png","sha256":"c3831250ea4855cbea6e637e7305b614888dd2f87c25061a2f868102a4bd0b0d"}},{"id":"tts:emojivoice","slug":"emojivoice","name":"EmojiVoice","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"EmojiVoice adds interpretable emoji prompts to the text encoder and flow predictor of Matcha-TTS. Changing prompts across phrases varies expression during longer robot utterances, providing a lightweight control interface for expressive synthesis.","architecture":"Emoji-conditioned Matcha-TTS acoustic generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-18","dateKind":"paper","links":{"code":"https://github.com/rosielab/emojivoice","weights":null,"paper":"https://arxiv.org/abs/2506.15085","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#emojivoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"EmojiVoice: Towards long-term controllable expressivity in robot speech","url":"https://arxiv.org/abs/2506.15085","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"EmojiVoice author repository","url":"https://github.com/rosielab/emojivoice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--emojivoice.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2506.15085","originUrl":"https://arxiv.org/html/2506.15085v2/Dessin_sans_titre-22.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emojivoice.png","sha256":"8371bf5da5cc9b731d03f11c4b7d689f16c14ffb1a1b915299ea3738a746f080"}},{"id":"tts:zipvoice","slug":"zipvoice","name":"ZipVoice","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"ZipVoice uses efficient flow-matching acoustic modeling to synthesize speech from text and a voice reference. Its compact architecture and accelerated variants target faster inference while preserving multilingual pronunciation and reference-speaker characteristics.","architecture":"Compact Zipformer-based zero-shot acoustic flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-16","dateKind":"paper","links":{"code":"https://github.com/k2-fsa/ZipVoice","weights":null,"paper":"https://arxiv.org/abs/2506.13053","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#zipvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"ZipVoice: Fast and High-Quality Zero-Shot Text-to-Speech with Flow Matching","url":"https://arxiv.org/abs/2506.13053","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"ZipVoice author repository","url":"https://github.com/k2-fsa/ZipVoice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--zipvoice.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2506.13053","originUrl":"https://arxiv.org/html/2506.13053v3/training.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/zipvoice.png","sha256":"b63e4691ef5031152ebaf7f5618bf398ddca515bf8df2d2beff9a5da8ed9c978"}},{"id":"tts:streammel","slug":"streammel","name":"StreamMel","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"StreamMel alternates text tokens with continuous acoustic frames in one streaming synthesis model. This organization lets incoming text guide speech immediately while retaining reference-voice information across the generated audio stream.","architecture":"Interleaved text and continuous mel-frame autoregression","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.12570","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#streammel"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"StreamMel: Real-Time Zero-shot Text-to-Speech via Interleaved Continuous Autoregressive Modeling","url":"https://arxiv.org/abs/2506.12570","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--streammel.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2506.12570","originUrl":"https://arxiv.org/html/2506.12570v1/HuiWa1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/streammel.png","sha256":"69694262b4df07e5326c0fcb2d8b5df1d2562fbaf0669a65466f522cd398ce77"}},{"id":"tts:umbratts","slug":"umbratts","name":"UmbraTTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"UmbraTTS generates spoken text together with a background acoustic scene using reference audio as context. Its conditioning includes background-volume control, allowing the same speech to be rendered in coherent environments with different prominence.","architecture":"Joint speech and environmental-audio flow matching","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","A"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-11","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.09874","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#umbratts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UmbraTTS: Adapting Text-to-Speech to Environmental Contexts with Flow Matching","url":"https://arxiv.org/abs/2506.09874","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"UmbraTTS author demo","url":"https://aiola-lab.github.io/umbra-tts/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--umbratts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.09874","originUrl":"https://arxiv.org/html/2506.09874v2/figures/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/umbratts.png","sha256":"9c54d6600899985077cbe0007d0f8b8b5c8635bcb0ea986657369307099fe8e1"}},{"id":"tts:bittts","slug":"bittts","name":"BitTTS","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"BitTTS reduces storage and computation through extremely low-bit trained weights and indexed parameter sharing. It targets speech generation on constrained devices, preserving a full synthesis path while shrinking the model representation.","architecture":"Quantization-aware compact TTS with shared weight indexing","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-04","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.03515","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#bittts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"BitTTS: Highly Compact Text-to-Speech Using 1.58-bit Quantization and Weight Indexing","url":"https://arxiv.org/abs/2506.03515","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--bittts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.03515","originUrl":"https://arxiv.org/html/2506.03515v1/bitconv_v15.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/bittts.png","sha256":"bf22d57a9f56af64b6bb86223e117d809f542141fce284b4c16302fa37fd86e7"}},{"id":"tts:masked-style-tts","slug":"masked-style-tts","name":"Masked-style TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This controllable system first predicts a masked-autoencoder-derived speech style representation from text and controls. A second model generates codec tokens, allowing speaker characteristics and expressive attributes to be specified separately during synthesis.","architecture":"Two-stage language modeling over style-rich and acoustic codes","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-03","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.02997","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#masked-style-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Controllable Text-to-Speech Synthesis with Masked-Autoencoded Style-Rich Representation","url":"https://arxiv.org/abs/2506.02997","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--masked-style-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2506.02997","originUrl":"https://arxiv.org/html/2506.02997v1/main.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/masked-style-tts.png","sha256":"2aaf88ba050172e36d6758f65478c637077ed5097ea07714838f702968587db1"}},{"id":"tts:prompt-unseen-emotion","slug":"prompt-unseen-emotion","name":"Prompt-Unseen-Emotion","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Prompt-Unseen-Emotion learns the relationship between emotion descriptions and speech using a language-model synthesis backbone. Weighted combinations of known emotions and contextual language knowledge allow expressive delivery outside the original categorical training labels.","architecture":"Prompt-conditioned language-model TTS with mixed-emotion weighting","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-03","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.02742","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#prompt-unseen-emotion"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Prompt-Unseen-Emotion: Zero-shot Expressive Speech Synthesis with Prompt-LLM Contextual Knowledge for Mixed Emotions","url":"https://arxiv.org/abs/2506.02742","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--prompt-unseen-emotion.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2506.02742","originUrl":"https://arxiv.org/html/2506.02742v1/inter2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/prompt-unseen-emotion.png","sha256":"71fc7fe4ffc8b355ed7a1ea35405b3b00b988dbfd4d0cecea9dd2412e8fd574a"}},{"id":"tts:ds-tts","slug":"ds-tts","name":"DS-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"DS-TTS extracts complementary voice characteristics through two style encoders. Dynamic modulation conditions the acoustic generator on these representations, supporting unseen speakers and adapting synthesis across different sentence lengths.","architecture":"Dual-style reference encoding with dynamic acoustic feature modulation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-01","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2506.01020","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#ds-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DS-TTS: Zero-Shot Speaker Style Adaptation from Voice Clips via Dynamic Dual-Style Feature Modulation","url":"https://arxiv.org/abs/2506.01020","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--ds-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2506.01020","originUrl":"https://arxiv.org/html/2506.01020v1/zongtijiagou.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ds-tts.png","sha256":"c014f82985264990ef043c87c90b37a573a2eb53d3227b4351296a526506403b"}},{"id":"tts:covomix2","slug":"covomix2","name":"CoVoMix2","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"CoVoMix2 generates scripted dialogue directly with a flow-matching model, using reference voices without their transcripts. Speaker-disentangled text, sentence alignment and prompt masking support controlled timing and overlapping turns.","architecture":"Fully non-autoregressive multi-speaker dialogue flow matching","notes":"The linked author repository contains CoVoMix2 data preparation scripts, rather than the full synthesis implementation.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-06-01","dateKind":"paper","links":{"code":"https://github.com/vivian556123/covomix2-dataprep","weights":null,"paper":"https://arxiv.org/abs/2506.00885","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#covomix2"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"CoVoMix2: Advancing Zero-Shot Dialogue Generation with Fully Non-Autoregressive Flow Matching","url":"https://arxiv.org/abs/2506.00885","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"CoVoMix2 author repository","url":"https://github.com/vivian556123/covomix2-dataprep","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--covomix2.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2506.00885","originUrl":"https://arxiv.org/html/2506.00885v2/covomix2-arxiv-pipeline.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/covomix2.png","sha256":"575cc9858aaeb5a0ea2b65b42931f3ead4ae7b72e596bbed1c15309c7ec56693"}},{"id":"tts:compressed-to-fine-speech-lm","slug":"compressed-to-fine-speech-lm","name":"Compressed-to-fine speech LM","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This speech-language-model design keeps recent acoustic tokens and voice prompts at full detail while compressing distant context. The asymmetric representation reduces redundant long-sequence processing without discarding the local cues needed for pronunciation and vocal consistency.","architecture":"Compressed distant context with detailed local codec-token prediction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-30","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.24496","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#compressed-to-fine-speech-lm"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Speech Token Prediction via Compressed-to-fine Language Modeling for Speech Generation","url":"https://arxiv.org/abs/2505.24496","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--compressed-to-fine-speech-lm.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.24496","originUrl":"https://arxiv.org/html/2505.24496v1/fig/arch/all_attn.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/compressed-to-fine-speech-lm.png","sha256":"1560fbce7d0b8c08a79a5a9355745da8ce9de950ff75bdbc40bc43756b927aeb"}},{"id":"tts:diemo-tts","slug":"diemo-tts","name":"DiEmo-TTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"DiEmo-TTS distills emotion information from speech while suppressing unrelated speaker characteristics. Cluster-based sampling and representation perturbation improve cross-speaker emotion transfer, including situations where extensive emotion labels are unavailable.","architecture":"Self-supervised emotion distillation with disentangled voice conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-26","dateKind":"paper","links":{"code":"https://github.com/Choddeok/DiEmo-TTS","weights":null,"paper":"https://arxiv.org/abs/2505.19687","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#diemo-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DiEmo-TTS: Disentangled Emotion Representations via Self-Supervised Distillation for Cross-Speaker Emotion Transfer in Text-to-Speech","url":"https://arxiv.org/abs/2505.19687","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DiEmo-TTS author repository","url":"https://github.com/Choddeok/DiEmo-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--diemo-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.19687","originUrl":"https://arxiv.org/html/2505.19687v2/250219_main.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/diemo-tts.png","sha256":"f5bab16af55c2cae3c434a61712a9fd4995228025ff1e85b6e0aac8a55e6fb5a"}},{"id":"tts:smlle","slug":"smlle","name":"SMLLE","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"SMLLE uses a transducer to align incoming text with semantic speech tokens and duration information. A separate autoregressive stage generates acoustic frames, with controlled access to future text stabilizing incremental synthesis.","architecture":"Streaming text-to-semantic transduction with autoregressive acoustics","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.19669","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#smlle"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Zero-Shot Streaming Text to Speech Synthesis with Transducer and Auto-Regressive Modeling","url":"https://arxiv.org/abs/2505.19669","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"SMLLE author demo","url":"https://shy-98.github.io/SMLLE_demo_page/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--smlle.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.19669","originUrl":"https://arxiv.org/html/2505.19669v2/SMLLE_framework.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/smlle.png","sha256":"77d607301f8f1e32db350e7ae26a0e57fa5803d2cdbea9803f2974fcb0ca0d22"}},{"id":"tts:gsa-tts","slug":"gsa-tts","name":"GSA-TTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"GSA-TTS extracts local style information at successive levels and combines it through attention into a global reference condition. This richer style representation guides the acoustic model when synthesizing an unseen speaker's voice.","architecture":"Hierarchical gradual style encoding from reference speech","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.19384","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#gsa-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"GSA-TTS : Toward Zero-Shot Speech Synthesis based on Gradual Style Adaptor","url":"https://arxiv.org/abs/2505.19384","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--gsa-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.19384","originUrl":"https://arxiv.org/html/2505.19384v1/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/gsa-tts.png","sha256":"bccec98caf0960e5d2e13692ad432b7d002382331eb206cf2259c00e54b59bcc"}},{"id":"tts:speakstream","slug":"speakstream","name":"SpeakStream","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"SpeakStream trains on text interleaved with corresponding speech and generates audio as new text becomes available. The synthesis module remains compatible with an upstream text-streaming language model, supporting responsive conversational playback.","architecture":"Decoder-only streaming synthesis from interleaved text and audio","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-25","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.19206","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#speakstream"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SpeakStream: Streaming Text-to-Speech with Interleaved Data","url":"https://arxiv.org/abs/2505.19206","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"SpeakStream author demo","url":"https://apple.github.io/speakstream-demo","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--speakstream.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2505.19206","originUrl":"https://arxiv.org/html/2505.19206v1/main.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/speakstream.png","sha256":"75468627c79706d24b237ac1ee382ed11183f6b73329f65b5f4b06d8ce608675"}},{"id":"tts:rv-tts","slug":"rv-tts","name":"RV-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Revival with Voice learns voice identity from face images and delivery attributes from descriptions. Audio-only training data and stylized portrait augmentation broaden its input coverage, enabling controlled speech from real faces or artistic portraits.","architecture":"Face-conditioned voice generation with textual attribute control","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-25","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.18972","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#rv-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Revival with Voice: Multi-modal Controllable Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2505.18972","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--rv-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.18972","originUrl":"https://arxiv.org/html/2505.18972v1/Fig1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/rv-tts.png","sha256":"2358e204a198de919783ddd61528b146ff186e462c6fe6c9e87a9197dd71f699"}},{"id":"tts:mpe-tts","slug":"mpe-tts","name":"MPE-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MPE-TTS combines reference speech and textual prompts to specify an unseen speaker and the desired emotion. A prosody predictor and emotion-consistency objective carry those controls into the synthesized acoustic performance.","architecture":"Multimodal emotion prompting with predicted prosody","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-24","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.18453","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#mpe-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MPE-TTS: Customized Emotion Zero-Shot Text-To-Speech Using Multi-Modal Prompt","url":"https://arxiv.org/abs/2505.18453","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mpe-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.18453","originUrl":"https://arxiv.org/html/2505.18453v1/model.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mpe-tts.png","sha256":"8b7e034bfe6bfe074b90c2630e3e8248da417ebc86378885de8c9d0bf69d94f3"}},{"id":"tts:unitts","slug":"unitts","name":"UniTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"UniTTS predicts the dense single-stream representation learned by DistilCodec from a multicodebook teacher. Keeping acoustic and semantic information together provides its language model with a unified target for reference-conditioned text-to-speech.","architecture":"Single-codebook acoustic language modeling with DistilCodec","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-23","dateKind":"paper","links":{"code":"https://github.com/IDEA-Emdoor-Lab/UniTTS","weights":null,"paper":"https://arxiv.org/abs/2505.17426","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#unitts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniTTS: An end-to-end TTS system without decoupling of acoustic and semantic information","url":"https://arxiv.org/abs/2505.17426","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"UniTTS author repository","url":"https://github.com/IDEA-Emdoor-Lab/UniTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--unitts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.17426","originUrl":"https://arxiv.org/html/2505.17426v1/figure_1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/unitts.png","sha256":"9ecb572b56efa03e9395e96bcabdb9be85e5954ded584174fffffb61cb763dbe"}},{"id":"tts:mm-moviedubber","slug":"mm-moviedubber","name":"MM-MovieDubber","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"MM-MovieDubber interprets scene information to distinguish dialogue, narration and monologue delivery. A speech generator then uses the resulting multimodal conditions with the target content to render expressive movie dubbing.","architecture":"Visual-language analysis conditioning multimodal speech generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-22","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.16279","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#mm-moviedubber"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MM-MovieDubber: Towards Multi-Modal Learning for Multi-Modal Movie Dubbing","url":"https://arxiv.org/abs/2505.16279","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--mm-moviedubber.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2505.16279","originUrl":"https://arxiv.org/html/2505.16279v1/mmdubber.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mm-moviedubber.png","sha256":"18696e8c196197c7a699a9b2913fa708fb8b04b39d2a683b3fc4f5d9c78e9a63"}},{"id":"tts:nr-lauratts","slug":"nr-lauratts","name":"NR-LauraTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"NR-LauraTTS cleans the discrete representation of a noisy voice prompt before passing it to LauraTTS. Token prediction and embedding refinement reduce background contamination, supporting reference cloning when the available recording is acoustically imperfect.","architecture":"Codec-token denoising before reference-conditioned LauraTTS generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-20","dateKind":"paper","links":{"code":"https://github.com/yxlu-0102/NR-LauraTTS","weights":null,"paper":"https://arxiv.org/abs/2505.13830","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#nr-lauratts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Improving Noise Robustness of LLM-based Zero-shot TTS via Discrete Acoustic Token Denoising","url":"https://arxiv.org/abs/2505.13830","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"NR-LauraTTS author repository","url":"https://github.com/yxlu-0102/NR-LauraTTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"NR-LauraTTS author demo","url":"https://yxlu-0102.github.io/NR-LauraTTS","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--nr-lauratts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.13830","originUrl":"https://arxiv.org/html/2505.13830v2/nr-lauratts.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/nr-lauratts.png","sha256":"1b34f07cd036b31d30f279e275401f3bed3d728e914a822f11c9676f1a071fea"}},{"id":"tts:sled","slug":"sled","name":"SLED","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"SLED learns the conditional distribution of acoustic latents using an energy-distance objective rather than discrete token classification. Its autoregressive generator samples continuous speech representations, simplifying synthesis while retaining acoustic detail.","architecture":"Continuous-latent autoregressive speech modeling with energy distance","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-05-19","dateKind":"paper","links":{"code":"https://github.com/ictnlp/SLED-TTS","weights":null,"paper":"https://arxiv.org/abs/2505.13181","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#sled"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Efficient Speech Language Modeling via Energy Distance in Continuous Latent Space","url":"https://arxiv.org/abs/2505.13181","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"SLED author repository","url":"https://github.com/ictnlp/SLED-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--sled.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2505.13181","originUrl":"https://arxiv.org/html/2505.13181v2/stream.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/sled.png","sha256":"df4207b3666fe3e7df3be79b90950547df2e2ccadf2c2e8781f3e37258c4c24a"}},{"id":"tts:ozspeech","slug":"ozspeech","name":"OZSpeech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"OZSpeech generates disentangled speech components with a flow model conditioned on a learned prior. The design targets single-step zero-shot synthesis while separately modeling content, prosody and speaker-related information from the voice prompt.","architecture":"Learned-prior one-step flow generation over factorized speech codes","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-19","dateKind":"paper","links":{"code":"https://github.com/ozspeech/OZSpeech","weights":null,"paper":"https://arxiv.org/abs/2505.12800","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#ozspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"OZSpeech: One-step Zero-shot Speech Synthesis with Learned-Prior-Conditioned Flow Matching","url":"https://arxiv.org/abs/2505.12800","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"OZSpeech author repository","url":"https://github.com/ozspeech/OZSpeech","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"OZSpeech author demo","url":"https://ozspeech.github.io/OZSpeech_Web/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--ozspeech.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.12800","originUrl":"https://arxiv.org/html/2505.12800v1/overall-2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ozspeech.png","sha256":"9fe718b0325a0dfc41411a587e628d824c132034a0e9946a63760f22f49cfca3"}},{"id":"tts:chain-talker","slug":"chain-talker","name":"Chain-Talker","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Chain-Talker first derives an emotional description from dialogue history, then predicts semantic speech codes. A final rendering stage combines these plans to synthesize expressive responses whose delivery fits the conversational context.","architecture":"Contextual emotion reasoning followed by semantic and acoustic rendering","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-19","dateKind":"paper","links":{"code":"https://github.com/AI-S2-Lab/Chain-Talker","weights":null,"paper":"https://arxiv.org/abs/2505.12597","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#chain-talker"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Chain-Talker: Chain Understanding and Rendering for Empathetic Conversational Speech Synthesis","url":"https://arxiv.org/abs/2505.12597","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Chain-Talker author repository","url":"https://github.com/AI-S2-Lab/Chain-Talker","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--chain-talker.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2505.12597","originUrl":"https://arxiv.org/html/2505.12597v1/acl-model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/chain-talker.png","sha256":"bd3322592110cbd7fe47f8a9b986e1051b30428fb1cd433e7d791161fc80cb63"}},{"id":"tts:shallow-flow-matching-tts","slug":"shallow-flow-matching-tts","name":"Shallow Flow Matching TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"Shallow Flow Matching adds a lightweight head that predicts an intermediate acoustic state for a flow-based synthesizer. Starting refinement closer to the target reduces the remaining generation path, allowing coarse-to-fine speech synthesis with less iterative work.","architecture":"Coarse acoustic prediction initializing a shallow flow trajectory","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-18","dateKind":"paper","links":{"code":"https://github.com/ydqmkkx/ShallowFlowMatching-TTS","weights":null,"paper":"https://arxiv.org/abs/2505.12226","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#shallow-flow-matching-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Shallow Flow Matching for Coarse-to-Fine Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2505.12226","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Shallow Flow Matching TTS author repository","url":"https://github.com/ydqmkkx/ShallowFlowMatching-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--shallow-flow-matching-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2505.12226","originUrl":"https://arxiv.org/html/2505.12226v2/sfm_head.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/shallow-flow-matching-tts.png","sha256":"41ec816ff6d126e50e5413548de64bfec6f97346a5494ff45acbfd00f296460a"}},{"id":"tts:uddetts","slug":"uddetts","name":"UDDETTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"UDDETTS combines named emotion labels with continuous affect dimensions in one TTS model. A semi-supervised training scheme uses differently annotated speech collections, enabling more flexible emotion control than a fixed list of categories alone.","architecture":"Unified categorical and continuous emotion conditioning in a speech LM","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-15","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.10599","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#uddetts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UDDETTS: Unifying Discrete and Dimensional Emotions for Controllable Emotional Text-to-Speech","url":"https://arxiv.org/abs/2505.10599","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--uddetts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.10599","originUrl":"https://arxiv.org/html/2505.10599v2/pictures/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/uddetts.png","sha256":"f060d134f43c96ec83d84535ab14b7c2720c139820b3413bbd6ebdf17ffab75d"}},{"id":"tts:le2e-tts","slug":"le2e-tts","name":"LE2E-TTS","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"LE2E-TTS trains a compact text-to-waveform pipeline end to end rather than separately optimizing acoustic and waveform stages. It targets local devices where model size, response time and compute cost constrain deployment.","architecture":"Joint lightweight acoustic and waveform synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.07701","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#le2e-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Lightweight End-to-end Text-to-speech Synthesis for low resource on-device applications","url":"https://arxiv.org/abs/2505.07701","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--le2e-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.07701","originUrl":"https://arxiv.org/html/2505.07701v1/img/LE2E.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/le2e-tts.png","sha256":"f6e231e85a1c6e732bdf038e23f78f1a8be23b2a94837e6889c6cd452ed9a7a1"}},{"id":"tts:flexspeech","slug":"flexspeech","name":"FlexSpeech","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"FlexSpeech separates timing control from the acoustic synthesis component to balance stable pronunciation and natural expression. A small set of style examples can adapt the duration module without retraining the full generator, supporting efficient delivery customization.","architecture":"Separately adaptable duration planning and expressive acoustic generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-05-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2505.05159","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#flexspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FlexSpeech: Towards Stable, Controllable and Expressive Text-to-Speech","url":"https://arxiv.org/abs/2505.05159","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"FlexSpeech author demo","url":"https://flexspeech.github.io/DEMO/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--flexspeech.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2505.05159","originUrl":"https://arxiv.org/html/2505.05159v3/overview_new.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/flexspeech.png","sha256":"17f4d51c4b3738dfa9fa5fed781afb2b5dee41423901f2dd777987079b295c49"}},{"id":"tts:aligndit","slug":"aligndit","name":"AlignDiT","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"AlignDiT aligns text, visual information and acoustic conditions before diffusion-based speech generation. Modality-specific guidance balances these inputs, targeting synchronized, intelligible speech that follows the timing and expression of the supplied scene.","architecture":"Multimodal alignment and guidance in a speech diffusion Transformer","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-29","dateKind":"paper","links":{"code":"https://github.com/kaistmm/AlignDiT","weights":null,"paper":"https://arxiv.org/abs/2504.20629","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#aligndit"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"AlignDiT: Multimodal Aligned Diffusion Transformer for Synchronized Speech Generation","url":"https://arxiv.org/abs/2504.20629","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"AlignDiT author repository","url":"https://github.com/kaistmm/AlignDiT","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--aligndit.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2504.20629","originUrl":"https://arxiv.org/html/2504.20629v2/teaser.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/aligndit.png","sha256":"d56cd4a51d1b7ad96d9ab9424a8ace0aa89f3e7c27443328fdaa8dd84b867ae4"}},{"id":"tts:muyan-tts","slug":"muyan-tts","name":"Muyan-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Muyan-TTS trains a speech language model on a large podcast collection for expressive reference-based synthesis. The release documents data preparation, training and optimized inference, with an emphasis on reproducible podcast-style voice generation.","architecture":"Podcast-trained autoregressive zero-shot speech generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-27","dateKind":"paper","links":{"code":"https://github.com/MYZY-AI/Muyan-TTS","weights":null,"paper":"https://arxiv.org/abs/2504.19146","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#muyan-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Muyan-TTS: A Trainable Text-to-Speech Model Optimized for Podcast Scenarios with a $50K Budget","url":"https://arxiv.org/abs/2504.19146","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Muyan-TTS author repository","url":"https://github.com/MYZY-AI/Muyan-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--muyan-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2504.19146","originUrl":"https://arxiv.org/html/2504.19146v1/img/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/muyan-tts.png","sha256":"2647f5dcb5ae26df2ed29006e46a5a5baf6a1a5f92bc0ed1a69557a8b6e3dc43"}},{"id":"tts:emovoice","slug":"emovoice","name":"EmoVoice","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"EmoVoice interprets free-form textual descriptions of emotional delivery. Its phoneme-boost variant predicts phonetic and acoustic information together to improve content consistency while retaining expressive style control.","architecture":"Instruction-conditioned emotional speech LM with optional phoneme prediction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-17","dateKind":"paper","links":{"code":"https://github.com/yanghaha0908/EmoVoice","weights":null,"paper":"https://arxiv.org/abs/2504.12867","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#emovoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"EmoVoice: LLM-based Emotional Text-To-Speech Model with Freestyle Text Prompting","url":"https://arxiv.org/abs/2504.12867","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"EmoVoice author repository","url":"https://github.com/yanghaha0908/EmoVoice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--emovoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2504.12867","originUrl":"https://arxiv.org/html/2504.12867v4/emovoice2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/emovoice.png","sha256":"fd943fd929cc8b7a5aa284c41e1ae631617f5c46f677034e0e99416ccb7e8678"}},{"id":"tts:goat-tts","slug":"goat-tts","name":"GOAT-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"GOAT-TTS encodes continuous voice information in one branch and predicts speech tokens in another. Partial language-model adaptation preserves textual knowledge, while multi-token prediction supports streaming synthesis with reference-based paralinguistic conditioning.","architecture":"Separate modality-alignment and speech-generation language-model branches","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-15","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2504.12339","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#goat-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"GOAT-TTS: Expressive and Realistic Speech Generation via A Dual-Branch LLM","url":"https://arxiv.org/abs/2504.12339","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--goat-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2504.12339","originUrl":"https://arxiv.org/html/2504.12339v2/GOAT_TTS.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/goat-tts.png","sha256":"e456af91cd365789cfdb27bfc161e2163094fa1087c54b9ee7dfa203a5d8db4d"}},{"id":"tts:palle","slug":"palle","name":"PALLE","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"PALLE generates variable-length speech spans at fixed decoding steps, combining temporal planning with parallel token prediction. A second non-autoregressive stage refines the initial sequence, supporting efficient zero-shot synthesis.","architecture":"Pseudo-autoregressive span generation followed by parallel refinement","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-14","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2504.10352","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#palle"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Pseudo-Autoregressive Neural Codec Language Models for Efficient Zero-Shot Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2504.10352","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--palle.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2504.10352","originUrl":"https://arxiv.org/html/2504.10352v3/palle_detail.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/palle.png","sha256":"3016ff3db9560e2d5dd4f00b838769f8110c7ee7c740f279d15083636e6f8cc1"}},{"id":"tts:autostyle-tts","slug":"autostyle-tts","name":"AutoStyle-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"AutoStyle-TTS matches the target text against a collection of expressive speech examples using learned textual embeddings. The selected recording provides style conditioning for synthesis, automatically adapting delivery to the content.","architecture":"Text-driven reference-style retrieval for expressive synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-14","dateKind":"paper","links":{"code":"https://github.com/Chengyuann/AutoStyle-TTS","weights":null,"paper":"https://arxiv.org/abs/2504.10309","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#autostyle-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"AutoStyle-TTS: Retrieval-Augmented Generation based Automatic Style Matching Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2504.10309","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"AutoStyle-TTS author repository","url":"https://github.com/Chengyuann/AutoStyle-TTS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"AutoStyle-TTS author demo","url":"https://thuhcsi.github.io/icme2025-AutoStyle-TTS","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--autostyle-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2504.10309","originUrl":"https://arxiv.org/html/2504.10309v1/rag_arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/autostyle-tts.png","sha256":"bdc060ae33fe57d37d3b0c8b97bde65c830d24344d06bc63af0f53bcf0925958"}},{"id":"tts:amnet","slug":"amnet","name":"AMNet","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"AMNet adds phrase-structure information and local convolutional modeling to a parallel Mandarin acoustic model. These changes help capture contextual pauses, emphasis and intonation before the accompanying vocoder reconstructs the waveform.","architecture":"FastSpeech 2 with phrase annotations and local convolution","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-12","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2504.09225","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#amnet"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"AMNet: An Acoustic Model Network for Enhanced Mandarin Speech Synthesis","url":"https://arxiv.org/abs/2504.09225","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--amnet.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2504.09225","originUrl":"https://arxiv.org/html/2504.09225v1/figure/1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/amnet.png","sha256":"266d38ca7a26d080f0bc6eca3df1e15c3c2e3f6db5b046834817708d7d35d353"}},{"id":"tts:lanstyletts","slug":"lanstyletts","name":"LanStyleTTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"LanStyleTTS standardizes phonetic inputs and introduces local style conditioning across languages. The framework can augment several parallel acoustic backbones, allowing one multilingual model to vary delivery at phoneme level.","architecture":"Shared multilingual phonemes with language-aware style adaptation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-11","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2504.08274","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#lanstyletts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Generalized Multilingual Text-to-Speech Generation with Language-Aware Style Adaptation","url":"https://arxiv.org/abs/2504.08274","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--lanstyletts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2504.08274","originUrl":"https://arxiv.org/html/2504.08274v1/stylespeech2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/lanstyletts.png","sha256":"95645a67d9c7a41c439150b818be8565453f3295a2ad5935a8f717be1b4feebf"}},{"id":"tts:phoneme-tone-adaptive-thai-tts","slug":"phoneme-tone-adaptive-thai-tts","name":"Phoneme-tone adaptive Thai TTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"This Thai speech synthesizer encodes phonemes and tones with a language-specific BERT model, then predicts duration, pitch and energy for a GAN-trained waveform decoder. A reference-derived style vector supports voice cloning. Multilingual pretraining of acoustic feature extractors and Thai adaptation address limited language-specific data.","architecture":"Phoneme-tone BERT with duration, pitch and energy predictors","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2504.07858","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#phoneme-tone-adaptive-thai-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Empowering Global Voices: A Data-Efficient, Phoneme-Tone Adaptive Approach to High-Fidelity Speech Synthesis","url":"https://arxiv.org/abs/2504.07858","reviewed_on":"2026-09-13","review_level":"paper"}],"figure":{"path":"assets/figures/tts--phoneme-tone-adaptive-thai-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2504.07858","originUrl":"https://arxiv.org/html/2504.07858v1/pic1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/phoneme-tone-adaptive-thai-tts.png","sha256":"33e05588a3f75625f15d9c9450733b8fa221afb7334b68f25bdfb81d4a6412c9"}},{"id":"tts:slimspeech","slug":"slimspeech","name":"SlimSpeech","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"SlimSpeech reduces the parameter count of a rectified-flow TTS model and transfers knowledge into a lightweight generator. It targets efficient reference-conditioned speech synthesis while retaining the acoustic quality of a larger teacher.","architecture":"Distilled compact rectified-flow speech synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2504.07776","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#slimspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SlimSpeech: Lightweight and Efficient Text-to-Speech with Slim Rectified Flow","url":"https://arxiv.org/abs/2504.07776","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--slimspeech.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2504.07776","originUrl":"https://arxiv.org/html/2504.07776v2/fig1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/slimspeech.png","sha256":"8a1abf0a79f4d234263dbb9e0bb2282dc5900b0a8e5b0137e2bfeb5a7a3f579a"}},{"id":"tts:rwkvtts","slug":"rwkvtts","name":"RWKVTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"RWKVTTS uses the recurrent RWKV-7 architecture for speech synthesis in place of a conventional Transformer backbone. Its token-generation path targets efficient streaming and reduced state-management cost while conditioning audio on the supplied text.","architecture":"RWKV-7 recurrent speech-token generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-04-04","dateKind":"paper","links":{"code":"https://github.com/yynil/RWKVTTS","weights":null,"paper":"https://arxiv.org/abs/2504.03289","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#rwkvtts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"RWKVTTS: Yet another TTS based on RWKV-7","url":"https://arxiv.org/abs/2504.03289","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"RWKVTTS author repository","url":"https://github.com/yynil/RWKVTTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--rwkvtts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2504.03289","originUrl":"https://arxiv.org/html/2504.03289v1/pipeline1.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/rwkvtts.png","sha256":"9bd2a9b1f5812967029dbcd7aa7fd843b494ecdb8ba4373c7c7e0b4e3e0208cf"}},{"id":"tts:f5r-tts","slug":"f5r-tts","name":"F5R-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"F5R-TTS adapts a flow-based synthesizer to reinforcement learning through a probabilistic formulation of generation. Transcription accuracy and speaker-similarity rewards refine the model's ability to preserve both target content and the reference voice.","architecture":"Flow-matching synthesis refined with group-relative policy optimization","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-03","dateKind":"paper","links":{"code":"https://github.com/FrontierLabs/F5R-TTS","weights":null,"paper":"https://arxiv.org/abs/2504.02407","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#f5r-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"F5R-TTS: Improving Flow-Matching based Text-to-Speech with Group Relative Policy Optimization","url":"https://arxiv.org/abs/2504.02407","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"F5R-TTS author repository","url":"https://github.com/FrontierLabs/F5R-TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--f5r-tts.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2504.02407","originUrl":"https://arxiv.org/html/2504.02407v3/figures/overall.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/f5r-tts.png","sha256":"b181823d2a25ffd7a75fc9f2a791c3986035dc09f6fd503388f8f84d823aed33"}},{"id":"tts:voicecraft-dub","slug":"voicecraft-dub","name":"VoiceCraft-Dub","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"VoiceCraft-Dub extends text-conditioned codec synthesis with facial information from video. The added visual conditions coordinate speech timing and expression with the visible speaker while preserving the voice and natural prosody.","architecture":"Video-conditioned neural codec language modeling for dubbing","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-04-03","dateKind":"paper","links":{"code":"https://github.com/kaist-ami/voicecraft-dub","weights":null,"paper":"https://arxiv.org/abs/2504.02386","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#voicecraft-dub"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VoiceCraft-Dub: Automated Video Dubbing with Neural Codec Language Models","url":"https://arxiv.org/abs/2504.02386","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"VoiceCraft-Dub author repository","url":"https://github.com/kaist-ami/voicecraft-dub","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--voicecraft-dub.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2504.02386","originUrl":"https://arxiv.org/html/2504.02386v1/method.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/voicecraft-dub.png","sha256":"6e81940f17753edc1e08758150a9ff19ffa7cc303df1223ee8e1277b0875041e"}},{"id":"tts:deepdubber-v1","slug":"deepdubber-v1","name":"DeepDubber-V1","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"DeepDubber-V1 interprets visual scenes and dubbing requirements before generating the target speech. Its multimodal conditions distinguish narration, monologue and dialogue, guiding both expressive delivery and synchronization.","architecture":"Multimodal reasoning conditioning adaptive movie-speech generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-03-31","dateKind":"paper","links":{"code":"https://github.com/woka-0a/DeepDubber-V1","weights":null,"paper":"https://arxiv.org/abs/2503.23660","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#deepdubber-v1"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DeepDubber-V1: Towards High Quality and Dialogue, Narration, Monologue Adaptive Movie Dubbing Via Multi-Modal Chain-of-Thoughts Reasoning Guidance","url":"https://arxiv.org/abs/2503.23660","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DeepDubber-V1 author repository","url":"https://github.com/woka-0a/DeepDubber-V1","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--deepdubber-v1.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2503.23660","originUrl":"https://arxiv.org/html/2503.23660v1/model_comm.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/deepdubber-v1.png","sha256":"d3b2c39f27a8d654efcbdd52196db10e9a69849dc0795dcb5dbe7d264e130f0f"}},{"id":"tts:fireredtts-1s","slug":"fireredtts-1s","name":"FireRedTTS-1S","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"FireRedTTS-1S extends the FireRed synthesis line with incremental acoustic decoding. Its chunked flow-matching and frame-autoregressive multi-stream decoder options provide different trade-offs between initial latency and sustained generation speed.","architecture":"Streaming semantic speech generation with alternative acoustic decoders","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-03-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2503.20499","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#fireredtts-1s"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FireRedTTS-1S: An Upgraded Streamable Foundation Text-to-Speech System","url":"https://arxiv.org/abs/2503.20499","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--fireredtts-1s.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2503.20499","originUrl":"https://arxiv.org/html/2503.20499v3/image/fireredtts1s_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/fireredtts-1s.png","sha256":"88c0e722d2809b0fb69399ac2281b360ed24092f3c2feba288f732c47a3f444b"}},{"id":"tts:mooncast","slug":"mooncast","name":"MoonCast","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MoonCast combines podcast script preparation with a synthesizer trained for longer, spontaneous-sounding delivery. Voice references allow unseen speakers to render the resulting conversation, while discourse-level context supports more natural transitions than isolated sentence synthesis.","architecture":"Podcast-oriented long-form speech synthesis with contextual voice conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-03-18","dateKind":"paper","links":{"code":"https://github.com/jzq2000/MoonCast","weights":null,"paper":"https://arxiv.org/abs/2503.14345","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#mooncast"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MoonCast: High-Quality Zero-Shot Podcast Generation","url":"https://arxiv.org/abs/2503.14345","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MoonCast author repository","url":"https://github.com/jzq2000/MoonCast","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"MoonCast author demo","url":"https://mooncastdemo.github.io","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--mooncast.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2503.14345","originUrl":"https://arxiv.org/html/2503.14345v2/x1.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mooncast.png","sha256":"aa453a66264b01017b8bdc1dfb55cee581c46d6be6675995509b6947253d826f"}},{"id":"tts:llmvox","slug":"llmvox","name":"LLMVoX","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"LLMVoX connects to an upstream language model through a streaming queue interface. Its small speech generator renders incoming text incrementally, allowing long conversations without tightly coupling the synthesizer to one particular language-model backbone.","architecture":"Compact autoregressive TTS with independent token-stream queues","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"streaming","specs":{},"date":"2025-03-06","dateKind":"paper","links":{"code":"https://github.com/mbzuai-oryx/LLMVoX","weights":null,"paper":"https://arxiv.org/abs/2503.04724","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#llmvox"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"LLMVoX: Autoregressive Streaming Text-to-Speech Model for Any LLM","url":"https://arxiv.org/abs/2503.04724","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"LLMVoX author repository","url":"https://github.com/mbzuai-oryx/LLMVoX","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"LLMVoX author demo","url":"https://mbzuai-oryx.github.io/LLMVoX","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--llmvox.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2503.04724","originUrl":"https://arxiv.org/html/2503.04724v1/arch_diagram.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/llmvox.png","sha256":"ad302cff5bd3d6dd372f8e7910df68489e2ef0b333b6de6d082a45ec99e9dc3e"}},{"id":"tts:uniwav-tts","slug":"uniwav-tts","name":"UniWav (TTS)","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"UniWav jointly pretrains representations for recognition and speech generation. Its TTS adaptation uses the learned encoder-decoder foundation to render text, demonstrating a synthesis path alongside recognition and tokenization tasks.","architecture":"Shared masked speech encoder and generative audio decoder","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S","A"],"interaction":"generation","specs":{},"date":"2025-03-02","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2503.00733","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#uniwav-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"UniWav: Towards Unified Pre-training for Speech Representation Learning and Generation","url":"https://arxiv.org/abs/2503.00733","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"UniWav (TTS) author demo","url":"https://alexander-h-liu.github.io/uniwav-demo.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--uniwav-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2503.00733","originUrl":"https://arxiv.org/html/2503.00733v1/overview.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/uniwav-tts.png","sha256":"289eef1583888492af241bb28dfa20549ffa2a8e22674f27e05879dda693e0fe"}},{"id":"tts:diffcss","slug":"diffcss","name":"DiffCSS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"DiffCSS samples prosody representations from multimodal conversational context using a diffusion model. A prosody-conditioned speech language model renders those samples, allowing several expressive deliveries that remain consistent with the same dialogue.","architecture":"Diffusion-based contextual prosody sampling with a speech LM","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-27","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.19924","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#diffcss"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DiffCSS: Diverse and Expressive Conversational Speech Synthesis with Diffusion Models","url":"https://arxiv.org/abs/2502.19924","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--diffcss.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2502.19924","originUrl":"https://arxiv.org/html/2502.19924v1/backbone.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/diffcss.png","sha256":"c0944d6e38359e89ccae1ac056d8756673e3fa3bc8b9827790fd8f73dfc6f182"}},{"id":"tts:clip-tts","slug":"clip-tts","name":"Clip-TTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"Clip-TTS trains its textual representation against corresponding mel-spectrogram information through a contrastive objective. The acoustic Transformer uses the resulting context-aware features to improve prosodic interpretation during speech generation.","architecture":"Contrastively aligned text and mel features in a Transformer acoustic model","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-26","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.18889","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#clip-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Clip-TTS: Contrastive Text-content and Mel-spectrogram, A High-Quality Text-to-Speech Method based on Contextual Semantic Understanding","url":"https://arxiv.org/abs/2502.18889","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--clip-tts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2502.18889","originUrl":"https://arxiv.org/html/2502.18889v2/1739288899038.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/clip-tts.png","sha256":"a1a016d47ed85e8f3a6d997faff765acf11cc5c5d4b878894b93547eaaa701e0"}},{"id":"tts:felle","slug":"felle","name":"FELLE","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"FELLE generates continuous acoustic frames sequentially, using the preceding frame to shape the next flow-matching prior. A coarse-to-fine acoustic head refines each prediction, supporting reference-conditioned speech without discrete speech-token classification.","architecture":"Token-wise coarse-to-fine flow matching with continuous autoregression","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-16","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.11128","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#felle"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FELLE: Autoregressive Speech Synthesis with Token-Wise Coarse-to-Fine Flow Matching","url":"https://arxiv.org/abs/2502.11128","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--felle.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.11128","originUrl":"https://arxiv.org/html/2502.11128v2/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/felle.png","sha256":"47baf0a0cfb3eecf8f609f42241467a8cf68a9d2e388536583195485e4d6a8b7"}},{"id":"tts:syncspeech","slug":"syncspeech","name":"SyncSpeech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"SyncSpeech uses a temporal masking scheme to coordinate sequential speech structure with parallel token decoding. This hybrid organization targets faster first audio and higher throughput while retaining reference-conditioned synthesis quality.","architecture":"Temporal masked Transformer combining ordered and parallel generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-16","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.11094","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#syncspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"SyncSpeech: Efficient and Low-Latency Text-to-Speech based on Temporal Masked Transformer","url":"https://arxiv.org/abs/2502.11094","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"SyncSpeech author demo","url":"https://SyncSpeech.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--syncspeech.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.11094","originUrl":"https://arxiv.org/html/2502.11094v2/SyncSpeech.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/syncspeech.png","sha256":"f2d4b1bea28819486e4708dd9ec8818b41ecda873c0b9806e7dc5c0195dfe67e"}},{"id":"tts:lorp-tts","slug":"lorp-tts","name":"LoRP-TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"LoRP-TTS adapts a pretrained zero-shot synthesizer using small low-rank parameter updates. It focuses on preserving a target speaker from limited, potentially noisy or spontaneous recordings whose acoustic conditions differ from the original training data.","architecture":"Low-rank voice adaptation from limited spontaneous recordings","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-11","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.07562","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#lorp-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"LoRP-TTS: Low-Rank Personalized Text-To-Speech","url":"https://arxiv.org/abs/2502.07562","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--lorp-tts.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2502.07562","originUrl":"https://arxiv.org/html/2502.07562v1/voicebox_pipeline.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/lorp-tts.png","sha256":"655c5c3b777291bb4fe28bd6b7a31c0f18b649418d77feec9b0cb1d204204c3b"}},{"id":"tts:controllable-masked-speech-tts","slug":"controllable-masked-speech-tts","name":"Controllable masked-speech TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"This synthesizer separates reference voice information from acoustic background conditions. An explicit task control selects whether background sound is retained or removed, allowing personalized speech generation under different environmental requirements.","architecture":"Dual-speaker encoders and task-conditioned masked speech prediction","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","A"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-11","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.07345","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#controllable-masked-speech-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Advanced Zero-Shot Text-to-Speech for Background Removal and Preservation with Controllable Masked Speech Prediction","url":"https://arxiv.org/abs/2502.07345","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--controllable-masked-speech-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2502.07345","originUrl":"https://arxiv.org/html/2502.07345v1/20240912-arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/controllable-masked-speech-tts.png","sha256":"9da3855cea18cf11dd258f0ad982deed16b09ca41e3d49b7c7d7466761890ac9"}},{"id":"tts:vevo","slug":"vevo","name":"Vevo","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Vevo progressively separates linguistic content, delivery style and speaker identity from speech through self-supervised learning. Its generation path can use text or speech content, enabling zero-shot TTS with independent control over reference timbre and style.","architecture":"Content-style token prediction with separate timbre-conditioned acoustics","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-11","dateKind":"paper","links":{"code":"https://github.com/open-mmlab/Amphion","weights":null,"paper":"https://arxiv.org/abs/2502.07243","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#vevo"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Vevo: Controllable Zero-Shot Voice Imitation with Self-Supervised Disentanglement","url":"https://arxiv.org/abs/2502.07243","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Vevo author repository","url":"https://github.com/open-mmlab/Amphion","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--vevo.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.07243","originUrl":"https://arxiv.org/html/2502.07243v1/img/vevo-pipeline.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/vevo.png","sha256":"f9d0683a14baffbc894764858dbd056592d09f89b310a1ca38908ac16e52b64b"}},{"id":"tts:bntts","slug":"bntts","name":"BnTTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"BnTTS extends an XTTS-based multilingual pipeline to Bangla using language-specific phonetic adaptations. A small amount of target-speaker audio supports personalization, with the model designed for limited-resource speech synthesis.","architecture":"Bangla-adapted XTTS with few-shot speaker conditioning","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-09","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.05729","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#bntts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"BnTTS: Few-Shot Speaker Adaptation in Low-Resource Setting","url":"https://arxiv.org/abs/2502.05729","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--bntts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.05729","originUrl":"https://arxiv.org/html/2502.05729v1/resources/hqtts.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/bntts.png","sha256":"7bb6baab0146899a623098aa697cd43b12c4ee9ca7cc672361d169bd20ea83ec"}},{"id":"tts:koel-tts","slug":"koel-tts","name":"Koel-TTS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Koel-TTS explores several ways to condition a Transformer synthesizer on text and reference audio. Automatic speech-recognition and speaker-verification feedback, together with classifier-free guidance, improve adherence to the requested words and voice.","architecture":"Encoder-decoder codec TTS with reference guidance and preference training","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-07","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.05236","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#koel-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Koel-TTS: Enhancing LLM based Speech Generation with Preference Alignment and Classifier Free Guidance","url":"https://arxiv.org/abs/2502.05236","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"Koel-TTS author demo","url":"https://koeltts.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--koel-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.05236","originUrl":"https://arxiv.org/html/2502.05236v2/ICML_Model_Diagram.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/koel-tts.png","sha256":"a303ca373fef7fdeb152b365710feef2982ee5be92942b6bfa4aa10808b04046"}},{"id":"tts:ditar","slug":"ditar","name":"DiTAR","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"DiTAR predicts a sequence of compressed acoustic patches using a language model, then generates each patch's detail through diffusion. Separating global temporal planning from local reconstruction supports zero-shot speech synthesis with controllable sampling diversity.","architecture":"Patch-autoregressive language modeling with local diffusion Transformers","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-06","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.03930","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#ditar"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DiTAR: Diffusion Transformer Autoregressive Modeling for Speech Generation","url":"https://arxiv.org/abs/2502.03930","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"DiTAR author demo","url":"https://spicyresearch.github.io/ditar/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--ditar.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.03930","originUrl":"https://arxiv.org/html/2502.03930v4/x1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ditar.png","sha256":"160469ff9696227a11258f1c275288534f50245b429b3c24e604552a7aa7b1e5"}},{"id":"tts:metis","slug":"metis","name":"Metis","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"Metis pretrains on unlabeled speech before adapting to task-specific conditions such as text. Self-supervised semantic tokens and acoustic codes support a shared foundation for reference-based TTS, conversion and other speech-generation tasks.","architecture":"Masked generative pretraining over semantic and acoustic speech codes","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-05","dateKind":"paper","links":{"code":"https://github.com/open-mmlab/Amphion","weights":null,"paper":"https://arxiv.org/abs/2502.03128","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#metis"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Metis: A Foundation Speech Generation Model with Masked Generative Pre-training","url":"https://arxiv.org/abs/2502.03128","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Metis author repository","url":"https://github.com/open-mmlab/Amphion","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"Metis author demo","url":"https://metis-demo.github.io/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--metis.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2502.03128","originUrl":"https://arxiv.org/html/2502.03128v1/figs/speech_gen.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/metis.png","sha256":"8a5a051189b0e40a27bb4b7dbe4091643b61ffba2f088b9f881c1cdebbc9abc4"}},{"id":"tts:ojibwe-mi-kmaq-maliseet-tts","slug":"ojibwe-mi-kmaq-maliseet-tts","name":"Ojibwe-Mi'kmaq-Maliseet TTS","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"This model family shares speech-synthesis training across three related Indigenous languages. The paper compares attention-based and attention-free flow architectures, demonstrating how joint linguistic coverage can help languages with limited recordings.","architecture":"Multilingual low-resource flow matching with efficient acoustic backbones","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-04","dateKind":"paper","links":{"code":"https://github.com/ShenranTomWang/TTS","weights":null,"paper":"https://arxiv.org/abs/2502.02703","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#ojibwe-mi-kmaq-maliseet-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Developing multilingual speech synthesis system for Ojibwe, Mi'kmaq, and Maliseet","url":"https://arxiv.org/abs/2502.02703","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"Ojibwe-Mi'kmaq-Maliseet TTS author repository","url":"https://github.com/ShenranTomWang/TTS","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--ojibwe-mi-kmaq-maliseet-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.02703","originUrl":"https://arxiv.org/html/2502.02703v1/throughput.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/ojibwe-mi-kmaq-maliseet-tts.png","sha256":"4698e8108afbc442bc38871327e1cc1ca7eba1e130aafd63ae678121b9c17ca3"}},{"id":"tts:stochastic-alignment-continuous-tts","slug":"stochastic-alignment-continuous-tts","name":"Stochastic-alignment continuous TTS","collection":"tts","category":"tts:continuous-lm","categoryName":"Continuous speech LMs","summary":"This synthesizer predicts continuous speech latents using a Gaussian-mixture conditional distribution. A stochastic monotonic alignment mechanism keeps the acoustic sequence ordered against the text, offering an alternative to autoregressive discrete-codec modeling.","architecture":"VAE latents with Gaussian-mixture autoregression and monotonic alignment","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-02-03","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2502.01084","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/continuous-lm.md#stochastic-alignment-continuous-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Continuous Autoregressive Modeling with Stochastic Monotonic Alignment for Speech Synthesis","url":"https://arxiv.org/abs/2502.01084","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--stochastic-alignment-continuous-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2502.01084","originUrl":"https://arxiv.org/html/2502.01084v2/codec_comp.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/stochastic-alignment-continuous-tts.png","sha256":"7644bd8b8d9bfb368a588b6f32b9665ec0bd8c2fe883ea72d607c9964f78b8bc"}},{"id":"tts:visualspeech","slug":"visualspeech","name":"VisualSpeech","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"VisualSpeech adds video-derived context to a text-to-speech model to reduce ambiguity in delivery. The visual representation guides prosodic choices so the same text can be rendered in a way that fits its scene.","architecture":"Visual-textual context conditioning for prosodic synthesis","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","V"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-31","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.19258","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#visualspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"VisualSpeech: Enhancing Prosody Modeling in TTS Using Video","url":"https://arxiv.org/abs/2501.19258","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"VisualSpeech author demo","url":"https://ariameetgit.github.io/VISUALSPEECH-SAMPLES/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--visualspeech.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2501.19258","originUrl":"https://arxiv.org/html/2501.19258v2/241117_model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/visualspeech.png","sha256":"8e5b21fee69b228ba219db8656bc3ef93370c50822b94093408d04f633409efd"}},{"id":"tts:breezyvoice","slug":"breezyvoice","name":"BreezyVoice","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"BreezyVoice combines supervised speech tokens, a language model and flow-based acoustics with a pronunciation frontend. Its Taiwanese Mandarin adaptation provides explicit phonetic control for characters with multiple readings while retaining reference-based voice synthesis.","architecture":"Taiwanese Mandarin CosyVoice adaptation with pronunciation disambiguation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-29","dateKind":"paper","links":{"code":"https://github.com/mtkresearch/BreezyVoice","weights":null,"paper":"https://arxiv.org/abs/2501.17790","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#breezyvoice"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"BreezyVoice: Adapting TTS for Taiwanese Mandarin with Enhanced Polyphone Disambiguation -- Challenges and Insights","url":"https://arxiv.org/abs/2501.17790","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"BreezyVoice author repository","url":"https://github.com/mtkresearch/BreezyVoice","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--breezyvoice.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2501.17790","originUrl":"https://arxiv.org/html/2501.17790v1/Data_Augmentation_Pipeline_v2.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/breezyvoice.png","sha256":"155e96e3dd283fcc6de23ec5692862b22ff7fbc031833138d63e5a826c0b714f"}},{"id":"tts:compact-neural-accessibility-tts","slug":"compact-neural-accessibility-tts","name":"Compact neural accessibility TTS","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"This compact synthesis system combines a shared-parameter text frontend with an efficient recurrent waveform generator. It targets responsive accessibility voices on low-power devices, where small storage requirements and immediate playback matter alongside naturalness.","architecture":"Parameter-shared acoustic frontend with a subscale WaveRNN backend","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-28","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.17332","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#compact-neural-accessibility-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Compact Neural TTS Voices for Accessibility","url":"https://arxiv.org/abs/2501.17332","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--compact-neural-accessibility-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2501.17332","originUrl":"https://arxiv.org/html/2501.17332v1/Frontend_formatted.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/compact-neural-accessibility-tts.png","sha256":"5c9c5518c26be8b013b39e9dc1541830cb6379e9c60a100fd65fb529cc0b8f0e"}},{"id":"tts:joint-non-autoregressive-stt-tts","slug":"joint-non-autoregressive-stt-tts","name":"Joint non-autoregressive STT-TTS","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"This model handles text and speech within one non-autoregressive architecture. Its TTS path predicts acoustic output from text, and feeding partial predictions back into the model improves generation through iterative refinement while also supporting recognition training.","architecture":"Shared multimodal encoder-decoder with iterative output refinement","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-15","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.09104","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#joint-non-autoregressive-stt-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"A Non-autoregressive Model for Joint STT and TTS","url":"https://arxiv.org/abs/2501.09104","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--joint-non-autoregressive-stt-tts.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2501.09104","originUrl":"https://arxiv.org/html/2501.09104v2/model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/joint-non-autoregressive-stt-tts.png","sha256":"5f511541deef15f7c754cc1dbd8267a4e936dd0b33061f4e6e0537c8aab04d85"}},{"id":"tts:self-distilled-zero-shot-tts","slug":"self-distilled-zero-shot-tts","name":"Self-distilled zero-shot TTS","collection":"tts","category":"tts:compact","categoryName":"Compact & on-device","summary":"This zero-shot synthesizer learns linguistic content and reference-speaker attributes through separate representations. Two-stage self-distillation creates aligned examples that strengthen their separation, targeting stable voice cloning with a small inference footprint.","architecture":"Lightweight content-speaker disentanglement through self-distillation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-15","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.08566","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/compact.md#self-distilled-zero-shot-tts"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Towards Lightweight and Stable Zero-shot TTS with Self-distilled Representation Disentanglement","url":"https://arxiv.org/abs/2501.08566","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--self-distilled-zero-shot-tts.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2501.08566","originUrl":"https://arxiv.org/html/2501.08566v1/model_arch.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/self-distilled-zero-shot-tts.png","sha256":"2b06274ff6c0d7fb7eb2cd134caafb9e0ecff1699398969711a14b42e2ea2190"}},{"id":"tts:radka-css","slug":"radka-css","name":"RADKA-CSS","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"RADKA-CSS retrieves dialogue examples related to the current conversation in both meaning and delivery. A graph-based aggregation mechanism combines their style information with current context, conditioning expressive conversational speech synthesis.","architecture":"Retrieved dialogue knowledge with multigranularity style graphs","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-11","dateKind":"paper","links":{"code":"https://github.com/Coder-jzq/RADKA-CSS","weights":null,"paper":"https://arxiv.org/abs/2501.06467","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#radka-css"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Retrieval-Augmented Dialogue Knowledge Aggregation for Expressive Conversational Speech Synthesis","url":"https://arxiv.org/abs/2501.06467","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"RADKA-CSS author repository","url":"https://github.com/Coder-jzq/RADKA-CSS","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"RADKA-CSS author demo","url":"https://coder-jzq.github.io/RADKA-CSS-Website/index.html#case-study","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--radka-css.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2501.06467","originUrl":"https://arxiv.org/html/2501.06467v1/figs/RADKA-CSS-main-c.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/radka-css.png","sha256":"7e9d1cb6f813602a935b27f5c5095413693fc2e3e8c4f1ea1f33209cc3290879"}},{"id":"tts:unispeaker","slug":"unispeaker","name":"UniSpeaker","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"UniSpeaker maps several kinds of voice descriptions into a common speaker space using a KV-Former-based aggregator. The shared representation conditions speech generation, allowing a portrait, description or recording to influence the synthesized voice.","architecture":"Unified voice aggregation from textual, acoustic and visual prompts","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-11","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.06394","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#unispeaker"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"Unispeaker: A Unified Approach for Multimodality-driven Speaker Generation","url":"https://arxiv.org/abs/2501.06394","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"UniSpeaker author demo","url":"https://UniSpeaker.github.io","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--unispeaker.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2501.06394","originUrl":"https://arxiv.org/html/2501.06394v1/pipeline.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/unispeaker.png","sha256":"dff1464c87ef9c37ccd92b6f1955dfb9c545b9531a6fcfcf4eae6e94a09086b2"}},{"id":"tts:tts-transducer","slug":"tts-transducer","name":"TTS-Transducer","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"TTS-Transducer learns an ordered mapping from tokenized text to the first speech codebook with a neural transducer. Subsequent acoustic-code generation reconstructs the waveform, combining explicit monotonic alignment with reference-conditioned speech synthesis.","architecture":"Neural transduction for monotonic text-to-codec alignment","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.06320","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#tts-transducer"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"TTS-Transducer: End-to-End Speech Synthesis with Neural Transducer","url":"https://arxiv.org/abs/2501.06320","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"TTS-Transducer author demo","url":"https://tts-transducer.github.io","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--tts-transducer.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2501.06320","originUrl":"https://arxiv.org/html/2501.06320v1/TTS-Transducer_Arch_v4.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/tts-transducer.png","sha256":"c1b70b9501bfb1da9cf3c1784da7306747b02ad1f616bde403069d3b89f21a04"}},{"id":"tts:proemo","slug":"proemo","name":"PROEMO","collection":"tts","category":"tts:parallel","categoryName":"Parallel acoustic","summary":"PROEMO combines emotional prompts with an explicit intensity control in a multi-speaker synthesizer. The conditioning adjusts delivery strength and prosodic variation, allowing the same spoken text to be rendered with different emotional performances.","architecture":"Prompt-conditioned multi-speaker emotion and intensity modeling","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-10","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.06276","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/parallel.md#proemo"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"PROEMO: Prompt-Driven Text-to-Speech Synthesis Based on Emotion and Intensity Control","url":"https://arxiv.org/abs/2501.06276","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--proemo.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2501.06276","originUrl":"https://arxiv.org/html/2501.06276v1/Figures/Model_Arc_480.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/proemo.png","sha256":"0492af48b42107ab7250fde286fa4e78b8dbb37c29b6c720b7fa8de79f92849b"}},{"id":"tts:mars6","slug":"mars6","name":"MARS6","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"MARS6 encodes text and a speaker representation before generating hierarchical acoustic codes. Its compact encoder-decoder design targets expressive speech and reference-voice cloning with reduced inference cost.","architecture":"Hierarchical-codec encoder-decoder Transformer","notes":"The author repository is named mars6-turbo; its README presents the MARS6 paper and hierarchical codec model.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-10","dateKind":"paper","links":{"code":"https://github.com/Camb-ai/mars6-turbo","weights":null,"paper":"https://arxiv.org/abs/2501.05787","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#mars6"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"MARS6: A Small and Robust Hierarchical-Codec Text-to-Speech Model","url":"https://arxiv.org/abs/2501.05787","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"MARS6 author repository","url":"https://github.com/Camb-ai/mars6-turbo","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"MARS6 author demo","url":"https://camb-ai.github.io/mars6-turbo/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--mars6.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2501.05787","originUrl":"https://arxiv.org/html/2501.05787v1/mars6_diagram.drawio.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/mars6.png","sha256":"9fd65c746efb540990969ca2cbf91fb1964668dd383e162fe6f3000e15449247"}},{"id":"tts:jelly","slug":"jelly","name":"JELLY","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"JELLY combines an emotion-aware Q-Former with several partially adapted language-model modules. Joint emotion recognition and contextual reasoning guide a speech synthesizer toward responses whose delivery matches the conversation.","architecture":"Emotion-aware speech encoding and context-specific language-model adapters","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-09","dateKind":"paper","links":{"code":"https://github.com/jh-cha-prml/JELLY","weights":null,"paper":"https://arxiv.org/abs/2501.04904","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#jelly"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"JELLY: Joint Emotion Recognition and Context Reasoning with LLMs for Conversational Speech Synthesis","url":"https://arxiv.org/abs/2501.04904","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"JELLY author repository","url":"https://github.com/jh-cha-prml/JELLY","reviewed_on":"2026-09-13","review_level":"readme"},{"kind":"project","title":"JELLY author demo","url":"https://jh-cha-prml.github.io/JELLY","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--jelly.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2501.04904","originUrl":"https://arxiv.org/html/2501.04904v1/JELLY_overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/jelly.png","sha256":"785fea4796d65ad2dd03b95e55a8ffc674ea89af799173e89a85c774036be217"}},{"id":"tts:flespeech","slug":"flespeech","name":"FleSpeech","collection":"tts","category":"tts:token-lm","categoryName":"Speech-token LMs","summary":"FleSpeech unifies text, voice recordings and visual prompts into a common conditioning representation. Its multistage generator uses those controls to manipulate voice and delivery attributes flexibly rather than requiring one fixed prompt modality.","architecture":"Multistage synthesis with a shared multimodal prompt encoder","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","S","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-08","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.04644","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/token-lm.md#flespeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FleSpeech: Flexibly Controllable Speech Generation with Various Prompts","url":"https://arxiv.org/abs/2501.04644","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"project","title":"FleSpeech author demo","url":"https://kkksuper.github.io/FleSpeech/","reviewed_on":"2026-09-13","review_level":"project"}],"figure":{"path":"assets/figures/tts--flespeech.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2501.04644","originUrl":"https://arxiv.org/html/2501.04644v2/241214_model_1.drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/flespeech.png","sha256":"75c134078831de0f8798d792f9dc0d8aea3da03b46f54837b4a54f466ae88f2c"}},{"id":"tts:drawspeech","slug":"drawspeech","name":"DrawSpeech","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"DrawSpeech turns user-drawn prosodic curves into detailed pitch and energy conditions for synthesis. A diffusion model fills in the acoustic detail, enabling localized expressive control beyond an overall style description.","architecture":"Sketch-conditioned diffusion for detailed pitch and energy generation","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-08","dateKind":"paper","links":{"code":"https://github.com/HappyColor/DrawSpeech_PyTorch","weights":null,"paper":"https://arxiv.org/abs/2501.04256","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#drawspeech"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"DrawSpeech: Expressive Speech Synthesis Using Prosodic Sketches as Control Conditions","url":"https://arxiv.org/abs/2501.04256","reviewed_on":"2026-09-13","review_level":"abstract"},{"kind":"repository","title":"DrawSpeech author repository","url":"https://github.com/HappyColor/DrawSpeech_PyTorch","reviewed_on":"2026-09-13","review_level":"readme"}],"figure":{"path":"assets/figures/tts--drawspeech.webp","kind":"source","label":"Paper figure","sourceUrl":"https://arxiv.org/abs/2501.04256","originUrl":"https://arxiv.org/html/2501.04256v1/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/drawspeech.png","sha256":"7a23dcb24ba4daba22fac3329de9114add32391867d2d7b4efcdd32c22e7580c"}},{"id":"tts:facespeak","slug":"facespeak","name":"FaceSpeak","collection":"tts","category":"tts:diffusion","categoryName":"Diffusion & flow matching","summary":"FaceSpeak extracts speaker-related and expressive information from real or stylized portraits. These visual conditions guide text-to-speech generation, allowing the requested words to be rendered in a plausible voice and emotion associated with the image.","architecture":"Separate identity and emotion extraction from portrait prompts","notes":"The first-submission date refers to the linked research paper. Reference and control options depend on the synthesis setting described there.","variants":[],"inputs":["T","I"],"outputs":["S"],"interaction":"generation","specs":{},"date":"2025-01-02","dateKind":"paper","links":{"code":null,"weights":null,"paper":"https://arxiv.org/abs/2501.03181","catalog":"https://github.com/kadirnar/awesome-tts-architectures/blob/caa36857411c1b5c5b5a491c18855a4a27fe04ad/models/diffusion.md#facespeak"},"license":{"status":"not-recorded","code":null,"weights":null,"codeUrl":null,"weightsUrl":null,"notes":"","components":[]},"sources":[{"kind":"paper","title":"FaceSpeak: Expressive and High-Quality Speech Synthesis from Human Portraits of Different Styles","url":"https://arxiv.org/abs/2501.03181","reviewed_on":"2026-09-13","review_level":"abstract"}],"figure":{"path":"assets/figures/tts--facespeak.webp","kind":"source","label":"Figure 3","sourceUrl":"https://arxiv.org/abs/2501.03181","originUrl":"https://arxiv.org/html/2501.03181v2/FaceSpeak_new.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-tts-architectures/caa36857411c1b5c5b5a491c18855a4a27fe04ad/assets/architectures/facespeak.png","sha256":"fcd64ee5a9b2e13bf647e1c557919ad8ad0b262eda46ad102af08cf72c62b80f"}},{"id":"codec:encodec","slug":"encodec","name":"EnCodec","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Compresses general audio with selectable bandwidths and a separate stereo music variant.","architecture":"Convolutional/LSTM encoder and decoder with residual vector quantization and multiscale spectrogram adversarial training.","notes":"Optional entropy models further compress the codes. The reference Python API processes entire files; causal architecture does not mean a ready-made chunked API.","variants":["24 kHz mono","48 kHz stereo"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono; 48 kHz stereo","frame_rate":"75 (24 kHz); 150 (48 kHz)","bitrate":"1.5–24 kbps (24 kHz); 3–24 kbps (48 kHz)","representation":"RVQ; variable active codebooks","streaming":"24 kHz causal; 48 kHz noncausal"},"date":"2022-10-24","dateKind":"paper","links":{"code":"https://github.com/facebookresearch/encodec","weights":"https://huggingface.co/facebook/encodec_24khz","paper":"https://arxiv.org/abs/2210.13438","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#encodec"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/facebookresearch/encodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/facebook/encodec_24khz/blob/c1dbe2ae3f1de713481a3b3e7c47f357092ee040/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/facebookresearch/encodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/facebook/encodec_24khz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/facebookresearch/encodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/facebook/encodec_24khz/blob/c1dbe2ae3f1de713481a3b3e7c47f357092ee040/README.md","review_level":"license file or model-card declaration"},{"title":"High Fidelity Neural Audio Compression","url":"https://arxiv.org/abs/2210.13438","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--encodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/facebookresearch/encodec","originUrl":"https://raw.githubusercontent.com/facebookresearch/encodec/main/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/encodec.png","sha256":"0a75e180efb3f43dad64796295922ce6b14e9c20ce65524182edf45b169368f8"}},{"id":"codec:dac","slug":"dac","name":"Descript Audio Codec (DAC)","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Encodes speech, music and environmental audio into residual discrete codes.","architecture":"Convolutional autoencoder with Snake activations, factorized low-dimensional RVQ and adversarial waveform reconstruction.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["44.1 kHz","24 kHz","16 kHz"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz (also 16 / 24 kHz)","frame_rate":"~86 (44.1 kHz)","bitrate":"~8 kbps (44.1 kHz)","representation":"RVQ","streaming":"Noncausal"},"date":"2023-06-11","dateKind":"paper","links":{"code":"https://github.com/descriptinc/descript-audio-codec","weights":"https://huggingface.co/descript/dac_44khz","paper":"https://arxiv.org/abs/2306.06546","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#dac"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/descriptinc/descript-audio-codec/blob/main/LICENSE","weightsUrl":"https://github.com/descriptinc/descript-audio-codec/blob/main/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/descriptinc/descript-audio-codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/descript/dac_44khz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/descriptinc/descript-audio-codec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://github.com/descriptinc/descript-audio-codec/blob/main/README.md","review_level":"license file or model-card declaration"},{"title":"High-Fidelity Audio Compression with Improved RVQGAN","url":"https://arxiv.org/abs/2306.06546","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--dac.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/descriptinc/descript-audio-codec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/dac.svg","sha256":"627cd358e38284727b6c2c2efdb7f410fec2a99b49b75f94c4695ed104a0e237"}},{"id":"codec:snac","slug":"snac","name":"SNAC","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Uses coarse and fine token streams at different time scales to shorten audio token sequences.","architecture":"Convolutional encoder/decoder with temporally downsampled hierarchical residual quantization.","notes":"The 24 kHz checkpoint is the reviewed release. Frame rates differ by quantizer; do not read the coarsest rate as the total token rate.","variants":["24 kHz speech","32 kHz music / sound","44.1 kHz music / sound"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz speech; 32 / 44.1 kHz audio","frame_rate":"Multiple temporal scales","bitrate":"0.98 / 1.9 / 2.6 kbps by variant","representation":"Multiscale RVQ","streaming":"Not specified"},"date":"2024-10-18","dateKind":"paper","links":{"code":"https://github.com/hubertsiuzdak/snac","weights":"https://huggingface.co/hubertsiuzdak/snac_24khz","paper":"https://arxiv.org/abs/2410.14411","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#snac"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/hubertsiuzdak/snac/blob/main/LICENSE","weightsUrl":"https://huggingface.co/hubertsiuzdak/snac_24khz/blob/d73ad176a12188fcf4f360ba3bf2c2fbbe8f58ec/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/hubertsiuzdak/snac","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/hubertsiuzdak/snac_24khz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/hubertsiuzdak/snac/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/hubertsiuzdak/snac_24khz/blob/d73ad176a12188fcf4f360ba3bf2c2fbbe8f58ec/README.md","review_level":"license file or model-card declaration"},{"title":"SNAC: Multi-Scale Neural Audio Codec","url":"https://arxiv.org/abs/2410.14411","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--snac.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/hubertsiuzdak/snac","originUrl":"https://raw.githubusercontent.com/hubertsiuzdak/snac/main/img/snac.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/snac.png","sha256":"c11698c806e77ebf946bfe8f60f1c1fb02b6cdd491a6e608a3ca1ac6c7cf576a"}},{"id":"codec:mimi","slug":"mimi","name":"Mimi","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Provides streaming semantic and acoustic tokens for real-time spoken dialogue.","architecture":"Convolutional and Transformer encoder/decoder with split residual quantization and WavLM distillation.","notes":"The quoted bandwidth uses eight 2,048-entry codebooks. The standalone checkpoint supports additional quantizers; 12.5 frames/s means 100 tokens/s with eight active codebooks.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono","frame_rate":"12.5","bitrate":"1.1 kbps (8 codebooks)","representation":"Split RVQ","streaming":"Streaming encoder and decoder"},"date":"2024-09-17","dateKind":"paper","links":{"code":"https://github.com/kyutai-labs/moshi","weights":"https://huggingface.co/kyutai/mimi","paper":"https://arxiv.org/abs/2410.00037","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#mimi"},"license":{"status":"open","code":"MIT (Python); Apache-2.0 (Rust)","weights":"CC-BY-4.0","codeUrl":"https://github.com/kyutai-labs/moshi/blob/main/README.md","weightsUrl":"https://huggingface.co/kyutai/mimi/blob/89091b3e466eb6a9d11e537bf26b144f194978f7/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/kyutai-labs/moshi","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/kyutai/mimi","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/kyutai-labs/moshi/blob/main/README.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/kyutai/mimi/blob/89091b3e466eb6a9d11e537bf26b144f194978f7/README.md","review_level":"license file or model-card declaration"},{"title":"Moshi: a speech-text foundation model for real-time dialogue","url":"https://arxiv.org/abs/2410.00037","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--mimi.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/kyutai-labs/moshi","originUrl":"https://raw.githubusercontent.com/kyutai-labs/moshi/main/mimi.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/mimi.png","sha256":"d6988c98a4126cba9d731cbb8decda3c8d954a892f1d8cd6f301c857ea52db4b"}},{"id":"codec:speechtokenizer","slug":"speechtokenizer","name":"SpeechTokenizer","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Separates a content-oriented first token stream from residual acoustic detail.","architecture":"Encoder/decoder with RVQ and HuBERT feature distillation into the first quantizer.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["HuBERT-average teacher","Snake-activation variant"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz mono","frame_rate":"50","bitrate":"4 kbps (8 codebooks)","representation":"8-layer RVQ","streaming":"Not specified"},"date":"2023-08-31","dateKind":"paper","links":{"code":"https://github.com/ZhangXInFD/SpeechTokenizer","weights":"https://huggingface.co/OpenMOSS-Team/SpeechTokenizer","paper":"https://arxiv.org/abs/2308.16692","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#speechtokenizer"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Not stated","codeUrl":"https://github.com/ZhangXInFD/SpeechTokenizer/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenMOSS-Team/SpeechTokenizer/blob/4d54939beef00572fa7bfe41ee35a335c3732f51/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/ZhangXInFD/SpeechTokenizer","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/OpenMOSS-Team/SpeechTokenizer","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/ZhangXInFD/SpeechTokenizer/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/OpenMOSS-Team/SpeechTokenizer/blob/4d54939beef00572fa7bfe41ee35a335c3732f51/README.md","review_level":"license file or model-card declaration"},{"title":"SpeechTokenizer: Unified Speech Tokenizer for Speech Large Language Models","url":"https://arxiv.org/abs/2308.16692","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--speechtokenizer.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/ZhangXInFD/SpeechTokenizer","originUrl":"https://raw.githubusercontent.com/ZhangXInFD/SpeechTokenizer/main/images/speechtokenizer_framework.jpg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/speechtokenizer.jpg","sha256":"e865ac647542825838cf9515b124747d56b7bd9174af959421f514f7e2dcd57a"}},{"id":"codec:hifi-codec","slug":"hifi-codec","name":"HiFi-Codec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Reconstructs speech using grouped residual quantization with few codebooks.","architecture":"HiFi-GAN-style encoder/decoder with group-residual vector quantization (GRVQ).","notes":"HiFi-Codec is the model; AcademiCodec is the hosting toolkit. SoundStream and EnCodec implementations in the toolkit are not counted as additional original models.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 / 24 kHz","frame_rate":"Not specified","bitrate":"Not specified","representation":"GRVQ; four codebooks","streaming":"Not specified"},"date":"2023-05-04","dateKind":"paper","links":{"code":"https://github.com/yangdongchao/AcademiCodec","weights":"https://huggingface.co/Dongchao/AcademiCodec","paper":"https://arxiv.org/abs/2305.02765","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#hifi-codec"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/yangdongchao/AcademiCodec/blob/master/readme.md","weightsUrl":"https://huggingface.co/Dongchao/AcademiCodec/blob/827e793904fe43cac45157efc26d0b6920cfda14/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/yangdongchao/AcademiCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Dongchao/AcademiCodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/yangdongchao/AcademiCodec/blob/master/readme.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Dongchao/AcademiCodec/blob/827e793904fe43cac45157efc26d0b6920cfda14/README.md","review_level":"license file or model-card declaration"},{"title":"HiFi-Codec: Group-residual Vector quantization for High Fidelity Audio Codec","url":"https://arxiv.org/abs/2305.02765","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--hifi-codec.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2305.02765","originUrl":"https://arxiv.org/html/2305.02765v2/academiCodec.drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/hifi-codec.png","sha256":"1aaa4b1f0993fae367d3b33146d3b780ecd3b9df0811165203777d5c959f6859"}},{"id":"codec:funcodec","slug":"funcodec","name":"FunCodec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Provides reproducible time-domain and frequency-domain neural speech coding recipes.","architecture":"Encoder/decoder toolkit with residual quantization and selectable waveform or magnitude/phase representations.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["LibriTTS ds320 / ds640","Chinese-English general audio","Frequency-domain codec"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"50 (ds320); 25 (ds640)","bitrate":"Not specified","representation":"RVQ; time/frequency-domain variants","streaming":"Not specified"},"date":"2023-09-14","dateKind":"paper","links":{"code":"https://github.com/modelscope/FunCodec","weights":"https://huggingface.co/alibaba-damo/audio_codec-encodec-en-libritts-16k-nq32ds320-pytorch","paper":"https://arxiv.org/abs/2309.07405","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#funcodec"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/modelscope/FunCodec/blob/master/LICENSE","weightsUrl":"https://huggingface.co/alibaba-damo/audio_codec-encodec-en-libritts-16k-nq32ds320-pytorch/blob/313f69d23b57459bc3d743ee987dadbb8681aca9/LICENSE","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/modelscope/FunCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/alibaba-damo/audio_codec-encodec-en-libritts-16k-nq32ds320-pytorch","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/modelscope/FunCodec/blob/master/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/alibaba-damo/audio_codec-encodec-en-libritts-16k-nq32ds320-pytorch/blob/313f69d23b57459bc3d743ee987dadbb8681aca9/LICENSE","review_level":"license file or model-card declaration"},{"title":"FunCodec: A Fundamental, Reproducible and Integrable Open-source Toolkit for Neural Speech Codec","url":"https://arxiv.org/abs/2309.07405","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--funcodec.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2309.07405","originUrl":"https://arxiv.org/html/2309.07405v2/framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/funcodec.png","sha256":"15636f4b905d4b6db225791abe3441b3ec3c2542b2c94c4f52b0c4c69647bb7f"}},{"id":"codec:bigcodec","slug":"bigcodec","name":"BigCodec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Scales a single-codebook speech codec to improve reconstruction at low bitrates.","architecture":"Large convolutional/sequential autoencoder with low-dimensional single-vector quantization.","notes":"The code repository states MIT; the published checkpoint card states CC-BY-SA-4.0. Both sources are linked separately.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"80","bitrate":"1.04 kbps","representation":"1 × 8,192-entry VQ","streaming":"Nonstreaming"},"date":"2024-09-09","dateKind":"paper","links":{"code":"https://github.com/Aria-K-Alethia/BigCodec","weights":"https://huggingface.co/Alethia/BigCodec","paper":"https://arxiv.org/abs/2409.05377","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#bigcodec"},"license":{"status":"open","code":"MIT","weights":"CC-BY-SA-4.0","codeUrl":"https://github.com/Aria-K-Alethia/BigCodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Alethia/BigCodec/blob/c6548832eda95a09dcc3485d9b8ef097ce15f387/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Aria-K-Alethia/BigCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Alethia/BigCodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Aria-K-Alethia/BigCodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Alethia/BigCodec/blob/c6548832eda95a09dcc3485d9b8ef097ce15f387/README.md","review_level":"license file or model-card declaration"},{"title":"BigCodec: Pushing the Limits of Low-Bitrate Neural Speech Codec","url":"https://arxiv.org/abs/2409.05377","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--bigcodec.webp","kind":"source","label":"Fig. 1","sourceUrl":"https://arxiv.org/abs/2409.05377","originUrl":"https://arxiv.org/html/2409.05377v1/generator.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/bigcodec.png","sha256":"441e99baafa08c516f5a59236fa383f3297a3744d3a2959d01465ef02a9d95c8"}},{"id":"codec:wavtokenizer","slug":"wavtokenizer","name":"WavTokenizer","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Represents speech, music and sounds with a single low-rate discrete stream.","architecture":"Single-quantizer acoustic encoder with an expanded codebook and attention-enhanced waveform reconstruction.","notes":"The bitrate shown is calculated from the author-documented 40 Hz and 4,096-code configuration before transport overhead.","variants":["40 tokens/s general-audio large","75 tokens/s speech large","Small and medium variants"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"40 (also 75)","bitrate":"0.48 kbps (40 Hz × 12 bits)","representation":"1 × 4,096-entry VQ","streaming":"Not specified"},"date":"2024-08-29","dateKind":"paper","links":{"code":"https://github.com/jishengpeng/WavTokenizer","weights":"https://huggingface.co/novateur/WavTokenizer-large-unify-40token","paper":"https://arxiv.org/abs/2408.16532","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#wavtokenizer"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/jishengpeng/WavTokenizer/blob/main/LICENSE","weightsUrl":"https://huggingface.co/novateur/WavTokenizer-large-unify-40token/blob/7dc47a9450ce71cab736434b7645604013ca028b/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/jishengpeng/WavTokenizer","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/novateur/WavTokenizer-large-unify-40token","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/jishengpeng/WavTokenizer/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/novateur/WavTokenizer-large-unify-40token/blob/7dc47a9450ce71cab736434b7645604013ca028b/README.md","review_level":"license file or model-card declaration"},{"title":"WavTokenizer: an Efficient Acoustic Discrete Codec Tokenizer for Audio Language Modeling","url":"https://arxiv.org/abs/2408.16532","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--wavtokenizer.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/jishengpeng/WavTokenizer","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/wavtokenizer.svg","sha256":"ad58f91e04f62cfae4193d07b0f66dcc2d567733f087c1c09c6a364174f6de39"}},{"id":"codec:x-codec","slug":"x-codec","name":"X-Codec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Combines pretrained semantic features with acoustic features before quantization.","architecture":"Parallel semantic and acoustic encoders, feature fusion, RVQ and reconstruction of both feature types.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["HuBERT LibriSpeech","HuBERT general audio","WavLM variants"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"Not specified","bitrate":"Not specified","representation":"Semantic/acoustic fusion + RVQ","streaming":"Not specified"},"date":"2024-08-30","dateKind":"paper","links":{"code":"https://github.com/zhenye234/xcodec","weights":"https://huggingface.co/ZhenYe234/xcodec","paper":"https://arxiv.org/abs/2408.17175","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#x-codec"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/zhenye234/xcodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ZhenYe234/xcodec/blob/de323edde57975f8eb0053c7494cc3daabbe7816/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/zhenye234/xcodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/ZhenYe234/xcodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/zhenye234/xcodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/ZhenYe234/xcodec/blob/de323edde57975f8eb0053c7494cc3daabbe7816/README.md","review_level":"license file or model-card declaration"},{"title":"Codec Does Matter: Exploring the Semantic Shortcoming of Codec for Audio Language Model","url":"https://arxiv.org/abs/2408.17175","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--x-codec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/zhenye234/xcodec","originUrl":"https://raw.githubusercontent.com/zhenye234/xcodec/main/fig1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/x-codec.png","sha256":"550476c19ccec8863fbad00b827e7bba38cbd9ed0feb35a7f709ca9411b20529"}},{"id":"codec:x-codec-2","slug":"x-codec-2","name":"X-Codec 2","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Encodes multilingual speech into a single stream for speech language modeling.","architecture":"Wav2Vec2-BERT semantic features fused with acoustic features and finite scalar quantization.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["Original checkpoint","Transformers-native export"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"50","bitrate":"0.8 kbps","representation":"FSQ; 65,536 combinations","streaming":"Not specified"},"date":"2025-02-06","dateKind":"paper","links":{"code":"https://github.com/zhenye234/X-Codec-2.0","weights":"https://huggingface.co/HKUSTAudio/xcodec2","paper":"https://arxiv.org/abs/2502.04128","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#x-codec-2"},"license":{"status":"restricted","code":"MIT","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/zhenye234/X-Codec-2.0/blob/main/LICENSE","weightsUrl":"https://huggingface.co/HKUSTAudio/xcodec2/blob/e412427ed30f0cf9d5e3c95562113deb10a32d03/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/zhenye234/X-Codec-2.0","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/HKUSTAudio/xcodec2","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/zhenye234/X-Codec-2.0/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/HKUSTAudio/xcodec2/blob/e412427ed30f0cf9d5e3c95562113deb10a32d03/README.md","review_level":"license file or model-card declaration"},{"title":"Llasa: Scaling Train-Time and Inference-Time Compute for Llama-based Speech Synthesis","url":"https://arxiv.org/abs/2502.04128","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--x-codec-2.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/zhenye234/X-Codec-2.0","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/x-codec-2.svg","sha256":"449779b795be3e9a82f728c27a08b73649c76b050be815b7624a903bf70453fc"}},{"id":"codec:focalcodec","slug":"focalcodec","name":"FocalCodec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Compresses speech into one codebook at several low token rates.","architecture":"WavLM features, focal modulation compression, binary spherical quantization and waveform synthesis.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["12.5 Hz","25 Hz","50 Hz"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"12.5 / 25 / 50","bitrate":"~0.16 / 0.33 / 0.65 kbps","representation":"1 × 8,192-entry codebook","streaming":"Nonstreaming"},"date":"2025-02-06","dateKind":"paper","links":{"code":"https://github.com/lucadellalib/focalcodec","weights":"https://huggingface.co/lucadellalib/focalcodec_50hz","paper":"https://arxiv.org/abs/2502.04465","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#focalcodec"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/lucadellalib/focalcodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/lucadellalib/focalcodec_50hz/blob/d6d6d9524e52155c85193c2c3b8da1cf8842f019/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/lucadellalib/focalcodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/lucadellalib/focalcodec_50hz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/lucadellalib/focalcodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/lucadellalib/focalcodec_50hz/blob/d6d6d9524e52155c85193c2c3b8da1cf8842f019/README.md","review_level":"license file or model-card declaration"},{"title":"FocalCodec: Low-Bitrate Speech Coding via Focal Modulation Networks","url":"https://arxiv.org/abs/2502.04465","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--focalcodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/lucadellalib/focalcodec","originUrl":"https://raw.githubusercontent.com/lucadellalib/focalcodec/main/focalcodec.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/focalcodec.png","sha256":"93eefb4b78b4ee860c678e8408456516082ef4f6fcf9cce9a831e234ea260b84"}},{"id":"codec:focalcodec-stream","slug":"focalcodec-stream","name":"FocalCodec-Stream","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Adds causal, incremental speech coding with a single token stream.","architecture":"Causal distillation of the FocalCodec pipeline with a streaming speech decoder.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["2k causal","4k causal","65k causal"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz input → 24 kHz output","frame_rate":"50","bitrate":"0.55 / 0.60 / 0.80 kbps","representation":"1 × 2,048 / 4,096 / 65,536 codes","streaming":"Streaming encoder and decoder"},"date":"2025-09-19","dateKind":"paper","links":{"code":"https://github.com/lucadellalib/focalcodec","weights":"https://huggingface.co/lucadellalib/focalcodec_50hz_65k_causal","paper":"https://arxiv.org/abs/2509.16195","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#focalcodec-stream"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/lucadellalib/focalcodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/lucadellalib/focalcodec_50hz_65k_causal/blob/4c709e2fa03df261906b6d48927f02b5d3ce0686/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/lucadellalib/focalcodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/lucadellalib/focalcodec_50hz_65k_causal","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/lucadellalib/focalcodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/lucadellalib/focalcodec_50hz_65k_causal/blob/4c709e2fa03df261906b6d48927f02b5d3ce0686/README.md","review_level":"license file or model-card declaration"},{"title":"FocalCodec-Stream: Streaming Low-Bitrate Speech Coding via Causal Distillation","url":"https://arxiv.org/abs/2509.16195","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--focalcodec-stream.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/lucadellalib/focalcodec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/focalcodec-stream.svg","sha256":"ea4ac46575d0d7ea6d23f1cda95de2fc4fbb841ecf1d85ff5da349eb2b11b231"}},{"id":"codec:magicodec","slug":"magicodec","name":"MagiCodec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Produces low-rate speech tokens using masked Gaussian-injection training.","architecture":"Causal Transformer codec with a single quantizer and masked Gaussian-injected latent training.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"50","bitrate":"≤0.85 kbps","representation":"Single-layer VQ","streaming":"Streaming; documented look-ahead ≤40 ms"},"date":"2025-05-31","dateKind":"paper","links":{"code":"https://github.com/Ereboas/MagiCodec","weights":"https://huggingface.co/Ereboas/MagiCodec_16k_50hz","paper":"https://arxiv.org/abs/2506.00385","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#magicodec"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/Ereboas/MagiCodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Ereboas/MagiCodec_16k_50hz/blob/36b011f5e36591c1f28213d882547bbb92851774/LICENSE","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Ereboas/MagiCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Ereboas/MagiCodec_16k_50hz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Ereboas/MagiCodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Ereboas/MagiCodec_16k_50hz/blob/36b011f5e36591c1f28213d882547bbb92851774/LICENSE","review_level":"license file or model-card declaration"},{"title":"MagiCodec: Simple Masked Gaussian-Injected Codec for High-Fidelity Reconstruction and Generation","url":"https://arxiv.org/abs/2506.00385","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--magicodec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/Ereboas/MagiCodec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/magicodec.svg","sha256":"67a84f0cf4c69fcb322833b8c9eff06e4b3c271fbe9d0223a57f4043020d2e86"}},{"id":"codec:dualcodec","slug":"dualcodec","name":"DualCodec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Combines a semantic first codebook with acoustic residual streams at low frame rates.","architecture":"Dual semantic/acoustic feature processing with a semantic-enriched first RVQ layer and residual reconstruction.","notes":"The first codebook has 16,384 entries; residual books use 4,096 entries at 12.5 Hz or 1,024 at 25 Hz. The encoder also requires the documented Wav2Vec2-BERT component.","variants":["12hz_v1: 1–8 quantizers","25hz_v1: 1–12 quantizers"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"12.5 / 25","bitrate":"Not specified","representation":"RVQ; 8 or 12 layers","streaming":"Not specified"},"date":"2025-05-19","dateKind":"paper","links":{"code":"https://github.com/jiaqili3/DualCodec","weights":"https://huggingface.co/amphion/dualcodec","paper":"https://arxiv.org/abs/2505.13000","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#dualcodec"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/jiaqili3/DualCodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/amphion/dualcodec/blob/e4984cd7ed7cffda2df5ea5563d9062589afa08a/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/jiaqili3/DualCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/amphion/dualcodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/jiaqili3/DualCodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/amphion/dualcodec/blob/e4984cd7ed7cffda2df5ea5563d9062589afa08a/README.md","review_level":"license file or model-card declaration"},{"title":"DualCodec: A Low-Frame-Rate, Semantically-Enhanced Neural Audio Codec for Speech Generation","url":"https://arxiv.org/abs/2505.13000","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--dualcodec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/jiaqili3/DualCodec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/dualcodec.svg","sha256":"c67c9247b6a1667bc96ecb4552d365df0794cb502d078f95f30c53499fff0df0"}},{"id":"codec:flexicodec","slug":"flexicodec","name":"FlexiCodec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Adapts speech token duration by merging semantically similar frames.","architecture":"ASR-assisted dual-stream encoding, Transformer bottlenecks and adaptive frame merging with RVQ.","notes":"Token durations are part of reconstruction; a fixed bitrate cannot be inferred from the maximum frame rate alone. Requires SenseVoiceSmall features.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"Dynamic: ~3–12.5","bitrate":"Not specified","representation":"RVQ; 1–24 quantizers","streaming":"Not specified"},"date":"2025-10-01","dateKind":"paper","links":{"code":"https://github.com/AmphionTeam/FlexiCodec","weights":"https://huggingface.co/jiaqili3/flexicodec","paper":"https://arxiv.org/abs/2510.00981","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#flexicodec"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/AmphionTeam/FlexiCodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/jiaqili3/flexicodec/blob/80610b35de3e8ff7d7fa497bfc12b1635a54ebd8/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/AmphionTeam/FlexiCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/jiaqili3/flexicodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/AmphionTeam/FlexiCodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/jiaqili3/flexicodec/blob/80610b35de3e8ff7d7fa497bfc12b1635a54ebd8/README.md","review_level":"license file or model-card declaration"},{"title":"FlexiCodec: A Dynamic Neural Audio Codec for Low Frame Rates","url":"https://arxiv.org/abs/2510.00981","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--flexicodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/AmphionTeam/FlexiCodec","originUrl":"https://raw.githubusercontent.com/amphionspace/flexicodec/main/.github/flexicodec.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/flexicodec.png","sha256":"2468e3da23c25ca0dd873c4594d45fdd04fa0521be190cce76f3608c09802d7f"}},{"id":"codec:unicodec","slug":"unicodec","name":"UniCodec (domain-adaptive)","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Uses one domain-adaptive codebook across speech, music and other sounds.","architecture":"Partitioned codebook with domain mixture-of-experts and masked self-supervised prediction.","notes":"This is the ACL 2025 unified-audio UniCodec, distinct from the similarly named universal speech-token learning paper.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Not specified","frame_rate":"Not specified","bitrate":"Not specified","representation":"Single domain-adaptive codebook","streaming":"Not specified"},"date":"2025-02-27","dateKind":"paper","links":{"code":"https://github.com/Jiang-Yidi/UniCodec","weights":"https://huggingface.co/Yidiii/UniCodec_ckpt","paper":"https://arxiv.org/abs/2502.20067","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#unicodec"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/Jiang-Yidi/UniCodec/blob/main/README.md","weightsUrl":"https://huggingface.co/Yidiii/UniCodec_ckpt/blob/4aa066c627c33728da013f2f7db3d1beecb0b34c/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Jiang-Yidi/UniCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Yidiii/UniCodec_ckpt","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Jiang-Yidi/UniCodec/blob/main/README.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Yidiii/UniCodec_ckpt/blob/4aa066c627c33728da013f2f7db3d1beecb0b34c/README.md","review_level":"license file or model-card declaration"},{"title":"UniCodec: Unified Audio Codec with Single Domain-Adaptive Codebook","url":"https://arxiv.org/abs/2502.20067","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--unicodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/Jiang-Yidi/UniCodec","originUrl":"https://raw.githubusercontent.com/Jiang-Yidi/UniCodec/main/overview.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/unicodec.png","sha256":"6cd7fb0515d588e3fd48dc6f499b9dc23e9a4150a70c101a61443c00c6a7cc74"}},{"id":"codec:semanticodec","slug":"semanticodec","name":"SemantiCodec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Compresses general audio into semantic and acoustic tokens with diffusion reconstruction.","architecture":"Semantic feature quantization plus acoustic residual quantization and a latent diffusion decoder.","notes":"The published API parameter is total token rate, not a universal frame rate. The diffusion decoder and required audio components affect compute cost.","variants":["25 / 50 / 100 total tokens per second","4k–32k semantic vocabulary"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"See total token-rate variants","bitrate":"~0.31–1.40 kbps","representation":"Semantic + acoustic code streams","streaming":"Not specified"},"date":"2024-04-30","dateKind":"paper","links":{"code":"https://github.com/haoheliu/SemantiCodec-inference","weights":"https://huggingface.co/haoheliu/SemantiCodec","paper":"https://arxiv.org/abs/2405.00233","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#semanticodec"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/haoheliu/SemantiCodec-inference/blob/main/LICENSE","weightsUrl":"https://huggingface.co/haoheliu/SemantiCodec/blob/3b6e7408ee4febb738db732875e8ece31cf5bdd2/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/haoheliu/SemantiCodec-inference","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/haoheliu/SemantiCodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/haoheliu/SemantiCodec-inference/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/haoheliu/SemantiCodec/blob/3b6e7408ee4febb738db732875e8ece31cf5bdd2/README.md","review_level":"license file or model-card declaration"},{"title":"SemantiCodec: An Ultra Low Bitrate Semantic Audio Codec for General Sound","url":"https://arxiv.org/abs/2405.00233","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--semanticodec.webp","kind":"source","label":"Fig. 2","sourceUrl":"https://arxiv.org/abs/2405.00233","originUrl":"https://arxiv.org/html/2405.00233v2/main-figure.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/semanticodec.png","sha256":"b25b47fe71ca1d61211696745d38cc17f832a16f72364d38f432a0cb979bd503"}},{"id":"codec:facodec","slug":"facodec","name":"FACodec","collection":"codec","category":"codec:disentangled","categoryName":"Disentangled codecs","summary":"Separates speech into content, prosody, timbre and acoustic-detail representations.","architecture":"Factorized speech autoencoder with separate residual quantizers and a timbre representation.","notes":"The linked Amphion implementation and checkpoint reproduce the codec component of NaturalSpeech 3; they are not the complete original TTS system.","variants":["FACodec v1","FACodec v2","Redecoder"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"80","bitrate":"Not specified","representation":"Factorized RVQ; 6 codebooks","streaming":"Not specified"},"date":"2024-03-05","dateKind":"paper","links":{"code":"https://github.com/open-mmlab/Amphion/tree/main/models/codec/ns3_codec","weights":"https://huggingface.co/amphion/naturalspeech3_facodec","paper":"https://arxiv.org/abs/2403.03100","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/disentangled.md#facodec"},"license":{"status":"open","code":"MIT","weights":"Apache-2.0","codeUrl":"https://github.com/open-mmlab/Amphion/blob/main/LICENSE","weightsUrl":"https://huggingface.co/amphion/naturalspeech3_facodec/blob/314afc3ea1455ba881a0e484ef9408b6cb996736/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/open-mmlab/Amphion/tree/main/models/codec/ns3_codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/amphion/naturalspeech3_facodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/open-mmlab/Amphion/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/amphion/naturalspeech3_facodec/blob/314afc3ea1455ba881a0e484ef9408b6cb996736/README.md","review_level":"license file or model-card declaration"},{"title":"NaturalSpeech 3: Zero-Shot Speech Synthesis with Factorized Codec and Diffusion Models","url":"https://arxiv.org/abs/2403.03100","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--facodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/open-mmlab/Amphion/tree/main/models/codec/ns3_codec","originUrl":"https://raw.githubusercontent.com/open-mmlab/Amphion/main/imgs/ns3/ns3_facodec.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/facodec.png","sha256":"7b7e0da6843a718d908856cf94a974b3ccfbd074e8abafc82e481fa3c563138f"}},{"id":"codec:bicodec","slug":"bicodec","name":"BiCodec","collection":"codec","category":"codec:disentangled","categoryName":"Disentangled codecs","summary":"Reconstructs speech from separate global speaker and time-varying semantic tokens.","architecture":"Bi-level speech tokenization with global and semantic token paths and a waveform decoder.","notes":"The codec is bundled inside Spark-TTS. This entry counts BiCodec only; the text language model is not a codec. Checkpoint terms differ from the code license. The configuration uses 32 global tokens per utterance; the calculated 0.65 kbps semantic rate excludes that overhead.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"50 semantic frames/s","bitrate":"0.65 kbps semantic stream + global tokens","representation":"8,192-entry semantic VQ + global FSQ tokens","streaming":"Not specified"},"date":"2025-03-03","dateKind":"paper","links":{"code":"https://github.com/SparkAudio/Spark-TTS","weights":"https://huggingface.co/SparkAudio/Spark-TTS-0.5B","paper":"https://arxiv.org/abs/2503.01710","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/disentangled.md#bicodec"},"license":{"status":"restricted","code":"Apache-2.0","weights":"CC-BY-NC-SA-4.0","codeUrl":"https://github.com/SparkAudio/Spark-TTS/blob/main/LICENSE","weightsUrl":"https://huggingface.co/SparkAudio/Spark-TTS-0.5B/blob/642071559bfc6346c2359d19dcb6be3f9dd8a05d/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/SparkAudio/Spark-TTS","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/SparkAudio/Spark-TTS-0.5B","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/SparkAudio/Spark-TTS/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/SparkAudio/Spark-TTS-0.5B/blob/642071559bfc6346c2359d19dcb6be3f9dd8a05d/README.md","review_level":"license file or model-card declaration"},{"title":"Spark-TTS: An Efficient LLM-Based Text-to-Speech Model with Single-Stream Decoupled Speech Tokens","url":"https://arxiv.org/abs/2503.01710","review_level":"primary paper abstract"},{"title":"BiCodec quantizer and sample-rate configuration","url":"https://huggingface.co/SparkAudio/Spark-TTS-0.5B/blob/642071559bfc6346c2359d19dcb6be3f9dd8a05d/BiCodec/config.yaml","review_level":"inspected model configuration"}],"figure":{"path":"assets/figures/codec--bicodec.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2503.01710","originUrl":"https://arxiv.org/html/2503.01710v1/Bicodec_overview3.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/bicodec.png","sha256":"ee797cf4a097b0a2a4bba5c8c93310f6361b39e857281361288fcbc20c393946"}},{"id":"codec:ticodec","slug":"ticodec","name":"TiCodec","collection":"codec","category":"codec:disentangled","categoryName":"Disentangled codecs","summary":"Moves time-invariant speech information into a separate code to reduce frame-level tokens.","architecture":"Separate time-invariant and time-varying representations with quantization and waveform reconstruction.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["1 global + 1 residual stream","1 global + 2 residual streams","1 global + 4 residual streams"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Not specified","frame_rate":"Not specified","bitrate":"Not specified","representation":"Time-invariant + frame-level codes","streaming":"Not specified"},"date":"2023-09-15","dateKind":"paper","links":{"code":"https://github.com/y-ren16/TiCodec","weights":"https://huggingface.co/y-ren16/TiCodec","paper":"https://arxiv.org/abs/2310.00014","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/disentangled.md#ticodec"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/y-ren16/TiCodec/blob/main/README.md","weightsUrl":"https://huggingface.co/y-ren16/TiCodec/blob/2117e04dc2493fb242a8486286fb3dab2350e0a2/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/y-ren16/TiCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/y-ren16/TiCodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/y-ren16/TiCodec/blob/main/README.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/y-ren16/TiCodec/blob/2117e04dc2493fb242a8486286fb3dab2350e0a2/README.md","review_level":"license file or model-card declaration"},{"title":"Fewer-token Neural Speech Codec with Time-invariant Codes","url":"https://arxiv.org/abs/2310.00014","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--ticodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/y-ren16/TiCodec","originUrl":"https://raw.githubusercontent.com/y-ren16/TiCodec/main/imgs/Ticodec-all.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/ticodec.png","sha256":"24cb33ad562e62f137690a67ff2b10585ec1fb49f9a1eecef154d21aa11121b6"}},{"id":"codec:past","slug":"past","name":"PAST","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Learns phonetic and acoustic speech tokens jointly with waveform reconstruction.","architecture":"End-to-end speech autoencoder with phonetic supervision and auxiliary tasks; also released as a causal variant.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["Full","Streamable","No-Transformer ablation"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Not specified","frame_rate":"Not specified","bitrate":"Not specified","representation":"Phonetic/acoustic discrete tokens","streaming":"Causal variant; 20 ms look-ahead"},"date":"2025-05-20","dateKind":"paper","links":{"code":"https://github.com/slp-rl/PAST","weights":"https://huggingface.co/slprl/PAST","paper":"https://arxiv.org/abs/2505.14470","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#past"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/slp-rl/PAST/blob/main/LICENSE","weightsUrl":"https://huggingface.co/slprl/PAST/blob/07329e810dd1e5e68eae97824e167bdb508ae7d9/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/slp-rl/PAST","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/slprl/PAST","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/slp-rl/PAST/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/slprl/PAST/blob/07329e810dd1e5e68eae97824e167bdb508ae7d9/README.md","review_level":"license file or model-card declaration"},{"title":"PAST: Phonetic-Acoustic Speech Tokenizer","url":"https://arxiv.org/abs/2505.14470","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--past.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/slp-rl/PAST","originUrl":"https://raw.githubusercontent.com/slp-rl/PAST/main/assets/PAST_figure.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/past.png","sha256":"6485923832d727182920dc5d1fae9fd3f4e80cd9beb11d73c5dd9f2d4fe16660"}},{"id":"codec:xy-tokenizer","slug":"xy-tokenizer","name":"XY-Tokenizer","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Aligns speech content and acoustics in low-frame-rate tokens.","architecture":"Semantic/acoustic speech codec with joint alignment and eight residual quantization layers.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"12.5","bitrate":"1 kbps","representation":"8-layer RVQ","streaming":"Not specified"},"date":"2025-06-29","dateKind":"paper","links":{"code":"https://github.com/gyt1145028706/XY-Tokenizer","weights":"https://huggingface.co/fdugyt/XY_Tokenizer","paper":"https://arxiv.org/abs/2506.23325","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#xy-tokenizer"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/gyt1145028706/XY-Tokenizer/blob/master/readme.md","weightsUrl":"https://huggingface.co/fdugyt/XY_Tokenizer/blob/117eec32c980273c86402bc4f5792bc766291195/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/gyt1145028706/XY-Tokenizer","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/fdugyt/XY_Tokenizer","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/gyt1145028706/XY-Tokenizer/blob/master/readme.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/fdugyt/XY_Tokenizer/blob/117eec32c980273c86402bc4f5792bc766291195/README.md","review_level":"license file or model-card declaration"},{"title":"XY-Tokenizer: Mitigating the Semantic-Acoustic Conflict in Low-Bitrate Speech Codecs","url":"https://arxiv.org/abs/2506.23325","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--xy-tokenizer.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/gyt1145028706/XY-Tokenizer","originUrl":"https://raw.githubusercontent.com/gyt1145028706/XY-Tokenizer/master/assets/XY-Tokenizer-Architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/xy-tokenizer.png","sha256":"3794b2ec3bd367f22046cff2838afcc23966d558d6854ef054669980aa165d33"}},{"id":"codec:sac","slug":"sac","name":"SAC","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Uses separate semantic and acoustic quantization streams for speech reconstruction.","architecture":"Dual-stream codec with semantic guidance from GLM-4-Voice and an acoustic reconstruction path.","notes":"The release also requires GLM-4-Voice-Tokenizer and the documented speaker-encoder assets; their terms must be checked separately.","variants":["37.5 Hz","62.5 Hz"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"37.5 / 62.5","bitrate":"0.525 / 0.875 kbps","representation":"Semantic + acoustic quantization","streaming":"Not specified"},"date":"2025-10-19","dateKind":"paper","links":{"code":"https://github.com/Soul-AILab/SAC","weights":"https://huggingface.co/Soul-AILab/SAC-16k-37_5Hz","paper":"https://arxiv.org/abs/2510.16841","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#sac"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/Soul-AILab/SAC/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Soul-AILab/SAC-16k-37_5Hz/blob/9715ce1875b82a9e9c6b10672c685a617a9a0758/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Soul-AILab/SAC","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Soul-AILab/SAC-16k-37_5Hz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Soul-AILab/SAC/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Soul-AILab/SAC-16k-37_5Hz/blob/9715ce1875b82a9e9c6b10672c685a617a9a0758/README.md","review_level":"license file or model-card declaration"},{"title":"SAC: Neural Speech Codec with Semantic-Acoustic Dual-Stream Quantization","url":"https://arxiv.org/abs/2510.16841","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--sac.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2510.16841","originUrl":"https://arxiv.org/html/2510.16841v2/SAC.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/sac.png","sha256":"a9b932f99c180509a01a692743a0cfc9531689b08a10c58f6403dcb3f932e0cd"}},{"id":"codec:socodec","slug":"socodec","name":"SoCodec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Orders speech token streams by semantic content for efficient speech language modeling.","architecture":"Semantic-ordered multistream codec with a separately released mel-to-waveform decoder.","notes":"The public checkpoint is the four-stream Chinese model. The repository lists the single-stream version as unreleased. The documented Chinese HuBERT encoder is an additional dependency.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"8.33 (120 ms frames)","bitrate":"~0.47 kbps","representation":"4 × 16,384-entry streams","streaming":"Not specified"},"date":"2024-09-02","dateKind":"paper","links":{"code":"https://github.com/hhguo/SoCodec","weights":"https://huggingface.co/hhguo/SoCodec","paper":"https://arxiv.org/abs/2409.00933","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#socodec"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/hhguo/SoCodec/blob/master/LICENSE","weightsUrl":"https://huggingface.co/hhguo/SoCodec/blob/8e5643efaa8dc5d6cfe14329268da1ff75bda72d/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/hhguo/SoCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/hhguo/SoCodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/hhguo/SoCodec/blob/master/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/hhguo/SoCodec/blob/8e5643efaa8dc5d6cfe14329268da1ff75bda72d/README.md","review_level":"license file or model-card declaration"},{"title":"SoCodec: A Semantic-Ordered Multi-Stream Speech Codec for Efficient Language Model Based Text-to-Speech Synthesis","url":"https://arxiv.org/abs/2409.00933","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--socodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/hhguo/SoCodec","originUrl":"https://raw.githubusercontent.com/hhguo/SoCodec/master/assets/socodec.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/socodec.png","sha256":"b25b80e9658139455395c24b7202f3c569f1b8af76583a583a089d4b5804100e"}},{"id":"codec:tadicodec","slug":"tadicodec","name":"TaDiCodec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Uses text-guided diffusion reconstruction to reduce the speech-token frame rate.","architecture":"Speech encoder and single quantizer paired with a text-aware diffusion Transformer decoder.","notes":"The reported token bitrate excludes the text guidance supplied to the diffusion decoder. It should not be read as a self-contained waveform-only transport bitrate.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"6.25","bitrate":"0.0875 kbps (tokens only)","representation":"Single codebook","streaming":"Not specified"},"date":"2025-08-22","dateKind":"paper","links":{"code":"https://github.com/AmphionTeam/TaDiCodec","weights":"https://huggingface.co/amphion/TaDiCodec","paper":"https://arxiv.org/abs/2508.16790","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#tadicodec"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/AmphionTeam/TaDiCodec/blob/main/README.md","weightsUrl":"https://huggingface.co/amphion/TaDiCodec/blob/e93c6666168407297a6b0a85b33f93aa07ec08ae/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/AmphionTeam/TaDiCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/amphion/TaDiCodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/AmphionTeam/TaDiCodec/blob/main/README.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/amphion/TaDiCodec/blob/e93c6666168407297a6b0a85b33f93aa07ec08ae/README.md","review_level":"license file or model-card declaration"},{"title":"TaDiCodec: Text-aware Diffusion Speech Tokenizer for Speech Language Modeling","url":"https://arxiv.org/abs/2508.16790","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--tadicodec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/AmphionTeam/TaDiCodec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/tadicodec.svg","sha256":"da1c6d50d75f11d22d5873e37bc9ebf898367e069481ff299811a6ac66a52ea8"}},{"id":"codec:u-codec","slug":"u-codec","name":"U-Codec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Reduces the speech representation to an ultra-low frame rate for speech generation.","architecture":"CodecFormer-based speech encoder/decoder with a compressed discrete bottleneck.","notes":"Use the author checkpoint and reconstruction configuration together; prompt and decoder settings affect the reconstruction protocol.","variants":["5 Hz / 16 RVQ / 4,096 entries","5 Hz / 32 RVQ / 256 entries"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"5","bitrate":"Not specified","representation":"16 × 4,096 or 32 × 256 RVQ","streaming":"Not specified"},"date":"2025-10-19","dateKind":"paper","links":{"code":"https://github.com/YangXusheng-yxs/CodecFormer_5Hz","weights":"https://huggingface.co/shaunxsyang/U-Codec","paper":"https://arxiv.org/abs/2510.16718","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#u-codec"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/YangXusheng-yxs/CodecFormer_5Hz/blob/main/readme.md","weightsUrl":"https://huggingface.co/shaunxsyang/U-Codec/blob/8c1aecaf2a30a83ae63531bcfef611db82b92ab4/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/YangXusheng-yxs/CodecFormer_5Hz","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/shaunxsyang/U-Codec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/YangXusheng-yxs/CodecFormer_5Hz/blob/main/readme.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/shaunxsyang/U-Codec/blob/8c1aecaf2a30a83ae63531bcfef611db82b92ab4/README.md","review_level":"license file or model-card declaration"},{"title":"U-Codec: Ultra Low Frame-rate Neural Speech Codec for Fast High-fidelity Speech Generation","url":"https://arxiv.org/abs/2510.16718","review_level":"primary paper abstract"},{"title":"U-Codec 16-RVQ configuration","url":"https://huggingface.co/shaunxsyang/U-Codec/blob/8c1aecaf2a30a83ae63531bcfef611db82b92ab4/5Hz_16rvq_c4096_config.json","review_level":"inspected model configuration"},{"title":"U-Codec 32-RVQ configuration","url":"https://huggingface.co/shaunxsyang/U-Codec/blob/8c1aecaf2a30a83ae63531bcfef611db82b92ab4/5Hz_32rvq_c256_config.json","review_level":"inspected model configuration"}],"figure":{"path":"assets/figures/codec--u-codec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/YangXusheng-yxs/CodecFormer_5Hz","originUrl":"https://raw.githubusercontent.com/YangXusheng-yxs/CodecFormer_5Hz/main/fig/fig2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/u-codec.png","sha256":"98a1be460e5ade3b9e0a3b84779491a78e8886a517b4ae565c55e8789091d16d"}},{"id":"codec:simwhisper-codec","slug":"simwhisper-codec","name":"SimWhisper-Codec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Uses a simplified Whisper-based representation for low-bitrate speech coding.","architecture":"Whisper-derived speech encoding with a discrete bottleneck and audio reconstruction decoder.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Not specified","frame_rate":"Not specified","bitrate":"Not specified","representation":"Whisper-based discrete speech codes","streaming":"Not specified"},"date":"2025-10-23","dateKind":"paper","links":{"code":"https://github.com/ZhangXinWhut/SimWhisper-Codec","weights":"https://huggingface.co/ZhangXinWhut/SimWhisper_Codec","paper":"https://arxiv.org/abs/2510.20504","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#simwhisper-codec"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/ZhangXinWhut/SimWhisper-Codec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ZhangXinWhut/SimWhisper_Codec/blob/8aad9dbf9ff3e1a8b2d2acc013e8d3cc84c31f94/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/ZhangXinWhut/SimWhisper-Codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/ZhangXinWhut/SimWhisper_Codec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/ZhangXinWhut/SimWhisper-Codec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/ZhangXinWhut/SimWhisper_Codec/blob/8aad9dbf9ff3e1a8b2d2acc013e8d3cc84c31f94/README.md","review_level":"license file or model-card declaration"},{"title":"Speaking Clearly: A Simplified Whisper-Based Codec for Low-Bitrate Speech Coding","url":"https://arxiv.org/abs/2510.20504","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--simwhisper-codec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/ZhangXinWhut/SimWhisper-Codec","originUrl":"https://raw.githubusercontent.com/ZhangXinWhut/SimWhisper-Codec/main/docs/assets/SimWhisper-Codec.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/simwhisper-codec.png","sha256":"521a63b37ec2208256d4be4db524f68bc7619c07e8029a069590eed5caccd1df"}},{"id":"codec:neucodec","slug":"neucodec","name":"NeuCodec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Compresses speech into one token stream and reconstructs it at a higher sample rate.","architecture":"Acoustic and Wav2Vec2-BERT encoders fused before finite scalar quantization and an upsampling decoder.","notes":"The distilled checkpoint is linked by the author repository as a family variant. The base codec uses a semantic feature encoder. The checkpoint download requires accepting the publisher's access terms.","variants":["NeuCodec","Distill-NeuCodec"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz input → 24 kHz output","frame_rate":"50","bitrate":"0.8 kbps","representation":"FSQ; 65,536 combinations","streaming":"Not specified"},"date":"2025-09-11","dateKind":"paper","links":{"code":"https://github.com/neuphonic/neucodec","weights":"https://huggingface.co/neuphonic/neucodec","paper":"https://arxiv.org/abs/2509.09550","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#neucodec"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/neuphonic/neucodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/neuphonic/neucodec/blob/a940021324df13a07f44185f0a17c6dac3698179/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/neuphonic/neucodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/neuphonic/neucodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/neuphonic/neucodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/neuphonic/neucodec/blob/a940021324df13a07f44185f0a17c6dac3698179/README.md","review_level":"license file or model-card declaration"},{"title":"Finite Scalar Quantization Enables Redundant and Transmission-Robust Neural Audio Compression at Low Bit-rates","url":"https://arxiv.org/abs/2509.09550","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--neucodec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/neuphonic/neucodec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/neucodec.svg","sha256":"16ba80dbaaa03e526bb61ccaf9deaa39fef40affe26d6bcef0526506bf736113"}},{"id":"codec:nanocodec","slug":"nanocodec","name":"NanoCodec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Provides low-frame-rate speech tokens with a compact, fast decoder.","architecture":"Noncausal convolutional encoder, finite scalar quantization and causal HiFi-GAN-style decoder with speech-model adversarial training.","notes":"The 0.6 kbps variant is intended for limited-speaker fine-tuning according to the model card. A causal decoder does not make the full codec causal.","variants":["1.78 kbps / 12.5 fps","1.89 kbps / 21.5 fps","0.6 kbps specialized fine-tuning variant"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"22.05 kHz","frame_rate":"12.5 (also 21.5)","bitrate":"1.78 kbps (also 1.89)","representation":"FSQ; 13 × 2,016 combinations","streaming":"Causal decoder; noncausal encoder"},"date":"2025-08-07","dateKind":"paper","links":{"code":"https://github.com/NVIDIA-NeMo/Speech","weights":"https://huggingface.co/nvidia/nemo-nano-codec-22khz-1.78kbps-12.5fps","paper":"https://arxiv.org/abs/2508.05835","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#nanocodec"},"license":{"status":"restricted","code":"Apache-2.0","weights":"NVIDIA Open Model License","codeUrl":"https://github.com/NVIDIA-NeMo/Speech/blob/main/LICENSE","weightsUrl":"https://huggingface.co/nvidia/nemo-nano-codec-22khz-1.78kbps-12.5fps/blob/c4ab84a92c8d36a8b5a79eaea807cfaf7f03ed86/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/NVIDIA-NeMo/Speech","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/nvidia/nemo-nano-codec-22khz-1.78kbps-12.5fps","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/NVIDIA-NeMo/Speech/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/nvidia/nemo-nano-codec-22khz-1.78kbps-12.5fps/blob/c4ab84a92c8d36a8b5a79eaea807cfaf7f03ed86/README.md","review_level":"license file or model-card declaration"},{"title":"NanoCodec: Towards High-Quality Ultra Fast Speech LLM Inference","url":"https://arxiv.org/abs/2508.05835","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--nanocodec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/NVIDIA-NeMo/Speech","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/nanocodec.svg","sha256":"4fe2fc7c96b1ac5a6b7d1d798b6cb0a553154027851f59cbc0f917e769922ba6"}},{"id":"codec:lfsc","slug":"lfsc","name":"Low Frame-rate Speech Codec (LFSC)","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Reduces acoustic frame rate for speech language-model training and inference.","architecture":"Convolutional speech encoder/decoder with quantization and a pretrained speech-model discriminator.","notes":"The linked release is the NeMo 22 kHz codec. Decoder throughput and full-codec streaming are separate properties.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"22.05 kHz","frame_rate":"21.53","bitrate":"Not specified","representation":"Finite scalar quantization","streaming":"Not specified"},"date":"2024-09-18","dateKind":"paper","links":{"code":"https://github.com/NVIDIA-NeMo/Speech","weights":"https://huggingface.co/nvidia/low-frame-rate-speech-codec-22khz","paper":"https://arxiv.org/abs/2409.12117","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#lfsc"},"license":{"status":"restricted","code":"Apache-2.0","weights":"NVIDIA Open Model License","codeUrl":"https://github.com/NVIDIA-NeMo/Speech/blob/main/LICENSE","weightsUrl":"https://huggingface.co/nvidia/low-frame-rate-speech-codec-22khz/blob/933d7a82d932bb58187c3a75ae10178004325e85/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/NVIDIA-NeMo/Speech","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/nvidia/low-frame-rate-speech-codec-22khz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/NVIDIA-NeMo/Speech/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/nvidia/low-frame-rate-speech-codec-22khz/blob/933d7a82d932bb58187c3a75ae10178004325e85/README.md","review_level":"license file or model-card declaration"},{"title":"Low Frame-rate Speech Codec: a Codec Designed for Fast High-quality Speech LLM Training and Inference","url":"https://arxiv.org/abs/2409.12117","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--lfsc.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/NVIDIA-NeMo/Speech","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/lfsc.svg","sha256":"0e04344a95ef99e2a2715099fda5393d60b80469a35043cc51fe4b40e16b590b"}},{"id":"codec:stable-codec","slug":"stable-codec","name":"Stable Codec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Uses large Transformer autoencoders for low-bitrate speech coding.","architecture":"Transformer-based speech autoencoder with finite scalar quantization and a post-hoc bottleneck.","notes":"Code and model weights have different licenses; checkpoint access is gated. The paper studies scaling and is not a license declaration. The checkpoint download requires accepting the publisher's access terms.","variants":["stable-codec-speech-16k","Continuous base variant"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"Not specified","bitrate":"Not specified","representation":"FSQ / continuous base variant","streaming":"Not specified"},"date":"2024-11-29","dateKind":"paper","links":{"code":"https://github.com/Stability-AI/stable-codec","weights":"https://huggingface.co/stabilityai/stable-codec-speech-16k","paper":"https://arxiv.org/abs/2411.19842","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#stable-codec"},"license":{"status":"restricted","code":"MIT","weights":"Stability AI Community License","codeUrl":"https://github.com/Stability-AI/stable-codec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stabilityai/stable-codec-speech-16k/blob/15c3ee1d7b5e60071f4f4a847f96804fbb963d3a/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Stability-AI/stable-codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/stabilityai/stable-codec-speech-16k","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Stability-AI/stable-codec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/stabilityai/stable-codec-speech-16k/blob/15c3ee1d7b5e60071f4f4a847f96804fbb963d3a/README.md","review_level":"license file or model-card declaration"},{"title":"Scaling Transformers for Low-Bitrate High-Quality Speech Coding","url":"https://arxiv.org/abs/2411.19842","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--stable-codec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/Stability-AI/stable-codec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/stable-codec.svg","sha256":"dbee80e2c0e33746855f91f0c7a164f25c3f5452d3f9cfc885a8d348ceb9112b"}},{"id":"codec:qwen3-tts-tokenizer","slug":"qwen3-tts-tokenizer","name":"Qwen3-TTS-Tokenizer-12Hz","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Encodes and reconstructs speech using low-rate, multi-codebook tokens.","architecture":"Semantic/acoustic tokenization with a 16-layer codebook stack and a lightweight causal convolutional decoder.","notes":"The released 12Hz model actually uses 12.5 frames/s. The separately reported 25Hz tokenizer is tracked in the research audit until a matching public codec release is verified.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"12.5","bitrate":"Not specified","representation":"16-layer multi-codebook representation","streaming":"Streaming waveform decoding"},"date":"2026-01-22","dateKind":"release announcement","links":{"code":"https://github.com/QwenLM/Qwen3-TTS","weights":"https://huggingface.co/Qwen/Qwen3-TTS-Tokenizer-12Hz","paper":"https://arxiv.org/abs/2601.15621","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#qwen3-tts-tokenizer"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/QwenLM/Qwen3-TTS/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Qwen/Qwen3-TTS-Tokenizer-12Hz/blob/7dd38ad4e9bad454aae9cd937d0cd577604fe229/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/QwenLM/Qwen3-TTS","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Qwen/Qwen3-TTS-Tokenizer-12Hz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/QwenLM/Qwen3-TTS/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Qwen/Qwen3-TTS-Tokenizer-12Hz/blob/7dd38ad4e9bad454aae9cd937d0cd577604fe229/README.md","review_level":"license file or model-card declaration"},{"title":"Qwen3-TTS Technical Report","url":"https://arxiv.org/abs/2601.15621","review_level":"primary paper abstract"},{"title":"Tokenizer sample rates, stride and codebooks","url":"https://huggingface.co/Qwen/Qwen3-TTS-Tokenizer-12Hz/blob/7dd38ad4e9bad454aae9cd937d0cd577604fe229/config.json","review_level":"inspected model configuration"}],"figure":{"path":"assets/figures/codec--qwen3-tts-tokenizer.webp","kind":"source","label":"Figure 2 — tokenizer family overview; 12.5 Hz variant","sourceUrl":"https://arxiv.org/abs/2601.15621","originUrl":"https://arxiv.org/html/2601.15621v1/figures/Tokenizer.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/qwen3-tts-tokenizer.png","sha256":"9cdde9a7d47ae02f2b83e3584d009dbc57fe9b8201a2af1dd4fbd758cbbcc30a"}},{"id":"codec:moss-audio-tokenizer","slug":"moss-audio-tokenizer","name":"MOSS-Audio-Tokenizer","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Provides low-rate semantic/acoustic tokens across speech, music and sound effects.","architecture":"CAT: causal Transformer encoder and decoder with variable-depth RVQ and joint semantic/reconstruction training.","notes":"Configuration and reconstruction behavior refer to the linked author release. Other family variants may differ.","variants":["PyTorch","ONNX / TensorRT exports"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono","frame_rate":"12.5","bitrate":"0.125–4 kbps","representation":"1–32 RVQ codebooks","streaming":"Streaming encoder and decoder"},"date":"2026-02-09","dateKind":"release announcement","links":{"code":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","weights":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer","paper":"https://arxiv.org/abs/2602.10934","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#moss-audio-tokenizer"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer/blob/3cd226ba2947efa357ef453bcad111b6eafba782/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer/blob/3cd226ba2947efa357ef453bcad111b6eafba782/README.md","review_level":"license file or model-card declaration"},{"title":"MOSS-Audio-Tokenizer: Scaling Audio Tokenizers for Future Audio Foundation Models","url":"https://arxiv.org/abs/2602.10934","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--moss-audio-tokenizer.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","originUrl":"https://raw.githubusercontent.com/OpenMOSS/MOSS-Audio-Tokenizer/main/images/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/moss-audio-tokenizer.png","sha256":"6a7108429fc230ead608a22837a27615aae3458f86e5e47d12fd3bd5d95c7058"}},{"id":"codec:moss-audio-tokenizer-v2","slug":"moss-audio-tokenizer-v2","name":"MOSS-Audio-Tokenizer v2","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Extends the MOSS codec interface to native stereo audio at 48 kHz.","architecture":"Larger stereo CAT encoder/decoder with causal Transformers and a 32-layer RVQ stack.","notes":"A named stereo generation with its own checkpoint and model card; the shared family paper describes the underlying CAT design.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz stereo","frame_rate":"12.5","bitrate":"Not specified","representation":"Up to 32 RVQ codebooks","streaming":"Streaming encoder and decoder"},"date":"2026-06-07","dateKind":"release announcement","links":{"code":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","weights":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2","paper":"https://arxiv.org/abs/2602.10934","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#moss-audio-tokenizer-v2"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2/blob/f6e20e543b33d2c252a7ef71bdf8aa71e5ff9169/LICENSE","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-v2/blob/f6e20e543b33d2c252a7ef71bdf8aa71e5ff9169/LICENSE","review_level":"license file or model-card declaration"},{"title":"MOSS-Audio-Tokenizer: Scaling Audio Tokenizers for Future Audio Foundation Models","url":"https://arxiv.org/abs/2602.10934","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--moss-audio-tokenizer-v2.webp","kind":"source","label":"CAT family architecture (shared; variant details in model card)","sourceUrl":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","originUrl":"https://raw.githubusercontent.com/OpenMOSS/MOSS-Audio-Tokenizer/main/images/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/moss-audio-tokenizer-v2.png","sha256":"6a7108429fc230ead608a22837a27615aae3458f86e5e47d12fd3bd5d95c7058"}},{"id":"codec:moss-audio-tokenizer-nano","slug":"moss-audio-tokenizer-nano","name":"MOSS-Audio-Tokenizer Nano","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Provides a compact stereo audio codec for lower-cost deployment.","architecture":"Compact causal Transformer CAT encoder/decoder with a 16-codebook residual quantizer.","notes":"The model card reports approximately 22M parameters. This named compact release has its own checkpoint and a different quantizer depth from the full-size family.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz stereo","frame_rate":"12.5","bitrate":"0.125–2 kbps","representation":"1–16 × 1,024-entry RVQ","streaming":"Streaming encoder and decoder"},"date":"2026-04-13","dateKind":"release announcement","links":{"code":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","weights":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano","paper":"https://arxiv.org/abs/2602.10934","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#moss-audio-tokenizer-nano"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano/blob/6aa02b01e445cc585582cf0ba480bc3ea6c8dd68/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano/blob/6aa02b01e445cc585582cf0ba480bc3ea6c8dd68/README.md","review_level":"license file or model-card declaration"},{"title":"MOSS-Audio-Tokenizer: Scaling Audio Tokenizers for Future Audio Foundation Models","url":"https://arxiv.org/abs/2602.10934","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--moss-audio-tokenizer-nano.webp","kind":"source","label":"CAT family architecture (shared; variant details in model card)","sourceUrl":"https://github.com/OpenMOSS/MOSS-Audio-Tokenizer","originUrl":"https://raw.githubusercontent.com/OpenMOSS/MOSS-Audio-Tokenizer/main/images/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/moss-audio-tokenizer-nano.png","sha256":"6a7108429fc230ead608a22837a27615aae3458f86e5e47d12fd3bd5d95c7058"}},{"id":"codec:higgs-audio-tokenizer","slug":"higgs-audio-tokenizer","name":"Higgs Audio Tokenizer v2","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Compresses speech, music and sound events into low-frame-rate audio tokens.","architecture":"Semantic/acoustic discrete audio tokenizer with a non-diffusion encoder/decoder.","notes":"The tokenizer checkpoint has its own community license. Fast batch inference is not treated as evidence of causal streaming.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"25","bitrate":"Not specified","representation":"Multi-codebook audio tokens","streaming":"Not specified"},"date":"","dateKind":"","links":{"code":"https://github.com/boson-ai/higgs-audio","weights":"https://huggingface.co/bosonai/higgs-audio-v2-tokenizer","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#higgs-audio-tokenizer"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Boson Higgs Audio 2 Community License","codeUrl":"https://github.com/boson-ai/higgs-audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/bosonai/higgs-audio-v2-tokenizer/blob/403fbacf2f60caaa102f893fdfabb694619b2417/LICENSE","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/boson-ai/higgs-audio","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/bosonai/higgs-audio-v2-tokenizer","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/boson-ai/higgs-audio/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/bosonai/higgs-audio-v2-tokenizer/blob/403fbacf2f60caaa102f893fdfabb694619b2417/LICENSE","review_level":"license file or model-card declaration"}],"figure":{"path":"assets/figures/codec--higgs-audio-tokenizer.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/boson-ai/higgs-audio","originUrl":"https://raw.githubusercontent.com/boson-ai/higgs-audio/main/figures/higgs_audio_tokenizer_architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/higgs-audio-tokenizer.png","sha256":"f7c0177e38dd9c873acf8ac55c159ce65ba50970cbeba9663582da4698037447"}},{"id":"codec:mimo-audio-tokenizer","slug":"mimo-audio-tokenizer","name":"MiMo-Audio-Tokenizer","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Learns reconstructable audio tokens with joint semantic and acoustic objectives.","architecture":"Transformer audio encoder/decoder with an eight-layer residual vector quantizer.","notes":"The downstream MiMo language model groups four codec frames into 6.25 Hz patches. That patch rate is not the codec frame rate. The model card describes eight active layers (200 tokens/s), while the released configuration declares a maximum of 20 quantizers. Use the documented active-codebook setting.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"25","bitrate":"Not specified","representation":"RVQ; report uses 8 active layers","streaming":"Causal decoder; noncausal encoder"},"date":"2025-12-29","dateKind":"paper","links":{"code":"https://github.com/XiaomiMiMo/MiMo-Audio","weights":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-Tokenizer","paper":"https://arxiv.org/abs/2512.23808","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#mimo-audio-tokenizer"},"license":{"status":"open","code":"Apache-2.0","weights":"MIT","codeUrl":"https://github.com/XiaomiMiMo/MiMo-Audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-Tokenizer/blob/5df9914f72d3acda1320d7fecde7d91622edb0c1/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/XiaomiMiMo/MiMo-Audio","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-Tokenizer","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/XiaomiMiMo/MiMo-Audio/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-Tokenizer/blob/5df9914f72d3acda1320d7fecde7d91622edb0c1/README.md","review_level":"license file or model-card declaration"},{"title":"MiMo-Audio: Audio Language Models are Few-Shot Learners","url":"https://arxiv.org/abs/2512.23808","review_level":"primary paper abstract"},{"title":"Tokenizer architecture and quantizer configuration","url":"https://huggingface.co/XiaomiMiMo/MiMo-Audio-Tokenizer/blob/5df9914f72d3acda1320d7fecde7d91622edb0c1/config.json","review_level":"inspected model configuration"}],"figure":{"path":"assets/figures/codec--mimo-audio-tokenizer.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/XiaomiMiMo/MiMo-Audio","originUrl":"https://raw.githubusercontent.com/Xiaomimimo/MiMo-Audio/main/assets/tokenizer.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/mimo-audio-tokenizer.png","sha256":"d8e4d26fea4419e3f3d09d1c44454d9d46b3d62bca6f4fb15189f0c8268f1a6a"}},{"id":"codec:longcat-audio-codec","slug":"longcat-audio-codec","name":"LongCat-Audio-Codec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Extracts semantic and acoustic speech tokens in parallel with selectable decoders.","architecture":"Parallel semantic/acoustic encoding and dedicated 16 kHz or 24 kHz waveform decoders.","notes":"The two-codebook 24 kHz decoder is fine-tuned on limited speakers. Select the decoder and active codebook count documented for the checkpoint.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz input; 16 / 24 kHz output","frame_rate":"~16.6","bitrate":"Not specified","representation":"1 semantic + up to 3 acoustic codebooks","streaming":"Not specified"},"date":"2025-10-17","dateKind":"paper","links":{"code":"https://github.com/meituan-longcat/LongCat-Audio-Codec","weights":"https://huggingface.co/meituan-longcat/LongCat-Audio-Codec","paper":"https://arxiv.org/abs/2510.15227","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#longcat-audio-codec"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/meituan-longcat/LongCat-Audio-Codec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/meituan-longcat/LongCat-Audio-Codec/blob/5c2493c6c68959bd9ad4b06cc3df9292fc1d670f/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/meituan-longcat/LongCat-Audio-Codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/meituan-longcat/LongCat-Audio-Codec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/meituan-longcat/LongCat-Audio-Codec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/meituan-longcat/LongCat-Audio-Codec/blob/5c2493c6c68959bd9ad4b06cc3df9292fc1d670f/README.md","review_level":"license file or model-card declaration"},{"title":"LongCat-Audio-Codec: An Audio Tokenizer and Detokenizer Solution Designed for Speech Large Language Models","url":"https://arxiv.org/abs/2510.15227","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--longcat-audio-codec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/meituan-longcat/LongCat-Audio-Codec","originUrl":"https://raw.githubusercontent.com/meituan-longcat/LongCat-Audio-Codec/main/images/codec-Frame.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/longcat-audio-codec.png","sha256":"96f7e143d66b6d417697f4d46baa3bbe0799b9bd92479ecc6405109646b6b4fc"}},{"id":"codec:jhcodec","slug":"jhcodec","name":"JHCodec","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Uses self-supervised representation reconstruction to improve streaming speech tokens.","architecture":"Residual quantization with a Transformer-based codec and representation-reconstruction supervision.","notes":"The nominal bitrate is calculated for the documented eight-codebook configuration. The 1.4M-step checkpoint is selected, following the updated reference implementation.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz mono","frame_rate":"50","bitrate":"4 kbps (8 × 1,024 codes at 50 Hz)","representation":"RVQ; 1,024-entry codebooks","streaming":"Streaming encoder and decoder"},"date":"2026-03-06","dateKind":"paper","links":{"code":"https://github.com/jhcodec843/jhcodec","weights":"https://huggingface.co/jhcodec/jhcodec_1.4m","paper":"https://arxiv.org/abs/2603.05887","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#jhcodec"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/jhcodec843/jhcodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/jhcodec/jhcodec_1.4m/blob/24910ef4f1e4739395340fee1fe1bb386dee952c/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/jhcodec843/jhcodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/jhcodec/jhcodec_1.4m","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/jhcodec843/jhcodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/jhcodec/jhcodec_1.4m/blob/24910ef4f1e4739395340fee1fe1bb386dee952c/README.md","review_level":"license file or model-card declaration"},{"title":"Reconstruct! Don't Encode: Self-Supervised Representation Reconstruction Loss for High-Intelligibility and Low-Latency Streaming Neural Audio Codec","url":"https://arxiv.org/abs/2603.05887","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--jhcodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/jhcodec843/jhcodec","originUrl":"https://raw.githubusercontent.com/jhcodec843/jhcodec/main/asset/overall.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/jhcodec.png","sha256":"c2598bc4cd8be24359f265477d90142b6a45b3dc3b572e8b7009551f30ed2ee6"}},{"id":"codec:harp","slug":"harp","name":"HARP","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Allocates residual codebook capacity across harmonic frequency bands.","architecture":"Harmonic-aware residual partitioning with frequency-band codebook groups for variable-bitrate reconstruction.","notes":"The rates are the author-reported approximations for nine 1,024-entry codebooks and progressively enabled groups.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Not specified","frame_rate":"~86","bitrate":"~2.6 / 4.3 / 6.0 / 7.7 kbps","representation":"3 / 5 / 7 / 9 RVQ codebooks","streaming":"Not specified"},"date":"2026-07-18","dateKind":"paper","links":{"code":"https://github.com/QiaoyuYang/harp-codec","weights":"https://huggingface.co/KelvinYang/harp-codec","paper":"https://arxiv.org/abs/2607.16657","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#harp"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/QiaoyuYang/harp-codec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/KelvinYang/harp-codec/blob/220ee5677707dc77297962d3654c18c4863887a1/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/QiaoyuYang/harp-codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/KelvinYang/harp-codec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/QiaoyuYang/harp-codec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/KelvinYang/harp-codec/blob/220ee5677707dc77297962d3654c18c4863887a1/README.md","review_level":"license file or model-card declaration"},{"title":"HARP: Harmonic-Aware Residual Partitioning for Neural Audio Codecs","url":"https://arxiv.org/abs/2607.16657","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--harp.webp","kind":"source","label":"Figure 1","sourceUrl":"https://arxiv.org/abs/2607.16657","originUrl":"https://arxiv.org/html/2607.16657v1/harp_title.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/harp.png","sha256":"d1bd6f7ae46362e7762cc06edac077098ce10e7144ef7f262123cec74ac74530"}},{"id":"codec:mucodec","slug":"mucodec","name":"MuCodec","collection":"codec","category":"codec:music","categoryName":"Music codecs","summary":"Reconstructs stereo music from an ultra-low-bitrate representation.","architecture":"Music feature extraction and discrete compression with generative audio reconstruction.","notes":"The authors release the 0.35 kbps configuration. Reconstruction requires mucodec.pt, muq.pt and the AudioLDM 48 kHz checkpoint; the latter has its own noncommercial terms.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz stereo","frame_rate":"Not specified","bitrate":"0.35 kbps (released configuration)","representation":"Discrete music representation","streaming":"Not specified"},"date":"2024-09-20","dateKind":"paper","links":{"code":"https://github.com/tencent-ailab/MuCodec","weights":"https://huggingface.co/yaoxunxu/mucodec","paper":"https://arxiv.org/abs/2409.13216","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/music.md#mucodec"},"license":{"status":"restricted","code":"MIT","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/tencent-ailab/MuCodec/blob/master/LICENSE","weightsUrl":"https://github.com/xuyaoxun/MuCodec/blob/main/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/tencent-ailab/MuCodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/yaoxunxu/mucodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/tencent-ailab/MuCodec/blob/master/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/yaoxunxu/mucodec/blob/0f023842c0248fb20c45f2114f286e0074f83f0a/README.md","review_level":"license file or model-card declaration"},{"title":"MuCodec: Ultra Low-Bitrate Music Codec","url":"https://arxiv.org/abs/2409.13216","review_level":"primary paper abstract"},{"title":"Author checkpoint license declaration","url":"https://github.com/xuyaoxun/MuCodec/blob/main/README.md","review_level":"explicit CC-BY-NC-4.0 declaration for muq.pt and mucodec.pt"}],"figure":{"path":"assets/figures/codec--mucodec.webp","kind":"source","label":"Fig. 1","sourceUrl":"https://arxiv.org/abs/2409.13216","originUrl":"https://arxiv.org/html/2409.13216v3/1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/mucodec.png","sha256":"71ad9046a5cee3a8194deb040c0af87d7fcc654acf87203cd4a541ac64544218"}},{"id":"codec:heartcodec","slug":"heartcodec","name":"HeartCodec","collection":"codec","category":"codec:music","categoryName":"Music codecs","summary":"Encodes music into tokens for the HeartMuLa music-generation family.","architecture":"Semantic-rich encoder, low-frame-rate compressor and generative high-fidelity reconstruction decoder.","notes":"This entry identifies the codec checkpoint only. HeartMuLa music generation and HeartTranscriptor are separate models. The selected codec release is HeartCodec-oss-20260123.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz stereo","frame_rate":"Not specified","bitrate":"Not specified","representation":"8 × 8,192-entry RVQ","streaming":"Not specified"},"date":"2026-01-15","dateKind":"paper","links":{"code":"https://github.com/HeartMuLa/heartlib","weights":"https://huggingface.co/HeartMuLa/HeartCodec-oss-20260123","paper":"https://arxiv.org/abs/2601.10547","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/music.md#heartcodec"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/HeartMuLa/heartlib/blob/main/LICENSE","weightsUrl":"https://huggingface.co/HeartMuLa/HeartCodec-oss-20260123/blob/f889dab0532cfa4bf459f2a3367eb6d346b8eeda/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/HeartMuLa/heartlib","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/HeartMuLa/HeartCodec-oss-20260123","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/HeartMuLa/heartlib/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/HeartMuLa/HeartCodec-oss-20260123/blob/f889dab0532cfa4bf459f2a3367eb6d346b8eeda/README.md","review_level":"license file or model-card declaration"},{"title":"HeartMuLa: A Family of Open Sourced Music Foundation Models","url":"https://arxiv.org/abs/2601.10547","review_level":"primary paper abstract"},{"title":"HeartCodec sample-rate and quantizer configuration","url":"https://huggingface.co/HeartMuLa/HeartCodec-oss-20260123/blob/f889dab0532cfa4bf459f2a3367eb6d346b8eeda/config.json","review_level":"inspected model configuration"}],"figure":{"path":"assets/figures/codec--heartcodec.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2601.10547","originUrl":"https://arxiv.org/html/2601.10547v3/HeartCodec.drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/heartcodec.png","sha256":"25b1ef0310d687258b551db67e56a932a0bd99cf5c5542465c5cfb7e8f492791"}},{"id":"codec:codicodec","slug":"codicodec","name":"CoDiCodec","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Offers continuous latents or discrete tokens through a shared stereo audio codec.","architecture":"Generative audio autoencoder with optional finite scalar quantization.","notes":"The continuous representation has 64 channels. The nominal discrete bitrate and latent frame rate refer to different documented interfaces.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 / 48 kHz stereo","frame_rate":"~11 (continuous latent sequence)","bitrate":"2.38 kbps (discrete mode)","representation":"Continuous latents or FSQ","streaming":"Not specified"},"date":"2025-09-11","dateKind":"paper","links":{"code":"https://github.com/SonyCSLParis/codicodec","weights":"https://huggingface.co/SonyCSLParis/codicodec","paper":"https://arxiv.org/abs/2509.09836","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#codicodec"},"license":{"status":"restricted","code":"CC-BY-NC-4.0","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/SonyCSLParis/codicodec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/SonyCSLParis/codicodec/blob/f6aa343c06104f743563e86485e69e55dfd30d58/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/SonyCSLParis/codicodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/SonyCSLParis/codicodec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/SonyCSLParis/codicodec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/SonyCSLParis/codicodec/blob/f6aa343c06104f743563e86485e69e55dfd30d58/README.md","review_level":"license file or model-card declaration"},{"title":"CoDiCodec: Unifying Continuous and Discrete Compressed Representations of Audio","url":"https://arxiv.org/abs/2509.09836","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--codicodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/SonyCSLParis/codicodec","originUrl":"https://raw.githubusercontent.com/SonyCSLParis/codicodec/main/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/codicodec.png","sha256":"0fe97cde5c13ae5ec8d26ff264973169865795f88a71a57faee25b99603bd1ae"}},{"id":"codec:music2latent","slug":"music2latent","name":"Music2Latent","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes music and speech into compact continuous latents and reconstructs the waveform.","architecture":"Consistency autoencoder for audio compression and generative modeling.","notes":"A generative audio autoencoder rather than a discrete token transport codec. A bitrate would require an additional quantization/serialization scheme.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz (also 48 kHz)","frame_rate":"~10 (44.1 kHz); ~12 (48 kHz)","bitrate":"Not applicable: continuous latents","representation":"64-channel continuous latents","streaming":"Not specified"},"date":"2024-08-12","dateKind":"paper","links":{"code":"https://github.com/SonyCSLParis/music2latent","weights":"https://huggingface.co/SonyCSLParis/music2latent","paper":"https://arxiv.org/abs/2408.06500","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#music2latent"},"license":{"status":"restricted","code":"CC-BY-NC-4.0","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/SonyCSLParis/music2latent/blob/master/LICENSE","weightsUrl":"https://huggingface.co/SonyCSLParis/music2latent/blob/faab47a447a2b663c77df2969e984e411e1d0c8d/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/SonyCSLParis/music2latent","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/SonyCSLParis/music2latent","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/SonyCSLParis/music2latent/blob/master/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/SonyCSLParis/music2latent/blob/faab47a447a2b663c77df2969e984e411e1d0c8d/README.md","review_level":"license file or model-card declaration"},{"title":"Music2Latent: Consistency Autoencoders for Latent Audio Compression","url":"https://arxiv.org/abs/2408.06500","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--music2latent.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/SonyCSLParis/music2latent","originUrl":"https://raw.githubusercontent.com/SonyCSLParis/music2latent/master/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/music2latent.png","sha256":"d77108d6a5af3ebb70bd968b669f0d7e5445904f5fc85b27801917c02848a7c0"}},{"id":"codec:semantic-vae","slug":"semantic-vae","name":"Semantic-VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Aligns continuous speech latents with semantic information for speech synthesis.","architecture":"Variational speech autoencoder with semantic alignment during representation learning.","notes":"The author repository states MIT for code; the checkpoint listing does not provide complete weight terms. The pretrained VAE is a component for waveform reconstruction. The semantic_vae_1000k configuration uses encoder strides [4, 4, 5, 5] (400 samples) and a 64-dimensional VAE/BigVGAN reconstruction path; acoustic-only ablations are grouped as variants.","variants":["Semantic-VAE 1000k","Semantic-VAE 600k","Acoustic VAE 16d","Acoustic VAE 64d"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz mono","frame_rate":"40","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents (semantic checkpoint)","streaming":"Not specified"},"date":"2025-09-26","dateKind":"paper","links":{"code":"https://github.com/ZhikangNiu/Semantic-VAE","weights":"https://huggingface.co/zkniu/Semantic-VAE","paper":"https://arxiv.org/abs/2509.22167","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#semantic-vae"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/ZhikangNiu/Semantic-VAE/blob/main/README.md","weightsUrl":"https://huggingface.co/zkniu/Semantic-VAE","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/ZhikangNiu/Semantic-VAE","review_level":"repository README"},{"title":"Checkpoint file listing","url":"https://huggingface.co/zkniu/Semantic-VAE","review_level":"checkpoint listing; no model card or separate weight license found"},{"title":"Code license evidence","url":"https://github.com/ZhikangNiu/Semantic-VAE/blob/main/README.md","review_level":"license file or explicit author declaration"},{"title":"Semantic-VAE: Semantic-Alignment Latent Representation for Better Speech Synthesis","url":"https://arxiv.org/abs/2509.22167","review_level":"primary paper abstract"},{"title":"Checkpoint configuration","url":"https://huggingface.co/zkniu/Semantic-VAE/blob/17921c81751470dd996935535929b2cc4413b606/semantic_vae_1000k/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--semantic-vae.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2509.22167","originUrl":"https://arxiv.org/html/2509.22167v3/svae0821.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/semantic-vae.png","sha256":"0e7597b58c666168f5e62e17b0bfa30850c2b0d1119ac7141ba65676b91d2696"}},{"id":"codec:stable-audio-autoencoder","slug":"stable-audio-autoencoder","name":"Stable Audio Open autoencoder (Oobleck)","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses stereo waveforms into continuous latents for audio diffusion models.","architecture":"Convolutional variational audio autoencoder used as the waveform encoder/decoder of Stable Audio Open.","notes":"Only the standalone vae_model.ckpt component is cataloged. The text encoder and diffusion generation model are outside the codec entry. The checkpoint download requires accepting the publisher's access terms.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz stereo","frame_rate":"Not specified","bitrate":"Not applicable: continuous latents","representation":"Continuous VAE latents","streaming":"Not specified"},"date":"2024-07-19","dateKind":"paper","links":{"code":"https://github.com/Stability-AI/stable-audio-tools","weights":"https://huggingface.co/stabilityai/stable-audio-open-1.0","paper":"https://arxiv.org/abs/2407.14358","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#stable-audio-autoencoder"},"license":{"status":"restricted","code":"MIT","weights":"Stability AI Community License","codeUrl":"https://github.com/Stability-AI/stable-audio-tools/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stabilityai/stable-audio-open-1.0/blob/f21265c1e2710b3bd2386596943f0007f55f802e/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Stability-AI/stable-audio-tools","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/stabilityai/stable-audio-open-1.0","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Stability-AI/stable-audio-tools/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/stabilityai/stable-audio-open-1.0/blob/f21265c1e2710b3bd2386596943f0007f55f802e/README.md","review_level":"license file or model-card declaration"},{"title":"Stable Audio Open","url":"https://arxiv.org/abs/2407.14358","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--stable-audio-autoencoder.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/Stability-AI/stable-audio-tools","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/stable-audio-autoencoder.svg","sha256":"6904b104b69ec254b40f16c6aceead8430fcab690d971f888c012e49b3953516"}},{"id":"codec:vibevoice-acoustic-tokenizer","slug":"vibevoice-acoustic-tokenizer","name":"VibeVoice acoustic tokenizer","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses speech into low-frame-rate continuous acoustic representations.","architecture":"Continuous acoustic encoder/decoder paired with a separate semantic tokenizer in VibeVoice.","notes":"The acoustic encoder and decoder now have a standalone publisher checkpoint. This entry excludes the separate semantic encoder and the parent TTS model. The fixed-noise variational bottleneck is documented by the tokenizer configuration.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"7.5","bitrate":"Not applicable: continuous latents","representation":"64-dimensional continuous acoustic latents; fixed-noise VAE","streaming":"Not specified"},"date":"2025-08-26","dateKind":"paper","links":{"code":"https://github.com/microsoft/VibeVoice","weights":"https://huggingface.co/microsoft/VibeVoice-AcousticTokenizer","paper":"https://arxiv.org/abs/2508.19205","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#vibevoice-acoustic-tokenizer"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/microsoft/VibeVoice/blob/main/LICENSE","weightsUrl":"https://huggingface.co/microsoft/VibeVoice-AcousticTokenizer/blob/f0643be7cab5b9c91c6f0038e3b8dde13e3e107e/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/microsoft/VibeVoice","review_level":"repository README"},{"title":"Code license evidence","url":"https://github.com/microsoft/VibeVoice/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"VibeVoice Technical Report","url":"https://arxiv.org/abs/2508.19205","review_level":"primary paper abstract"},{"title":"Standalone acoustic tokenizer","url":"https://huggingface.co/microsoft/VibeVoice-AcousticTokenizer","review_level":"model card and checkpoint file listing"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/microsoft/VibeVoice-AcousticTokenizer/blob/f0643be7cab5b9c91c6f0038e3b8dde13e3e107e/README.md","review_level":"model-card declaration"},{"title":"Checkpoint configuration","url":"https://huggingface.co/microsoft/VibeVoice-AcousticTokenizer/blob/f0643be7cab5b9c91c6f0038e3b8dde13e3e107e/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--vibevoice-acoustic-tokenizer.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/microsoft/VibeVoice","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/vibevoice-acoustic-tokenizer.svg","sha256":"3f39311519b22b1a82bf01040b52cb3d2252d7923583ee5a476107e414058b16"}},{"id":"codec:l3ac","slug":"l3ac","name":"L3AC / SQCodec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Uses a single quantizer in a lightweight speech reconstruction model.","architecture":"Lightweight convolutional codec with a single large scalar-quantized bottleneck.","notes":"The authors direct SQCodec users to L3AC under the same paper identifier. They are grouped here to avoid counting a renamed project twice. The repository title uses the word lossless, but this catalog makes no bit-exact waveform-losslessness claim.","variants":["L3AC low-bitrate releases","Earlier SQCodec 3–24 kbps family"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"44.44 / 59.26 / 88.89 / 166.67","bitrate":"~0.75 / 1 / 1.5 / 3 kbps","representation":"Single quantizer","streaming":"Not specified"},"date":"2025-04-07","dateKind":"paper","links":{"code":"https://github.com/zhai-lw/L3AC","weights":"https://huggingface.co/zhai-lw/L3AC","paper":"https://arxiv.org/abs/2504.04949","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#l3ac"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/zhai-lw/L3AC/blob/main/LICENSE","weightsUrl":"https://huggingface.co/zhai-lw/L3AC/blob/7f3513a0f2ee65e4527b1f2d21baf929c0990023/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/zhai-lw/L3AC","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/zhai-lw/L3AC","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/zhai-lw/L3AC/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/zhai-lw/L3AC/blob/7f3513a0f2ee65e4527b1f2d21baf929c0990023/README.md","review_level":"license file or model-card declaration"},{"title":"L3AC: Towards a Lightweight and Lossless Audio Codec","url":"https://arxiv.org/abs/2504.04949","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--l3ac.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/zhai-lw/L3AC","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/l3ac.svg","sha256":"9b314ec238905235cce0da47f16fdcec459d570bd45e8478d0283068061c7580"}},{"id":"codec:lilac","slug":"lilac","name":"LILAC","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Targets low-bitrate speech compression with stable codes under repeated decode/re-encode cycles.","architecture":"Reversible chart, projective stages, finite scalar quantization and coarse-to-fine coordinate filling.","notes":"The repository is a public PyTorch reproduction of the reported JAX checkpoint. Integer-code idempotence is a documented property; waveform losslessness is not claimed.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono","frame_rate":"9.375","bitrate":"0.75 kbps","representation":"20 scalar coordinates × 16 levels","streaming":"Not specified"},"date":"2026-08-06","dateKind":"paper","links":{"code":"https://github.com/Rick-McCoy/lilac-codec","weights":"https://huggingface.co/julianyi1/lilac","paper":"https://arxiv.org/abs/2608.05727","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#lilac"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/Rick-McCoy/lilac-codec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/julianyi1/lilac/blob/e65bc9974a54511aaba615df73e143e5f468564c/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Rick-McCoy/lilac-codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/julianyi1/lilac","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Rick-McCoy/lilac-codec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/julianyi1/lilac/blob/e65bc9974a54511aaba615df73e143e5f468564c/README.md","review_level":"license file or model-card declaration"},{"title":"LILAC: An Idempotent Neural Speech Codec","url":"https://arxiv.org/abs/2608.05727","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--lilac.svg","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/Rick-McCoy/lilac-codec","originUrl":"https://raw.githubusercontent.com/Rick-McCoy/lilac-codec/main/docs/lilac-architecture.svg","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/lilac.svg","sha256":"4761500c4c184e57424aeb02deebb6e2d5b9a7f649d56b760032824a41bd4956"}},{"id":"codec:spine","slug":"spine","name":"Spine","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Encodes expressive speech with token streams at several temporal scales.","architecture":"Multiscale FSQ bottleneck with a band-limited convolutional decoder and high-band synthesis branches.","notes":"The total token count sums all temporal scales. The high-frequency synthesis branch is part of the documented reconstruction design.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono","frame_rate":"~6 / 12 / 23 / 47 by scale","bitrate":"1.57 kbps","representation":"Multiscale FSQ; ~88 total tokens/s","streaming":"Not specified"},"date":"","dateKind":"","links":{"code":"https://github.com/twangodev/spine-codec","weights":"https://huggingface.co/twangodev/spine-codec","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#spine"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/twangodev/spine-codec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/twangodev/spine-codec/blob/3fb5c8d4e6d3411c19bb3c432d75563f7c45ea43/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/twangodev/spine-codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/twangodev/spine-codec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/twangodev/spine-codec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/twangodev/spine-codec/blob/3fb5c8d4e6d3411c19bb3c432d75563f7c45ea43/README.md","review_level":"license file or model-card declaration"}],"figure":{"path":"assets/figures/codec--spine.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/twangodev/spine-codec","originUrl":"https://raw.githubusercontent.com/twangodev/spine-codec/main/.github/assets/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/spine.png","sha256":"ba5e4e52563dd8c619ae0378cf5f5c39813a57e52238e23ebf265f68ef6050cd"}},{"id":"codec:soviamate-codec","slug":"soviamate-codec","name":"SoviaMate-Codec","collection":"codec","category":"codec:disentangled","categoryName":"Disentangled codecs","summary":"Provides reconstruction and speaker-conditioned codec checkpoints with enhancement training.","architecture":"Audio encoder/quantizer/decoder; speaker-adaptation variant adds an ASR head and a post-quantization speaker adapter.","notes":"Use the base checkpoint for plain reconstruction. The speaker-conditioned checkpoint requires a reference prompt; bundled speaker-verification models retain separate upstream terms.","variants":["Base reconstruction checkpoint","Speaker-adaptation checkpoint"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Not specified","frame_rate":"Not specified","bitrate":"Not specified","representation":"Discrete acoustic bottleneck","streaming":"Not specified"},"date":"","dateKind":"","links":{"code":"https://github.com/samson-ailabs/SoviaMate","weights":"https://huggingface.co/samson-ailabs/SoviaMate-Codec","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/disentangled.md#soviamate-codec"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/samson-ailabs/SoviaMate/blob/main/LICENSE","weightsUrl":"https://huggingface.co/samson-ailabs/SoviaMate-Codec/blob/8d8f3c3e4ab9009ccc25622afae460bf6f52bf97/README.md","notes":"The model card adds usage restrictions alongside its Apache-2.0 declaration.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/samson-ailabs/SoviaMate","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/samson-ailabs/SoviaMate-Codec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/samson-ailabs/SoviaMate/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/samson-ailabs/SoviaMate-Codec/blob/8d8f3c3e4ab9009ccc25622afae460bf6f52bf97/README.md","review_level":"license file or model-card declaration"}],"figure":{"path":"assets/figures/codec--soviamate-codec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/samson-ailabs/SoviaMate","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/soviamate-codec.svg","sha256":"420027d7fef2beed220915404cd6cdae409d49d6e50abefad2a0a0cec84975fb"}},{"id":"codec:firefly-codec","slug":"firefly-codec","name":"Firefly codec (Fish Speech 1.5)","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Encodes speech into grouped finite-scalar tokens for Fish Speech.","architecture":"Firefly convolutional encoder/decoder with grouped residual finite scalar quantization.","notes":"The implementation is the v1.5.1 release and the checkpoint is the bundled Firefly generator. Later OpenAudio/Fish S2 codecs are separate releases.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz","frame_rate":"~21.5","bitrate":"Not specified","representation":"Grouped residual FSQ","streaming":"Not specified"},"date":"","dateKind":"","links":{"code":"https://github.com/fishaudio/fish-speech/tree/v1.5.1","weights":"https://huggingface.co/fishaudio/fish-speech-1.5","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#firefly-codec"},"license":{"status":"restricted","code":"Apache-2.0","weights":"CC-BY-NC-SA-4.0","codeUrl":"https://github.com/fishaudio/fish-speech/blob/v1.5.1/LICENSE","weightsUrl":"https://huggingface.co/fishaudio/fish-speech-1.5/blob/275a984d33c33659e39eed41ff5bcd6e67517f4c/README.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/fishaudio/fish-speech","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/fishaudio/fish-speech-1.5","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/fishaudio/fish-speech/blob/v1.5.1/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/fishaudio/fish-speech-1.5/blob/275a984d33c33659e39eed41ff5bcd6e67517f4c/README.md","review_level":"license file or model-card declaration"},{"title":"Versioned Firefly implementation","url":"https://github.com/fishaudio/fish-speech/tree/v1.5.1","review_level":"v1.5.1 implementation and README"}],"figure":{"path":"assets/figures/codec--firefly-codec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/fishaudio/fish-speech","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/firefly-codec.svg","sha256":"271190f4e85162022c6a4838ba078add2677df8ef32342e65bc5c6b82fd3760d"}},{"id":"codec:fish-audio-codec","slug":"fish-audio-codec","name":"Fish Audio codec (OpenAudio S1 / S2)","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Provides the audio tokenization and reconstruction component used by OpenAudio and Fish S2.","architecture":"DAC-derived audio codec with grouped quantization and waveform reconstruction.","notes":"Only codec.pth is counted. The selected Fish S2 release has custom research terms; the older OpenAudio S1 checkpoint has separate noncommercial terms.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz","frame_rate":"Not specified","bitrate":"Not specified","representation":"Discrete audio tokens","streaming":"Not specified"},"date":"","dateKind":"","links":{"code":"https://github.com/fishaudio/fish-speech","weights":"https://huggingface.co/fishaudio/s2-pro","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#fish-audio-codec"},"license":{"status":"restricted","code":"Fish Audio Research License","weights":"Fish Audio Research License","codeUrl":"https://github.com/fishaudio/fish-speech/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fishaudio/s2-pro/blob/1de9996b6be38b745688de084d87a5633f714e4e/LICENSE.md","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/fishaudio/fish-speech","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/fishaudio/s2-pro","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/fishaudio/fish-speech/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/fishaudio/s2-pro/blob/1de9996b6be38b745688de084d87a5633f714e4e/LICENSE.md","review_level":"license file or model-card declaration"},{"title":"Fish Audio codec configuration","url":"https://github.com/fishaudio/fish-speech/blob/main/fish_speech/configs/modded_dac_vq.yaml","review_level":"inspected codec configuration"}],"figure":{"path":"assets/figures/codec--fish-audio-codec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/fishaudio/fish-speech","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/fish-audio-codec.svg","sha256":"87ce081be853b18c400d46a557f9f953e079b561b5be10329f1da46d31493055"}},{"id":"codec:firered-tokenizer","slug":"firered-tokenizer","name":"FireRedTTS-2 speech tokenizer","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Provides streaming speech tokens for long conversational speech generation.","architecture":"Low-frame-rate speech codec packaged as the encoder/decoder component of FireRedTTS-2.","notes":"Only codec.pt is included in the entry. The two language-model checkpoints in the release are not additional codecs. Bitrate is calculated from 12.5 frames/s × 16 codebooks × 11 bits.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"12.5","bitrate":"2.2 kbps (16 codebooks)","representation":"16 × 2,048-entry RVQ","streaming":"Streaming codec"},"date":"2025-09-02","dateKind":"paper","links":{"code":"https://github.com/FireRedTeam/FireRedTTS2","weights":"https://huggingface.co/FireRedTeam/FireRedTTS2","paper":"https://arxiv.org/abs/2509.02020","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#firered-tokenizer"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/FireRedTeam/FireRedTTS2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/FireRedTeam/FireRedTTS2/blob/4af3f5cc4963373b86b52d750220d4de85261f05/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/FireRedTeam/FireRedTTS2","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/FireRedTeam/FireRedTTS2","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/FireRedTeam/FireRedTTS2/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/FireRedTeam/FireRedTTS2/blob/4af3f5cc4963373b86b52d750220d4de85261f05/README.md","review_level":"license file or model-card declaration"},{"title":"FireRedTTS-2: Towards Long Conversational Speech Generation for Podcast and Chatbot","url":"https://arxiv.org/abs/2509.02020","review_level":"primary paper abstract"},{"title":"FireRed codec configuration","url":"https://huggingface.co/FireRedTeam/FireRedTTS2/blob/4af3f5cc4963373b86b52d750220d4de85261f05/config_codec.json","review_level":"inspected model configuration"}],"figure":{"path":"assets/figures/codec--firered-tokenizer.webp","kind":"source","label":"Figure 1 — system overview, including the tokenizer","sourceUrl":"https://arxiv.org/abs/2509.02020","originUrl":"https://arxiv.org/html/2509.02020v2/image/tts_model_framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/firered-tokenizer.png","sha256":"a3806ef1b934f010a12e868bd107f06d3dcf7f3d5b1541b7a4fa78a1771f78c5"}},{"id":"codec:llm-codec-uniaudio","slug":"llm-codec-uniaudio","name":"LLM-Codec (UniAudio 1.5)","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Maps audio into an existing language-model vocabulary while preserving reconstruction.","architecture":"Audio encoder and decoder trained around a language-model token space.","notes":"The author usage also requires Llama 2 weights under their own terms. This is the 2024 UniAudio 1.5 codec, distinct from the 2026 LM-objective adaptation framework with the same short name.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Not specified","frame_rate":"Not specified","bitrate":"Not specified","representation":"LLM-vocabulary audio codes","streaming":"Not specified"},"date":"2024-06-14","dateKind":"paper","links":{"code":"https://github.com/yangdongchao/LLM-Codec","weights":"https://huggingface.co/Dongchao/2024","paper":"https://arxiv.org/abs/2406.10056","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#llm-codec-uniaudio"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/yangdongchao/LLM-Codec/blob/main/readme.md","weightsUrl":"https://huggingface.co/Dongchao/2024/blob/d603d7ca1d640f399245bc953ee5d3c1addb4f90/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/yangdongchao/LLM-Codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Dongchao/2024","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/yangdongchao/LLM-Codec/blob/main/readme.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Dongchao/2024/blob/d603d7ca1d640f399245bc953ee5d3c1addb4f90/README.md","review_level":"license file or model-card declaration"},{"title":"UniAudio 1.5: Large Language Model-driven Audio Codec is A Few-shot Audio Task Learner","url":"https://arxiv.org/abs/2406.10056","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--llm-codec-uniaudio.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/yangdongchao/LLM-Codec","originUrl":"https://raw.githubusercontent.com/yangdongchao/LLM-Codec/main/fig/llama_code.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/llm-codec-uniaudio.png","sha256":"03baf72e5f0a15bbd9430fdb855f9598f3e0f9fa4b840c977f36570b4c78c8d5"}},{"id":"codec:llm-codec-objectives","slug":"llm-codec-objectives","name":"LLM-Codec (LM objectives)","collection":"codec","category":"codec:semantic","categoryName":"Semantic & acoustic codecs","summary":"Adapts codec tokens to be easier for autoregressive language models to predict.","architecture":"Reconstruction-preserving codec adaptation with future-token prediction, semantic alignment and a differentiable discrete bridge.","notes":"This is the separately released adapted codec checkpoint, not the bundled language-model weights. Downstream TTS/evaluation scripts are not included in the trimmed repository.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Not specified","frame_rate":"Not specified","bitrate":"Not specified","representation":"Adapted discrete codec tokens","streaming":"Not specified"},"date":"2026-04-20","dateKind":"paper","links":{"code":"https://github.com/voidful/llm-codec","weights":"https://huggingface.co/voidful/llm-codec","paper":"https://arxiv.org/abs/2604.17852","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/semantic.md#llm-codec-objectives"},"license":{"status":"unclear","code":"Not stated","weights":"Not stated","codeUrl":"https://github.com/voidful/llm-codec/blob/main/README.md","weightsUrl":"https://huggingface.co/voidful/llm-codec/blob/a0ef01c0c2720fbadc2f8df7f30505c31d76d695/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/voidful/llm-codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/voidful/llm-codec","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/voidful/llm-codec/blob/main/README.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/voidful/llm-codec/blob/a0ef01c0c2720fbadc2f8df7f30505c31d76d695/README.md","review_level":"license file or model-card declaration"},{"title":"LLM-Codec: Neural Audio Codec Meets Language Model Objectives","url":"https://arxiv.org/abs/2604.17852","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--llm-codec-objectives.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/voidful/llm-codec","originUrl":"https://raw.githubusercontent.com/voidful/llm-codec/main/webpage/assets/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/llm-codec-objectives.png","sha256":"1e47e1203cf82dd746b41dc69924de2705dd9a39c5fb8ce4082d0cb9a7d1d80b"}},{"id":"codec:audiodec","slug":"audiodec","name":"AudioDec","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Reconstructs high-sample-rate speech with a streamable two-stage codec.","architecture":"Streaming autoencoder and quantizer with a separately trained waveform decoder.","notes":"The README lists 12.8 kbps for the released hop300 configuration, whereas the paper abstract rounds to 12 kbps. The author archive includes multiple generator configurations.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz (also 24 kHz)","frame_rate":"160 (48 kHz / hop300)","bitrate":"12.8 kbps (selected 48 kHz model)","representation":"Residual vector quantization","streaming":"Streaming encoder and decoder"},"date":"2023-05-26","dateKind":"paper","links":{"code":"https://github.com/facebookresearch/AudioDec","weights":"https://github.com/facebookresearch/AudioDec/releases/tag/pretrain_models_v02","paper":"https://arxiv.org/abs/2305.16608","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#audiodec"},"license":{"status":"restricted","code":"CC-BY-NC-4.0","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/facebookresearch/AudioDec/blob/main/LICENSE","weightsUrl":"https://github.com/facebookresearch/AudioDec/blob/main/LICENSE","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Published checkpoint archive","url":"https://github.com/facebookresearch/AudioDec/releases/download/pretrain_models_v02/exp.zip","review_level":"GitHub release asset name and size"},{"title":"Official implementation and usage","url":"https://github.com/facebookresearch/AudioDec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://github.com/facebookresearch/AudioDec/releases/tag/pretrain_models_v02","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/facebookresearch/AudioDec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"AudioDec: An Open-source Streaming High-fidelity Neural Audio Codec","url":"https://arxiv.org/abs/2305.16608","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--audiodec.webp","kind":"source","label":"Author architecture (repository)","sourceUrl":"https://github.com/facebookresearch/AudioDec","originUrl":"https://raw.githubusercontent.com/facebookresearch/AudioDec/main/figs/architecture-1.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/audiodec.png","sha256":"567ef6ec46d719fa6074d7e5d3b4bff8e39b66e9557db9bb3eb0860f209955af"}},{"id":"codec:flowdec","slug":"flowdec","name":"FlowDec","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Combines non-adversarial audio compression with a generative flow-matching postfilter.","architecture":"NDAC audio codec followed by a stochastic conditional flow-matching reconstruction stage.","notes":"The release contains FlowDec-75m, FlowDec-25s and their underlying NDAC checkpoints. The stochastic postfilter changes the reconstruction/compute trade-off.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz","frame_rate":"Not specified","bitrate":"Not specified","representation":"Discrete NDAC codes + flow postfilter","streaming":"Not specified"},"date":"2025-03-03","dateKind":"paper","links":{"code":"https://github.com/facebookresearch/FlowDec","weights":"https://github.com/facebookresearch/FlowDec/releases/tag/checkpoints","paper":"https://arxiv.org/abs/2503.01485","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#flowdec"},"license":{"status":"restricted","code":"CC-BY-NC-4.0","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/facebookresearch/FlowDec/blob/main/LICENSE","weightsUrl":"https://github.com/facebookresearch/FlowDec/blob/main/LICENSE","notes":"The linked custom or noncommercial terms apply to this release.","components":[]},"sources":[{"title":"Published checkpoint archive","url":"https://github.com/facebookresearch/FlowDec/releases/download/checkpoints/checkpoints.zip","review_level":"GitHub release asset name and size"},{"title":"Official implementation and usage","url":"https://github.com/facebookresearch/FlowDec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://github.com/facebookresearch/FlowDec/releases/tag/checkpoints","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/facebookresearch/FlowDec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"FlowDec: A flow-based full-band general audio codec with high perceptual quality","url":"https://arxiv.org/abs/2503.01485","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--flowdec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/facebookresearch/FlowDec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/flowdec.svg","sha256":"bcc4c2ec3e4725672286215bd6ea7910f9f891b8f067c404e0832cac4af98f9c"}},{"id":"codec:codecslime","slug":"codecslime","name":"CodecSlime","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Removes redundant speech frames to support a controllable token rate.","architecture":"Speech codec with dynamic-programming frame merging and VQ or FSQ quantization.","notes":"The release processes one utterance at a time. Token durations must accompany the codes, so token-index bitrate alone is not a full transport budget.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz mono","frame_rate":"36–80","bitrate":"Not specified","representation":"8,192-entry VQ or 18,225-combination FSQ","streaming":"Offline utterance processing"},"date":"2025-06-26","dateKind":"paper","links":{"code":"https://github.com/X-LANCE/codecslime","weights":"https://github.com/X-LANCE/codecslime/releases/tag/v0.1.0","paper":"https://arxiv.org/abs/2506.21074","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#codecslime"},"license":{"status":"open","code":"MIT","weights":"CC-BY-4.0","codeUrl":"https://github.com/X-LANCE/codecslime/blob/main/LICENSE","weightsUrl":"https://github.com/X-LANCE/codecslime/blob/main/MODEL_LICENSE","notes":"","components":[]},"sources":[{"title":"Published checkpoint archive","url":"https://github.com/X-LANCE/codecslime/releases/download/v0.1.0/codecslime-fsq18k.safetensors","review_level":"GitHub release asset name and size"},{"title":"Published checkpoint archive","url":"https://github.com/X-LANCE/codecslime/releases/download/v0.1.0/codecslime-vq8k.safetensors","review_level":"GitHub release asset name and size"},{"title":"Official implementation and usage","url":"https://github.com/X-LANCE/codecslime","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://github.com/X-LANCE/codecslime/releases/tag/v0.1.0","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/X-LANCE/codecslime/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://github.com/X-LANCE/codecslime/blob/main/MODEL_LICENSE","review_level":"license file or model-card declaration"},{"title":"CodecSlime: Temporal Redundancy Compression of Neural Speech Codec via Dynamic Frame Rate","url":"https://arxiv.org/abs/2506.21074","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--codecslime.webp","kind":"source","label":"Figure 2","sourceUrl":"https://arxiv.org/abs/2506.21074","originUrl":"https://arxiv.org/html/2506.21074v2/codecslime-old-updated.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/codecslime.png","sha256":"b9892155a8de5312d19ed4959c92c993cc9f1ae9f73684407cb85ba164e1f025"}},{"id":"codec:lyra","slug":"lyra","name":"Lyra v2 (SoundStream-based)","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Provides low-bitrate speech communication with bundled mobile inference models.","architecture":"SoundStream-based encoder, vector quantization and generative decoder packaged for real-time communication.","notes":"The selected repository is Lyra v2. SoundStream is the architecture lineage; this is not a claim that the original research checkpoint is the same as the Lyra release.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"50 (20 ms frames)","bitrate":"3.2 / 6 / 9.2 kbps","representation":"SoundStream-derived quantized features","streaming":"Streaming codec"},"date":"","dateKind":"","links":{"code":"https://github.com/google/lyra","weights":"https://github.com/google/lyra/tree/main/lyra/model_coeffs","paper":"https://arxiv.org/abs/2107.03312","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#lyra"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Not stated","codeUrl":"https://github.com/google/lyra/blob/main/LICENSE","weightsUrl":"https://github.com/google/lyra/blob/main/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/google/lyra","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://github.com/google/lyra/tree/main/lyra/model_coeffs","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/google/lyra/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://github.com/google/lyra/blob/main/README.md","review_level":"license file or model-card declaration"},{"title":"SoundStream: An End-to-End Neural Audio Codec","url":"https://arxiv.org/abs/2107.03312","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--lyra.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/google/lyra","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/lyra.svg","sha256":"d283981906b1ef4b3244f734f0aa36ae062f6293f6865485019ab644e0ea2a17"}},{"id":"codec:hilcodec","slug":"hilcodec","name":"HILCodec","collection":"codec","category":"codec:general","categoryName":"General audio codecs","summary":"Targets lightweight, high-fidelity neural audio coding with deployable inference graphs.","architecture":"Convolutional neural codec with residual quantization and waveform reconstruction.","notes":"Actual speech ONNX encoder, decoder and quantizer graphs are present in the source tree. PyTorch checkpoints are additionally linked through the author's Google Drive folder.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"Not specified","bitrate":"Not specified","representation":"Residual vector quantization","streaming":"Streaming ONNX implementation"},"date":"2024-05-08","dateKind":"paper","links":{"code":"https://github.com/aask1357/hilcodec","weights":"https://github.com/aask1357/hilcodec/tree/master/onnx","paper":"https://arxiv.org/abs/2405.04752","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/general.md#hilcodec"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/aask1357/hilcodec/blob/master/LICENSE","weightsUrl":"https://github.com/aask1357/hilcodec/blob/master/README.md","notes":"The reviewed sources do not state complete code and checkpoint terms.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/aask1357/hilcodec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://github.com/aask1357/hilcodec/tree/master/onnx","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/aask1357/hilcodec/blob/master/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://github.com/aask1357/hilcodec/blob/master/README.md","review_level":"license file or model-card declaration"},{"title":"HILCodec: High-Fidelity and Lightweight Neural Audio Codec","url":"https://arxiv.org/abs/2405.04752","review_level":"primary paper abstract"},{"title":"HILCodec speech configuration","url":"https://github.com/aask1357/hilcodec/blob/master/configs/hilcodec_speech.yaml","review_level":"inspected sample-rate and quantizer configuration"}],"figure":{"path":"assets/figures/codec--hilcodec.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/aask1357/hilcodec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/hilcodec.svg","sha256":"d0534bcd3128cb424f45d15f0a6414687ff6b06ad01ef07c13761cc7e7e3e58d"}},{"id":"codec:speech-dac","slug":"speech-dac","name":"Speech DAC (IBM)","collection":"codec","category":"codec:speech","categoryName":"Speech codecs","summary":"Fine-tunes DAC for compact, high-quality speech representations.","architecture":"Speech-adapted RVQGAN encoder and decoder with two or four residual codebooks.","notes":"The publisher model-card table labels bitrate units as kHz; its prose and paper identify 1,500–3,000 bits/s. Total token rate differs from the 75 Hz codec frame rate.","variants":["1.5 kbps","3 kbps"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz","frame_rate":"75","bitrate":"1.5 / 3 kbps","representation":"2 / 4 codebooks; 150 / 300 total tokens/s","streaming":"Not specified"},"date":"2024-10-10","dateKind":"paper","links":{"code":"https://github.com/descriptinc/descript-audio-codec","weights":"https://huggingface.co/ibm-research/DAC.speech.v1.0","paper":"https://arxiv.org/abs/2410.08325","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/speech.md#speech-dac"},"license":{"status":"open","code":"MIT","weights":"CDLA-Permissive-2.0","codeUrl":"https://github.com/descriptinc/descript-audio-codec/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ibm-research/DAC.speech.v1.0/blob/1ea7f64cd0678415e2d8c32d67b190722cb9b149/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/descriptinc/descript-audio-codec","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/ibm-research/DAC.speech.v1.0","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/descriptinc/descript-audio-codec/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/ibm-research/DAC.speech.v1.0/blob/1ea7f64cd0678415e2d8c32d67b190722cb9b149/README.md","review_level":"license file or model-card declaration"},{"title":"Low Bitrate High-Quality RVQGAN-based Discrete Speech Tokenizer","url":"https://arxiv.org/abs/2410.08325","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--speech-dac.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/descriptinc/descript-audio-codec","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/speech-dac.svg","sha256":"d4c67b0696270585003e4be26d9baaa9c3fee5ea68f6b6234a8ffcf8a4bec9e9"}},{"id":"codec:dacvae","slug":"dacvae","name":"DACVAE (Movie Gen)","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Replaces DAC residual vector quantization with a continuous variational bottleneck for general-audio reconstruction.","architecture":"DAC-derived convolutional encoder and decoder with a Gaussian VAE bottleneck; the public decoder integrates AudioSeal watermarking.","notes":"The numerical configuration is reported in Movie Gen, Section 6.1.3. The reviewed public checkpoint is dacvae-watermarked: its decoder embeds a watermark, and its model card does not separately enumerate the stored architecture parameters. It is a separate VAE release from the discrete Descript Audio Codec.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz mono (Movie Gen paper)","frame_rate":"25 (Movie Gen paper)","bitrate":"Not applicable: continuous latents","representation":"128-dimensional VAE latents (paper)","streaming":"Not specified"},"date":"2024-10-17","dateKind":"paper","links":{"code":"https://github.com/facebookresearch/dacvae","weights":"https://huggingface.co/facebook/dacvae-watermarked","paper":"https://arxiv.org/abs/2410.13720","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#dacvae"},"license":{"status":"unclear","code":"Apache-2.0","weights":"Conflicting: Apache-2.0 / SAM License","codeUrl":"https://github.com/facebookresearch/dacvae/blob/main/LICENSE","weightsUrl":"https://huggingface.co/facebook/dacvae-watermarked/blob/8680102d141858a21bd533543966a2eb2e569f92/README.md","notes":"The GitHub code is Apache-2.0. Checkpoint-card metadata says Apache-2.0, but its license paragraph says SAM License and references a LICENSE file absent from the inspected checkpoint tree; the weight terms need clarification.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/facebookresearch/dacvae","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/facebook/dacvae-watermarked","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/facebookresearch/dacvae/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/facebook/dacvae-watermarked/blob/8680102d141858a21bd533543966a2eb2e569f92/README.md","review_level":"license file or model-card declaration"},{"title":"Movie Gen: A Cast of Media Foundation Models","url":"https://arxiv.org/abs/2410.13720","review_level":"primary paper and codec architecture section"},{"title":"Codec implementation","url":"https://github.com/facebookresearch/dacvae/blob/main/dacvae/model/dacvae.py","review_level":"encoder/decoder implementation"},{"title":"Codec implementation","url":"https://github.com/facebookresearch/dacvae/blob/main/dacvae/nn/bottleneck.py","review_level":"encoder/decoder implementation"}],"figure":{"path":"assets/figures/codec--dacvae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/facebookresearch/dacvae","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/dacvae.svg","sha256":"4f865c914e8713c98ed1bedfb68ab94b5ea46b7a60e173ee111dca60110fcd36"}},{"id":"codec:descript-audio-vae","slug":"descript-audio-vae","name":"Descript Audio VAE (community)","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Provides a community DAC-to-VAE adaptation for continuous audio latents.","architecture":"DAC-derived VAE-GAN that replaces RVQ with a variational bottleneck.","notes":"Independent community implementation by innnky; it is not the Meta DACVAE release. The checkpoint announcement specifies 44.1 kHz, 87 Hz and 64 dimensions. Inherited DAC examples and discrete-bitstream claims in the README do not describe this VAE checkpoint.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz","frame_rate":"87 (author release label)","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Not specified"},"date":"","dateKind":"","links":{"code":"https://github.com/innnky/descript-audio-vae","weights":"https://huggingface.co/innnky/vae-dac","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#descript-audio-vae"},"license":{"status":"unclear","code":"MIT","weights":"Not stated","codeUrl":"https://github.com/innnky/descript-audio-vae/blob/main/LICENSE","weightsUrl":"https://huggingface.co/innnky/vae-dac","notes":"The code is MIT. The checkpoint has no model-card license; generic MIT weight wording inherited in the DAC README is not treated as unambiguous evidence for the modified VAE checkpoint.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/innnky/descript-audio-vae","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/innnky/vae-dac","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/innnky/descript-audio-vae/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/innnky/vae-dac","review_level":"license file or model-card declaration"}],"figure":{"path":"assets/figures/codec--descript-audio-vae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/innnky/descript-audio-vae","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/descript-audio-vae.svg","sha256":"c752ec97db86890d76d69d7896bdf3b4024b3c698bbe482a9282b5b1cded4cf3"}},{"id":"codec:stft-vae","slug":"stft-vae","name":"STFT-VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses complex spectrograms into very low-rate continuous audio latents and reconstructs phase with an inverse STFT.","architecture":"STFT front end, multistage transformer encoder, Gaussian VAE bottleneck, transformer decoder and inverse STFT.","notes":"The released encoder returns the posterior mean at inference. Five temporal downsampling stages follow a 240-sample STFT hop. STFT windowing and buffering must be included when evaluating latency.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono","frame_rate":"3.125","bitrate":"Not applicable: continuous latents","representation":"128-dimensional VAE latents","streaming":"Causal transformer configuration; end-to-end streaming not established"},"date":"","dateKind":"","links":{"code":"https://github.com/fluxions-ai/stftvae","weights":"https://huggingface.co/fluxions/stftvae","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#stft-vae"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/fluxions-ai/stftvae/blob/main/LICENSE","weightsUrl":"https://huggingface.co/fluxions/stftvae/blob/ed77728c00cac7f9f8ee240d9a9d398b31464320/LICENSE","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/fluxions-ai/stftvae","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/fluxions/stftvae","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/fluxions-ai/stftvae/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/fluxions/stftvae/blob/ed77728c00cac7f9f8ee240d9a9d398b31464320/LICENSE","review_level":"license file or model-card declaration"},{"title":"Checkpoint configuration","url":"https://huggingface.co/fluxions/stftvae/blob/ed77728c00cac7f9f8ee240d9a9d398b31464320/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--stft-vae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/fluxions-ai/stftvae","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/stft-vae.svg","sha256":"4c6a162c21293e9e958bcc5bd9c2029c158db42d656ccd60490b688d7786dee7"}},{"id":"codec:kvae-audio","slug":"kvae-audio","name":"KVAE-Audio","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes full-band speech, music and sound into continuous latents for reconstruction and generative modeling.","architecture":"Convolutional audio encoder/decoder with attention and a variational bottleneck; the API exposes posterior-mean or sampled encoding.","notes":"The audio member of the KVAE family is cataloged here. The 960-sample encoder stride gives 50 frames/s at 48 kHz. The internal encoder width is not the 64-dimensional exported bottleneck.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz","frame_rate":"50","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Not specified"},"date":"2026-08-06","dateKind":"paper","links":{"code":"https://github.com/kandinskylab/kvae-audio","weights":"https://huggingface.co/kandinskylab/KVAE-Audio","paper":"https://arxiv.org/abs/2608.05798","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#kvae-audio"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/kandinskylab/kvae-audio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/kandinskylab/KVAE-Audio/blob/80edd2bccefecf0c69b90163da693adba8fa449d/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/kandinskylab/kvae-audio","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/kandinskylab/KVAE-Audio","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/kandinskylab/kvae-audio/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/kandinskylab/KVAE-Audio/blob/80edd2bccefecf0c69b90163da693adba8fa449d/README.md","review_level":"license file or model-card declaration"},{"title":"KVAE: Family of Tokenizers for Multimodal Generative Models","url":"https://arxiv.org/abs/2608.05798","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/kandinskylab/KVAE-Audio/blob/80edd2bccefecf0c69b90163da693adba8fa449d/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--kvae-audio.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/kandinskylab/kvae-audio","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/kvae-audio.svg","sha256":"efd0c8d2269070efa1aebb51024c3e2908248d7585fd116cd1d4aca97714b51b"}},{"id":"codec:voxcpm-audiovae","slug":"voxcpm-audiovae","name":"VoxCPM AudioVAE (1.0 / 1.5)","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Reconstructs speech from continuous latents used by VoxCPM and VoxCPM1.5.","architecture":"DAC-derived causal convolutional encoder/decoder with variational mean and variance projections.","notes":"Only audiovae.pth is counted. VoxCPM1.5 uses a 1764-sample stride at 44.1 kHz; the original uses 640 samples at 16 kHz. Both yield 25 codec frames/s. The downstream 6.25/12.5 Hz LM patch rates are different measurements.","variants":["VoxCPM-0.5B AudioVAE","VoxCPM1.5 AudioVAE"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz (1.5); 16 kHz (1.0)","frame_rate":"25","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Causal encoder/decoder; chunked decoder implementation"},"date":"2025-09-29","dateKind":"paper","links":{"code":"https://github.com/OpenBMB/VoxCPM","weights":"https://huggingface.co/openbmb/VoxCPM1.5","paper":"https://arxiv.org/abs/2509.24650","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#voxcpm-audiovae"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/OpenBMB/VoxCPM/blob/main/LICENSE","weightsUrl":"https://huggingface.co/openbmb/VoxCPM1.5/blob/8cdc403854cda0e3af12252d27da038fda5982ac/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/OpenBMB/VoxCPM","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/openbmb/VoxCPM1.5","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/OpenBMB/VoxCPM/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/openbmb/VoxCPM1.5/blob/8cdc403854cda0e3af12252d27da038fda5982ac/README.md","review_level":"license file or model-card declaration"},{"title":"VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning","url":"https://arxiv.org/abs/2509.24650","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/openbmb/VoxCPM1.5/blob/8cdc403854cda0e3af12252d27da038fda5982ac/config.json","review_level":"configuration file"},{"title":"Codec implementation","url":"https://github.com/OpenBMB/VoxCPM/blob/main/src/voxcpm/modules/audiovae/audio_vae.py","review_level":"encoder/decoder implementation"},{"title":"Original VoxCPM AudioVAE weights","url":"https://huggingface.co/openbmb/VoxCPM-0.5B","review_level":"checkpoint listing and model card"}],"figure":{"path":"assets/figures/codec--voxcpm-audiovae.webp","kind":"source","label":"Parent VoxCPM architecture with AudioVAE","sourceUrl":"https://github.com/OpenBMB/VoxCPM","originUrl":"https://raw.githubusercontent.com/OpenBMB/VoxCPM/main/assets/voxcpm_model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/voxcpm-audiovae.png","sha256":"49f6eb7998135ad49f5dd0ee1fa2c099d79a016ab59fe29fc039f7f32ef8f5ca"}},{"id":"codec:voxcpm2-audiovae","slug":"voxcpm2-audiovae","name":"VoxCPM2 AudioVAE V2","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes 16 kHz reference speech and decodes continuous latents into 48 kHz speech.","architecture":"Asymmetric causal convolutional VAE with separate encoder and decoder strides and sample-rate conditioning.","notes":"The published configuration has a 640-sample encoder stride and a 1920-sample decoder stride. Higher-rate output is generated by the decoder; it does not mean high-frequency input information was losslessly encoded.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz input → 48 kHz output","frame_rate":"25","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Causal audio VAE; streaming decode support"},"date":"2026-06-05","dateKind":"paper","links":{"code":"https://github.com/OpenBMB/VoxCPM","weights":"https://huggingface.co/openbmb/VoxCPM2","paper":"https://arxiv.org/abs/2606.06928","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#voxcpm2-audiovae"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/OpenBMB/VoxCPM/blob/main/LICENSE","weightsUrl":"https://huggingface.co/openbmb/VoxCPM2/blob/32279effe8c19989596f05d353d1447f51d9e915/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/OpenBMB/VoxCPM","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/openbmb/VoxCPM2","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/OpenBMB/VoxCPM/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/openbmb/VoxCPM2/blob/32279effe8c19989596f05d353d1447f51d9e915/README.md","review_level":"license file or model-card declaration"},{"title":"VoxCPM2 Technical Report","url":"https://arxiv.org/abs/2606.06928","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/openbmb/VoxCPM2/blob/32279effe8c19989596f05d353d1447f51d9e915/config.json","review_level":"configuration file"},{"title":"Codec implementation","url":"https://github.com/OpenBMB/VoxCPM/blob/main/src/voxcpm/modules/audiovae/audio_vae_v2.py","review_level":"encoder/decoder implementation"}],"figure":{"path":"assets/figures/codec--voxcpm2-audiovae.webp","kind":"source","label":"Parent VoxCPM2 architecture with AudioVAE V2","sourceUrl":"https://github.com/OpenBMB/VoxCPM","originUrl":"https://raw.githubusercontent.com/OpenBMB/VoxCPM/main/assets/voxcpm2_model.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/voxcpm2-audiovae.png","sha256":"437d37d53aac0603af16f55eff48a23197b597e01d36b56997c9578c33db749b"}},{"id":"codec:same","slug":"same","name":"SAME (S / L)","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses stereo audio into semantically aligned continuous latents with transformer-based encoders and decoders.","architecture":"Patched-waveform transformer autoencoder with a softnorm bottleneck and semantic regularization.","notes":"SAME-S and SAME-L share a 4096-sample temporal compression ratio. The released configuration uses softnorm rather than a Gaussian KL VAE bottleneck, so this entry is labeled a continuous autoencoder. The primary inspected checkpoint is SAME-L.","variants":["SAME-L","SAME-S"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz stereo","frame_rate":"~10.77","bitrate":"Not applicable: continuous latents","representation":"256-dimensional continuous latents; softnorm bottleneck","streaming":"Not specified"},"date":"2026-05-18","dateKind":"paper","links":{"code":"https://github.com/Stability-AI/stable-audio-3","weights":"https://huggingface.co/stabilityai/SAME-L","paper":"https://arxiv.org/abs/2605.18613","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#same"},"license":{"status":"restricted","code":"MIT","weights":"Stability AI Community License","codeUrl":"https://github.com/Stability-AI/stable-audio-3/blob/main/LICENSE","weightsUrl":"https://huggingface.co/stabilityai/SAME-L/blob/41acf79dd242877d6499a1108ca5dba5d5eecfc5/LICENSE.md","notes":"The linked noncommercial or custom terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Stability-AI/stable-audio-3","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/stabilityai/SAME-L","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Stability-AI/stable-audio-3/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/stabilityai/SAME-L/blob/41acf79dd242877d6499a1108ca5dba5d5eecfc5/LICENSE.md","review_level":"license file or model-card declaration"},{"title":"SAME: A Semantically-Aligned Music Autoencoder","url":"https://arxiv.org/abs/2605.18613","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/stabilityai/SAME-L/blob/41acf79dd242877d6499a1108ca5dba5d5eecfc5/model_config.json","review_level":"configuration file"},{"title":"SAME-S release","url":"https://huggingface.co/stabilityai/SAME-S","review_level":"model card, configuration and file listing"}],"figure":{"path":"assets/figures/codec--same.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/Stability-AI/stable-audio-3","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/same.svg","sha256":"a2fa30f6b5cd629ff1635276ceb3553548206acc0746ac413fb61c828d19d63e"}},{"id":"codec:ear-vae","slug":"ear-vae","name":"εar-VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Learns stereo music latents with objectives that preserve phase and spatial structure.","architecture":"Oobleck-style convolutional VAE-GAN with transformer bottleneck layers and phase-aware reconstruction training.","notes":"The paper reports a 1024-sample stride for the 44.1 kHz configuration. The repository also releases a separately trained 48 kHz checkpoint; do not transfer all original-paper measurements to that variant.","variants":["44.1 kHz release","48 kHz release"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz stereo; 48 kHz variant","frame_rate":"~43.07 (44.1 kHz / 1024)","bitrate":"Not applicable: continuous latents","representation":"128-dimensional VAE latents (paper configuration)","streaming":"Not specified"},"date":"2025-09-18","dateKind":"paper","links":{"code":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE","weights":"https://huggingface.co/earlab/EAR_VAE","paper":"https://arxiv.org/abs/2509.14912","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#ear-vae"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE/blob/main/LICENSE","weightsUrl":"https://huggingface.co/earlab/EAR_VAE/blob/5950873f1bca6d105fb873af8b68f43b05380093/LICENSE","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/earlab/EAR_VAE","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/earlab/EAR_VAE/blob/5950873f1bca6d105fb873af8b68f43b05380093/LICENSE","review_level":"license file or model-card declaration"},{"title":"Back to Ear: Perceptually Driven High Fidelity Music Reconstruction","url":"https://arxiv.org/abs/2509.14912","review_level":"primary paper and codec architecture section"}],"figure":{"path":"assets/figures/codec--ear-vae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/ear-vae.svg","sha256":"d4af6f50fe097a4134130ffc8b3261c5678b89b68fa8a1f5eba77a3d7c917355"}},{"id":"codec:ear-vae2","slug":"ear-vae2","name":"εar-VAE2","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Reconstructs stereo music using a variational representation learned in the complex spectral domain.","architecture":"STFT-domain autoencoder with frequency-aware Spec-SnakeBeta activations and a band-dependent magnitude/phase refiner.","notes":"The published model card documents a 1920-sample temporal stride. Public weights were retrained on public datasets and are not the same training run used for the paper headline results. Chunked encode/decode is documented; this alone does not establish causal streaming.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz stereo","frame_rate":"25","bitrate":"Not applicable: continuous latents","representation":"128-dimensional continuous VAE latents","streaming":"Not specified"},"date":"2026-08-20","dateKind":"paper","links":{"code":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE2","weights":"https://huggingface.co/earlab/EAR_VAE2","paper":"https://arxiv.org/abs/2608.19843","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#ear-vae2"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/earlab/EAR_VAE2/blob/b6556e5b28c5a9806c64fdb9d9c00787af5f8473/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE2","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/earlab/EAR_VAE2","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Eps-Acoustic-Revolution-Lab/EAR_VAE2/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/earlab/EAR_VAE2/blob/b6556e5b28c5a9806c64fdb9d9c00787af5f8473/README.md","review_level":"license file or model-card declaration"},{"title":"Checkpoint configuration","url":"https://huggingface.co/earlab/EAR_VAE2/blob/b6556e5b28c5a9806c64fdb9d9c00787af5f8473/config.json","review_level":"configuration file"},{"title":"Technical paper","url":"https://arxiv.org/abs/2608.19843","review_level":"primary paper abstract"}],"figure":{"path":"assets/figures/codec--ear-vae2.webp","kind":"source","label":"Figure 1 — spectral audio VAE architecture","sourceUrl":"https://arxiv.org/abs/2608.19843","originUrl":"https://arxiv.org/html/2608.19843v1/ear-vae2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/ear-vae2.png","sha256":"ae5b4e4473d6063eba8943e6ad25d6b6c4a5aac357004a31cc762d5c9d78dbdd"}},{"id":"codec:ezaudio-vae","slug":"ezaudio-vae","name":"EzAudio waveform VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Provides the waveform compression stage used by EzAudio text-to-audio models.","architecture":"One-dimensional Oobleck-style convolutional VAE with direct waveform reconstruction.","notes":"Only the independently identified ckpts/vae/1m.pt component is listed. Its configuration specifies a 480-sample stride; the text encoder and diffusion generator are separate components.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono","frame_rate":"50","bitrate":"Not applicable: continuous latents","representation":"128-dimensional VAE latents","streaming":"Not specified"},"date":"2024-09-17","dateKind":"paper","links":{"code":"https://github.com/haidog-yaqub/EzAudio","weights":"https://huggingface.co/OpenSound/EzAudio","paper":"https://arxiv.org/abs/2409.10819","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#ezaudio-vae"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/haidog-yaqub/EzAudio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenSound/EzAudio/blob/60370bf1866b50e3ef07660a68bb6389738dd77a/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/haidog-yaqub/EzAudio","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/OpenSound/EzAudio","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/haidog-yaqub/EzAudio/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/OpenSound/EzAudio/blob/60370bf1866b50e3ef07660a68bb6389738dd77a/README.md","review_level":"license file or model-card declaration"},{"title":"EzAudio: Enhancing Text-to-Audio Generation with Efficient Diffusion Transformer","url":"https://arxiv.org/abs/2409.10819","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/OpenSound/EzAudio/blob/60370bf1866b50e3ef07660a68bb6389738dd77a/ckpts/vae/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--ezaudio-vae.webp","kind":"source","label":"Figure 1 — parent EzAudio system with waveform VAE","sourceUrl":"https://arxiv.org/abs/2409.10819","originUrl":"https://arxiv.org/html/2409.10819v2/framework.drawio.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/ezaudio-vae.png","sha256":"932ea1151131313dc493984c0e524c731345c4ca1053e34090ed6d0001103a39"}},{"id":"codec:diffrhythm-vae","slug":"diffrhythm-vae","name":"DiffRhythm VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses stereo songs into continuous waveform latents for DiffRhythm.","architecture":"Fine-tuned Stable Audio Open/Oobleck waveform VAE with convolutional encoder and decoder.","notes":"The released configuration has a 2048-sample stride. This is the DiffRhythm VAE release, not evidence for the separate 5 Hz MusicVAE described in DiffRhythm 2.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz stereo","frame_rate":"~21.53","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Not specified"},"date":"2025-03-03","dateKind":"paper","links":{"code":"https://github.com/ASLP-lab/DiffRhythm","weights":"https://huggingface.co/ASLP-lab/DiffRhythm-vae","paper":"https://arxiv.org/abs/2503.01183","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#diffrhythm-vae"},"license":{"status":"restricted","code":"Apache-2.0","weights":"Stability AI Community License","codeUrl":"https://github.com/ASLP-lab/DiffRhythm/blob/main/LICENSE.md","weightsUrl":"https://huggingface.co/ASLP-lab/DiffRhythm-vae/blob/74e2afacfd91dd1b96662c96dcef763c1258768b/LICENSE.md","notes":"The linked noncommercial or custom terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/ASLP-lab/DiffRhythm","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/ASLP-lab/DiffRhythm-vae","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/ASLP-lab/DiffRhythm/blob/main/LICENSE.md","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/ASLP-lab/DiffRhythm-vae/blob/74e2afacfd91dd1b96662c96dcef763c1258768b/LICENSE.md","review_level":"license file or model-card declaration"},{"title":"DiffRhythm: Blazingly Fast and Embarrassingly Simple End-to-End Full-Length Song Generation with Latent Diffusion","url":"https://arxiv.org/abs/2503.01183","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/ASLP-lab/DiffRhythm-vae/blob/74e2afacfd91dd1b96662c96dcef763c1258768b/vae_config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--diffrhythm-vae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/ASLP-lab/DiffRhythm","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/diffrhythm-vae.svg","sha256":"06b951186dcd64983be73309367ae539ee9803e636cefe1965480c931fde4b87"}},{"id":"codec:acestep15-vae","slug":"acestep15-vae","name":"ACE-Step 1.5 VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes stereo music directly into continuous waveform latents for ACE-Step 1.5.","architecture":"One-dimensional Oobleck-style variational waveform autoencoder with a 1920-sample temporal stride.","notes":"This waveform VAE replaces the earlier ACE-Step spectrogram DCAE pipeline. The music language model and its discrete semantic tokens are separate from the VAE representation.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz stereo","frame_rate":"25","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Not specified"},"date":"2026-01-31","dateKind":"paper","links":{"code":"https://github.com/ace-step/ACE-Step-1.5","weights":"https://huggingface.co/ACE-Step/Ace-Step1.5","paper":"https://arxiv.org/abs/2602.00744","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#acestep15-vae"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/ace-step/ACE-Step-1.5/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ACE-Step/Ace-Step1.5/blob/19671f406d603126926c1b7e2adc169acbcade22/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/ace-step/ACE-Step-1.5","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/ACE-Step/Ace-Step1.5","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/ace-step/ACE-Step-1.5/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/ACE-Step/Ace-Step1.5/blob/19671f406d603126926c1b7e2adc169acbcade22/README.md","review_level":"license file or model-card declaration"},{"title":"ACE-Step 1.5: Pushing the Boundaries of Open-Source Music Generation","url":"https://arxiv.org/abs/2602.00744","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/ACE-Step/Ace-Step1.5/blob/19671f406d603126926c1b7e2adc169acbcade22/vae/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--acestep15-vae.webp","kind":"source","label":"Figure 2 — parent ACE-Step 1.5 framework with VAE","sourceUrl":"https://arxiv.org/abs/2602.00744","originUrl":"https://arxiv.org/html/2602.00744v3/figures/ace-stepv1.5_framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/acestep15-vae.png","sha256":"8987c2c8dedcc423cec636010b70b0518a8f47675f15d326b19058764a214c38"}},{"id":"codec:acestep-dcae","slug":"acestep-dcae","name":"ACE-Step music DCAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses music spectrograms into continuous latents and reconstructs audio through a matching vocoder.","architecture":"Convolutional/EfficientViT deep-compression spectrogram autoencoder followed by an ADaMoS HiFi-GAN vocoder.","notes":"The release uses AutoencoderDC, a deterministic DCAE rather than a Gaussian VAE. The inspected vocoder config specifies 44.1 kHz and a 512-sample hop; the DCAE reduces time by 8. The original paper also discusses a 32 kHz monophonic vocoder limitation; the table refers to the inspected release configuration.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz (released vocoder config)","frame_rate":"~10.77","bitrate":"Not applicable: continuous latents","representation":"8 latent channels × 16 frequency positions (128 values/time step)","streaming":"Not specified"},"date":"2025-05-28","dateKind":"paper","links":{"code":"https://github.com/ace-step/ACE-Step","weights":"https://huggingface.co/ACE-Step/ACE-Step-v1-3.5B","paper":"https://arxiv.org/abs/2506.00045","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#acestep-dcae"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/ace-step/ACE-Step/blob/main/LICENSE","weightsUrl":"https://huggingface.co/ACE-Step/ACE-Step-v1-3.5B/blob/82cd0d7b6322bd28cd4e830fe675ddb6180ce36c/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/ace-step/ACE-Step","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/ACE-Step/ACE-Step-v1-3.5B","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/ace-step/ACE-Step/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/ACE-Step/ACE-Step-v1-3.5B/blob/82cd0d7b6322bd28cd4e830fe675ddb6180ce36c/README.md","review_level":"license file or model-card declaration"},{"title":"ACE-Step: A Step Towards Music Generation Foundation Model","url":"https://arxiv.org/abs/2506.00045","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/ACE-Step/ACE-Step-v1-3.5B/blob/82cd0d7b6322bd28cd4e830fe675ddb6180ce36c/music_dcae_f8c8/config.json","review_level":"configuration file"},{"title":"Checkpoint configuration","url":"https://huggingface.co/ACE-Step/ACE-Step-v1-3.5B/blob/82cd0d7b6322bd28cd4e830fe675ddb6180ce36c/music_vocoder/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--acestep-dcae.webp","kind":"source","label":"Figure 1 — parent ACE-Step system with DCAE and vocoder","sourceUrl":"https://arxiv.org/abs/2506.00045","originUrl":"https://arxiv.org/html/2506.00045v1/figures/ACE-Step_framework.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/acestep-dcae.png","sha256":"013314e06bc28c3473ecc58a884a7e8b9bdb28133ad6bce0f8329dce887b6316"}},{"id":"codec:mmaudio-vae","slug":"mmaudio-vae","name":"MMAudio VAE (16 / 44.1 kHz)","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes mel spectrograms into continuous audio latents and reconstructs waveforms with BigVGAN.","architecture":"Convolutional spectrogram VAE using magnitude-preserving blocks, paired with a sample-rate-specific BigVGAN vocoder.","notes":"The VAE checkpoint includes the encoder, although generation-only runtime paths can remove it. Waveform reconstruction also needs a vocoder: best_netG.pt for 16 kHz, or nvidia/bigvgan_v2_44khz_128band_512x for 44.1 kHz. The NVIDIA component retains its own terms.","variants":["16 kHz VAE + released BigVGAN","44.1 kHz VAE + NVIDIA BigVGAN v2"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 / 44.1 kHz","frame_rate":"31.25 (16 kHz); ~43.07 (44.1 kHz)","bitrate":"Not applicable: continuous latents","representation":"20 / 40-dimensional VAE latents","streaming":"Not specified"},"date":"2024-12-19","dateKind":"paper","links":{"code":"https://github.com/hkchengrex/MMAudio","weights":"https://huggingface.co/hkchengrex/MMAudio","paper":"https://arxiv.org/abs/2412.15322","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#mmaudio-vae"},"license":{"status":"restricted","code":"MIT","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/hkchengrex/MMAudio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/hkchengrex/MMAudio/blob/eb13a1a98fdbec91753775c57b074ccdfc60587c/README.md","notes":"The linked noncommercial or custom terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/hkchengrex/MMAudio","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/hkchengrex/MMAudio","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/hkchengrex/MMAudio/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/hkchengrex/MMAudio/blob/eb13a1a98fdbec91753775c57b074ccdfc60587c/README.md","review_level":"license file or model-card declaration"},{"title":"MMAudio: Taming Multimodal Joint Training for High-Quality Video-to-Audio Synthesis","url":"https://arxiv.org/abs/2412.15322","review_level":"primary paper and codec architecture section"},{"title":"Codec implementation","url":"https://github.com/hkchengrex/MMAudio/blob/main/mmaudio/ext/autoencoder/autoencoder.py","review_level":"encoder/decoder implementation"},{"title":"44.1 kHz vocoder","url":"https://huggingface.co/nvidia/bigvgan_v2_44khz_128band_512x","review_level":"official model card and checkpoint listing"}],"figure":{"path":"assets/figures/codec--mmaudio-vae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/hkchengrex/MMAudio","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/mmaudio-vae.svg","sha256":"841b2e575f31a8dc7d65dc3c5147ab690b697058b63a847f25d17d46798a5a38"}},{"id":"codec:audioldm-vae","slug":"audioldm-vae","name":"AudioLDM / AudioLDM 2 VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses mel spectrograms with a variational autoencoder and reconstructs audio using HiFi-GAN.","architecture":"Two-dimensional convolutional AutoencoderKL with a mel-spectrogram HiFi-GAN reconstruction stage.","notes":"The main evidence refers to the AudioLDM 2 VAE and vocoder subfolders. The original AudioLDM release is linked as a related configuration, not counted again merely because the generation backbone differs. The latent has a frequency axis; 128 is a flattened per-time-step width, not the latent-channel count.","variants":["AudioLDM VAE","AudioLDM 2 VAE"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz mono","frame_rate":"25","bitrate":"Not applicable: continuous latents","representation":"8 latent channels × 16 frequency positions (128 values/time step)","streaming":"Not specified"},"date":"2023-08-10","dateKind":"paper","links":{"code":"https://github.com/haoheliu/AudioLDM2","weights":"https://huggingface.co/cvssp/audioldm2","paper":"https://arxiv.org/abs/2308.05734","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#audioldm-vae"},"license":{"status":"restricted","code":"CC-BY-NC-SA-4.0","weights":"CC-BY-NC-SA-4.0","codeUrl":"https://github.com/haoheliu/AudioLDM2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/cvssp/audioldm2/blob/c8e7e189d324425c05c4c2f81214041ef4107983/README.md","notes":"The linked noncommercial or custom terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/haoheliu/AudioLDM2","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/cvssp/audioldm2","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/haoheliu/AudioLDM2/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/cvssp/audioldm2/blob/c8e7e189d324425c05c4c2f81214041ef4107983/README.md","review_level":"license file or model-card declaration"},{"title":"AudioLDM 2: Learning Holistic Audio Generation with Self-supervised Pretraining","url":"https://arxiv.org/abs/2308.05734","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/cvssp/audioldm2/blob/c8e7e189d324425c05c4c2f81214041ef4107983/vae/config.json","review_level":"configuration file"},{"title":"Checkpoint configuration","url":"https://huggingface.co/cvssp/audioldm2/blob/c8e7e189d324425c05c4c2f81214041ef4107983/vocoder/config.json","review_level":"configuration file"},{"title":"Original AudioLDM release","url":"https://huggingface.co/cvssp/audioldm-s-full-v2","review_level":"VAE/vocoder configuration and checkpoint listing"},{"title":"AudioLDM paper","url":"https://arxiv.org/abs/2301.12503","review_level":"primary paper and VAE description"}],"figure":{"path":"assets/figures/codec--audioldm-vae.webp","kind":"source","label":"Figure 1 — parent AudioLDM system with VAE and vocoder","sourceUrl":"https://arxiv.org/abs/2301.12503","originUrl":"https://arxiv.org/html/2301.12503v3/main_graph_v2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/audioldm-vae.png","sha256":"796d52f43f9d1ccfe724b8e65c7314b8d35e7cc3023aedeea0cf6ef2099dc05a"}},{"id":"codec:auffusion-vae","slug":"auffusion-vae","name":"Auffusion spectrogram VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Uses an image-style variational autoencoder on audio spectrograms with a released audio reconstruction path.","architecture":"Two-dimensional convolutional AutoencoderKL for three-channel spectrogram images, followed by an audio vocoder.","notes":"This entry covers the audio spectrogram preprocessing, VAE and paired vocoder together. The VAE alone decodes an image-like spectrogram rather than a waveform. Its latent frequency axis should be retained when comparing latent dimensionality.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"12.5 (paper configuration)","bitrate":"Not applicable: continuous latents","representation":"4-channel spectrogram VAE latents","streaming":"Not specified"},"date":"2024-01-02","dateKind":"paper","links":{"code":"https://github.com/happylittlecat2333/Auffusion","weights":"https://huggingface.co/auffusion/auffusion","paper":"https://arxiv.org/abs/2401.01044","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#auffusion-vae"},"license":{"status":"restricted","code":"CC-BY-NC-SA-4.0","weights":"CC-BY-NC-SA-4.0","codeUrl":"https://github.com/happylittlecat2333/Auffusion/blob/main/LICENSE","weightsUrl":"https://huggingface.co/auffusion/auffusion/blob/211aeb853fe565a306bb164d1227505e70558529/README.md","notes":"The linked noncommercial or custom terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/happylittlecat2333/Auffusion","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/auffusion/auffusion","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/happylittlecat2333/Auffusion/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/auffusion/auffusion/blob/211aeb853fe565a306bb164d1227505e70558529/README.md","review_level":"license file or model-card declaration"},{"title":"Auffusion: Leveraging the Power of Diffusion and Large Language Models for Text-to-Audio Generation","url":"https://arxiv.org/abs/2401.01044","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/auffusion/auffusion/blob/211aeb853fe565a306bb164d1227505e70558529/vae/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--auffusion-vae.webp","kind":"source","label":"Figure 1 — spectrogram, VAE and waveform reconstruction path","sourceUrl":"https://arxiv.org/abs/2401.01044","originUrl":"https://arxiv.org/html/2401.01044v1/pic/overview_v4.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/auffusion-vae.png","sha256":"0209a23cdeccb71395704ed1553a27e98b52bd2fb9d75685a707cf8caab6c992"}},{"id":"codec:musika-ae","slug":"musika-ae","name":"Musika autoencoder","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Builds a compact, hierarchical continuous representation for waveform music reconstruction.","architecture":"Two separately trained hierarchical audio autoencoder stages with spectral processing.","notes":"The inspected author model card specifies a final 4096× temporal compression ratio for general 44.1 kHz waveforms. Both encoder/decoder stages are included. These release specifications need not equal every configuration in the original paper.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz (released model card)","frame_rate":"~10.77 (final stage)","bitrate":"Not applicable: continuous latents","representation":"Hierarchical continuous latents; non-VAE autoencoder","streaming":"Not specified"},"date":"2022-08-18","dateKind":"paper","links":{"code":"https://github.com/marcoppasini/musika","weights":"https://huggingface.co/marcop/musika_ae","paper":"https://arxiv.org/abs/2208.08706","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#musika-ae"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/marcoppasini/musika/blob/main/LICENSE","weightsUrl":"https://huggingface.co/marcop/musika_ae/blob/4572cc3d89d024e14db630b1796c93afdf1186ff/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/marcoppasini/musika","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/marcop/musika_ae","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/marcoppasini/musika/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/marcop/musika_ae/blob/4572cc3d89d024e14db630b1796c93afdf1186ff/README.md","review_level":"license file or model-card declaration"},{"title":"Musika! Fast Infinite Waveform Music Generation","url":"https://arxiv.org/abs/2208.08706","review_level":"primary paper and codec architecture section"}],"figure":{"path":"assets/figures/codec--musika-ae.webp","kind":"source","label":"Figure 1 — two-stage audio autoencoder","sourceUrl":"https://arxiv.org/abs/2208.08706","originUrl":"https://arxiv.org/html/2208.08706v1/ae2.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/musika-ae.png","sha256":"ccf1f0b2bcc8a363b43921625037354856b06f9aab773d2e4bf3c46cf860126e"}},{"id":"codec:longcat-wavvae","slug":"longcat-wavvae","name":"LongCat Wav-VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes speech directly into continuous waveform latents for LongCat-AudioDiT.","architecture":"Convolutional waveform VAE with Snake activations and a 2048-sample downsampling ratio.","notes":"VAE parameters are bundled in the full AudioDiT checkpoint and exposed as model.vae. The listed file size is the complete container, not the VAE parameter size. The codec is distinct from the discrete LongCat-Audio-Codec.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono","frame_rate":"~11.72","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Not specified"},"date":"2026-03-31","dateKind":"paper","links":{"code":"https://github.com/meituan-longcat/LongCat-AudioDiT","weights":"https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B","paper":"https://arxiv.org/abs/2603.29339","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#longcat-wavvae"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/meituan-longcat/LongCat-AudioDiT/blob/main/LICENSE","weightsUrl":"https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/LICENSE","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/meituan-longcat/LongCat-AudioDiT","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/meituan-longcat/LongCat-AudioDiT/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/LICENSE","review_level":"license file or model-card declaration"},{"title":"LongCat-AudioDiT: High-Fidelity Diffusion Text-to-Speech in the Waveform Latent Space","url":"https://arxiv.org/abs/2603.29339","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/meituan-longcat/LongCat-AudioDiT-3.5B/blob/5d3e6dc007872d756c3a69b44951598d7d9bdf6d/config.json","review_level":"configuration file"},{"title":"Codec implementation","url":"https://github.com/meituan-longcat/LongCat-AudioDiT/blob/main/audiodit/modeling_audiodit.py","review_level":"encoder/decoder implementation"}],"figure":{"path":"assets/figures/codec--longcat-wavvae.webp","kind":"source","label":"Parent LongCat-AudioDiT architecture with Wav-VAE","sourceUrl":"https://github.com/meituan-longcat/LongCat-AudioDiT","originUrl":"https://raw.githubusercontent.com/meituan-longcat/LongCat-AudioDiT/main/assets/architecture.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/longcat-wavvae.png","sha256":"dbcc8448913dd5b7e634bb7405fd4321ce720463d275c900007378a7a18bf608"}},{"id":"codec:holitok","slug":"holitok","name":"HoliTok","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Learns continuous speech latents that can support both audio reconstruction and semantic feature extraction.","architecture":"Convolutional encoder, temporal LSTM variational bottleneck and BigVGAN-style waveform decoder.","notes":"Only the reconstructable VAE tokenizer is counted. The separately published semantic.pt module supplies understanding features and is not necessary for basic VAE encode/decode. Posterior sampling and posterior-mean reconstruction are exposed by the runtime.","variants":["HoliTok-Base","HoliTok-Unite"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz mono","frame_rate":"25","bitrate":"Not applicable: continuous latents","representation":"128-dimensional VAE latents","streaming":"Mostly causal; two-frame lookahead in encoder and decoder"},"date":"2026-05-28","dateKind":"paper","links":{"code":"https://github.com/bovod-sjtu/HoliTok","weights":"https://huggingface.co/bovod-sjtu/HoliTok","paper":"https://arxiv.org/abs/2605.29948","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#holitok"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/bovod-sjtu/HoliTok/blob/main/LICENSE","weightsUrl":"https://huggingface.co/bovod-sjtu/HoliTok/blob/d9ddf774c8eb4b03d25b33dd9bbd8798066c865b/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/bovod-sjtu/HoliTok","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/bovod-sjtu/HoliTok","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/bovod-sjtu/HoliTok/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/bovod-sjtu/HoliTok/blob/d9ddf774c8eb4b03d25b33dd9bbd8798066c865b/README.md","review_level":"license file or model-card declaration"},{"title":"HoliTok: A Continuous Holistic Tokenization with Robust Dual Capabilities of Speech Generation and Understanding","url":"https://arxiv.org/abs/2605.29948","review_level":"primary paper and codec architecture section"},{"title":"Codec implementation","url":"https://github.com/bovod-sjtu/HoliTok/blob/main/holitok/model.py","review_level":"encoder/decoder implementation"}],"figure":{"path":"assets/figures/codec--holitok.webp","kind":"source","label":"Figure 1 — HoliTok tokenizer and training framework","sourceUrl":"https://arxiv.org/abs/2605.29948","originUrl":"https://arxiv.org/html/2605.29948v3/holitok.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/holitok.png","sha256":"8ff0b83e0d7301b9dcd2451668a6ab5d701db385e564126ef7d627fcbc9fa413"}},{"id":"codec:dots-audiovae","slug":"dots-audiovae","name":"dots.tts AudioVAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes and decodes the continuous speech representation used by dots.tts.","architecture":"Convolutional variational audio encoder with a BigVGAN-style causal decoder and recurrent bottleneck processing.","notes":"Despite its filename, vocoder.safetensors contains the AudioVAE component used for both latent extraction and decoding. The encoder/decoder stride is 1920 samples. Four-frame patches in the speech generator are a downstream grouping, not the codec frame rate.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz mono","frame_rate":"25","bitrate":"Not applicable: continuous latents","representation":"128-dimensional VAE latents","streaming":"Causal configuration with a stateful decoder implementation"},"date":"2026-06-05","dateKind":"paper","links":{"code":"https://github.com/studio-dots-ai/dots.tts","weights":"https://huggingface.co/dots-studio/dots.tts-base","paper":"https://arxiv.org/abs/2606.07080","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#dots-audiovae"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/studio-dots-ai/dots.tts/blob/main/LICENSE","weightsUrl":"https://huggingface.co/dots-studio/dots.tts-base/blob/b5aac65a395e04a60a8dacd4e2f4aadc5ad3dbe1/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/studio-dots-ai/dots.tts","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/dots-studio/dots.tts-base","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/studio-dots-ai/dots.tts/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/dots-studio/dots.tts-base/blob/b5aac65a395e04a60a8dacd4e2f4aadc5ad3dbe1/README.md","review_level":"license file or model-card declaration"},{"title":"dots.tts Technical Report","url":"https://arxiv.org/abs/2606.07080","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/dots-studio/dots.tts-base/blob/b5aac65a395e04a60a8dacd4e2f4aadc5ad3dbe1/config.json","review_level":"configuration file"},{"title":"Codec implementation","url":"https://github.com/studio-dots-ai/dots.tts/blob/main/src/dots_tts/modules/vocoder/vocoder_inference.py","review_level":"encoder/decoder implementation"}],"figure":{"path":"assets/figures/codec--dots-audiovae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/studio-dots-ai/dots.tts","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/dots-audiovae.svg","sha256":"4e0a5e7c90af7661a1e93dc7ecb7f9b2c31d53057ebfaeed937fe3e76cb05b08"}},{"id":"codec:mingtok-audio","slug":"mingtok-audio","name":"MingTok-Audio","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Provides continuous audio latents for joint speech understanding, generation and editing.","architecture":"Transformer-based audio VAE with acoustic reconstruction and semantic supervision.","notes":"The 320-sample input hop corresponds to 50 frames/s. The checkpoint also includes semantic supervision components; the codec entry is the audio encoder/decoder pathway.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"50","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Causal windowed transformer configuration"},"date":"2025-10-26","dateKind":"paper","links":{"code":"https://github.com/inclusionAI/Ming-UniAudio","weights":"https://huggingface.co/inclusionAI/MingTok-Audio","paper":"https://arxiv.org/abs/2511.05516","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#mingtok-audio"},"license":{"status":"open","code":"MIT","weights":"Apache-2.0","codeUrl":"https://github.com/inclusionAI/Ming-UniAudio/blob/main/LICENSE","weightsUrl":"https://huggingface.co/inclusionAI/MingTok-Audio/blob/44c61fbf2e03e58dc2d4c9b86d8d9f814e6cec9e/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/inclusionAI/Ming-UniAudio","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/inclusionAI/MingTok-Audio","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/inclusionAI/Ming-UniAudio/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/inclusionAI/MingTok-Audio/blob/44c61fbf2e03e58dc2d4c9b86d8d9f814e6cec9e/README.md","review_level":"license file or model-card declaration"},{"title":"Ming-UniAudio: Speech LLM for Joint Understanding, Generation and Editing with Unified Representation","url":"https://arxiv.org/abs/2511.05516","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/inclusionAI/MingTok-Audio/blob/44c61fbf2e03e58dc2d4c9b86d8d9f814e6cec9e/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--mingtok-audio.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/inclusionAI/Ming-UniAudio","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/mingtok-audio.svg","sha256":"17f010471f10c24b69f875d6f690cc91a10b49926af3c9663a2a20e80c90f5f8"}},{"id":"codec:ming-omni-tts-vae","slug":"ming-omni-tts-vae","name":"Ming-omni-tts continuous tokenizer","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses speech, music and environmental audio into a shared continuous latent sequence.","architecture":"Causal windowed transformer VAE with patch-based temporal compression.","notes":"The tokenizer combines an 882-sample base hop with four-frame compression, yielding 12.5 codec frames/s. The parent TTS system further groups codec frames to approximately 3.1 LM steps/s.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"44.1 kHz","frame_rate":"12.5","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Causal configuration; incremental decoder interface"},"date":"","dateKind":"","links":{"code":"https://github.com/inclusionAI/Ming-omni-tts","weights":"https://huggingface.co/inclusionAI/Ming-omni-tts-tokenizer-12Hz","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#ming-omni-tts-vae"},"license":{"status":"open","code":"MIT","weights":"Apache-2.0","codeUrl":"https://github.com/inclusionAI/Ming-omni-tts/blob/main/LICENSE","weightsUrl":"https://huggingface.co/inclusionAI/Ming-omni-tts-tokenizer-12Hz/blob/ee75c60e0c86c0d39e7bf55516eed991b39548a1/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/inclusionAI/Ming-omni-tts","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/inclusionAI/Ming-omni-tts-tokenizer-12Hz","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/inclusionAI/Ming-omni-tts/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/inclusionAI/Ming-omni-tts-tokenizer-12Hz/blob/ee75c60e0c86c0d39e7bf55516eed991b39548a1/README.md","review_level":"license file or model-card declaration"},{"title":"Checkpoint configuration","url":"https://huggingface.co/inclusionAI/Ming-omni-tts-tokenizer-12Hz/blob/ee75c60e0c86c0d39e7bf55516eed991b39548a1/config.json","review_level":"configuration file"},{"title":"Codec implementation","url":"https://github.com/inclusionAI/Ming-omni-tts/blob/main/audio_tokenizer/modeling_audio_vae.py","review_level":"encoder/decoder implementation"}],"figure":{"path":"assets/figures/codec--ming-omni-tts-vae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/inclusionAI/Ming-omni-tts","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/ming-omni-tts-vae.svg","sha256":"9eb5411cc3bd6d7bf4a4af7e934f60d00b9d54028b22a3177e0c0eaab9df0bbb"}},{"id":"codec:hunyuan-foley-vae","slug":"hunyuan-foley-vae","name":"HunyuanVideo-Foley audio VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Reconstructs general audio using the waveform VAE released with HunyuanVideo-Foley.","architecture":"Enhanced DAC-derived variational waveform autoencoder for 48 kHz speech, music and sound.","notes":"The independently named vae_128d_48k.pth file identifies the reviewed codec. This is Tencent’s separately trained audio VAE, not Meta’s 25 Hz DACVAE checkpoint. The video and text generation components are not part of this entry.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz","frame_rate":"50","bitrate":"Not applicable: continuous latents","representation":"128-dimensional VAE latents","streaming":"Not specified"},"date":"2025-08-23","dateKind":"paper","links":{"code":"https://github.com/Tencent-Hunyuan/HunyuanVideo-Foley","weights":"https://huggingface.co/tencent/HunyuanVideo-Foley","paper":"https://arxiv.org/abs/2508.16930","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#hunyuan-foley-vae"},"license":{"status":"restricted","code":"Tencent Hunyuan Community License","weights":"Tencent Hunyuan Community License","codeUrl":"https://github.com/Tencent-Hunyuan/HunyuanVideo-Foley/blob/main/LICENSE","weightsUrl":"https://huggingface.co/tencent/HunyuanVideo-Foley/blob/3abd4e833b95b8db0fc9c687afc52483a48e9a97/LICENSE","notes":"The linked noncommercial or custom terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Tencent-Hunyuan/HunyuanVideo-Foley","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/tencent/HunyuanVideo-Foley","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Tencent-Hunyuan/HunyuanVideo-Foley/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/tencent/HunyuanVideo-Foley/blob/3abd4e833b95b8db0fc9c687afc52483a48e9a97/LICENSE","review_level":"license file or model-card declaration"},{"title":"HunyuanVideo-Foley: Multimodal Diffusion with Representation Alignment for High-Fidelity Foley Audio Generation","url":"https://arxiv.org/abs/2508.16930","review_level":"primary paper and codec architecture section"}],"figure":{"path":"assets/figures/codec--hunyuan-foley-vae.webp","kind":"source","label":"Figure 2 — parent HunyuanVideo-Foley system with DAC-VAE","sourceUrl":"https://arxiv.org/abs/2508.16930","originUrl":"https://arxiv.org/html/2508.16930v1/TV2A-arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/hunyuan-foley-vae.png","sha256":"a580388ebca5131442ac6005ba169ec0a1d9e30f76a43028d0335b62a2419e81"}},{"id":"codec:omni2sound-vae","slug":"omni2sound-vae","name":"Omni2Sound OOB/Wav VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Provides a waveform latent representation for the audio path of Omni2Sound.","architecture":"Convolutional Oobleck-style variational waveform autoencoder.","notes":"The standalone OOB VAE checkpoint is separately named in the author release. Access is gated by the publisher. The joint video/text generation model is not counted as another codec.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz","frame_rate":"25 (paper configuration)","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents (paper configuration)","streaming":"Not specified"},"date":"2026-01-06","dateKind":"paper","links":{"code":"https://github.com/omni2sound/Omni2Sound","weights":"https://huggingface.co/Dalision/Omni2Sound","paper":"https://arxiv.org/abs/2601.02731","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#omni2sound-vae"},"license":{"status":"restricted","code":"CC-BY-NC-4.0","weights":"CC-BY-NC-4.0","codeUrl":"https://github.com/omni2sound/Omni2Sound/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Dalision/Omni2Sound","notes":"The linked noncommercial or custom terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/omni2sound/Omni2Sound","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Dalision/Omni2Sound","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/omni2sound/Omni2Sound/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Dalision/Omni2Sound","review_level":"license file or model-card declaration"},{"title":"Omni2Sound: Towards Unified Video-Text-to-Audio Generation","url":"https://arxiv.org/abs/2601.02731","review_level":"primary paper and codec architecture section"}],"figure":{"path":"assets/figures/codec--omni2sound-vae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/omni2sound/Omni2Sound","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/omni2sound-vae.svg","sha256":"2250349c5d7edb8d9946511242865bdde8e917f7befbd2856da13b8902bc9a08"}},{"id":"codec:minimax-h3-audiovae","slug":"minimax-h3-audiovae","name":"MiniMax H3 AudioVAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Compresses stereo audio into a continuous representation for MiniMax H3 audio generation.","architecture":"DAC-derived convolutional encoder with attention, Gaussian posterior projections and a BigVGAN waveform decoder.","notes":"The inspected audio_vae component has an 800-sample encoder stride, giving 40 frames/s. The autoencoder itself is mono: H3 handles stereo by putting left and right channels in separate batch items. The internal 2048-dimensional feature width is different from the 32 exported latent channels. The linked Diffusers implementation exposes both encode and decode.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"32 kHz mono; stereo channels processed separately","frame_rate":"40","bitrate":"Not applicable: continuous latents","representation":"32-dimensional VAE latents per audio channel","streaming":"Not specified"},"date":"","dateKind":"","links":{"code":"https://github.com/huggingface/diffusers/blob/c419dac0152186060246c93a095bc1bfaea342b3/src/diffusers/models/autoencoders/autoencoder_kl_minimax_h3_audio.py","weights":"https://huggingface.co/MiniMaxAI/MiniMax-H3","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#minimax-h3-audiovae"},"license":{"status":"restricted","code":"Apache-2.0","weights":"MiniMax H3 Community License","codeUrl":"https://github.com/huggingface/diffusers/blob/c419dac0152186060246c93a095bc1bfaea342b3/src/diffusers/models/autoencoders/autoencoder_kl_minimax_h3_audio.py","weightsUrl":"https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/42ed227ee7df40d41602854ae760620d6eb651fe/LICENSE","notes":"The reviewed Diffusers codec implementation is Apache-2.0; the publisher’s checkpoint uses the MiniMax H3 Community License.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/MiniMax-AI/MiniMax-H3","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/MiniMaxAI/MiniMax-H3","review_level":"file listing and model card"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/42ed227ee7df40d41602854ae760620d6eb651fe/LICENSE","review_level":"license file or model-card declaration"},{"title":"Checkpoint configuration","url":"https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/42ed227ee7df40d41602854ae760620d6eb651fe/audio_vae/config.json","review_level":"configuration file"},{"title":"Diffusers audio VAE implementation and Apache-2.0 notice","url":"https://github.com/huggingface/diffusers/blob/c419dac0152186060246c93a095bc1bfaea342b3/src/diffusers/models/autoencoders/autoencoder_kl_minimax_h3_audio.py","review_level":"encode/decode implementation and license header"}],"figure":{"path":"assets/figures/codec--minimax-h3-audiovae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/huggingface/diffusers/blob/c419dac0152186060246c93a095bc1bfaea342b3/src/diffusers/models/autoencoders/autoencoder_kl_minimax_h3_audio.py","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/minimax-h3-audiovae.svg","sha256":"21c080a61ce97b6e733db2ab64e99454ae3eee52f870a9ea7d6b65f75b134ca9"}},{"id":"codec:omnivae-audio","slug":"omnivae-audio","name":"OmniVAE audio-only","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Provides an independently usable audio VAE within the OmniVAE release.","architecture":"Convolutional variational waveform encoder/decoder; the broader family also studies joint audio-video alignment.","notes":"This card covers the audio-only reconstruction checkpoint. The configuration sets continuous=true and a 960-sample encoder stride; inactive quantizer fields do not make this release a discrete codec. Video-only and joint audiovisual checkpoints are outside this entry.","variants":["audio_only/recon","audio_only/recon_distill"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"48 kHz","frame_rate":"50","bitrate":"Not applicable: continuous latents","representation":"128-dimensional VAE latents","streaming":"Not specified"},"date":"2026-07-26","dateKind":"paper","links":{"code":"https://github.com/OpenMOSS/OmniVAE","weights":"https://huggingface.co/OpenMOSS-Team/OmniVAE","paper":"https://arxiv.org/abs/2607.23855","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#omnivae-audio"},"license":{"status":"open","code":"Apache-2.0","weights":"Apache-2.0","codeUrl":"https://github.com/OpenMOSS/OmniVAE/blob/main/LICENSE","weightsUrl":"https://huggingface.co/OpenMOSS-Team/OmniVAE/blob/e87bfcc1747942972680ba5cbabfea6822d79bc7/README.md","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/OpenMOSS/OmniVAE","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/OpenMOSS-Team/OmniVAE","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/OpenMOSS/OmniVAE/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/OpenMOSS-Team/OmniVAE/blob/e87bfcc1747942972680ba5cbabfea6822d79bc7/README.md","review_level":"license file or model-card declaration"},{"title":"OmniVAE: An Audio-Video VAE with Cross-Modal Alignment for Joint Generation","url":"https://arxiv.org/abs/2607.23855","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/OpenMOSS-Team/OmniVAE/blob/e87bfcc1747942972680ba5cbabfea6822d79bc7/models/vae/audio_only/recon/config.yaml","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--omnivae-audio.webp","kind":"source","label":"OmniVAE family architecture; this card covers the audio-only checkpoint","sourceUrl":"https://github.com/OpenMOSS/OmniVAE","originUrl":"https://raw.githubusercontent.com/OpenMOSS/OmniVAE/main/assets/architecture/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/omnivae-audio.png","sha256":"7762422be8564ab15bf268c98fd958b9b8fddb6ee5d31b4d2482b988343209aa"}},{"id":"codec:ltx2-audiovae","slug":"ltx2-audiovae","name":"LTX-2 audio VAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes audio spectrograms into continuous latents and reconstructs stereo audio through the LTX-2 vocoder.","architecture":"Causal two-dimensional mel-spectrogram VAE with a separate neural waveform vocoder.","notes":"The evidence is the original LTX-2 audio_vae and vocoder subfolders, not a video VAE. The mel hop is 160 samples and the VAE reduces time by four. The inspected vocoder outputs 24 kHz; this must not be confused with the 16 kHz analysis rate or assumed to describe every later LTX-2.x release.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"16 kHz analysis → 24 kHz stereo output","frame_rate":"25","bitrate":"Not applicable: continuous latents","representation":"8 latent channels × 16 frequency positions (128 values/time step)","streaming":"Causal VAE configuration; full streaming path not established"},"date":"","dateKind":"","links":{"code":"https://github.com/Lightricks/LTX-2","weights":"https://huggingface.co/Lightricks/LTX-2","paper":null,"catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#ltx2-audiovae"},"license":{"status":"restricted","code":"LTX Community License (version-specific)","weights":"LTX-2 Community License","codeUrl":"https://github.com/Lightricks/LTX-2/blob/main/LICENSE","weightsUrl":"https://huggingface.co/Lightricks/LTX-2/blob/dfcc2108383fe1aaa0584bdf55d368a4bdadd90c/LICENSE","notes":"The linked noncommercial or custom terms apply to this release.","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Lightricks/LTX-2","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/Lightricks/LTX-2","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Lightricks/LTX-2/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/Lightricks/LTX-2/blob/dfcc2108383fe1aaa0584bdf55d368a4bdadd90c/LICENSE","review_level":"license file or model-card declaration"},{"title":"Checkpoint configuration","url":"https://huggingface.co/Lightricks/LTX-2/blob/dfcc2108383fe1aaa0584bdf55d368a4bdadd90c/audio_vae/config.json","review_level":"configuration file"},{"title":"Checkpoint configuration","url":"https://huggingface.co/Lightricks/LTX-2/blob/dfcc2108383fe1aaa0584bdf55d368a4bdadd90c/vocoder/config.json","review_level":"configuration file"}],"figure":{"path":"assets/figures/codec--ltx2-audiovae.svg","kind":"editorial","label":"Editorial overview","sourceUrl":"https://github.com/Lightricks/LTX-2","originUrl":"","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/ltx2-audiovae.svg","sha256":"7ce29c3a2af0cb0ebaf23d75982d79c8a59458436172b439188b95b3ad21a9c2"}},{"id":"codec:auk-vae","slug":"auk-vae","name":"AuK BigVGANFlowVAE","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Encodes speech into continuous variational latents for generation and audio editing.","architecture":"Convolutional audio encoder with a variational posterior, residual coupling flow during training and a BigVGAN-style waveform decoder.","notes":"The VAE is independently stored in vae.safetensors; the full diffusion backbone is a separate file. A 480-sample stride gives 50 frames/s. The code has causal settings but also explicitly noncausal convolutions, so a strict end-to-end streaming claim is not made.","variants":[],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"24 kHz mono","frame_rate":"50","bitrate":"Not applicable: continuous latents","representation":"64-dimensional VAE latents","streaming":"Causal components; some noncausal convolutions remain"},"date":"2026-09-08","dateKind":"paper","links":{"code":"https://github.com/Tencent-Hunyuan/AuK","weights":"https://huggingface.co/tencent/AuK","paper":"https://arxiv.org/abs/2609.08936","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#auk-vae"},"license":{"status":"open","code":"MIT","weights":"MIT","codeUrl":"https://github.com/Tencent-Hunyuan/AuK/blob/main/LICENSE","weightsUrl":"https://huggingface.co/tencent/AuK/blob/790742b71a4430120daf2b2099192abae449eb9f/LICENSE","notes":"","components":[]},"sources":[{"title":"Official implementation and usage","url":"https://github.com/Tencent-Hunyuan/AuK","review_level":"repository README"},{"title":"Checkpoint files and model card","url":"https://huggingface.co/tencent/AuK","review_level":"file listing and model card"},{"title":"Code license evidence","url":"https://github.com/Tencent-Hunyuan/AuK/blob/main/LICENSE","review_level":"license file or explicit author declaration"},{"title":"Checkpoint license evidence","url":"https://huggingface.co/tencent/AuK/blob/790742b71a4430120daf2b2099192abae449eb9f/LICENSE","review_level":"license file or model-card declaration"},{"title":"AuK Technical Report: An Open-Source Foundational Model for Speech Generation and Editing","url":"https://arxiv.org/abs/2609.08936","review_level":"primary paper and codec architecture section"},{"title":"Checkpoint configuration","url":"https://huggingface.co/tencent/AuK/blob/790742b71a4430120daf2b2099192abae449eb9f/config.yaml","review_level":"configuration file"},{"title":"Codec implementation","url":"https://github.com/Tencent-Hunyuan/AuK/blob/main/src/auk/model/vae/bigvgan_flow_vae.py","review_level":"encoder/decoder implementation"}],"figure":{"path":"assets/figures/codec--auk-vae.webp","kind":"source","label":"Parent AuK architecture with audio VAE","sourceUrl":"https://github.com/Tencent-Hunyuan/AuK","originUrl":"https://raw.githubusercontent.com/Tencent-Hunyuan/AuK/main/assets/arch.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/auk-vae.png","sha256":"1512b37ee0a642be4ec67a79b37152b87900f63dc9176585a891d42ebaef891a"}},{"id":"codec:rave","slug":"rave","name":"RAVE (v1 / v2)","collection":"codec","category":"codec:continuous","categoryName":"VAE & continuous codecs","summary":"Provides variational audio encoders and decoders for reconstruction, timbre transfer and real-time audio processing.","architecture":"Convolutional variational audio autoencoder with multiband processing and adversarial fine-tuning; later configurations support streaming exports.","notes":"The inspected author-hosted checkpoint is the MusicNet RAVE v2 TorchScript export. It contains the reconstruction model rather than only a generative prior. Model-specific sample rate, latent dimension and frame rate are not inferred from the original v1 paper or a community mirror. The optional discrete RAVE configuration is not the checkpoint reviewed here.","variants":["RAVE v1","RAVE v2; MusicNet checkpoint reviewed"],"inputs":[],"outputs":[],"interaction":"","specs":{"sample_rate":"Checkpoint-specific; not independently established","frame_rate":"Not specified","bitrate":"Not applicable: continuous latents","representation":"Continuous variational latents; checkpoint-dependent width","streaming":"Author-provided streaming TorchScript exports"},"date":"2021-11-09","dateKind":"paper","links":{"code":"https://github.com/acids-ircam/RAVE","weights":"https://acids-ircam.github.io/rave_models_download","paper":"https://arxiv.org/abs/2111.05011","catalog":"https://github.com/kadirnar/awesome-codec-architectures/blob/59a1baab61a719c55d8df2150665d2243076c116/models/continuous.md#rave"},"license":{"status":"restricted","code":"CC-BY-NC-4.0","weights":"Not stated on the inspected download catalog","codeUrl":"https://github.com/acids-ircam/RAVE/blob/master/LICENSE","weightsUrl":"https://acids-ircam.github.io/rave_models_download","notes":"The code has noncommercial terms. The inspected author download catalog does not separately state the MusicNet checkpoint license.","components":[]},"sources":[{"title":"Official implementation and streaming exports","url":"https://github.com/acids-ircam/RAVE","review_level":"repository README"},{"title":"Official checkpoint catalog","url":"https://acids-ircam.github.io/rave_models_download","review_level":"author-maintained download page"},{"title":"Official model metadata","url":"https://play.forum.ircam.fr/rave-vst-api/get_available_models","review_level":"author API: MusicNet v2 entry"},{"title":"MusicNet checkpoint","url":"https://play.forum.ircam.fr/rave-vst-api/get_model/musicnet","review_level":"HTTP HEAD: filename, Content-Length and Last-Modified"},{"title":"Code license","url":"https://github.com/acids-ircam/RAVE/blob/master/LICENSE","review_level":"license file"},{"title":"RAVE: A variational autoencoder for fast and high-quality neural audio synthesis","url":"https://arxiv.org/abs/2111.05011","review_level":"primary paper and architecture figure"}],"figure":{"path":"assets/figures/codec--rave.webp","kind":"source","label":"Figure 1 — original RAVE family architecture","sourceUrl":"https://arxiv.org/abs/2111.05011","originUrl":"https://arxiv.org/html/2111.05011v2/first_page_figure.png","originalUrl":"https://raw.githubusercontent.com/kadirnar/awesome-codec-architectures/59a1baab61a719c55d8df2150665d2243076c116/assets/architectures/rave.png","sha256":"b02ad256dfa1daf59aae90d56dd77d77c638b6376d63c5bca614d32934584561"}}],"asOf":"2026-09-13"}
