Skip to content

Commit 982bf06

Browse files
GuikingoneGuikingone
committed
feat(platform): add Speech
1 parent 95751b8 commit 982bf06

File tree

61 files changed

+1454
-19
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+1454
-19
lines changed

‎demo/tests/Blog/Command/StreamCommandTest.php‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,13 @@
1616
use Symfony\AI\Agent\AgentInterface;
1717
use Symfony\AI\Platform\Message\MessageBag;
1818
use Symfony\AI\Platform\Metadata\Metadata;
19+
use Symfony\AI\Platform\Result\DeferredResult;
20+
use Symfony\AI\Platform\Result\InMemoryRawResult;
1921
use Symfony\AI\Platform\Result\RawResultInterface;
2022
use Symfony\AI\Platform\Result\ResultInterface;
23+
use Symfony\AI\Platform\Result\TextResult;
24+
use Symfony\AI\Platform\Speech\Speech;
25+
use Symfony\AI\Platform\Test\PlainConverter;
2126
use Symfony\Component\Console\Input\ArrayInput;
2227
use Symfony\Component\Console\Output\BufferedOutput;
2328
use Symfony\Component\Console\Style\SymfonyStyle;
@@ -52,6 +57,15 @@ public function getRawResult(): ?RawResultInterface
5257
public function setRawResult(RawResultInterface $rawResult): void
5358
{
5459
}
60+
61+
public function addSpeech(Speech $speech): void
62+
{
63+
}
64+
65+
public function getSpeech(string $identifier): Speech
66+
{
67+
return new Speech([], new DeferredResult(new PlainConverter(new TextResult('foo')), new InMemoryRawResult()), 'bar');
68+
}
5569
});
5670

5771
$input = new ArrayInput([]);

‎docs/components/platform.rst‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,63 @@ This allows fast and isolated testing of AI-powered features without relying on
502502

503503
This requires `cURL` and the `ext-curl` extension to be installed.
504504

505+
Speech support
506+
~~~~~~~~~~~~~~
507+
508+
Using speech to send messages / receive answers as audio is a common use case when integrating agents and/or chats.
509+
510+
Speech support can be enable using ``Symfony\AI\Platform\Speech\SpeechProviderListener``::
511+
512+
use Symfony\AI\Agent\Agent;
513+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
514+
use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
515+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
516+
use Symfony\AI\Platform\Message\Message;
517+
use Symfony\AI\Platform\Message\MessageBag;
518+
use Symfony\AI\Platform\Speech\SpeechConfiguration;
519+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
520+
use Symfony\Component\EventDispatcher\EventDispatcher;
521+
522+
$eventDispatcher = new EventDispatcher();
523+
$eventDispatcher->addSubscriber(new SpeechProviderListener([
524+
new ElevenLabsSpeechProvider(PlatformFactory::create(
525+
apiKey: $elevenLabsApiKey,
526+
httpClient: http_client(),
527+
speechConfiguration: new SpeechConfiguration(
528+
ttsModel: 'eleven_multilingual_v2',
529+
ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
530+
sttModel: 'eleven_multilingual_v2'
531+
)),
532+
),
533+
], []));
534+
535+
$platform = OpenAiPlatformFactory::create($openAiApiKey, httpClient: HttpClient::create(), eventDispatcher: $eventDispatcher);
536+
537+
$agent = new Agent($platform, 'gpt-4o');
538+
$answer = $agent->call(new MessageBag(
539+
Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'),
540+
));
541+
542+
echo $answer->getSpeech('eleven_labs')->asBinary();
543+
544+
When using the bundle, the configuration allows to configure models and voices::
545+
546+
ai:
547+
platform:
548+
eleven_labs:
549+
api_key: '%env(ELEVEN_LABS_API_KEY)%'
550+
551+
speech:
552+
eleven_labs:
553+
tts_model: 'eleven_multilingual_v2'
554+
tts_voice: '%env(ELEVEN_LABS_VOICE_IDENTIFIER)%'
555+
tts_extra_options:
556+
foo: bar
557+
558+
.. note::
559+
560+
Please be aware that enabling speech support requires to define corresponding platforms.
561+
505562
Code Examples
506563
~~~~~~~~~~~~~
507564

‎examples/speech/README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Speech Examples
2+
3+
Speech is mainly used to transform text to audio and vice versa, it can also be used to create an audio to audio pipeline.
4+
5+
To run the examples, you can use additional tools like (mpg123)[https://www.mpg123.de/]:
6+
7+
```bash
8+
php speech/agent-eleven-labs-speech-tts.php | mpg123 -
9+
php speech/agent-eleven-labs-speech-sts.php | mpg123 -
10+
```

‎examples/speech/agent-eleven-labs-speech-sts.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechListener;
14+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
15+
use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
16+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
17+
use Symfony\AI\Platform\Message\Content\Audio;
18+
use Symfony\AI\Platform\Message\Message;
19+
use Symfony\AI\Platform\Message\MessageBag;
20+
use Symfony\AI\Platform\Speech\SpeechConfiguration;
21+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
22+
use Symfony\Component\EventDispatcher\EventDispatcher;
23+
24+
require_once dirname(__DIR__).'/bootstrap.php';
25+
26+
$eventDispatcher = new EventDispatcher();
27+
$eventDispatcher->addSubscriber(new SpeechProviderListener([
28+
new ElevenLabsSpeechProvider(PlatformFactory::create(
29+
apiKey: env('ELEVEN_LABS_API_KEY'),
30+
httpClient: http_client(),
31+
speechConfiguration: new SpeechConfiguration(
32+
ttsModel: 'eleven_multilingual_v2',
33+
ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
34+
sttModel: 'eleven_multilingual_v2'
35+
)),
36+
),
37+
], [
38+
new ElevenLabsSpeechListener(PlatformFactory::create(
39+
apiKey: env('ELEVEN_LABS_API_KEY'),
40+
httpClient: http_client(),
41+
speechConfiguration: new SpeechConfiguration(
42+
sttModel: 'scribe_v1'
43+
)),
44+
),
45+
]));
46+
47+
$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
48+
49+
$agent = new Agent($platform, 'gpt-4o');
50+
$answer = $agent->call(new MessageBag(
51+
Message::ofUser(Audio::fromFile(dirname(__DIR__, 2).'/fixtures/audio.mp3'))
52+
));
53+
54+
echo $answer->getSpeech('eleven_labs')->asBinary();

‎examples/speech/agent-eleven-labs-speech-stt.php‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechListener;
14+
use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
15+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
16+
use Symfony\AI\Platform\Message\Content\Audio;
17+
use Symfony\AI\Platform\Message\Message;
18+
use Symfony\AI\Platform\Message\MessageBag;
19+
use Symfony\AI\Platform\Speech\SpeechConfiguration;
20+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
21+
use Symfony\Component\EventDispatcher\EventDispatcher;
22+
23+
require_once dirname(__DIR__).'/bootstrap.php';
24+
25+
$eventDispatcher = new EventDispatcher();
26+
$eventDispatcher->addSubscriber(new SpeechProviderListener([], [
27+
new ElevenLabsSpeechListener(PlatformFactory::create(
28+
apiKey: env('ELEVEN_LABS_API_KEY'),
29+
httpClient: http_client(),
30+
speechConfiguration: new SpeechConfiguration(
31+
sttModel: 'scribe_v1'
32+
)),
33+
),
34+
]));
35+
36+
$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
37+
38+
$agent = new Agent($platform, 'gpt-4o');
39+
$answer = $agent->call(new MessageBag(
40+
Message::ofUser(Audio::fromFile(dirname(__DIR__, 2).'/fixtures/audio.mp3'))
41+
));
42+
43+
echo $answer->getContent();

‎examples/speech/agent-eleven-labs-speech-tts.php‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Platform\Bridge\ElevenLabs\ElevenLabsSpeechProvider;
14+
use Symfony\AI\Platform\Bridge\ElevenLabs\PlatformFactory;
15+
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory;
16+
use Symfony\AI\Platform\Message\Message;
17+
use Symfony\AI\Platform\Message\MessageBag;
18+
use Symfony\AI\Platform\Speech\SpeechConfiguration;
19+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
20+
use Symfony\Component\EventDispatcher\EventDispatcher;
21+
22+
require_once dirname(__DIR__).'/bootstrap.php';
23+
24+
$eventDispatcher = new EventDispatcher();
25+
$eventDispatcher->addSubscriber(new SpeechProviderListener([
26+
new ElevenLabsSpeechProvider(PlatformFactory::create(
27+
apiKey: env('ELEVEN_LABS_API_KEY'),
28+
httpClient: http_client(),
29+
speechConfiguration: new SpeechConfiguration(
30+
ttsModel: 'eleven_multilingual_v2',
31+
ttsVoice: 'Dslrhjl3ZpzrctukrQSN', // Brad (https://elevenlabs.io/app/voice-library?voiceId=Dslrhjl3ZpzrctukrQSN)
32+
sttModel: 'eleven_multilingual_v2'
33+
)),
34+
),
35+
], []));
36+
37+
$platform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), httpClient: http_client(), eventDispatcher: $eventDispatcher);
38+
39+
$agent = new Agent($platform, 'gpt-4o');
40+
$answer = $agent->call(new MessageBag(
41+
Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'),
42+
));
43+
44+
echo $answer->getSpeech('eleven_labs')->asBinary();

‎src/agent/src/Output.php‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
use Symfony\AI\Platform\Message\MessageBag;
1515
use Symfony\AI\Platform\Result\ResultInterface;
16+
use Symfony\AI\Platform\Speech\Speech;
1617

1718
/**
1819
* @author Christopher Hertel <mail@christopher-hertel.de>
@@ -27,6 +28,7 @@ public function __construct(
2728
private ResultInterface $result,
2829
private readonly MessageBag $messageBag,
2930
private readonly array $options = [],
31+
private ?Speech $speech = null,
3032
) {
3133
}
3234

@@ -57,4 +59,14 @@ public function getOptions(): array
5759
{
5860
return $this->options;
5961
}
62+
63+
public function setSpeech(?Speech $speech): void
64+
{
65+
$this->speech = $speech;
66+
}
67+
68+
public function getSpeech(): ?Speech
69+
{
70+
return $this->speech;
71+
}
6072
}

‎src/ai-bundle/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,4 @@ CHANGELOG
3434
- Token usage metadata in agent results including prompt, completion, total, cached, and thinking tokens
3535
- Rate limit information tracking for supported platforms
3636
* Add support for configuring chats and message stores
37+
* Add support for configuring speeches

‎src/ai-bundle/config/options.php‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,6 +1035,23 @@
10351035
->end()
10361036
->end()
10371037
->end()
1038+
->arrayNode('speech')
1039+
->children()
1040+
->arrayNode('elevenlabs')
1041+
->children()
1042+
->stringNode('tts_model')->end()
1043+
->stringNode('tts_voice')->end()
1044+
->arrayNode('tts_extra_options')
1045+
->scalarPrototype()->end()
1046+
->end()
1047+
->stringNode('stt_model')->end()
1048+
->arrayNode('stt_extra_options')
1049+
->scalarPrototype()->end()
1050+
->end()
1051+
->end()
1052+
->end()
1053+
->end()
1054+
->end()
10381055
->arrayNode('vectorizer')
10391056
->info('Vectorizers for converting strings to Vector objects and transforming TextDocument arrays to VectorDocument arrays')
10401057
->useAttributeAsKey('name')

‎src/ai-bundle/config/services.php‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
use Symfony\AI\Platform\Contract\JsonSchema\DescriptionParser;
6464
use Symfony\AI\Platform\Contract\JsonSchema\Factory as SchemaFactory;
6565
use Symfony\AI\Platform\Serializer\StructuredOutputSerializer;
66+
use Symfony\AI\Platform\Speech\SpeechProviderListener;
6667
use Symfony\AI\Platform\StructuredOutput\PlatformSubscriber;
6768
use Symfony\AI\Platform\StructuredOutput\ResponseFormatFactory;
6869
use Symfony\AI\Platform\StructuredOutput\ResponseFormatFactoryInterface;
@@ -241,5 +242,13 @@
241242
tagged_locator('ai.message_store', 'name'),
242243
])
243244
->tag('console.command')
245+
246+
// listeners
247+
->set('ai.speech_provider.listener', SpeechProviderListener::class)
248+
->args([
249+
tagged_iterator('ai.speech_provider', 'name'),
250+
tagged_iterator('ai.speech_listener', 'name'),
251+
])
252+
->tag('kernel.event_subscriber')
244253
;
245254
};

0 commit comments

Comments
 (0)