From 6f303a3c13344b2e43fe634017483db648fd9568 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 14:53:35 +0200 Subject: [PATCH 1/7] add realtime text to speech client, WIP --- orchestration/pom.xml | 5 + .../orchestration/OrchestrationClient.java | 165 ++++++++++++++- .../orchestration/RealtimeClientFactory.java | 200 ++++++++++++++++++ .../TextToSpeechRealtimeClient.java | 53 +++++ .../WSSOpenAIRealtimeClient.java | 157 ++++++++++++++ .../orchestration/WebsocketListenerGlue.java | 31 +++ .../java/com/sap/ai/sdk/app/Application.java | 25 ++- .../app/services/OrchestrationService.java | 9 + 8 files changed, 640 insertions(+), 5 deletions(-) create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java diff --git a/orchestration/pom.xml b/orchestration/pom.xml index f8027fe4a..a1ef764f1 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -122,6 +122,11 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml + + com.openai + openai-java + 4.41.0 + org.projectlombok diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 73c7161e0..4e901e966 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,6 +7,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import com.openai.models.beta.realtime.sessions.Session; +import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; @@ -17,16 +21,27 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import io.vavr.collection.Array; import io.vavr.control.Try; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -34,9 +49,12 @@ public class OrchestrationClient { private static final String DEFAULT_SCENARIO = "orchestration"; private static final String COMPLETION_ENDPOINT = "/v2/completion"; + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + private final OrchestrationHttpExecutor executor; private final List
customHeaders = new ArrayList<>(); @@ -103,6 +121,147 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } + /** + * Voices input text into output audio + * + * @param input text to voice + * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + */ + public Stream speech(String input) { + + var deploymentID = ""; + final List result = new ArrayList<>(); + try { + HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable + WebSocket ws = + client + .newWebSocketBuilder() + .header( + "Authorization", + "Bearer .......") + .header("AI-Resource-Group", "default") + .buildAsync( + URI.create(REALTIME_API_PATH + deploymentID), + new WebSocket.Listener() { + + @Override + public void onOpen(WebSocket webSocket) { + log.warn("Websocket connection opened"); + String sessionUpdateJson = + """ + { + "type": "session.update", + "session": { + "type": "realtime", + "output_modalities": ["audio"], + "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", + "audio": { + "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, + "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } + } + } + } + """; + + // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": + // {\"modalities\": [\"audio\", \"text\"]}}"; + webSocket.sendText(sessionUpdateJson, true); + webSocket.request(1); + + String textMessage = + """ +{ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "system", + "content": [ + { + "type": "input_text", + "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" + } + ] + } +} +"""; + //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input + log.warn("Sending message to websocket: {}", textMessage); + webSocket.sendText(textMessage, true); + webSocket.request(1); + + String voiceMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Eat more of these fresh french baguettes" + } + ] + } + } + """; + + webSocket.sendText(voiceMessage, true); + webSocket.request(1); + + + + + String askForResponse = "{ \"type\": \"response.create\" }"; + webSocket.sendText(askForResponse, true); + + webSocket.request(80); + } + + @Override + public CompletionStage onText( + WebSocket webSocket, CharSequence data, boolean last) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + log.warn("text received: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + String type = event.get("type").asText(); + if ("response.output_audio.delta".equals(type)) { + String base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + result.add(audio); + } else if ("response.audio_transcript.delta".equals(type)) { + log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); + } else if ("response.output_audio.done".equals(type)) { + // TODO: we should not wait last ack, we should return stream immediately, + // this requires + // TODO: thread management, factory and thread safety of all structs + webSocket.sendClose(1000, "done"); + return null; + } else { + log.warn("Unhandled event type: {}", data); + } + return CompletableFuture.completedStage(null); + } + }) + .join(); + Thread.sleep(25000); + return result.stream(); + } catch (Exception e) { + //log.error(e.getMessage(), e); + throw new OrchestrationClientException("Failed to call realtime API", e); + } + } + /** * Generate a completion for the given prompt. * diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java new file mode 100644 index 000000000..d48dd7b9f --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java @@ -0,0 +1,200 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.responses.Response; +import com.sap.cloud.sdk.cloudplatform.connectivity.Header; +import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nonnull; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Stream; + +@Slf4j +@RequiredArgsConstructor(access = lombok.AccessLevel.PRIVATE) +public class RealtimeClientFactory { + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; + + private final Supplier destinationSupplier; + private final Supplier deploymentIdSupplier; + + /** + * Voices input text into output audio + * + * @param input text to voice + * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + */ + public Stream textToSpeech(@Nonnull final Consumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + + var deploymentId = deploymentIdSupplier.get(); + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + sb.append(destination.getUri().getHost()); + // set port explicitly if defined in the uri + if (destination.getUri().getPort() != -1) { + sb.append(":").append(destination.getUri().getPort()); + } + sb.append(String.format(REALTIME_API_PATH_SUFFIX, deploymentId)); + var endpoint = sb.toString(); + + + final List result = new ArrayList<>(); + try { + HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable + WebSocket ws = + client + .newWebSocketBuilder() + .header( + "Authorization", + "Bearer ....") + .header("AI-Resource-Group", "default") + .buildAsync( + URI.create(endpoint), + new WebSocket.Listener() { + + @Override + public void onOpen(WebSocket webSocket) { + log.warn("Websocket connection opened"); + String sessionUpdateJson = + """ + { + "type": "session.update", + "session": { + "type": "realtime", + "output_modalities": ["audio"], + "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", + "audio": { + "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, + "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } + } + } + } + """; + + // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": + // {\"modalities\": [\"audio\", \"text\"]}}"; + webSocket.sendText(sessionUpdateJson, true); + webSocket.request(1); + + String textMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "system", + "content": [ + { + "type": "input_text", + "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" + } + ] + } + } + """; + //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input + log.warn("Sending message to websocket: {}", textMessage); + webSocket.sendText(textMessage, true); + webSocket.request(1); + + String voiceMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Eat more of these fresh french baguettes" + } + ] + } + } + """; + + webSocket.sendText(voiceMessage, true); + webSocket.request(1); + + + + + String askForResponse = "{ \"type\": \"response.create\" }"; + webSocket.sendText(askForResponse, true); + + webSocket.request(80); + } + + @Override + public CompletionStage onText( + WebSocket webSocket, CharSequence data, boolean last) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + log.warn("text received: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + Response r = null; + + String type = event.get("type").asText(); + if ("response.output_audio.delta".equals(type)) { + String base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + result.add(audio); + } else if ("response.audio_transcript.delta".equals(type)) { + log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); + } else if ("response.output_audio.done".equals(type)) { + // TODO: we should not wait last ack, we should return stream immediately, + // this requires + // TODO: thread management, factory and thread safety of all structs + webSocket.sendClose(1000, "done"); + return null; + } else { + log.warn("Unhandled event type: {}", data); + } + return CompletableFuture.completedStage(null); + } + }) + .join(); + Thread.sleep(25000); + return result.stream(); + } catch (Exception e) { + //log.error(e.getMessage(), e); + throw new OrchestrationClientException("Failed to call realtime API", e); + } + } + +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java new file mode 100644 index 000000000..7df1881da --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -0,0 +1,53 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.RealtimeSessionCreateRequest; +import io.vavr.collection.Array; + +import java.util.*; +import java.util.function.BiConsumer; + +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient { + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + + + private final BiConsumer outputConsumer; + + public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected List getOutputModalities() { + return OUTPUT_MODALITIES; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.accept(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java new file mode 100644 index 000000000..588e5853b --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -0,0 +1,157 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; + +import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; + +@Slf4j +@AllArgsConstructor(access = AccessLevel.PACKAGE) +public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { + + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider + * in cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of inactivity, + * this is normally not configurable and would leave realtime conversation unusable on a pause. + *

+ * In mobile networks this period is even shorter and typical CGNAT session can only idle for 10-15 seconds + * before it gets terminated + *

+ * In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required + * to prevent unintended closing of connections while conversation is still used + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); + } + this.ws = wsBuilder.buildAsync(URI.create(url), new WebsocketListenerGlue(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); + } + + public synchronized void askForResponse() { + WebSocket ws = this.ws.join(); + ws.sendText(JACKSON.valueToTree(ResponseCreateEvent.builder().build()), true); + ws.request(1); + } + + @Override + public synchronized void close() { + WebSocket ws = this.ws.join(); + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + this.client.close(); + } + + protected abstract String getSystemPrompt(); + + protected abstract List getOutputModalities(); + + protected abstract void onResponse(String eventType, JsonNode event); + + protected synchronized void sendMessage(RealtimeConversationItemUserMessage message) { + WebSocket ws = this.ws.join(); + ws.sendText(JACKSON.valueToTree(message), true); + ws.request(1); + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); + } + + webSocket.request(1); + } + + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; + } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } + + private void configureSession(WebSocket ws) { + SessionUpdateEvent sue = SessionUpdateEvent.builder() + .session(ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(getOutputModalities()) + .audio(RealtimeAudioConfig.builder() + .input(RealtimeAudioConfigInput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .build()) + .output(RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .voice(RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + ws.sendText(JACKSON.valueToTree(sue), true); + ws.request(1); + } + + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + var message = new RealtimeConversationItemSystemMessage.Builder(). + addContent(RealtimeConversationItemSystemMessage.Content.builder().text(systemPrompt).build()); + ws.sendText(JACKSON.valueToTree(message), true); + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = new TimerTask(){ + @Override + public void run() { + sendPing(ws); + } + + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java new file mode 100644 index 000000000..28bb102ec --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java @@ -0,0 +1,31 @@ +package com.sap.ai.sdk.orchestration; + +import java.net.http.WebSocket; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public class WebsocketListenerGlue implements WebSocket.Listener { + + private final Consumer onOpen; + private final BiConsumer onText; + + public WebsocketListenerGlue(Consumer onOpen, BiConsumer onText) { + this.onOpen = onOpen; + this.onText = onText; + } + + @Override + public void onOpen(WebSocket webSocket) { + this.onOpen.accept(webSocket); + webSocket.request(1); + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + this.onText.accept(webSocket, data); + webSocket.request(1); + return CompletableFuture.completedStage(null); + } +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index 0f4ed8348..01269478d 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -6,6 +6,9 @@ import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.Nonnull; + +import com.sap.ai.sdk.app.services.OrchestrationService; +import com.sap.ai.sdk.core.AiCoreService; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -13,6 +16,10 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; +import java.io.*; +import java.nio.file.Files; +import java.util.stream.Collectors; + /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -37,7 +44,21 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) { - SpringApplication.run(Application.class, args); + public static void main(final String[] args) throws IOException { + var ctx = SpringApplication.run(Application.class, args); + var svc = ctx.getBean(OrchestrationService.class); + var res = svc.speech("Eat more of these fresh french baguettes").toList(); + String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; + File file = new File(path); + try (FileOutputStream in = new FileOutputStream(file)) { + var ttl = 0; + for (byte[] chunk: res) { + System.out.println("Writing bytes: " + chunk.length); + in.write(chunk); + ttl += chunk.length; + } + System.out.println("Written bytes: " + ttl); + } + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index bf8d82b9d..4ef188372 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -82,6 +82,15 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) return client.chatCompletion(prompt, config); } + /** + * TODO: document + * @param input + * @return + */ + public Stream speech(@Nonnull final String input) { + return client.speech(input); + } + /** * Chat request to OpenAI through the Orchestration service with an image. * From ecd953e472948e64b9ccd96176ea7ede8f7142a5 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 19:42:08 +0200 Subject: [PATCH 2/7] polish minimap text-to-speech realtime poc --- .../foundationmodels/openai/OpenAiModel.java | 2 +- .../orchestration/OrchestrationAiModel.java | 3 + .../orchestration/OrchestrationClient.java | 152 ++----------- .../orchestration/RealtimeChannelFactory.java | 58 +++++ .../orchestration/RealtimeClientFactory.java | 200 ------------------ .../sdk/orchestration/TextInputChannel.java | 5 + .../TextToSpeechRealtimeClient.java | 22 +- .../WSSOpenAIRealtimeClient.java | 121 +++++++---- .../java/com/sap/ai/sdk/app/Application.java | 28 ++- .../app/services/OrchestrationService.java | 34 +-- 10 files changed, 225 insertions(+), 400 deletions(-) create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java index 1926f56ce..62c435bcc 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java @@ -121,7 +121,7 @@ public record OpenAiModel(@Nonnull String name, @Nullable String version) implem /** Azure OpenAI GPT-5-nano model */ public static final OpenAiModel GPT_5_NANO = new OpenAiModel("gpt-5-nano", null); - /** Azure OpenAI GPT-5-nano model */ + /** Azure OpenAI GPT-realtime model */ public static final OpenAiModel GPT_REALTIME = new OpenAiModel("gpt-realtime", null); /** Azure OpenAI GPT-5.2 model */ diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java index 06b6beb08..1c6cdf7b3 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java @@ -330,6 +330,9 @@ public class OrchestrationAiModel { /** Azure OpenAI GPT-5-mini model */ public static final OrchestrationAiModel GPT_5_MINI = new OrchestrationAiModel("gpt-5-mini"); + /** Azure OpenAI GPT-realtime model */ + public static final OrchestrationAiModel GPT_REALTIME = new OrchestrationAiModel("gpt-realtime"); + /** Azure OpenAI GPT-5-nano model */ public static final OrchestrationAiModel GPT_5_NANO = new OrchestrationAiModel("gpt-5-nano"); diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 4e901e966..ad082065d 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -12,6 +12,7 @@ import com.openai.models.beta.realtime.sessions.Session; import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; +import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.CompletionRequestConfiguration; @@ -32,6 +33,7 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; @@ -42,6 +44,8 @@ import lombok.val; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -124,142 +128,26 @@ public OrchestrationChatResponse chatCompletion( /** * Voices input text into output audio * - * @param input text to voice - * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, + * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore */ - public Stream speech(String input) { - - var deploymentID = ""; - final List result = new ArrayList<>(); - try { - HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable - WebSocket ws = - client - .newWebSocketBuilder() - .header( - "Authorization", - "Bearer .......") - .header("AI-Resource-Group", "default") - .buildAsync( - URI.create(REALTIME_API_PATH + deploymentID), - new WebSocket.Listener() { - - @Override - public void onOpen(WebSocket webSocket) { - log.warn("Websocket connection opened"); - String sessionUpdateJson = - """ - { - "type": "session.update", - "session": { - "type": "realtime", - "output_modalities": ["audio"], - "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", - "audio": { - "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, - "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } - } - } - } - """; - - // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": - // {\"modalities\": [\"audio\", \"text\"]}}"; - webSocket.sendText(sessionUpdateJson, true); - webSocket.request(1); - - String textMessage = - """ -{ - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "system", - "content": [ - { - "type": "input_text", - "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" - } - ] - } -} -"""; - //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input - log.warn("Sending message to websocket: {}", textMessage); - webSocket.sendText(textMessage, true); - webSocket.request(1); - - String voiceMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Eat more of these fresh french baguettes" - } - ] - } - } - """; - - webSocket.sendText(voiceMessage, true); - webSocket.request(1); - - - + public TextInputChannel textToSpeech(BiConsumer output) { + var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { + @Override + public @NonNull String name() { + return OrchestrationAiModel.GPT_REALTIME.getName(); + } - String askForResponse = "{ \"type\": \"response.create\" }"; - webSocket.sendText(askForResponse, true); + @Override + public @Nullable String version() { + return OrchestrationAiModel.GPT_REALTIME.getVersion(); + } + }); - webSocket.request(80); - } + var factory = new RealtimeChannelFactory(() -> destination); - @Override - public CompletionStage onText( - WebSocket webSocket, CharSequence data, boolean last) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - log.warn("text received: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - String type = event.get("type").asText(); - if ("response.output_audio.delta".equals(type)) { - String base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - result.add(audio); - } else if ("response.audio_transcript.delta".equals(type)) { - log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); - } else if ("response.output_audio.done".equals(type)) { - // TODO: we should not wait last ack, we should return stream immediately, - // this requires - // TODO: thread management, factory and thread safety of all structs - webSocket.sendClose(1000, "done"); - return null; - } else { - log.warn("Unhandled event type: {}", data); - } - return CompletableFuture.completedStage(null); - } - }) - .join(); - Thread.sleep(25000); - return result.stream(); - } catch (Exception e) { - //log.error(e.getMessage(), e); - throw new OrchestrationClientException("Failed to call realtime API", e); - } + return factory.textToSpeech(output); } /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java new file mode 100644 index 000000000..cffba0e1f --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -0,0 +1,58 @@ +package com.sap.ai.sdk.orchestration; + +import com.sap.cloud.sdk.cloudplatform.connectivity.Header; +import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nonnull; +import java.util.*; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +@Slf4j +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +public class RealtimeChannelFactory { + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; + + private final Supplier destinationSupplier; + + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and + * Boolean - marker if current audio reply has been finished (current response ended) + * @return input channel, allowing for text input + */ + public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + var endpoint = sb.toString(); + + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } + +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java deleted file mode 100644 index d48dd7b9f..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java +++ /dev/null @@ -1,200 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.responses.Response; -import com.sap.cloud.sdk.cloudplatform.connectivity.Header; -import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Flow; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Stream; - -@Slf4j -@RequiredArgsConstructor(access = lombok.AccessLevel.PRIVATE) -public class RealtimeClientFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; - - private final Supplier destinationSupplier; - private final Supplier deploymentIdSupplier; - - /** - * Voices input text into output audio - * - * @param input text to voice - * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian - */ - public Stream textToSpeech(@Nonnull final Consumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var deploymentId = deploymentIdSupplier.get(); - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - sb.append(destination.getUri().getHost()); - // set port explicitly if defined in the uri - if (destination.getUri().getPort() != -1) { - sb.append(":").append(destination.getUri().getPort()); - } - sb.append(String.format(REALTIME_API_PATH_SUFFIX, deploymentId)); - var endpoint = sb.toString(); - - - final List result = new ArrayList<>(); - try { - HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable - WebSocket ws = - client - .newWebSocketBuilder() - .header( - "Authorization", - "Bearer ....") - .header("AI-Resource-Group", "default") - .buildAsync( - URI.create(endpoint), - new WebSocket.Listener() { - - @Override - public void onOpen(WebSocket webSocket) { - log.warn("Websocket connection opened"); - String sessionUpdateJson = - """ - { - "type": "session.update", - "session": { - "type": "realtime", - "output_modalities": ["audio"], - "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", - "audio": { - "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, - "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } - } - } - } - """; - - // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": - // {\"modalities\": [\"audio\", \"text\"]}}"; - webSocket.sendText(sessionUpdateJson, true); - webSocket.request(1); - - String textMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "system", - "content": [ - { - "type": "input_text", - "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" - } - ] - } - } - """; - //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input - log.warn("Sending message to websocket: {}", textMessage); - webSocket.sendText(textMessage, true); - webSocket.request(1); - - String voiceMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Eat more of these fresh french baguettes" - } - ] - } - } - """; - - webSocket.sendText(voiceMessage, true); - webSocket.request(1); - - - - - String askForResponse = "{ \"type\": \"response.create\" }"; - webSocket.sendText(askForResponse, true); - - webSocket.request(80); - } - - @Override - public CompletionStage onText( - WebSocket webSocket, CharSequence data, boolean last) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - log.warn("text received: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - Response r = null; - - String type = event.get("type").asText(); - if ("response.output_audio.delta".equals(type)) { - String base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - result.add(audio); - } else if ("response.audio_transcript.delta".equals(type)) { - log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); - } else if ("response.output_audio.done".equals(type)) { - // TODO: we should not wait last ack, we should return stream immediately, - // this requires - // TODO: thread management, factory and thread safety of all structs - webSocket.sendClose(1000, "done"); - return null; - } else { - log.warn("Unhandled event type: {}", data); - } - return CompletableFuture.completedStage(null); - } - }) - .join(); - Thread.sleep(25000); - return result.stream(); - } catch (Exception e) { - //log.error(e.getMessage(), e); - throw new OrchestrationClientException("Failed to call realtime API", e); - } - } - -} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java new file mode 100644 index 000000000..20e6fcc66 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java @@ -0,0 +1,5 @@ +package com.sap.ai.sdk.orchestration; + +public interface TextInputChannel extends AutoCloseable { + void sendText(String text); +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java index 7df1881da..45ea9335a 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -1,13 +1,14 @@ package com.sap.ai.sdk.orchestration; import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.RealtimeSessionCreateRequest; -import io.vavr.collection.Array; +import com.openai.models.realtime.*; +import lombok.extern.slf4j.Slf4j; import java.util.*; import java.util.function.BiConsumer; -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient { +@Slf4j +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -30,6 +31,18 @@ public TextToSpeechRealtimeClient(String url, Map httpHeaders, B this.outputConsumer = outputConsumer; } + public void sendText(String text) { + var message = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent(RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + super.sendMessage(message); + } + @Override protected String getSystemPrompt() { return TASK; @@ -46,8 +59,11 @@ protected void onResponse(String eventType, JsonNode event) { var base64Audio = event.get("delta").asText(); byte[] audio = Base64.getDecoder().decode(base64Audio); this.outputConsumer.accept(audio, Boolean.FALSE); + log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); } else if ("response.output_audio.done".equals(eventType)) { this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); } } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 588e5853b..8770baf5f 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -1,12 +1,16 @@ package com.sap.ai.sdk.orchestration; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.conversations.Message; +import com.openai.models.conversations.TextContent; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import lombok.AccessLevel; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.net.URI; @@ -19,11 +23,12 @@ import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; @Slf4j -@AllArgsConstructor(access = AccessLevel.PACKAGE) +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() + .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); /** * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider @@ -34,7 +39,7 @@ public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { * before it gets terminated *

* In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required - * to prevent unintended closing of connections while conversation is still used + * to prevent unintended closing of connections while conversation is still expected to continue */ private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; @@ -55,22 +60,30 @@ public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set< this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); } - public synchronized void askForResponse() { + public void askForResponse() { WebSocket ws = this.ws.join(); - ws.sendText(JACKSON.valueToTree(ResponseCreateEvent.builder().build()), true); - ws.request(1); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); + } } @Override - public synchronized void close() { + public void close() { WebSocket ws = this.ws.join(); - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 } - this.client.close(); } protected abstract String getSystemPrompt(); @@ -79,10 +92,19 @@ public synchronized void close() { protected abstract void onResponse(String eventType, JsonNode event); - protected synchronized void sendMessage(RealtimeConversationItemUserMessage message) { + protected void sendMessage(ConversationItemCreateEvent message) { WebSocket ws = this.ws.join(); - ws.sendText(JACKSON.valueToTree(message), true); - ws.request(1); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); + askForResponse(); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } + + } } private synchronized void onSocketOpen(WebSocket ws) { @@ -92,6 +114,7 @@ private synchronized void onSocketOpen(WebSocket ws) { } private void onText(WebSocket webSocket, CharSequence data) { + log.warn("WSS received message: {}", data); final JsonNode event; try { event = JACKSON.readTree(data.toString()); @@ -116,31 +139,57 @@ private synchronized void sendPing(WebSocket ws) { } private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = SessionUpdateEvent.builder() - .session(ClientSecretCreateParams.Session.ofRealtime( + SessionUpdateEvent sue = + SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( RealtimeSessionCreateRequest.builder() - .outputModalities(getOutputModalities()) - .audio(RealtimeAudioConfig.builder() - .input(RealtimeAudioConfigInput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .build()) - .output(RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .voice(RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - ws.sendText(JACKSON.valueToTree(sue), true); + .outputModalities(getOutputModalities()) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize session request", e); + } + ws.request(1); } private void configureConversation(WebSocket ws) { var systemPrompt = getSystemPrompt(); - var message = new RealtimeConversationItemSystemMessage.Builder(). - addContent(RealtimeConversationItemSystemMessage.Content.builder().text(systemPrompt).build()); - ws.sendText(JACKSON.valueToTree(message), true); + var systemConversationItem = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent(RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + log.warn("WSS conversation config sent: {}", json); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } ws.request(1); } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index 01269478d..aced5c87e 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -18,6 +18,8 @@ import java.io.*; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; /** Main class to start the Spring Boot application. */ @@ -44,15 +46,35 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) throws IOException { + public static void main(final String[] args) throws Exception { var ctx = SpringApplication.run(Application.class, args); var svc = ctx.getBean(OrchestrationService.class); - var res = svc.speech("Eat more of these fresh french baguettes").toList(); + var outputBytes = new ArrayList(); + var done = new AtomicBoolean(false); + try (var channel = svc.textToSpeech((bytes, last) -> { + System.err.println("appended output bytes: " + bytes.length); + outputBytes.add(bytes); + if (last) { + done.set(true); + } + })) { + System.err.println("BEFORE SENDING TEXT"); + channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); + System.err.println("AFTER SENDING TEXT"); + long timeout = System.currentTimeMillis() + 20000; + while (!done.get() && System.currentTimeMillis() < timeout) { + Thread.yield(); + System.err.println("Waiting for messages..."); + Thread.sleep(500); + } + } + + //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; File file = new File(path); try (FileOutputStream in = new FileOutputStream(file)) { var ttl = 0; - for (byte[] chunk: res) { + for (byte[] chunk: outputBytes) { System.out.println("Writing bytes: " + chunk.length); in.write(chunk); ttl += chunk.length; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 4ef188372..49fbaa594 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -10,26 +10,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.orchestration.AzureContentFilter; -import com.sap.ai.sdk.orchestration.AzureFilterThreshold; -import com.sap.ai.sdk.orchestration.DpiMasking; -import com.sap.ai.sdk.orchestration.Grounding; -import com.sap.ai.sdk.orchestration.ImageItem; -import com.sap.ai.sdk.orchestration.LlamaGuardFilter; -import com.sap.ai.sdk.orchestration.Message; -import com.sap.ai.sdk.orchestration.OrchestrationAiModel; -import com.sap.ai.sdk.orchestration.OrchestrationChatResponse; -import com.sap.ai.sdk.orchestration.OrchestrationClient; -import com.sap.ai.sdk.orchestration.OrchestrationClientException; -import com.sap.ai.sdk.orchestration.OrchestrationConfigReference; -import com.sap.ai.sdk.orchestration.OrchestrationEmbeddingRequest; -import com.sap.ai.sdk.orchestration.OrchestrationEmbeddingResponse; -import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig; -import com.sap.ai.sdk.orchestration.OrchestrationPrompt; -import com.sap.ai.sdk.orchestration.ResponseJsonSchema; -import com.sap.ai.sdk.orchestration.SystemMessage; -import com.sap.ai.sdk.orchestration.TemplateConfig; -import com.sap.ai.sdk.orchestration.TranslationConfig; +import com.sap.ai.sdk.orchestration.*; import com.sap.ai.sdk.orchestration.model.DPIEntities; import com.sap.ai.sdk.orchestration.model.DataRepositoryType; import com.sap.ai.sdk.orchestration.model.DocumentGroundingFilter; @@ -52,6 +33,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -83,12 +65,14 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) } /** - * TODO: document - * @param input - * @return + * Voices input text into output audio + * + * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, + * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore */ - public Stream speech(@Nonnull final String input) { - return client.speech(input); + public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { + return client.textToSpeech(outputConsumer); } /** From 71d4a4cc2b5cd9db00edaf14546030dbdd0e7938 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 20:03:22 +0200 Subject: [PATCH 3/7] add incomplete interim packets buffering and glueing --- ...{WebsocketListenerGlue.java => BufferedWebSocketListener.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename orchestration/src/main/java/com/sap/ai/sdk/orchestration/{WebsocketListenerGlue.java => BufferedWebSocketListener.java} (100%) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java similarity index 100% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java rename to orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java From 232725d79a04f8b1b162b41b97a01cbca4da1eb0 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 20:06:44 +0200 Subject: [PATCH 4/7] remove obsolete debug constants --- .../orchestration/BufferedWebSocketListener.java | 14 +++++++++++--- .../ai/sdk/orchestration/OrchestrationClient.java | 2 -- .../sdk/orchestration/RealtimeChannelFactory.java | 5 ----- .../sdk/orchestration/WSSOpenAIRealtimeClient.java | 5 ++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java index 28bb102ec..118b2d5ce 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java @@ -6,14 +6,16 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; -public class WebsocketListenerGlue implements WebSocket.Listener { +public class BufferedWebSocketListener implements WebSocket.Listener { private final Consumer onOpen; private final BiConsumer onText; + private final StringBuilder buffer; - public WebsocketListenerGlue(Consumer onOpen, BiConsumer onText) { + public BufferedWebSocketListener(Consumer onOpen, BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; + this.buffer = new StringBuilder(1024 * 1024); } @Override @@ -24,8 +26,14 @@ public void onOpen(WebSocket webSocket) { @Override public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { - this.onText.accept(webSocket, data); + buffer.append(data); webSocket.request(1); + if (last) { + var completeMessage = buffer.toString(); + buffer.setLength(0); + this.onText.accept(webSocket, completeMessage); + } + return CompletableFuture.completedStage(null); } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index ad082065d..7c52d772b 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -53,8 +53,6 @@ public class OrchestrationClient { private static final String DEFAULT_SCENARIO = "orchestration"; private static final String COMPLETION_ENDPOINT = "/v2/completion"; - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java index cffba0e1f..28edd05c4 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -16,13 +16,8 @@ public class RealtimeChannelFactory { private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; - private final Supplier destinationSupplier; /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 8770baf5f..678bff93a 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -5,8 +5,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.openai.models.conversations.Message; -import com.openai.models.conversations.TextContent; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import lombok.AccessLevel; @@ -55,7 +53,7 @@ public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set< for (Map.Entry entry : httpHeaders.entrySet()) { wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } - this.ws = wsBuilder.buildAsync(URI.create(url), new WebsocketListenerGlue(this::onSocketOpen, this::onText)); + this.ws = wsBuilder.buildAsync(URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); this.handleMessageTypes = handleMessageTypes; this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); } @@ -149,6 +147,7 @@ private void configureSession(WebSocket ws) { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) .format( RealtimeAudioFormats.AudioPcm.builder() .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) From 6b872d4b76a02976b4f4ad98ef0606bc7681704f Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Tue, 7 Jul 2026 18:08:02 +0000 Subject: [PATCH 5/7] Formatting --- .../BufferedWebSocketListener.java | 49 +-- .../orchestration/OrchestrationClient.java | 65 ++-- .../orchestration/RealtimeChannelFactory.java | 77 ++--- .../sdk/orchestration/TextInputChannel.java | 2 +- .../TextToSpeechRealtimeClient.java | 99 +++--- .../WSSOpenAIRealtimeClient.java | 297 +++++++++--------- .../java/com/sap/ai/sdk/app/Application.java | 42 +-- .../app/services/OrchestrationService.java | 5 +- 8 files changed, 321 insertions(+), 315 deletions(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java index 118b2d5ce..6116baeea 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java @@ -8,32 +8,33 @@ public class BufferedWebSocketListener implements WebSocket.Listener { - private final Consumer onOpen; - private final BiConsumer onText; - private final StringBuilder buffer; + private final Consumer onOpen; + private final BiConsumer onText; + private final StringBuilder buffer; - public BufferedWebSocketListener(Consumer onOpen, BiConsumer onText) { - this.onOpen = onOpen; - this.onText = onText; - this.buffer = new StringBuilder(1024 * 1024); - } - - @Override - public void onOpen(WebSocket webSocket) { - this.onOpen.accept(webSocket); - webSocket.request(1); - } + public BufferedWebSocketListener( + Consumer onOpen, BiConsumer onText) { + this.onOpen = onOpen; + this.onText = onText; + this.buffer = new StringBuilder(1024 * 1024); + } - @Override - public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { - buffer.append(data); - webSocket.request(1); - if (last) { - var completeMessage = buffer.toString(); - buffer.setLength(0); - this.onText.accept(webSocket, completeMessage); - } + @Override + public void onOpen(WebSocket webSocket) { + this.onOpen.accept(webSocket); + webSocket.request(1); + } - return CompletableFuture.completedStage(null); + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + buffer.append(data); + webSocket.request(1); + if (last) { + var completeMessage = buffer.toString(); + buffer.setLength(0); + this.onText.accept(webSocket, completeMessage); } + + return CompletableFuture.completedStage(null); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 7c52d772b..b03548e49 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,10 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; -import com.openai.client.OpenAIClient; -import com.openai.client.okhttp.OpenAIOkHttpClient; -import com.openai.models.beta.realtime.sessions.Session; -import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; @@ -22,28 +18,15 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import io.vavr.collection.Array; import io.vavr.control.Try; - -import java.io.ByteArrayOutputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClients; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; @@ -56,7 +39,6 @@ public class OrchestrationClient { static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); - private final OrchestrationHttpExecutor executor; private final List

customHeaders = new ArrayList<>(); @@ -123,30 +105,35 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } - /** - * Voices input text into output audio - * - * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(BiConsumer output) { - var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { - @Override - public @NonNull String name() { - return OrchestrationAiModel.GPT_REALTIME.getName(); - } + /** + * Voices input text into output audio + * + * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, + * little-endian, boolean flag signifies end of the current reply (only true for last chunk of + * current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore + */ + public TextInputChannel textToSpeech(BiConsumer output) { + var destination = + new AiCoreService() + .getInferenceDestination() + .forModel( + new AiModel() { + @Override + public @NonNull String name() { + return OrchestrationAiModel.GPT_REALTIME.getName(); + } - @Override - public @Nullable String version() { - return OrchestrationAiModel.GPT_REALTIME.getVersion(); - } - }); + @Override + public @Nullable String version() { + return OrchestrationAiModel.GPT_REALTIME.getVersion(); + } + }); - var factory = new RealtimeChannelFactory(() -> destination); + var factory = new RealtimeChannelFactory(() -> destination); - return factory.textToSpeech(output); - } + return factory.textToSpeech(output); + } /** * Generate a completion for the given prompt. diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java index 28edd05c4..6eb076f12 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -2,52 +2,53 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; import java.util.*; import java.util.function.BiConsumer; import java.util.function.Supplier; +import javax.annotation.Nonnull; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public class RealtimeChannelFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private final Supplier destinationSupplier; - - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and - * Boolean - marker if current audio reply has been finished (current response ended) - * @return input channel, allowing for text input - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - var endpoint = sb.toString(); + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + private final Supplier destinationSupplier; + + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 + * Hz output and Boolean - marker if current audio reply has been finished (current response + * ended) + * @return input channel, allowing for text input + */ + public TextInputChannel textToSpeech( + @Nonnull final BiConsumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + var endpoint = sb.toString(); + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java index 20e6fcc66..eff6a15df 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java @@ -1,5 +1,5 @@ package com.sap.ai.sdk.orchestration; public interface TextInputChannel extends AutoCloseable { - void sendText(String text); + void sendText(String text); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java index 45ea9335a..74552bb68 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -2,68 +2,71 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; -import lombok.extern.slf4j.Slf4j; - import java.util.*; import java.util.function.BiConsumer; +import lombok.extern.slf4j.Slf4j; @Slf4j -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient + implements TextInputChannel { - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; - private final BiConsumer outputConsumer; + private final BiConsumer outputConsumer; - public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - } + public TextToSpeechRealtimeClient( + String url, Map httpHeaders, BiConsumer outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } - public void sendText(String text) { - var message = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemUserMessage( - RealtimeConversationItemUserMessage.builder() - .addContent(RealtimeConversationItemUserMessage.Content.builder() - .text(text) - .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); + public void sendText(String text) { + var message = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent( + RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); - super.sendMessage(message); - } + super.sendMessage(message); + } - @Override - protected String getSystemPrompt() { - return TASK; - } + @Override + protected String getSystemPrompt() { + return TASK; + } - @Override - protected List getOutputModalities() { - return OUTPUT_MODALITIES; - } + @Override + protected List getOutputModalities() { + return OUTPUT_MODALITIES; + } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.accept(audio, Boolean.FALSE); - log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.accept(audio, Boolean.FALSE); + log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); } + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 678bff93a..9415f1840 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -1,5 +1,7 @@ package com.sap.ai.sdk.orchestration; +import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonProcessingException; @@ -7,136 +9,138 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.CompletableFuture; - -import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { - private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() - .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); - - /** - * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider - * in cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of inactivity, - * this is normally not configurable and would leave realtime conversation unusable on a pause. - *

- * In mobile networks this period is even shorter and typical CGNAT session can only idle for 10-15 seconds - * before it gets terminated - *

- * In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required - * to prevent unintended closing of connections while conversation is still expected to continue - */ - private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; - private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; - - private final HttpClient client; - private final CompletableFuture ws; - private final Timer heartbeatTimer; - private final Set handleMessageTypes; - - public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set handleMessageTypes) { - this.client = HttpClient.newHttpClient(); - var wsBuilder = this.client.newWebSocketBuilder(); - for (Map.Entry entry : httpHeaders.entrySet()) { - wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); - } - this.ws = wsBuilder.buildAsync(URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); - this.handleMessageTypes = handleMessageTypes; - this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = + getOrchestrationObjectMapper() + .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in + * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of + * inactivity, this is normally not configurable and would leave realtime conversation unusable on + * a pause. + * + *

In mobile networks this period is even shorter and typical CGNAT session can only idle for + * 10-15 seconds before it gets terminated + * + *

In realtime client we rely on explicit lifetime management (create/close), heartbeat + * mechanism is required to prevent unintended closing of connections while conversation is still + * expected to continue + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAIRealtimeClient( + String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } - - public void askForResponse() { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to serialize ask for response", e); - } - ws.request(1); - } + this.ws = + wsBuilder.buildAsync( + URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); + } + + public void askForResponse() { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); } - - @Override - public void close() { - WebSocket ws = this.ws.join(); - synchronized (this) { - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); - } - // this.client.close(); // exists only since java 21 - } + } + + @Override + public void close() { + WebSocket ws = this.ws.join(); + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 } + } - protected abstract String getSystemPrompt(); - - protected abstract List getOutputModalities(); + protected abstract String getSystemPrompt(); - protected abstract void onResponse(String eventType, JsonNode event); + protected abstract List getOutputModalities(); - protected void sendMessage(ConversationItemCreateEvent message) { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(message), true); - ws.request(1); - log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); - askForResponse(); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); - } + protected abstract void onResponse(String eventType, JsonNode event); - } + protected void sendMessage(ConversationItemCreateEvent message) { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); + askForResponse(); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } } - - private synchronized void onSocketOpen(WebSocket ws) { - configureSession(ws); - configureConversation(ws); - scheduleHeartbeat(ws); + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + log.warn("WSS received message: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Error parsing JSON response from speech API", e); } - - private void onText(WebSocket webSocket, CharSequence data) { - log.warn("WSS received message: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - var eventType = event.get("type").asText(); - if (handleMessageTypes.contains(eventType)) { - onResponse(eventType, event); - } - - webSocket.request(1); + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); } - private synchronized void sendPing(WebSocket ws) { - if (ws.isInputClosed()) { - return; - } - ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); - ws.request(1); + webSocket.request(1); + } + + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } - private void configureSession(WebSocket ws) { + private void configureSession(WebSocket ws) { SessionUpdateEvent sue = SessionUpdateEvent.builder() .session( @@ -147,10 +151,12 @@ private void configureSession(WebSocket ws) { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) + .turnDetection(Optional.empty()) .format( RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) .build()) .build()) .output( @@ -163,43 +169,48 @@ private void configureSession(WebSocket ws) { .build()) .asRealtime()) .build(); - try { - ws.sendText(JACKSON.writeValueAsString(sue), true); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize session request", e); - } - - ws.request(1); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize session request", e); } - private void configureConversation(WebSocket ws) { - var systemPrompt = getSystemPrompt(); - var systemConversationItem = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemSystemMessage( - RealtimeConversationItemSystemMessage.builder() - .addContent(RealtimeConversationItemSystemMessage.Content.builder() - .text(systemPrompt) - .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); - - try { - var json = JACKSON.writeValueAsString(systemConversationItem); - ws.sendText(json, true); - log.warn("WSS conversation config sent: {}", json); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); - } - ws.request(1); - } - - private void scheduleHeartbeat(WebSocket ws) { - TimerTask task = new TimerTask(){ - @Override - public void run() { - sendPing(ws); - } + ws.request(1); + } + + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + var systemConversationItem = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent( + RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); - }; - heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + log.warn("WSS conversation config sent: {}", json); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); } + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = + new TimerTask() { + @Override + public void run() { + sendPing(ws); + } + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index aced5c87e..61b5be494 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -5,10 +5,11 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; -import javax.annotation.Nonnull; - import com.sap.ai.sdk.app.services.OrchestrationService; -import com.sap.ai.sdk.core.AiCoreService; +import java.io.*; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nonnull; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -16,12 +17,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; -import java.io.*; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -51,13 +46,15 @@ public static void main(final String[] args) throws Exception { var svc = ctx.getBean(OrchestrationService.class); var outputBytes = new ArrayList(); var done = new AtomicBoolean(false); - try (var channel = svc.textToSpeech((bytes, last) -> { - System.err.println("appended output bytes: " + bytes.length); - outputBytes.add(bytes); - if (last) { - done.set(true); - } - })) { + try (var channel = + svc.textToSpeech( + (bytes, last) -> { + System.err.println("appended output bytes: " + bytes.length); + outputBytes.add(bytes); + if (last) { + done.set(true); + } + })) { System.err.println("BEFORE SENDING TEXT"); channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); System.err.println("AFTER SENDING TEXT"); @@ -69,18 +66,23 @@ public static void main(final String[] args) throws Exception { } } - //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); - String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; + // var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); + String path = + System.getProperty("user.home") + + File.separator + + "Documents" + + File.separator + + System.currentTimeMillis() + + ".pcm"; File file = new File(path); try (FileOutputStream in = new FileOutputStream(file)) { var ttl = 0; - for (byte[] chunk: outputBytes) { + for (byte[] chunk : outputBytes) { System.out.println("Writing bytes: " + chunk.length); in.write(chunk); ttl += chunk.length; } System.out.println("Written bytes: " + ttl); } - } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 49fbaa594..8a26d382b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -67,8 +67,9 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) /** * Voices input text into output audio * - * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, + * little-endian, boolean flag signifies end of the current reply (only true for last chunk of + * current response) * @return text input channel to supply text to voice, should be closed when not needed anymore */ public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { From 54b06a55a28df501253efb0050e5608e91c3e3a2 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 10 Jul 2026 14:56:26 +0200 Subject: [PATCH 6/7] add speech to speech realtime support and text to speech example --- foundation-models/openai/pom.xml | 4 + .../openai/AudioInputChannel.java | 17 +++ .../openai/AudioOutputChannel.java | 22 ++++ .../openai}/BufferedWebSocketListener.java | 17 ++- .../foundationmodels/openai/OpenAiClient.java | 73 ++++++++++++ .../openai/SpeechToSpeechRealtimeClient.java | 103 +++++++++++++++++ .../openai/SpeechToTextRealtimeClient.java | 93 +++++++++++++++ .../openai}/TextInputChannel.java | 2 +- .../openai/TextOutputChannel.java | 20 ++++ .../openai/TextToSpeechRealtimeClient.java | 100 ++++++++++++++++ .../openai}/WSSOpenAIRealtimeClient.java | 58 +++------- orchestration/pom.xml | 5 - .../orchestration/OrchestrationClient.java | 46 -------- .../orchestration/RealtimeChannelFactory.java | 53 --------- .../TextToSpeechRealtimeClient.java | 69 ----------- pom.xml | 6 + sample-code/spring-app/pom.xml | 4 + .../java/com/sap/ai/sdk/app/Application.java | 46 +------- .../com/sap/ai/sdk/app/WebsocketConfig.java | 23 ++++ .../TextToSpeechWebsocketHandler.java | 61 ++++++++++ .../ai/sdk/app/services/OpenAiService.java | 48 ++++++-- .../app/services/OrchestrationService.java | 12 -- .../src/main/resources/static/index.html | 107 ++++++++++++++++++ 23 files changed, 707 insertions(+), 282 deletions(-) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/BufferedWebSocketListener.java (69%) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/TextInputChannel.java (65%) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/WSSOpenAIRealtimeClient.java (70%) delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index 4e47adcd3..81ea4024f 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -66,6 +66,10 @@ com.sap.ai.sdk core + + com.openai + openai-java + com.google.code.findbugs jsr305 diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java new file mode 100644 index 000000000..9b4862802 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java @@ -0,0 +1,17 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +/** + * Functional interface representing audio input channel (audio data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface AudioInputChannel extends AutoCloseable { + + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context + * + * @param rawBytesChunk binary data in the depending on the use case format + */ + void inputAudio(byte[] rawBytesChunk); +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java new file mode 100644 index 000000000..e62e3c4cd --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -0,0 +1,22 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import java.io.IOException; + +/** + * Functional interface representing audio output channel (audio data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface AudioOutputChannel { + + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context + * + * @param rawBytesChunk binary data in the depending on the use case format + * @param isLast true if this call logically concludes previous and this passed bytes data into a + * single logical entity (e.g. gets called at the end when all byte parts of a single + * message get passed) + */ + void outputAudio(byte[] rawBytesChunk, boolean isLast); +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java similarity index 69% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 118b2d5ce..8377a2889 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -1,11 +1,15 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; + +import lombok.extern.slf4j.Slf4j; import java.net.http.WebSocket; +import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; import java.util.function.Consumer; +@Slf4j public class BufferedWebSocketListener implements WebSocket.Listener { private final Consumer onOpen; @@ -36,4 +40,15 @@ public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean return CompletableFuture.completedStage(null); } + + @Override + public void onError(WebSocket webSocket, Throwable error) { + log.error("Websocket error occurred during realtime communication", error); + } + + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + log.warn("Received unexpected binary bytes for WebSocket connection"); + return CompletableFuture.completedStage(null); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index ce981fa2a..72e162e6a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -33,6 +33,7 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -50,6 +51,8 @@ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class OpenAiClient { private static final String DEFAULT_API_VERSION = "2024-02-01"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + private static final ObjectMapper JACKSON = getOpenAiObjectMapper(); @@ -260,6 +263,76 @@ public Stream streamChatCompletion(@Nonnull final String prompt) .map(OpenAiChatCompletionDelta::getDeltaContent); } + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

The input channel should be used with a try-with-resources block to ensure that the underlying + * connection is closed. + * + *

Example: + * + * 

{@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }
+ * + * This API implements full duplex (input + output) communication channels. Application should logically + * synchronize their state and close input channel when it is appropriate (e.g. last part of the response + * has been received via output channel and application does not need to send any other input). When input + * channel is closed, output channel will be closed automatically and output consumer will not be called + * anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output + * @return input channel, allowing for text input + */ + @Nonnull + public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } + + public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint() + "?intent=transcription"; + + return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); + } + + public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel); + } + + private String getRealtimeEndpoint() { + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.asHttp().getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.asHttp().getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + return sb.toString(); + } + private static void throwOnContentFilter(@Nonnull final OpenAiChatCompletionDelta delta) { final String finishReason = delta.getFinishReason(); if (finishReason != null && finishReason.equals("content_filter")) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java new file mode 100644 index 000000000..3addfabfe --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -0,0 +1,103 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import java.util.*; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { + + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = ""; + + + private final AudioOutputChannel outputConsumer; + + public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void inputAudio(byte[] rawAudioChunk) { + if (rawAudioChunk.length == 0) { + return; + } + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)).build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; + } + + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java new file mode 100644 index 000000000..6e56866fb --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -0,0 +1,93 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import java.util.*; + +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { + + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio_transcript.done" + ); + + private static final String TASK = + "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; + + + private final TextOutputChannel outputConsumer; + + public SpeechToTextRealtimeClient(String url, Map httpHeaders, TextOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void inputAudio(byte[] rawAudioChunk){ + if (rawAudioChunk.length == 0) { + return; + } + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)).build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; + } + + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio_transcript.done".equals(eventType)) { + this.outputConsumer + .outputText(event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); + } else { + log.trace("skipping message type: {}", eventType); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = + SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofTranscription( + RealtimeTranscriptionSessionCreateRequest.builder() + .audio( + RealtimeTranscriptionSessionAudio.builder() + .input( + RealtimeTranscriptionSessionAudioInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .transcription( + AudioTranscription.builder() + .model(AudioTranscription.Model.GPT_4O_TRANSCRIBE) + .build()) + .build()) + .build()) + .build()) + .asTranscription()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java similarity index 65% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java index 20e6fcc66..8e2372024 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; public interface TextInputChannel extends AutoCloseable { void sendText(String text); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java new file mode 100644 index 000000000..1c0425bb4 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -0,0 +1,20 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +/** + * Functional interface representing test output channel (text data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface TextOutputChannel { + + /** + * This method is sequentially invoked by test data provider to supply implementer (consumer) + * with the test data. + * + * @param textChunk chunk of the text (possibly partial content) + * @param isLast true if this call logically concludes previous and this passed text data into a + * single logical entity (e.g. gets called at the end when all parts of a single + * message get passed) + */ + void outputText(CharSequence textChunk, boolean isLast); +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java new file mode 100644 index 000000000..5e42e5ac4 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -0,0 +1,100 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +@Slf4j +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + + + private final AudioOutputChannel outputConsumer; + + public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void sendText(String text) { + var message = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent(RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + super.sendMessage(message); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java similarity index 70% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java index 678bff93a..77e2d881b 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.sap.ai.sdk.core.common.ClientException; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -18,14 +19,12 @@ import java.util.*; import java.util.concurrent.CompletableFuture; -import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; - @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() + private static final ObjectMapper JACKSON = new ObjectMapper() .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); /** @@ -86,20 +85,18 @@ public void close() { protected abstract String getSystemPrompt(); - protected abstract List getOutputModalities(); - protected abstract void onResponse(String eventType, JsonNode event); - protected void sendMessage(ConversationItemCreateEvent message) { + protected abstract SessionUpdateEvent sessionConfiguration(); + + protected void sendMessage(Object message) { WebSocket ws = this.ws.join(); synchronized (this) { try { ws.sendText(JACKSON.writeValueAsString(message), true); ws.request(1); - log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); - askForResponse(); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); + throw new ClientException("Failed to serialize message", e); } } @@ -112,17 +109,17 @@ private synchronized void onSocketOpen(WebSocket ws) { } private void onText(WebSocket webSocket, CharSequence data) { - log.warn("WSS received message: {}", data); final JsonNode event; try { event = JACKSON.readTree(data.toString()); } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); + throw new ClientException("Error parsing JSON response from speech API", e); } var eventType = event.get("type").asText(); if (handleMessageTypes.contains(eventType)) { onResponse(eventType, event); + } else { + log.trace("Unhandled event type: {}", eventType); } webSocket.request(1); @@ -137,36 +134,11 @@ private synchronized void sendPing(WebSocket ws) { } private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = - SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() - .outputModalities(getOutputModalities()) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .voice( - RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); + SessionUpdateEvent sue = sessionConfiguration(); try { ws.sendText(JACKSON.writeValueAsString(sue), true); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize session request", e); + throw new ClientException("Failed to serialize session request", e); } ws.request(1); @@ -174,6 +146,9 @@ private void configureSession(WebSocket ws) { private void configureConversation(WebSocket ws) { var systemPrompt = getSystemPrompt(); + if (systemPrompt == null || systemPrompt.isEmpty()) { + return; + } var systemConversationItem = ConversationItemCreateEvent.builder() .item(ConversationItem.ofRealtimeConversationItemSystemMessage( RealtimeConversationItemSystemMessage.builder() @@ -185,9 +160,8 @@ private void configureConversation(WebSocket ws) { try { var json = JACKSON.writeValueAsString(systemConversationItem); ws.sendText(json, true); - log.warn("WSS conversation config sent: {}", json); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); + throw new ClientException("Failed to serialize message", e); } ws.request(1); } diff --git a/orchestration/pom.xml b/orchestration/pom.xml index a1ef764f1..f8027fe4a 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -122,11 +122,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - com.openai - openai-java - 4.41.0 - org.projectlombok diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 7c52d772b..78f347b56 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,12 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; -import com.openai.client.OpenAIClient; -import com.openai.client.okhttp.OpenAIOkHttpClient; -import com.openai.models.beta.realtime.sessions.Session; -import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.CompletionRequestConfiguration; @@ -22,30 +17,15 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import io.vavr.collection.Array; import io.vavr.control.Try; -import java.io.ByteArrayOutputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.function.BiConsumer; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.Nullable; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -56,7 +36,6 @@ public class OrchestrationClient { static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); - private final OrchestrationHttpExecutor executor; private final List

customHeaders = new ArrayList<>(); @@ -123,31 +102,6 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } - /** - * Voices input text into output audio - * - * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(BiConsumer output) { - var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { - @Override - public @NonNull String name() { - return OrchestrationAiModel.GPT_REALTIME.getName(); - } - - @Override - public @Nullable String version() { - return OrchestrationAiModel.GPT_REALTIME.getVersion(); - } - }); - - var factory = new RealtimeChannelFactory(() -> destination); - - return factory.textToSpeech(output); - } - /** * Generate a completion for the given prompt. * diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java deleted file mode 100644 index 28edd05c4..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.sap.cloud.sdk.cloudplatform.connectivity.Header; -import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import java.util.*; -import java.util.function.BiConsumer; -import java.util.function.Supplier; - -@Slf4j -@RequiredArgsConstructor(access = AccessLevel.PACKAGE) -public class RealtimeChannelFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private final Supplier destinationSupplier; - - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and - * Boolean - marker if current audio reply has been finished (current response ended) - * @return input channel, allowing for text input - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - var endpoint = sb.toString(); - - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); - } - -} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java deleted file mode 100644 index 45ea9335a..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.*; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; -import java.util.function.BiConsumer; - -@Slf4j -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); - - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); - - private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; - - - private final BiConsumer outputConsumer; - - public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - } - - public void sendText(String text) { - var message = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemUserMessage( - RealtimeConversationItemUserMessage.builder() - .addContent(RealtimeConversationItemUserMessage.Content.builder() - .text(text) - .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); - - super.sendMessage(message); - } - - @Override - protected String getSystemPrompt() { - return TASK; - } - - @Override - protected List getOutputModalities() { - return OUTPUT_MODALITIES; - } - - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.accept(audio, Boolean.FALSE); - log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } - } -} diff --git a/pom.xml b/pom.xml index 6cf8692d1..03781bc31 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,7 @@ 2.1.3 3.5.6 1.1.8 + 4.41.0 3.8.6 3.2.0 5.23.0 @@ -117,6 +118,11 @@ pom import + + com.openai + openai-java + ${com.openai.openai-java.version} + io.projectreactor diff --git a/sample-code/spring-app/pom.xml b/sample-code/spring-app/pom.xml index 41f069c14..c5ccd9c1f 100644 --- a/sample-code/spring-app/pom.xml +++ b/sample-code/spring-app/pom.xml @@ -130,6 +130,10 @@ org.springframework.ai spring-ai-commons + + org.springframework.boot + spring-boot-starter-websocket + org.springframework.ai spring-ai-model diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index aced5c87e..0f6b0e26c 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.Nonnull; -import com.sap.ai.sdk.app.services.OrchestrationService; -import com.sap.ai.sdk.core.AiCoreService; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -16,12 +14,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; -import java.io.*; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -46,41 +38,7 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) throws Exception { - var ctx = SpringApplication.run(Application.class, args); - var svc = ctx.getBean(OrchestrationService.class); - var outputBytes = new ArrayList(); - var done = new AtomicBoolean(false); - try (var channel = svc.textToSpeech((bytes, last) -> { - System.err.println("appended output bytes: " + bytes.length); - outputBytes.add(bytes); - if (last) { - done.set(true); - } - })) { - System.err.println("BEFORE SENDING TEXT"); - channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); - System.err.println("AFTER SENDING TEXT"); - long timeout = System.currentTimeMillis() + 20000; - while (!done.get() && System.currentTimeMillis() < timeout) { - Thread.yield(); - System.err.println("Waiting for messages..."); - Thread.sleep(500); - } - } - - //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); - String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; - File file = new File(path); - try (FileOutputStream in = new FileOutputStream(file)) { - var ttl = 0; - for (byte[] chunk: outputBytes) { - System.out.println("Writing bytes: " + chunk.length); - in.write(chunk); - ttl += chunk.length; - } - System.out.println("Written bytes: " + ttl); - } - + public static void main(final String[] args) { + SpringApplication.run(Application.class, args); } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java new file mode 100644 index 000000000..606b3bc4a --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -0,0 +1,23 @@ +package com.sap.ai.sdk.app; + +import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; + +@Configuration +@EnableWebSocket +public class WebsocketConfig implements WebSocketConfigurer { + + private final TextToSpeechWebsocketHandler textToSpeech; + + public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech) { + this.textToSpeech = textToSpeech; + } + + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); + } +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java new file mode 100644 index 000000000..27a667d3c --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -0,0 +1,61 @@ +package com.sap.ai.sdk.app.realtime; + +import com.sap.ai.sdk.app.services.OpenAiService; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.BinaryMessage; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.BinaryWebSocketHandler; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +@Slf4j +public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { + + private final OpenAiService service; + private final Map channels; + + @Autowired + public TextToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } + + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] textBytes = payload.array(); + TextInputChannel channel = channels.computeIfAbsent( + session.getId(), + sessionId -> service.textToSpeech((rawBytesChunk, isLast) -> { + try { + session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); + } catch (IOException e) { + log.error("failed to send text message to realtime api", e); + } + })); + channel.sendText(new String(textBytes)); + } + + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { + channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; + }); + super.afterConnectionClosed(session, status); + } + +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index 1b84e56f6..efb884097 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -5,15 +5,8 @@ import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.TEXT_EMBEDDING_3_SMALL; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionDelta; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionRequest; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionResponse; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingRequest; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingResponse; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiImageItem; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiMessage; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiTool; +import com.sap.ai.sdk.foundationmodels.openai.*; + import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -74,6 +67,43 @@ public Stream streamChatCompletionDeltas( return OpenAiClient.forModel(GPT_5_MINI).streamChatCompletionDeltas(request); } + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

The input channel should be used with a try-with-resources block to ensure that the underlying + * connection is closed. + * + *

Example: + * + * 

{@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }
+ * + * This API implements full duplex (input + output) communication channels. Application should logically + * synchronize their state and close input channel when it is appropriate (e.g. last part of the response + * has been received via output channel and application does not need to send any other input). When input + * channel is closed, output channel will be closed automatically and output consumer will not be called + * anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output + * @return input channel, allowing for text input + */ + @Nonnull + public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).textToSpeech(audioOutputConsumer); + } + + public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToText(textOutputConsumer); + } + + public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToSpeech(audioOutputConsumer); + } + /** * Asynchronous stream of an OpenAI chat request * diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 49fbaa594..f6fc24853 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -33,7 +33,6 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -64,17 +63,6 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) return client.chatCompletion(prompt, config); } - /** - * Voices input text into output audio - * - * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { - return client.textToSpeech(outputConsumer); - } - /** * Chat request to OpenAI through the Orchestration service with an image. * diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index cd874bc61..be0528612 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1355,6 +1355,113 @@

📂 Batch API

+ +
+
+
+
+

⏰ Realtime API

+
+ Realtime API allows for various real time interactions TBD Documentation +
+
+
    +
  • +
    +

    Text to speech

    + disabled +
    + + + +
    +
    + +
    • +
+
+
+
From 80a5d3eecd598f90d110ce2abcfbac1e716a9b10 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 10 Jul 2026 19:07:08 +0200 Subject: [PATCH 7/7] add speech to speech demo scenario --- .../openai/SpeechToSpeechRealtimeClient.java | 34 ++--- .../com/sap/ai/sdk/app/WebsocketConfig.java | 6 +- .../SpeechToSpeechWebsocketHandler.java | 61 +++++++++ .../src/main/resources/static/index.html | 91 ++------------ .../main/resources/static/speech-to-speech.js | 117 ++++++++++++++++++ .../main/resources/static/text-to-speech.js | 77 ++++++++++++ 6 files changed, 292 insertions(+), 94 deletions(-) create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java create mode 100644 sample-code/spring-app/src/main/resources/static/speech-to-speech.js create mode 100644 sample-code/spring-app/src/main/resources/static/text-to-speech.js diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3addfabfe..afbefea13 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -44,9 +44,10 @@ public void inputAudio(byte[] rawAudioChunk) { cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); + // TODO: make turn detection configurable and uncomment for explicit turn control + // var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + // super.sendMessage(commitAudioMessage); + // askForResponse(); } @Override @@ -67,7 +68,8 @@ protected void onResponse(String eventType, JsonNode event) { @Override protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + SessionUpdateEvent sessionUpdateEvent = + SessionUpdateEvent.builder() .session( ClientSecretCreateParams.Session.ofRealtime( RealtimeSessionCreateRequest.builder() @@ -76,21 +78,25 @@ protected SessionUpdateEvent sessionConfiguration() { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) + .turnDetection( + RealtimeAudioInputTurnDetection.ofSemanticVad(RealtimeAudioInputTurnDetection.SemanticVad.builder().build())) .format( - RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) .build()) .output( RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + .build()) .voice( RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) .build()) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index 606b3bc4a..ac7b07758 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -1,5 +1,6 @@ package com.sap.ai.sdk.app; +import com.sap.ai.sdk.app.realtime.SpeechToSpeechWebsocketHandler; import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; import org.springframework.context.annotation.Configuration; import org.springframework.web.socket.config.annotation.EnableWebSocket; @@ -11,13 +12,16 @@ public class WebsocketConfig implements WebSocketConfigurer { private final TextToSpeechWebsocketHandler textToSpeech; + private final SpeechToSpeechWebsocketHandler speechToSpeech; - public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech) { + public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { this.textToSpeech = textToSpeech; + this.speechToSpeech = speechToSpeech; } @Override public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); + registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java new file mode 100644 index 000000000..72f27fab0 --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -0,0 +1,61 @@ +package com.sap.ai.sdk.app.realtime; + +import com.sap.ai.sdk.app.services.OpenAiService; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.BinaryMessage; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.BinaryWebSocketHandler; + +@Component +@Slf4j +public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { + + private final OpenAiService service; + private final Map channels; + + @Autowired + public SpeechToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } + + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] chunkBytes = payload.array(); + AudioInputChannel channel = channels.computeIfAbsent( + session.getId(), + sessionId -> service.speechToSpeech((rawBytesChunk, isLast) -> { + try { + session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); + } catch (IOException e) { + log.error("failed to send audio data to realtime api", e); + } + })); + channel.inputAudio(chunkBytes); + } + + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { + channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; + }); + super.afterConnectionClosed(session, status); + } + +} diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index be0528612..a30067be7 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1378,85 +1378,18 @@

Text to speech

- + + +
  • +
    +

    Speech to speech

    +
    disconnected
    +
    disabled
    +
    + +
    +
    +
    • diff --git a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js new file mode 100644 index 000000000..e6cd3f9be --- /dev/null +++ b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js @@ -0,0 +1,117 @@ +const SPEECH_TO_SPEECH_URL = 'ws://localhost:8080/speech-to-speech'; +const SPEECH_TO_SPEECH_SAMPLE_RATE = 24000; + +const wsStatusEl = document.getElementById('speech-to-speech-websocket-status') +const micStatusEl = document.getElementById('speech-to-speech-mic-status') +const speechBtn = document.getElementById('speech-to-speech-btn') + +let sts_ws = null; +let sts_audioCtx = null; +let sts_mediaStream = null; +let sts_processor = null; +let sts_nextStartTime = 0; +let sts_started = false; + +async function startSession() { + sts_ws = new WebSocket(SPEECH_TO_SPEECH_URL); + sts_ws.binaryType = 'arraybuffer'; + + sts_ws.onopen = async () => { + wsStatusEl.textContent = `connected to ${SPEECH_TO_SPEECH_URL}`; + wsStatusEl.style.color = 'green'; + + await startMicrophone(); + speechBtn.innerText = 'Stop'; + sts_started = true; + } + + sts_ws.onmessage = (event) => { + if (event.data instanceof ArrayBuffer) { + sts_playPcmAudio(event.data) + } + } + + sts_ws.onclose = () => stopSession(); +} + +async function startMicrophone() { + try { + sts_mediaStream = await navigator.mediaDevices.getUserMedia({audio: true}); + console.log("mediaStream is: ", sts_mediaStream) + micStatusEl.textContent = 'active'; + micStatusEl.style.color = 'green'; + + sts_audioCtx = new (window.AudioContext || window.webkitAudioContext)({sampleRate: SPEECH_TO_SPEECH_SAMPLE_RATE}); + + const source = sts_audioCtx.createMediaStreamSource(sts_mediaStream); + sts_processor = sts_audioCtx.createScriptProcessor(2048, 1, 1); + sts_processor.onaudioprocess = (e) => { + if (sts_ws && sts_ws.readyState === WebSocket.OPEN) { + const float32Input = e.inputBuffer.getChannelData(0); + const int16Buffer = new Int16Array(float32Input.length); + + for (let i = 0; i < float32Input.length; i++) { + let s = Math.max(-1, Math.min(1, float32Input[i])); + int16Buffer[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + sts_ws.send(int16Buffer.buffer); + } + }; + + const gainNode = sts_audioCtx.createGain(); + gainNode.gain.value = 0; + + source.connect(sts_processor); + sts_processor.connect(gainNode); + gainNode.connect(sts_audioCtx.destination) + + } catch (err) { + console.error('failed to find or bind microphone: ', err); + micStatusEl.innerText = 'mic binding error'; + micStatusEl.style.color = 'red'; + } +} + +function sts_playPcmAudio(arrayBuffer) { + if (!sts_audioCtx) return; + + const int16Data = new Int16Array(arrayBuffer); + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + float32Data[i] = int16Data[i] / 32768.0; + } + + const audioBuffer = sts_audioCtx.createBuffer(1, float32Data.length, SPEECH_TO_SPEECH_SAMPLE_RATE); + audioBuffer.copyToChannel(float32Data, 0); + const source = sts_audioCtx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(sts_audioCtx.destination); + + if (sts_nextStartTime < sts_audioCtx.currentTime) { + sts_nextStartTime = sts_audioCtx.currentTime; + } + + source.start(sts_nextStartTime); + sts_nextStartTime += audioBuffer.duration; +} + +function stopSession() { + if (sts_ws) sts_ws.close(); + if (sts_mediaStream) sts_mediaStream.getTracks().forEach(track => track.stop()); + if (sts_processor) sts_processor.disconnect(); + if (sts_audioCtx) { + sts_audioCtx.close(); + sts_audioCtx = null; + } + + wsStatusEl.textContent = 'disconnected'; + wsStatusEl.style.color = 'red'; + micStatusEl.textContent = 'disabled'; + micStatusEl.style.color = 'red'; + speechBtn.innerText = 'Start'; + sts_started = false; +} + +speechBtn.addEventListener('click', () => { + sts_started ? stopSession() : startSession(); +}) \ No newline at end of file diff --git a/sample-code/spring-app/src/main/resources/static/text-to-speech.js b/sample-code/spring-app/src/main/resources/static/text-to-speech.js new file mode 100644 index 000000000..80bd75841 --- /dev/null +++ b/sample-code/spring-app/src/main/resources/static/text-to-speech.js @@ -0,0 +1,77 @@ +const WS_URL = 'ws://localhost:8080/text-to-speech'; +const SAMPLE_RATE = 24000; + +const statusEl = document.getElementById('text-to-speech-status') +const textInput = document.getElementById('text-to-speech-input') +const sendBtn = document.getElementById('text-to-speech-send-btn') + +let audioCtx; +let nextStartTime = 0; + +let ws = new WebSocket(WS_URL); +ws.binaryType = 'arraybuffer'; + +ws.onopen = () => { + statusEl.textContent = `connected to ${WS_URL}`; + statusEl.style.color = 'green'; + textInput.disabled = false; + sendBtn.disabled = false; +}; + +ws.onclose = () => { + statusEl.textContent = 'disconnected'; + statusEl.style.color = 'red'; + textInput.disabled = true; + sendBtn.disabled = true; +} + +ws.onerror = (error) => { + console.error('text-to-speech WebSocket error:', error); + statusEl.textContent = 'connection error' + statusEl.style.color = 'red'; +} + +ws.onmessage = (event) => { + if (event.data instanceof ArrayBuffer) { + playPcmAudio(event.data); + } else { + console.log('text data received:', event.data) + } +} + +sendBtn.addEventListener('click', () => { + const text = textInput.value.trim(); + if (text && ws.readyState === WebSocket.OPEN) { + if (!audioCtx) { + audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE}); + } + + const encoder = new TextEncoder(); + const binaryData = encoder.encode(text); + ws.send(binaryData); + textInput.value = ''; + } +}) + +function playPcmAudio(arrayBuffer) { + if (!audioCtx) return; + + const int16Data = new Int16Array(arrayBuffer); + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + float32Data[i] = int16Data[i] / 32768.0; + } + + const audioBuffer = audioCtx.createBuffer(1, float32Data.length, SAMPLE_RATE); + audioBuffer.copyToChannel(float32Data, 0); + const source = audioCtx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(audioCtx.destination); + + if (nextStartTime < audioCtx.currentTime) { + nextStartTime = audioCtx.currentTime; + } + + source.start(nextStartTime); + nextStartTime += audioBuffer.duration; +} \ No newline at end of file