diff --git a/README.md b/README.md index 1ae0f3db..1918efa0 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ This repository contains Swift community-maintained implementation over [OpenAI] - [Audio Create Speech](#audio-create-speech) - [Audio Transcriptions](#audio-transcriptions) - [Audio Translations](#audio-translations) + - [Audio input and output with Chat Completions](#audio-input-and-output-with-chat-completions) - [Structured Outputs](#structured-outputs) - [Specialized models](#specialized-models) - [Embeddings](#embeddings) @@ -736,6 +737,113 @@ openAI.audioTranslations(query: query) { result in let result = try await openAI.audioTranslations(query: query) ``` +### Audio input and output with Chat Completions + +Models like `gpt-audio-1.5` take audio as input and answer with spoken audio in a single Chat Completions call, so a separate transcription → chat → speech pipeline is not needed. + +There is no separate audio endpoint: use the same `ChatQuery` with `func chats(query:)` and `func chatsStream(query:)`. Two parameters opt a query into audio: + +- `modalities` — pass `[.text, .audio]` to get both a transcript and generated speech back +- `audioOptions` — the `voice` and the `format` of the generated audio + +Audio input is a content part on a user message, so it mixes freely with text and image parts. + +**Models:** `.gpt_audio_1_5` (recommended), `.gpt_audio`, `.gpt_audio_mini` + +**Formats:** + +- Input audio: `wav` and `mp3` +- Output audio: `mp3`, `opus`, `flac`, `wav` and `pcm16` — prefer `pcm16` when streaming, it is the cheapest to play back chunk by chunk + +**Example: audio in, audio out** + +```swift +let audioData = try Data(contentsOf: audioFileURL) + +let query = ChatQuery( + messages: [ + .system(.init(content: .textContent("You are a helpful voice assistant."))), + .user(.init(content: .contentParts([ + .audio(.init(inputAudio: .init(data: audioData, format: .wav))) + ]))) + ], + model: .gpt_audio_1_5, + modalities: [.text, .audio], + audioOptions: .init(format: .wav, voice: .alloy) +) + +let result = try await openAI.chats(query: query) + +if let audio = result.choices.first?.message.audio { + print(audio.transcript) + let spokenReply = Data(base64Encoded: audio.data) +} +``` + +`InputAudio` has an initializer that takes raw `Data` and base64-encodes it for you, and one that takes an already base64-encoded `String`. + +**Example: text in, audio out** + +Audio output does not require audio input — a plain text message works too. + +```swift +let query = ChatQuery( + messages: [ + .user(.init(content: .string("Tell me a joke about Swift"))) + ], + model: .gpt_audio_1_5, + modalities: [.text, .audio], + audioOptions: .init(format: .mp3, voice: .sage) +) + +let result = try await openAI.chats(query: query) +``` + +**Example: streaming** + +Streaming delivers the transcript and the audio in chunks on the delta. Every field of the delta audio is optional, since a chunk usually carries only one of them. + +```swift +for try await result in openAI.chatsStream(query: query) { + guard let audio = result.choices.first?.delta.audio else { continue } + + if let transcript = audio.transcript { + print(transcript, terminator: "") + } + + if let data = audio.data, let chunk = Data(base64Encoded: data) { + // append to your player buffer + } +} +``` + +**Example: multi-turn conversations** + +A generated audio response has an `id` that stays valid until `expiresAt`. Reference that id from an assistant message on the next turn instead of uploading the audio again. + +```swift +guard let previousAudio = result.choices.first?.message.audio else { return } + +let nextTurn = ChatQuery( + messages: [ + .user(.init(content: .contentParts([ + .audio(.init(inputAudio: .init(data: firstQuestionAudio, format: .wav))) + ]))), + .assistant(.init(audio: .init(id: previousAudio.id))), + .user(.init(content: .contentParts([ + .audio(.init(inputAudio: .init(data: secondQuestionAudio, format: .wav))) + ]))) + ], + model: .gpt_audio_1_5, + modalities: [.text, .audio], + audioOptions: .init(format: .wav, voice: .alloy) +) +``` + +Once `expiresAt` has passed the id is rejected, so keep the transcript around and send it as a text assistant message instead. + +Audio tokens are reported separately in `result.usage` — `promptTokensDetails.audioTokens` and `completionTokensDetails.audioTokens`. + Review [Audio Documentation](https://platform.openai.com/docs/api-reference/audio) for more info. ## Structured Outputs diff --git a/Sources/OpenAI/Public/Models/Models/Models.swift b/Sources/OpenAI/Public/Models/Models/Models.swift index c3794038..a794a997 100644 --- a/Sources/OpenAI/Public/Models/Models/Models.swift +++ b/Sources/OpenAI/Public/Models/Models/Models.swift @@ -103,6 +103,7 @@ public extension Model { static let gpt4_o = "gpt-4o" /// `gpt-4o-audio-preview`, this is a preview release of the GPT-4o Audio models. These models accept audio inputs and outputs, and can be used in the Chat Completions REST API. + @available(*, deprecated, message: "gpt-4o-audio-preview was shut down on May 7th, 2026. Recommended replacement: gpt-audio-1.5") static let gpt_4o_audio_preview = "gpt-4o-audio-preview" /// `chatgpt-4o-latest`: GPT-4o model used in ChatGPT @@ -119,6 +120,22 @@ public extension Model { /// `gpt-4o-mini-audio-preview`, this is a preview release of the smaller GPT-4o Audio mini model. It's designed to input audio or create audio outputs via the REST API. static let gpt_4o_mini_audio_preview = "gpt-4o-mini-audio-preview" + // MARK: - Audio models + // Models that accept audio inputs and produce audio outputs in the Chat Completions API. + + /// `gpt-audio-1.5`: OpenAI's recommended model for audio in, audio out over Chat Completions. + /// + /// Improves on the GPT-4o Audio preview models in instruction following, tool calling and multilingual accuracy. + /// + /// See the [audio guide](https://platform.openai.com/docs/guides/audio). + static let gpt_audio_1_5 = "gpt-audio-1.5" + + /// `gpt-audio`: audio model that accepts audio inputs and produces audio outputs in the Chat Completions API. + static let gpt_audio = "gpt-audio" + + /// `gpt-audio-mini`: smaller and cheaper audio model for audio inputs and outputs in the Chat Completions API. + static let gpt_audio_mini = "gpt-audio-mini" + // MARK: - Realtime models // Models capable of realtime text and audio inputs and outputs. @@ -280,7 +297,9 @@ public extension Model { // reasoning .o4_mini, o3, o3_mini, .o1, // flagship - .gpt5, .gpt5_mini, .gpt5_nano, .gpt5_chat, .gpt5_1, .gpt5_1_chat_latest, .gpt5_6_sol, .gpt5_6_terra, .gpt5_6_luna, .gpt4_1, .gpt4_o, .gpt_4o_audio_preview, chatgpt_4o_latest, + .gpt5, .gpt5_mini, .gpt5_nano, .gpt5_chat, .gpt5_1, .gpt5_1_chat_latest, .gpt5_6_sol, .gpt5_6_terra, .gpt5_6_luna, .gpt4_1, .gpt4_o, chatgpt_4o_latest, + // audio + .gpt_audio_1_5, .gpt_audio, .gpt_audio_mini, // cost-optimized .gpt4_1_mini, .gpt4_1_nano, .gpt4_o_mini, .gpt_4o_mini_audio_preview, // tool-specific diff --git a/Tests/OpenAITests/ChatQueryCodingTests.swift b/Tests/OpenAITests/ChatQueryCodingTests.swift index 5a0871e2..33d116ab 100644 --- a/Tests/OpenAITests/ChatQueryCodingTests.swift +++ b/Tests/OpenAITests/ChatQueryCodingTests.swift @@ -161,6 +161,80 @@ struct ChatQueryCodingTests { #expect(try equal(query, expected)) } + @Test func encodeUserMessageWithInputAudioContentPart() throws { + let query = ChatQuery( + messages: [ + .user(.init( + content: .contentParts([ + .text(.init(text: "What is this recording about?")), + .audio(.init(inputAudio: .init(data: Data("fake-audio-bytes".utf8), format: .wav))) + ]) + )) + ], + model: .gpt_audio_1_5, + modalities: [.text, .audio], + audioOptions: .init(format: .pcm16, voice: .alloy) + ) + + let expected = """ + { + "model": "gpt-audio-1.5", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is this recording about?" + }, + { + "type": "input_audio", + "input_audio": { + "data": "ZmFrZS1hdWRpby1ieXRlcw==", + "format": "wav" + } + } + ] + } + ], + "modalities": ["text", "audio"], + "audio": { + "format": "pcm16", + "voice": "alloy" + }, + "stream": false + } + """ + + #expect(try equal(query, expected)) + } + + @Test func encodeAssistantMessageReferencingPreviousAudio() throws { + let query = ChatQuery( + messages: [ + .assistant(.init(audio: .init(id: "audio_abc123"))) + ], + model: .gpt_audio_1_5 + ) + + let expected = """ + { + "model": "gpt-audio-1.5", + "messages": [ + { + "role": "assistant", + "audio": { + "id": "audio_abc123" + } + } + ], + "stream": false + } + """ + + #expect(try equal(query, expected)) + } + private func equal(_ query: Codable, _ expected: String) throws -> Bool { let encodedQuery = try encodedAndComparable(query) let decodedExpectation = try decodedAndComparable(expected) diff --git a/Tests/OpenAITests/ChatResultTests.swift b/Tests/OpenAITests/ChatResultTests.swift index 1cd68433..11f91393 100644 --- a/Tests/OpenAITests/ChatResultTests.swift +++ b/Tests/OpenAITests/ChatResultTests.swift @@ -153,6 +153,69 @@ final class ChatResultTests: XCTestCase { XCTAssertEqual(result.serviceTier, .flexTier) } + func testDecodeMessageAudio() throws { + let jsonString = """ + { + "id": "some_id", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-audio-1.5", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": null, + "refusal": null, + "audio": { + "id": "audio_abc123", + "data": "ZmFrZS1hdWRpby1ieXRlcw==", + "expires_at": 1677655888, + "transcript": "Hello there" + } + } + } + ] + } + """ + + let audio = try XCTUnwrap(decode(jsonString).choices.first?.message.audio) + XCTAssertEqual(audio.id, "audio_abc123") + XCTAssertEqual(audio.data, "ZmFrZS1hdWRpby1ieXRlcw==") + XCTAssertEqual(audio.expiresAt, 1677655888) + XCTAssertEqual(audio.transcript, "Hello there") + XCTAssertEqual(Data(base64Encoded: audio.data), Data("fake-audio-bytes".utf8)) + } + + func testDecodeAudioTokenUsage() throws { + let jsonString = """ + { + "id": "some_id", + "object": "chat.completion", + "created": 1677652288, + "model": "gpt-audio-1.5", + "choices": [], + "usage": { + "prompt_tokens": 60, + "completion_tokens": 40, + "total_tokens": 100, + "prompt_tokens_details": { + "audio_tokens": 55, + "cached_tokens": 0 + }, + "completion_tokens_details": { + "audio_tokens": 35 + } + } + } + """ + + let usage = try XCTUnwrap(decode(jsonString).usage) + XCTAssertEqual(usage.promptTokensDetails?.audioTokens, 55) + XCTAssertEqual(usage.completionTokensDetails?.audioTokens, 35) + } + private func decode(_ jsonString: String) throws -> ChatResult { let jsonData = jsonString.data(using: .utf8)! return try decoder.decode(ChatResult.self, from: jsonData) diff --git a/Tests/OpenAITests/ChatStreamResultTests.swift b/Tests/OpenAITests/ChatStreamResultTests.swift index dce87d42..9c3c5151 100644 --- a/Tests/OpenAITests/ChatStreamResultTests.swift +++ b/Tests/OpenAITests/ChatStreamResultTests.swift @@ -115,6 +115,65 @@ final class ChatStreamResultTests: XCTestCase { XCTAssertEqual(result.serviceTier, .flexTier) } + func testDecodeDeltaAudio() throws { + let jsonString = """ + { + "id": "some_id", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "gpt-audio-1.5", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "audio": { + "id": "audio_abc123", + "transcript": "Hello", + "expires_at": 1677655888, + "data": "ZmFrZS1hdWRpby1ieXRlcw==" + } + } + } + ] + } + """ + + let audio = try XCTUnwrap(decode(jsonString).choices.first?.delta.audio) + XCTAssertEqual(audio.id, "audio_abc123") + XCTAssertEqual(audio.transcript, "Hello") + XCTAssertEqual(audio.expiresAt, 1677655888) + XCTAssertEqual(audio.data, "ZmFrZS1hdWRpby1ieXRlcw==") + } + + /// Audio chunks in a stream usually carry a single field, so every field of the delta audio must be optional. + func testDecodeDeltaAudioWithDataOnly() throws { + let jsonString = """ + { + "id": "some_id", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "gpt-audio-1.5", + "choices": [ + { + "index": 0, + "delta": { + "audio": { + "data": "ZmFrZS1hdWRpby1ieXRlcw==" + } + } + } + ] + } + """ + + let audio = try XCTUnwrap(decode(jsonString).choices.first?.delta.audio) + XCTAssertEqual(audio.data, "ZmFrZS1hdWRpby1ieXRlcw==") + XCTAssertNil(audio.id) + XCTAssertNil(audio.transcript) + XCTAssertNil(audio.expiresAt) + } + private func decode(_ jsonString: String) throws -> ChatStreamResult { let jsonData = jsonString.data(using: .utf8)! return try decoder.decode(ChatStreamResult.self, from: jsonData)