Add audio chat API support (gpt-4o-audio-preview, gpt-audio-1.5)

README.md

@@ -33,6 +33,7 @@ This repository contains Swift community-maintained implementation over [OpenAI]
     - [Audio Create Speech](#audio-create-speech)
     - [Audio Transcriptions](#audio-transcriptions)
     - [Audio Translations](#audio-translations)
+    - [Audio input and output with Chat Completions](#audio-input-and-output-with-chat-completions)
 - [Structured Outputs](#structured-outputs)
 - [Specialized models](#specialized-models)
     - [Embeddings](#embeddings)
@@ -736,6 +737,113 @@ openAI.audioTranslations(query: query) { result in
 let result = try await openAI.audioTranslations(query: query)
 ```
 
+### Audio input and output with Chat Completions
+
+Models like `gpt-audio-1.5` take audio as input and answer with spoken audio in a single Chat Completions call, so a separate transcription → chat → speech pipeline is not needed.
+
+There is no separate audio endpoint: use the same `ChatQuery` with `func chats(query:)` and `func chatsStream(query:)`. Two parameters opt a query into audio:
+
+- `modalities` — pass `[.text, .audio]` to get both a transcript and generated speech back
+- `audioOptions` — the `voice` and the `format` of the generated audio
+
+Audio input is a content part on a user message, so it mixes freely with text and image parts.
+
+**Models:** `.gpt_audio_1_5` (recommended), `.gpt_audio`, `.gpt_audio_mini`
+
+**Formats:**
+
+- Input audio: `wav` and `mp3`
+- Output audio: `mp3`, `opus`, `flac`, `wav` and `pcm16` — prefer `pcm16` when streaming, it is the cheapest to play back chunk by chunk
+
+**Example: audio in, audio out**
+
+```swift
+let audioData = try Data(contentsOf: audioFileURL)
+
+let query = ChatQuery(
+    messages: [
+        .system(.init(content: .textContent("You are a helpful voice assistant."))),
+        .user(.init(content: .contentParts([
+            .audio(.init(inputAudio: .init(data: audioData, format: .wav)))
+        ])))
+    ],
+    model: .gpt_audio_1_5,
+    modalities: [.text, .audio],
+    audioOptions: .init(format: .wav, voice: .alloy)
+)
+
+let result = try await openAI.chats(query: query)
+
+if let audio = result.choices.first?.message.audio {
+    print(audio.transcript)
+    let spokenReply = Data(base64Encoded: audio.data)
+}
+```
+
+`InputAudio` has an initializer that takes raw `Data` and base64-encodes it for you, and one that takes an already base64-encoded `String`.
+
+**Example: text in, audio out**
+
+Audio output does not require audio input — a plain text message works too.
+
+```swift
+let query = ChatQuery(
+    messages: [
+        .user(.init(content: .string("Tell me a joke about Swift")))
+    ],
+    model: .gpt_audio_1_5,
+    modalities: [.text, .audio],
+    audioOptions: .init(format: .mp3, voice: .sage)
+)
+
+let result = try await openAI.chats(query: query)
+```
+
+**Example: streaming**
+
+Streaming delivers the transcript and the audio in chunks on the delta. Every field of the delta audio is optional, since a chunk usually carries only one of them.
+
+```swift
+for try await result in openAI.chatsStream(query: query) {
+    guard let audio = result.choices.first?.delta.audio else { continue }
+
+    if let transcript = audio.transcript {
+        print(transcript, terminator: "")
+    }
+
+    if let data = audio.data, let chunk = Data(base64Encoded: data) {
+        // append to your player buffer
+    }
+}
+```
+
+**Example: multi-turn conversations**
+
+A generated audio response has an `id` that stays valid until `expiresAt`. Reference that id from an assistant message on the next turn instead of uploading the audio again.
+
+```swift
+guard let previousAudio = result.choices.first?.message.audio else { return }
+
+let nextTurn = ChatQuery(
+    messages: [
+        .user(.init(content: .contentParts([
+            .audio(.init(inputAudio: .init(data: firstQuestionAudio, format: .wav)))
+        ]))),
+        .assistant(.init(audio: .init(id: previousAudio.id))),
+        .user(.init(content: .contentParts([
+            .audio(.init(inputAudio: .init(data: secondQuestionAudio, format: .wav)))
+        ])))
+    ],
+    model: .gpt_audio_1_5,
+    modalities: [.text, .audio],
+    audioOptions: .init(format: .wav, voice: .alloy)
+)
+```
+
+Once `expiresAt` has passed the id is rejected, so keep the transcript around and send it as a text assistant message instead.
+
+Audio tokens are reported separately in `result.usage` — `promptTokensDetails.audioTokens` and `completionTokensDetails.audioTokens`.
+
 Review [Audio Documentation](https://platform.openai.com/docs/api-reference/audio) for more info.
 
 ## Structured Outputs

Sources/OpenAI/Public/Models/Models/Models.swift

@@ -103,6 +103,7 @@ public extension Model {
     static let gpt4_o = "gpt-4o"
 
     /// `gpt-4o-audio-preview`, this is a preview release of the GPT-4o Audio models. These models accept audio inputs and outputs, and can be used in the Chat Completions REST API.
+    @available(*, deprecated, message: "gpt-4o-audio-preview was shut down on May 7th, 2026. Recommended replacement: gpt-audio-1.5")
     static let gpt_4o_audio_preview = "gpt-4o-audio-preview"
 
     /// `chatgpt-4o-latest`: GPT-4o model used in ChatGPT
@@ -119,6 +120,22 @@ public extension Model {
     /// `gpt-4o-mini-audio-preview`, this is a preview release of the smaller GPT-4o Audio mini model. It's designed to input audio or create audio outputs via the REST API.
     static let gpt_4o_mini_audio_preview = "gpt-4o-mini-audio-preview"
 
+    // MARK: - Audio models
+    // Models that accept audio inputs and produce audio outputs in the Chat Completions API.
+
+    /// `gpt-audio-1.5`: OpenAI's recommended model for audio in, audio out over Chat Completions.
+    ///
+    /// Improves on the GPT-4o Audio preview models in instruction following, tool calling and multilingual accuracy.
+    ///
+    /// See the [audio guide](https://platform.openai.com/docs/guides/audio).
+    static let gpt_audio_1_5 = "gpt-audio-1.5"
+
+    /// `gpt-audio`: audio model that accepts audio inputs and produces audio outputs in the Chat Completions API.
+    static let gpt_audio = "gpt-audio"
+
+    /// `gpt-audio-mini`: smaller and cheaper audio model for audio inputs and outputs in the Chat Completions API.
+    static let gpt_audio_mini = "gpt-audio-mini"
+
     // MARK: - Realtime models
     // Models capable of realtime text and audio inputs and outputs.
 
@@ -280,7 +297,9 @@ public extension Model {
             // reasoning
             .o4_mini, o3, o3_mini, .o1,
             // flagship
-            .gpt5, .gpt5_mini, .gpt5_nano, .gpt5_chat, .gpt5_1, .gpt5_1_chat_latest, .gpt5_6_sol, .gpt5_6_terra, .gpt5_6_luna, .gpt4_1, .gpt4_o, .gpt_4o_audio_preview, chatgpt_4o_latest,
+            .gpt5, .gpt5_mini, .gpt5_nano, .gpt5_chat, .gpt5_1, .gpt5_1_chat_latest, .gpt5_6_sol, .gpt5_6_terra, .gpt5_6_luna, .gpt4_1, .gpt4_o, chatgpt_4o_latest,
+            // audio
+            .gpt_audio_1_5, .gpt_audio, .gpt_audio_mini,
             // cost-optimized
             .gpt4_1_mini, .gpt4_1_nano, .gpt4_o_mini, .gpt_4o_mini_audio_preview,
             // tool-specific

Tests/OpenAITests/ChatQueryCodingTests.swift

@@ -161,6 +161,80 @@ struct ChatQueryCodingTests {
         #expect(try equal(query, expected))
     }
 
+    @Test func encodeUserMessageWithInputAudioContentPart() throws {
+        let query = ChatQuery(
+            messages: [
+                .user(.init(
+                    content: .contentParts([
+                        .text(.init(text: "What is this recording about?")),
+                        .audio(.init(inputAudio: .init(data: Data("fake-audio-bytes".utf8), format: .wav)))
+                    ])
+                ))
+            ],
+            model: .gpt_audio_1_5,
+            modalities: [.text, .audio],
+            audioOptions: .init(format: .pcm16, voice: .alloy)
+        )
+
+        let expected = """
+        {
+            "model": "gpt-audio-1.5",
+            "messages": [
+              {
+                "role": "user",
+                "content": [
+                  {
+                    "type": "text",
+                    "text": "What is this recording about?"
+                  },
+                  {
+                    "type": "input_audio",
+                    "input_audio": {
+                      "data": "ZmFrZS1hdWRpby1ieXRlcw==",
+                      "format": "wav"
+                    }
+                  }
+                ]
+              }
+            ],
+            "modalities": ["text", "audio"],
+            "audio": {
+                "format": "pcm16",
+                "voice": "alloy"
+            },
+            "stream": false
+        }
+        """
+
+        #expect(try equal(query, expected))
+    }
+
+    @Test func encodeAssistantMessageReferencingPreviousAudio() throws {
+        let query = ChatQuery(
+            messages: [
+                .assistant(.init(audio: .init(id: "audio_abc123")))
+            ],
+            model: .gpt_audio_1_5
+        )
+
+        let expected = """
+        {
+            "model": "gpt-audio-1.5",
+            "messages": [
+              {
+                "role": "assistant",
+                "audio": {
+                  "id": "audio_abc123"
+                }
+              }
+            ],
+            "stream": false
+        }
+        """
+
+        #expect(try equal(query, expected))
+    }
+
     private func equal(_ query: Codable, _ expected: String) throws -> Bool {
         let encodedQuery = try encodedAndComparable(query)
         let decodedExpectation = try decodedAndComparable(expected)

Tests/OpenAITests/ChatResultTests.swift

@@ -153,6 +153,69 @@ final class ChatResultTests: XCTestCase {
         XCTAssertEqual(result.serviceTier, .flexTier)
     }
 
+    func testDecodeMessageAudio() throws {
+        let jsonString = """
+            {
+                "id": "some_id",
+                "object": "chat.completion",
+                "created": 1677652288,
+                "model": "gpt-audio-1.5",
+                "choices": [
+                    {
+                        "index": 0,
+                        "finish_reason": "stop",
+                        "message": {
+                            "role": "assistant",
+                            "content": null,
+                            "refusal": null,
+                            "audio": {
+                                "id": "audio_abc123",
+                                "data": "ZmFrZS1hdWRpby1ieXRlcw==",
+                                "expires_at": 1677655888,
+                                "transcript": "Hello there"
+                            }
+                        }
+                    }
+                ]
+            }
+            """
+
+        let audio = try XCTUnwrap(decode(jsonString).choices.first?.message.audio)
+        XCTAssertEqual(audio.id, "audio_abc123")
+        XCTAssertEqual(audio.data, "ZmFrZS1hdWRpby1ieXRlcw==")
+        XCTAssertEqual(audio.expiresAt, 1677655888)
+        XCTAssertEqual(audio.transcript, "Hello there")
+        XCTAssertEqual(Data(base64Encoded: audio.data), Data("fake-audio-bytes".utf8))
+    }
+
+    func testDecodeAudioTokenUsage() throws {
+        let jsonString = """
+            {
+                "id": "some_id",
+                "object": "chat.completion",
+                "created": 1677652288,
+                "model": "gpt-audio-1.5",
+                "choices": [],
+                "usage": {
+                    "prompt_tokens": 60,
+                    "completion_tokens": 40,
+                    "total_tokens": 100,
+                    "prompt_tokens_details": {
+                        "audio_tokens": 55,
+                        "cached_tokens": 0
+                    },
+                    "completion_tokens_details": {
+                        "audio_tokens": 35
+                    }
+                }
+            }
+            """
+
+        let usage = try XCTUnwrap(decode(jsonString).usage)
+        XCTAssertEqual(usage.promptTokensDetails?.audioTokens, 55)
+        XCTAssertEqual(usage.completionTokensDetails?.audioTokens, 35)
+    }
+
     private func decode(_ jsonString: String) throws -> ChatResult {
         let jsonData = jsonString.data(using: .utf8)!
         return try decoder.decode(ChatResult.self, from: jsonData)

Tests/OpenAITests/ChatStreamResultTests.swift

@@ -115,6 +115,65 @@ final class ChatStreamResultTests: XCTestCase {
         XCTAssertEqual(result.serviceTier, .flexTier)
     }
 
+    func testDecodeDeltaAudio() throws {
+        let jsonString = """
+            {
+                "id": "some_id",
+                "object": "chat.completion.chunk",
+                "created": 1677652288,
+                "model": "gpt-audio-1.5",
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {
+                            "role": "assistant",
+                            "audio": {
+                                "id": "audio_abc123",
+                                "transcript": "Hello",
+                                "expires_at": 1677655888,
+                                "data": "ZmFrZS1hdWRpby1ieXRlcw=="
+                            }
+                        }
+                    }
+                ]
+            }
+            """
+
+        let audio = try XCTUnwrap(decode(jsonString).choices.first?.delta.audio)
+        XCTAssertEqual(audio.id, "audio_abc123")
+        XCTAssertEqual(audio.transcript, "Hello")
+        XCTAssertEqual(audio.expiresAt, 1677655888)
+        XCTAssertEqual(audio.data, "ZmFrZS1hdWRpby1ieXRlcw==")
+    }
+
+    /// Audio chunks in a stream usually carry a single field, so every field of the delta audio must be optional.
+    func testDecodeDeltaAudioWithDataOnly() throws {
+        let jsonString = """
+            {
+                "id": "some_id",
+                "object": "chat.completion.chunk",
+                "created": 1677652288,
+                "model": "gpt-audio-1.5",
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {
+                            "audio": {
+                                "data": "ZmFrZS1hdWRpby1ieXRlcw=="
+                            }
+                        }
+                    }
+                ]
+            }
+            """
+
+        let audio = try XCTUnwrap(decode(jsonString).choices.first?.delta.audio)
+        XCTAssertEqual(audio.data, "ZmFrZS1hdWRpby1ieXRlcw==")
+        XCTAssertNil(audio.id)
+        XCTAssertNil(audio.transcript)
+        XCTAssertNil(audio.expiresAt)
+    }
+
     private func decode(_ jsonString: String) throws -> ChatStreamResult {
         let jsonData = jsonString.data(using: .utf8)!
         return try decoder.decode(ChatStreamResult.self, from: jsonData)