ios: support safe assistant inline images

2026-10-20 14:31:15 -07:00
parent 9098fee32a
commit ee845c9fa3
13 changed files with 369 additions and 15 deletions
@@ -14,10 +14,15 @@ private func sampleAttachment(_ id: String = "opaque-server-id", data: Data = or
    AgentAttachment(id: id, kind: .file, filename: name, mimeType: "application/octet-stream", sizeBytes: data.count,
        sha256: SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined(), workspacePath: "/runtime/provided/\(name)")
 }
-private func descriptorData(capability: Bool = true) throws -> Data {
+private func descriptorData(capability: Bool = true, assistantImages: Bool = false) throws -> Data {
    var object: [String: Any] = ["id": "root", "title": "Sybil", "type": "persistent-agent", "model": "fixture", "protocol": "pi-room.v1", "websocket": "/v1/agent/ws",
        "activity": ["state": "idle", "isThinking": false, "queued": 0, "backgroundWorkers": 0, "tools": [], "queuedMessages": []]]
    if capability { object["capabilities"] = ["attachments": try JSONSerialization.jsonObject(with: JSONEncoder().encode(attachmentLimits)), "future": true] }
+    if assistantImages {
+        var capabilities = object["capabilities"] as? [String: Any] ?? [:]
+        capabilities["assistantImages"] = ["version": 1]
+        object["capabilities"] = capabilities
+    }
    object["unknown"] = ["additive": true]
    return try JSONSerialization.data(withJSONObject: object)
 }
@@ -124,6 +129,106 @@ private func requestBytes(_ request: URLRequest) -> Data {
 }

@Suite(.serialized) struct AgentAttachmentTests {
+    @Test @MainActor func assistantCapabilityDoesNotAdmitInputAndRequiresToken() async throws {
+        AttachmentURLProtocol.install { request in
+            #expect(request.url!.path == "/api/v1/agent") // Never upload or submit references.
+            return (200, [:], try descriptorData(capability: false, assistantImages: true))
+        }
+        let session = attachmentSession(); defer { session.invalidateAndCancel() }
+        for token in [nil, " ", "synthetic-test"] {
+            let client = SybilAPIClient(configuration: .init(baseURL: URL(string: "https://fixture.invalid/api")!, authToken: token), session: session)
+            let descriptor = try await client.agentDescriptor()
+            #expect(descriptor.capabilities?.attachments == nil)
+            #expect((descriptor.capabilities?.assistantImages?.supported == true) == (token == "synthetic-test"))
+            let draft = AgentAttachmentDraft()
+            draft.configure(client: client, storage: nil); draft.updateCapability(descriptor.capabilities?.attachments)
+            #expect(!draft.supportsAttachments)
+            #expect(draft.add(filename: "original.bin") { .init(filename: "original.bin", mimeType: "application/octet-stream", data: originalBytes) } == nil)
+            await #expect(throws: AgentAttachmentIssue.self) { try await client.agentUpload(file: .init(filename: "original.bin", mimeType: "application/octet-stream", data: originalBytes), uploadID: UUID().uuidString) }
+            await #expect(throws: AgentAttachmentIssue.self) { try await client.agentSubmit(AgentSubmission(content: "", requestID: "output-only", attachments: [sampleAttachment()])) }
+        }
+        for json in [#"{"assistantImages":{"version":2}}"#, #"{"assistantImages":{"version":"future"}}"#] {
+            #expect(try JSONDecoder().decode(AgentCapabilities.self, from: Data(json.utf8)).assistantImages?.supported != true)
+        }
+    }
+
+    @Test func assistantDescriptorsRequireCompletedAssistantImageAndSafeFields() throws {
+        let image = AgentAttachment(id: "output-image", kind: .image, filename: "landscape.png", mimeType: "image/png", sizeBytes: 12, sha256: String(repeating: "a", count: 64))
+        var row = Message(id: "outbound:1", createdAt: Date(), role: .assistant, content: "", metadata: .object([
+            "kind": .string("message"), "status": .string("complete"), "agentAttachments": .array([image.metadataValue])]))
+        #expect(row.agentAttachments == [image])
+        for role in [MessageRole.system, .tool] {
+            var wrong = row; wrong.role = role
+            #expect(wrong.agentAttachments.isEmpty)
+        }
+        for (key, value) in [("kind", "tool_call"), ("kind", "other"), ("status", "queued"), ("status", "streaming")] {
+            var wrong = row, fields = row.metadata!.objectValue!
+            fields[key] = .string(value); wrong.metadata = .object(fields)
+            #expect(wrong.agentAttachments.isEmpty)
+        }
+        var descriptors: [JSONValue] = [sampleAttachment().metadataValue]
+        for key in ["workspacePath", "sourcePath", "privatePath", "url", "sourceURL", "downloadUrl", "external", "future"] {
+            var fields = image.metadataValue.objectValue!; fields[key] = .string("/private/synthetic.png")
+            descriptors.append(.object(fields))
+        }
+        for (key, value) in [("filename", "../private.png"), ("filename", "https://example.invalid/a.png"), ("mimeType", "image/svg+xml"), ("sha256", "bad")] {
+            var fields = image.metadataValue.objectValue!; fields[key] = .string(value)
+            descriptors.append(.object(fields))
+        }
+        for descriptor in descriptors {
+            row.metadata = .object(["kind": .string("message"), "status": .string("complete"), "agentAttachments": .array([descriptor])])
+            #expect(row.agentAttachments.isEmpty)
+        }
+        row.role = .user
+        row.metadata = .object(["agentAttachments": .array([sampleAttachment().metadataValue])])
+        #expect(row.agentAttachments == [sampleAttachment()])
+    }
+
+    @Test @MainActor func imageOnlyAssistantLiveAndHistoryUseSameMergeAndAuthenticatedLoader() async throws {
+        let png = try #require(UIGraphicsImageRenderer(size: CGSize(width: 24, height: 16)).image { context in
+            UIColor.systemTeal.setFill(); context.fill(CGRect(x: 0, y: 0, width: 24, height: 16))
+        }.pngData())
+        let attachment = AgentAttachment(id: "output/opaque", kind: .image, filename: "landscape.png", mimeType: "image/png", sizeBytes: png.count,
+            sha256: SHA256.hash(data: png).map { String(format: "%02x", $0) }.joined())
+        let row = Message(id: "outbound:1", createdAt: Date(timeIntervalSince1970: 1), role: .assistant, content: "", metadata: .object([
+            "kind": .string("message"), "status": .string("complete"), "entryId": .number(1), "agentAttachments": .array([attachment.metadataValue])]))
+        let page = AgentPage(conversationId: "fixture-root", messages: [row], cursor: "1", hasMore: false)
+        let encoder = JSONEncoder(); encoder.dateEncodingStrategy = .iso8601
+        let bytes = try encoder.encode(page)
+        var envelope = try JSONSerialization.jsonObject(with: bytes) as! [String: Any]; envelope["type"] = "messages"
+        let decodedEvent = try AgentEvent.decode(JSONSerialization.data(withJSONObject: envelope))
+        let event = try #require(decodedEvent)
+        let room = SybilAgentRoom(); defer { room.stop() }
+        try room.receive(event); room.flush()
+        var history = AgentRoomState()
+        try history.apply(AgentCoding.decoder().decode(AgentPage.self, from: bytes))
+        // Replay must not duplicate a completed output; no current capability is needed.
+        try room.receive(event); room.flush()
+        #expect(AgentTranscript(messages: room.messages).displayed == AgentTranscript(messages: history.messages).displayed)
+        #expect(room.messages.count == 1); #expect(room.messages[0].content.isEmpty)
+        #expect(room.messages[0].agentAttachments == [attachment])
+        var presentation = AgentPresentation()
+        presentation.receive(event, before: AgentRoomState(), after: history, replay: false, now: Date())
+        #expect(presentation.suppressed)
+
+        AttachmentURLProtocol.install { request in
+            #expect(request.url!.absoluteString == "https://fixture.invalid/api/v1/agent/attachments/output%2Fopaque/content")
+            #expect(request.httpMethod == "GET")
+            #expect(request.value(forHTTPHeaderField: "Authorization") == "Bearer synthetic-test")
+            return (200, ["Content-Length": String(png.count)], png)
+        }
+        let session = attachmentSession(); defer { session.invalidateAndCancel() }
+        let client = SybilAPIClient(configuration: .init(baseURL: URL(string: "https://fixture.invalid/api")!, authToken: "synthetic-test"), session: session)
+        let downloads = AgentAttachmentDownloads()
+        #expect(try await downloads.data(for: attachment, client: client) == png)
+        AttachmentURLProtocol.install { _ in Issue.record("Verified output should be cached"); return (500, [:], Data()) }
+        #expect(try await downloads.data(for: attachment, client: client) == png)
+        AttachmentURLProtocol.install { _ in (200, [:], Data(repeating: 0, count: png.count)) }
+        await #expect(throws: AgentAttachmentIssue.self) { try await client.agentDownload(attachment) }
+        let unauthenticated = SybilAPIClient(configuration: .init(baseURL: URL(string: "https://fixture.invalid/api")!), session: session)
+        await #expect(throws: AgentAttachmentIssue.self) { try await unauthenticated.agentDownload(attachment) }
+    }
+
    @Test func additiveCapabilityAndDistinctMetadata() throws {
        let old = try AgentCoding.decoder().decode(AgentDescriptor.self, from: descriptorData(capability: false))
        #expect(old.capabilities == nil)
@@ -6,6 +6,57 @@ import UIKit

@Suite @MainActor
 struct MessagePresentationTests {
+    @Test func agentMarkdownKeepsAltAndFormattingWithoutImageOrHTMLNodes() {
+        let source = """
+        **Caption** ![Inline *landscape*](https://images.invalid/inline.png)
+
+        ![Reference landscape][photo]
+
+        [![Linked landscape](//images.invalid/linked.png)](https://example.invalid)
+
+        > ![Nested ![inner](file:///private/image.png)](data:image/png;base64,AAAA)
+
+        <picture><source srcset="https://images.invalid/source.png"><img src="https://images.invalid/html.png" alt="HTML landscape" title="a > b"></picture>
+
+        [Useful link](https://example.invalid)
+
+        [photo]: https://images.invalid/reference.png
+        """
+        let content = AgentMessageMarkdown.content(source)
+        let html = content.renderHTML(), plain = content.renderPlainText()
+        for label in ["Inline landscape", "Reference landscape", "Linked landscape", "Nested inner", "HTML landscape"] {
+            #expect(plain.contains(label))
+        }
+        #expect(!html.contains("<img")); #expect(!html.contains("<picture")); #expect(!html.contains("<source"))
+        #expect(!html.contains("images.invalid")); #expect(!html.contains("/private/"))
+        #expect(html.contains("<strong>Caption</strong>"))
+        #expect(html.contains(#"href="https://example.invalid""#))
+        // Normal chat still uses unmodified MarkdownUI content.
+        #expect(MarkdownContent(source).renderHTML().contains("<img"))
+        #expect(AgentMessageMarkdown.content("<img src='https://images.invalid/a.png' alt='HTML landscape'>\n").renderHTML() == "<p>HTML landscape</p>\n")
+    }
+
+    @Test func agentMarkdownRetainsCodeTablesAndEscapesImageLikeAltText() {
+        let source = """
+        `![literal](https://images.invalid/code.png)`
+
+        ```html
+        <img src="literal-code">
+        ```
+
+        | Image | Value |
+        | --- | --- |
+        | ![Table alt](https://images.invalid/table.png) | **Kept** |
+
+        <IMG ALT='![literal alt](https://images.invalid/alt.png)' SRC='ignored'>
+        """
+        let html = AgentMessageMarkdown.content(source).renderHTML()
+        #expect(html.contains("<table>")); #expect(html.contains("Table alt"))
+        #expect(html.contains("<strong>Kept</strong>"))
+        #expect(html.contains("&lt;img src=")); #expect(html.contains("![literal]"))
+        #expect(html.contains("![literal alt]")); #expect(!html.contains("<img"))
+    }
+
    @Test func userMarkupRemainsLiteral() {
        let source = "**bold** [link](https://example.invalid) `code`\n- item\n<script>literal</script>"
        let content = SybilMessageText.literalContent(source)