ios: support safe assistant inline images

This commit is contained in:
2026-10-20 14:31:15 -07:00
parent 9098fee32a
commit ee845c9fa3
13 changed files with 369 additions and 15 deletions
+6 -2
View File
@@ -15,7 +15,9 @@ let package = Package(
targets: ["Sybil"]),
],
dependencies: [
.package(url: "https://github.com/gonzalezreal/swift-markdown-ui.git", from: "2.4.1")
.package(url: "https://github.com/gonzalezreal/swift-markdown-ui.git", from: "2.4.1"),
// Use MarkdownUI's existing parser to remove Agent image/HTML nodes structurally.
.package(url: "https://github.com/swiftlang/swift-cmark", from: "0.7.1")
],
targets: [
// Targets are the basic building blocks of a package, defining a module or a test suite.
@@ -23,7 +25,9 @@ let package = Package(
.target(
name: "Sybil",
dependencies: [
.product(name: "MarkdownUI", package: "swift-markdown-ui")
.product(name: "MarkdownUI", package: "swift-markdown-ui"),
.product(name: "cmark-gfm", package: "swift-cmark"),
.product(name: "cmark-gfm-extensions", package: "swift-cmark")
],
resources: [.process("Resources")]),
.testTarget(
@@ -0,0 +1,87 @@
import Foundation
import MarkdownUI
import cmark_gfm
import cmark_gfm_extensions
/// Agent images come only from validated attachment descriptors. Rewrite parsed nodes,
/// including reference/nested images, so neither MarkdownUI image loader receives a URL.
enum AgentMessageMarkdown {
static func content(_ source: String) -> MarkdownContent {
cmark_gfm_core_extensions_ensure_registered()
guard let parser = cmark_parser_new(CMARK_OPT_DEFAULT) else { return literal(source) }
defer { cmark_parser_free(parser) }
for name in ["table", "strikethrough", "autolink", "tasklist"] {
if let ext = cmark_find_syntax_extension(name) { cmark_parser_attach_syntax_extension(parser, ext) }
}
source.withCString { cmark_parser_feed(parser, $0, source.utf8.count) }
guard let document = cmark_parser_finish(parser) else { return literal(source) }
defer { cmark_node_free(document) }
stripImagesAndHTML(document)
guard let markdown = cmark_render_commonmark(document, CMARK_OPT_DEFAULT, 0) else { return literal(source) }
defer { free(markdown) }
return MarkdownContent(String(cString: markdown))
}
private static func literal(_ source: String) -> MarkdownContent {
MarkdownContent { Paragraph { source } }
}
private static func stripImagesAndHTML(_ parent: UnsafeMutablePointer<cmark_node>) {
var child = cmark_node_first_child(parent)
while let node = child {
child = cmark_node_next(node)
let type = cmark_node_get_type(node)
if type == CMARK_NODE_IMAGE || type == CMARK_NODE_HTML_INLINE || type == CMARK_NODE_HTML_BLOCK {
let text = type == CMARK_NODE_IMAGE ? altText(node) : htmlText(cmark_node_get_literal(node).map { String(cString: $0) } ?? "")
let replacement = cmark_node_new(type == CMARK_NODE_HTML_BLOCK ? CMARK_NODE_PARAGRAPH : CMARK_NODE_TEXT)!
if type == CMARK_NODE_HTML_BLOCK {
let inline = cmark_node_new(CMARK_NODE_TEXT)!
// cmark includes the HTML block's terminating newline in its literal.
// It is a block delimiter, not an extra line in the visible alt text.
cmark_node_set_literal(inline, text.trimmingCharacters(in: .newlines))
cmark_node_append_child(replacement, inline)
} else {
cmark_node_set_literal(replacement, text)
}
cmark_node_replace(node, replacement)
cmark_node_free(node)
} else {
stripImagesAndHTML(node)
}
}
}
private static func altText(_ parent: UnsafeMutablePointer<cmark_node>) -> String {
var result = "", child = cmark_node_first_child(parent)
while let node = child {
let type = cmark_node_get_type(node)
if type == CMARK_NODE_TEXT || type == CMARK_NODE_CODE {
result += cmark_node_get_literal(node).map { String(cString: $0) } ?? ""
} else if type == CMARK_NODE_SOFTBREAK || type == CMARK_NODE_LINEBREAK {
result += " "
} else if type != CMARK_NODE_HTML_INLINE {
result += altText(node)
}
child = cmark_node_next(node)
}
return result
}
private static func htmlText(_ source: String) -> String {
// HTML is never rendered. Retain ordinary text and img alt attributes as text
// nodes; quoted '>' characters cannot expose the rest of an image attribute.
let tags = try! NSRegularExpression(pattern: #"(?s)<(?:[^>"']|"[^"]*"|'[^']*')*>"#)
let alt = try! NSRegularExpression(pattern: #"(?is)^<img\b.*?\s+alt\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))"#)
var result = source
for tag in tags.matches(in: source, range: NSRange(source.startIndex..., in: source)).reversed() {
let range = Range(tag.range, in: source)!
let value = String(source[range])
let match = alt.firstMatch(in: value, range: NSRange(value.startIndex..., in: value))
let label = match.flatMap { match in
(1...3).compactMap { Range(match.range(at: $0), in: value).map { String(value[$0]) } }.first
} ?? ""
result.replaceSubrange(Range(tag.range, in: result)!, with: label)
}
return result
}
}
@@ -46,7 +46,10 @@ actor SybilAPIClient: SybilAPIClienting, AgentAPIClienting, AgentPushAPIClientin
guard value.type == "persistent-agent", value.protocol == "pi-room.v1" else {
throw AgentFailure(message: "Unsupported Agent room protocol.")
}
if configuration.authToken?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty != false { value.capabilities?.attachments = nil }
if configuration.authToken?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty != false {
value.capabilities?.attachments = nil
value.capabilities?.assistantImages = nil
}
return value
}
@@ -174,9 +174,15 @@ struct AgentAttachmentCard: View {
if attachment.kind == .image {
ZStack {
SybilTheme.surface
if let image { Image(uiImage: image).resizable().scaledToFit() }
if let image {
Image(uiImage: image).resizable().scaledToFit()
.accessibilityLabel("Image: \(attachment.filename)")
.accessibilityIdentifier("agent-image-preview")
}
else { Image(systemName: "photo").foregroundStyle(SybilTheme.textMuted) }
}.frame(height: 180).clipped()
.accessibilityElement(children: .contain)
.accessibilityIdentifier("agent-image-frame")
}
if let text {
Text(text).font(.system(.caption, design: .monospaced)).lineLimit(6)
@@ -44,14 +44,24 @@ struct AgentAttachmentCapabilities: Codable, Equatable, Sendable {
}
}
struct AgentAssistantImageCapabilities: Decodable, Equatable, Sendable {
var version: Int
var supported: Bool { version == 1 }
}
struct AgentCapabilities: Decodable, Sendable {
var attachments: AgentAttachmentCapabilities?
init(attachments: AgentAttachmentCapabilities? = nil) { self.attachments = attachments }
private enum CodingKeys: String, CodingKey { case attachments }
var assistantImages: AgentAssistantImageCapabilities?
init(attachments: AgentAttachmentCapabilities? = nil, assistantImages: AgentAssistantImageCapabilities? = nil) {
self.attachments = attachments
self.assistantImages = assistantImages
}
private enum CodingKeys: String, CodingKey { case attachments, assistantImages }
init(from decoder: Decoder) throws {
let values = try decoder.container(keyedBy: CodingKeys.self)
// Unknown versions or malformed future capabilities must not break text chat.
attachments = try? values.decode(AgentAttachmentCapabilities.self, forKey: .attachments)
assistantImages = try? values.decode(AgentAssistantImageCapabilities.self, forKey: .assistantImages)
}
}
@@ -91,9 +101,21 @@ struct AgentAttachment: Codable, Hashable, Identifiable, Sendable {
extension Message {
var agentAttachments: [AgentAttachment] {
guard let entries = metadata?.objectValue?["agentAttachments"]?.arrayValue else { return [] }
return entries.compactMap {
guard let data = try? JSONEncoder().encode($0),
guard role == .user || role == .assistant,
let info = metadata?.objectValue,
let entries = info["agentAttachments"]?.arrayValue else { return [] }
if role == .assistant {
guard info["kind"]?.stringValue == "message", info["status"]?.stringValue == "complete" else { return [] }
}
return entries.compactMap { entry in
// Inspect the original JSON before Codable can discard unknown fields. Output
// descriptors are deliberately path/URL-free; legacy user uploads stay compatible.
if role == .assistant {
let safeFields: Set<String> = ["id", "kind", "filename", "mimeType", "sizeBytes", "sha256"]
guard let fields = entry.objectValue, Set(fields.keys) == safeFields,
fields["kind"]?.stringValue == "image" else { return nil }
}
guard let data = try? JSONEncoder().encode(entry),
let attachment = try? JSONDecoder().decode(AgentAttachment.self, from: data),
(try? attachment.validate()) != nil else { return nil }
return attachment
@@ -249,7 +249,7 @@ struct AgentPresentation {
.sorted { AgentCursor.less($0.agentEntryCursor ?? "0", $1.agentEntryCursor ?? "0") }
if !replay {
for message in fresh {
if message.isAgentPublicReply && !message.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { suppressed = true }
if message.isAgentPublicReply && (!message.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !message.agentAttachments.isEmpty) { suppressed = true }
else if message.role == .user { suppressed = false }
else if message.toolCallMetadata?.status == "initiated",
!Self.quietTools.contains(message.toolCallMetadata?.toolName ?? ""), !seenTools.contains(message.agentToolID) { suppressed = false }
@@ -275,7 +275,9 @@ private struct AgentMessageRow: View {
ForEach(message.agentAttachments) { attachment in
AgentAttachmentCard(attachment: attachment, room: room)
}
MessageBubble(message: message, isSending: false, streamingAssistantPresentation: nil, isForking: false, onFork: nil, agentStyle: true)
if !message.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || message.agentAttachments.isEmpty {
MessageBubble(message: message, isSending: false, streamingAssistantPresentation: nil, isForking: false, onFork: nil, agentStyle: true)
}
HStack(spacing: 6) {
Text(own ? "You" : "Sybil")
Text(message.createdAt, style: .time)
@@ -223,14 +223,14 @@ struct MessageBubble: View {
)
}
if isStreamingAssistant {
if isStreamingAssistant && !agentStyle {
StreamingAssistantContentView(
content: message.content,
isSending: isSending,
presentation: matchingStreamingAssistantPresentation
)
} else if !message.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
SybilMessageText(content: message.content, format: isUser ? .literal : .markdown)
SybilMessageText(content: message.content, format: isUser ? .literal : .markdown, agentStyle: agentStyle)
}
}
.padding(.horizontal, isUser || agentStyle ? 14 : 2)
@@ -7,6 +7,7 @@ struct SybilMessageText: View {
enum Format { case literal, markdown }
var content: String
var format: Format
var agentStyle = false
static func literalContent(_ text: String) -> MarkdownContent {
// Pasted CRLF/CR and blank lines containing spaces or tabs have the same
@@ -31,7 +32,7 @@ struct SybilMessageText: View {
#if DEBUG
let _ = AgentRenderDiagnostics.record("messageTextBuild")
#endif
Markdown(format == .literal ? Self.literalContent(content) : MarkdownContent(content))
Markdown(format == .literal ? Self.literalContent(content) : agentStyle ? AgentMessageMarkdown.content(content) : MarkdownContent(content))
.tint(SybilTheme.primary)
.foregroundStyle(format == .literal ? SybilTheme.text : SybilTheme.text.opacity(0.95))
.markdownTheme(.sybilReadable)
@@ -14,10 +14,15 @@ private func sampleAttachment(_ id: String = "opaque-server-id", data: Data = or
AgentAttachment(id: id, kind: .file, filename: name, mimeType: "application/octet-stream", sizeBytes: data.count,
sha256: SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined(), workspacePath: "/runtime/provided/\(name)")
}
private func descriptorData(capability: Bool = true) throws -> Data {
private func descriptorData(capability: Bool = true, assistantImages: Bool = false) throws -> Data {
var object: [String: Any] = ["id": "root", "title": "Sybil", "type": "persistent-agent", "model": "fixture", "protocol": "pi-room.v1", "websocket": "/v1/agent/ws",
"activity": ["state": "idle", "isThinking": false, "queued": 0, "backgroundWorkers": 0, "tools": [], "queuedMessages": []]]
if capability { object["capabilities"] = ["attachments": try JSONSerialization.jsonObject(with: JSONEncoder().encode(attachmentLimits)), "future": true] }
if assistantImages {
var capabilities = object["capabilities"] as? [String: Any] ?? [:]
capabilities["assistantImages"] = ["version": 1]
object["capabilities"] = capabilities
}
object["unknown"] = ["additive": true]
return try JSONSerialization.data(withJSONObject: object)
}
@@ -124,6 +129,106 @@ private func requestBytes(_ request: URLRequest) -> Data {
}
@Suite(.serialized) struct AgentAttachmentTests {
@Test @MainActor func assistantCapabilityDoesNotAdmitInputAndRequiresToken() async throws {
AttachmentURLProtocol.install { request in
#expect(request.url!.path == "/api/v1/agent") // Never upload or submit references.
return (200, [:], try descriptorData(capability: false, assistantImages: true))
}
let session = attachmentSession(); defer { session.invalidateAndCancel() }
for token in [nil, " ", "synthetic-test"] {
let client = SybilAPIClient(configuration: .init(baseURL: URL(string: "https://fixture.invalid/api")!, authToken: token), session: session)
let descriptor = try await client.agentDescriptor()
#expect(descriptor.capabilities?.attachments == nil)
#expect((descriptor.capabilities?.assistantImages?.supported == true) == (token == "synthetic-test"))
let draft = AgentAttachmentDraft()
draft.configure(client: client, storage: nil); draft.updateCapability(descriptor.capabilities?.attachments)
#expect(!draft.supportsAttachments)
#expect(draft.add(filename: "original.bin") { .init(filename: "original.bin", mimeType: "application/octet-stream", data: originalBytes) } == nil)
await #expect(throws: AgentAttachmentIssue.self) { try await client.agentUpload(file: .init(filename: "original.bin", mimeType: "application/octet-stream", data: originalBytes), uploadID: UUID().uuidString) }
await #expect(throws: AgentAttachmentIssue.self) { try await client.agentSubmit(AgentSubmission(content: "", requestID: "output-only", attachments: [sampleAttachment()])) }
}
for json in [#"{"assistantImages":{"version":2}}"#, #"{"assistantImages":{"version":"future"}}"#] {
#expect(try JSONDecoder().decode(AgentCapabilities.self, from: Data(json.utf8)).assistantImages?.supported != true)
}
}
@Test func assistantDescriptorsRequireCompletedAssistantImageAndSafeFields() throws {
let image = AgentAttachment(id: "output-image", kind: .image, filename: "landscape.png", mimeType: "image/png", sizeBytes: 12, sha256: String(repeating: "a", count: 64))
var row = Message(id: "outbound:1", createdAt: Date(), role: .assistant, content: "", metadata: .object([
"kind": .string("message"), "status": .string("complete"), "agentAttachments": .array([image.metadataValue])]))
#expect(row.agentAttachments == [image])
for role in [MessageRole.system, .tool] {
var wrong = row; wrong.role = role
#expect(wrong.agentAttachments.isEmpty)
}
for (key, value) in [("kind", "tool_call"), ("kind", "other"), ("status", "queued"), ("status", "streaming")] {
var wrong = row, fields = row.metadata!.objectValue!
fields[key] = .string(value); wrong.metadata = .object(fields)
#expect(wrong.agentAttachments.isEmpty)
}
var descriptors: [JSONValue] = [sampleAttachment().metadataValue]
for key in ["workspacePath", "sourcePath", "privatePath", "url", "sourceURL", "downloadUrl", "external", "future"] {
var fields = image.metadataValue.objectValue!; fields[key] = .string("/private/synthetic.png")
descriptors.append(.object(fields))
}
for (key, value) in [("filename", "../private.png"), ("filename", "https://example.invalid/a.png"), ("mimeType", "image/svg+xml"), ("sha256", "bad")] {
var fields = image.metadataValue.objectValue!; fields[key] = .string(value)
descriptors.append(.object(fields))
}
for descriptor in descriptors {
row.metadata = .object(["kind": .string("message"), "status": .string("complete"), "agentAttachments": .array([descriptor])])
#expect(row.agentAttachments.isEmpty)
}
row.role = .user
row.metadata = .object(["agentAttachments": .array([sampleAttachment().metadataValue])])
#expect(row.agentAttachments == [sampleAttachment()])
}
@Test @MainActor func imageOnlyAssistantLiveAndHistoryUseSameMergeAndAuthenticatedLoader() async throws {
let png = try #require(UIGraphicsImageRenderer(size: CGSize(width: 24, height: 16)).image { context in
UIColor.systemTeal.setFill(); context.fill(CGRect(x: 0, y: 0, width: 24, height: 16))
}.pngData())
let attachment = AgentAttachment(id: "output/opaque", kind: .image, filename: "landscape.png", mimeType: "image/png", sizeBytes: png.count,
sha256: SHA256.hash(data: png).map { String(format: "%02x", $0) }.joined())
let row = Message(id: "outbound:1", createdAt: Date(timeIntervalSince1970: 1), role: .assistant, content: "", metadata: .object([
"kind": .string("message"), "status": .string("complete"), "entryId": .number(1), "agentAttachments": .array([attachment.metadataValue])]))
let page = AgentPage(conversationId: "fixture-root", messages: [row], cursor: "1", hasMore: false)
let encoder = JSONEncoder(); encoder.dateEncodingStrategy = .iso8601
let bytes = try encoder.encode(page)
var envelope = try JSONSerialization.jsonObject(with: bytes) as! [String: Any]; envelope["type"] = "messages"
let decodedEvent = try AgentEvent.decode(JSONSerialization.data(withJSONObject: envelope))
let event = try #require(decodedEvent)
let room = SybilAgentRoom(); defer { room.stop() }
try room.receive(event); room.flush()
var history = AgentRoomState()
try history.apply(AgentCoding.decoder().decode(AgentPage.self, from: bytes))
// Replay must not duplicate a completed output; no current capability is needed.
try room.receive(event); room.flush()
#expect(AgentTranscript(messages: room.messages).displayed == AgentTranscript(messages: history.messages).displayed)
#expect(room.messages.count == 1); #expect(room.messages[0].content.isEmpty)
#expect(room.messages[0].agentAttachments == [attachment])
var presentation = AgentPresentation()
presentation.receive(event, before: AgentRoomState(), after: history, replay: false, now: Date())
#expect(presentation.suppressed)
AttachmentURLProtocol.install { request in
#expect(request.url!.absoluteString == "https://fixture.invalid/api/v1/agent/attachments/output%2Fopaque/content")
#expect(request.httpMethod == "GET")
#expect(request.value(forHTTPHeaderField: "Authorization") == "Bearer synthetic-test")
return (200, ["Content-Length": String(png.count)], png)
}
let session = attachmentSession(); defer { session.invalidateAndCancel() }
let client = SybilAPIClient(configuration: .init(baseURL: URL(string: "https://fixture.invalid/api")!, authToken: "synthetic-test"), session: session)
let downloads = AgentAttachmentDownloads()
#expect(try await downloads.data(for: attachment, client: client) == png)
AttachmentURLProtocol.install { _ in Issue.record("Verified output should be cached"); return (500, [:], Data()) }
#expect(try await downloads.data(for: attachment, client: client) == png)
AttachmentURLProtocol.install { _ in (200, [:], Data(repeating: 0, count: png.count)) }
await #expect(throws: AgentAttachmentIssue.self) { try await client.agentDownload(attachment) }
let unauthenticated = SybilAPIClient(configuration: .init(baseURL: URL(string: "https://fixture.invalid/api")!), session: session)
await #expect(throws: AgentAttachmentIssue.self) { try await unauthenticated.agentDownload(attachment) }
}
@Test func additiveCapabilityAndDistinctMetadata() throws {
let old = try AgentCoding.decoder().decode(AgentDescriptor.self, from: descriptorData(capability: false))
#expect(old.capabilities == nil)
@@ -6,6 +6,57 @@ import UIKit
@Suite @MainActor
struct MessagePresentationTests {
@Test func agentMarkdownKeepsAltAndFormattingWithoutImageOrHTMLNodes() {
let source = """
**Caption** ![Inline *landscape*](https://images.invalid/inline.png)
![Reference landscape][photo]
[![Linked landscape](//images.invalid/linked.png)](https://example.invalid)
> ![Nested ![inner](file:///private/image.png)](data:image/png;base64,AAAA)
<picture><source srcset="https://images.invalid/source.png"><img src="https://images.invalid/html.png" alt="HTML landscape" title="a > b"></picture>
[Useful link](https://example.invalid)
[photo]: https://images.invalid/reference.png
"""
let content = AgentMessageMarkdown.content(source)
let html = content.renderHTML(), plain = content.renderPlainText()
for label in ["Inline landscape", "Reference landscape", "Linked landscape", "Nested inner", "HTML landscape"] {
#expect(plain.contains(label))
}
#expect(!html.contains("<img")); #expect(!html.contains("<picture")); #expect(!html.contains("<source"))
#expect(!html.contains("images.invalid")); #expect(!html.contains("/private/"))
#expect(html.contains("<strong>Caption</strong>"))
#expect(html.contains(#"href="https://example.invalid""#))
// Normal chat still uses unmodified MarkdownUI content.
#expect(MarkdownContent(source).renderHTML().contains("<img"))
#expect(AgentMessageMarkdown.content("<img src='https://images.invalid/a.png' alt='HTML landscape'>\n").renderHTML() == "<p>HTML landscape</p>\n")
}
@Test func agentMarkdownRetainsCodeTablesAndEscapesImageLikeAltText() {
let source = """
`![literal](https://images.invalid/code.png)`
```html
<img src="literal-code">
```
| Image | Value |
| --- | --- |
| ![Table alt](https://images.invalid/table.png) | **Kept** |
<IMG ALT='![literal alt](https://images.invalid/alt.png)' SRC='ignored'>
"""
let html = AgentMessageMarkdown.content(source).renderHTML()
#expect(html.contains("<table>")); #expect(html.contains("Table alt"))
#expect(html.contains("<strong>Kept</strong>"))
#expect(html.contains("&lt;img src=")); #expect(html.contains("![literal]"))
#expect(html.contains("![literal alt]")); #expect(!html.contains("<img"))
}
@Test func userMarkupRemainsLiteral() {
let source = "**bold** [link](https://example.invalid) `code`\n- item\n<script>literal</script>"
let content = SybilMessageText.literalContent(source)