ios: support safe assistant inline images
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
import Foundation
|
||||
import MarkdownUI
|
||||
import cmark_gfm
|
||||
import cmark_gfm_extensions
|
||||
|
||||
/// Agent images come only from validated attachment descriptors. Rewrite parsed nodes,
|
||||
/// including reference/nested images, so neither MarkdownUI image loader receives a URL.
|
||||
enum AgentMessageMarkdown {
|
||||
static func content(_ source: String) -> MarkdownContent {
|
||||
cmark_gfm_core_extensions_ensure_registered()
|
||||
guard let parser = cmark_parser_new(CMARK_OPT_DEFAULT) else { return literal(source) }
|
||||
defer { cmark_parser_free(parser) }
|
||||
for name in ["table", "strikethrough", "autolink", "tasklist"] {
|
||||
if let ext = cmark_find_syntax_extension(name) { cmark_parser_attach_syntax_extension(parser, ext) }
|
||||
}
|
||||
source.withCString { cmark_parser_feed(parser, $0, source.utf8.count) }
|
||||
guard let document = cmark_parser_finish(parser) else { return literal(source) }
|
||||
defer { cmark_node_free(document) }
|
||||
stripImagesAndHTML(document)
|
||||
guard let markdown = cmark_render_commonmark(document, CMARK_OPT_DEFAULT, 0) else { return literal(source) }
|
||||
defer { free(markdown) }
|
||||
return MarkdownContent(String(cString: markdown))
|
||||
}
|
||||
|
||||
private static func literal(_ source: String) -> MarkdownContent {
|
||||
MarkdownContent { Paragraph { source } }
|
||||
}
|
||||
|
||||
private static func stripImagesAndHTML(_ parent: UnsafeMutablePointer<cmark_node>) {
|
||||
var child = cmark_node_first_child(parent)
|
||||
while let node = child {
|
||||
child = cmark_node_next(node)
|
||||
let type = cmark_node_get_type(node)
|
||||
if type == CMARK_NODE_IMAGE || type == CMARK_NODE_HTML_INLINE || type == CMARK_NODE_HTML_BLOCK {
|
||||
let text = type == CMARK_NODE_IMAGE ? altText(node) : htmlText(cmark_node_get_literal(node).map { String(cString: $0) } ?? "")
|
||||
let replacement = cmark_node_new(type == CMARK_NODE_HTML_BLOCK ? CMARK_NODE_PARAGRAPH : CMARK_NODE_TEXT)!
|
||||
if type == CMARK_NODE_HTML_BLOCK {
|
||||
let inline = cmark_node_new(CMARK_NODE_TEXT)!
|
||||
// cmark includes the HTML block's terminating newline in its literal.
|
||||
// It is a block delimiter, not an extra line in the visible alt text.
|
||||
cmark_node_set_literal(inline, text.trimmingCharacters(in: .newlines))
|
||||
cmark_node_append_child(replacement, inline)
|
||||
} else {
|
||||
cmark_node_set_literal(replacement, text)
|
||||
}
|
||||
cmark_node_replace(node, replacement)
|
||||
cmark_node_free(node)
|
||||
} else {
|
||||
stripImagesAndHTML(node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static func altText(_ parent: UnsafeMutablePointer<cmark_node>) -> String {
|
||||
var result = "", child = cmark_node_first_child(parent)
|
||||
while let node = child {
|
||||
let type = cmark_node_get_type(node)
|
||||
if type == CMARK_NODE_TEXT || type == CMARK_NODE_CODE {
|
||||
result += cmark_node_get_literal(node).map { String(cString: $0) } ?? ""
|
||||
} else if type == CMARK_NODE_SOFTBREAK || type == CMARK_NODE_LINEBREAK {
|
||||
result += " "
|
||||
} else if type != CMARK_NODE_HTML_INLINE {
|
||||
result += altText(node)
|
||||
}
|
||||
child = cmark_node_next(node)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private static func htmlText(_ source: String) -> String {
|
||||
// HTML is never rendered. Retain ordinary text and img alt attributes as text
|
||||
// nodes; quoted '>' characters cannot expose the rest of an image attribute.
|
||||
let tags = try! NSRegularExpression(pattern: #"(?s)<(?:[^>"']|"[^"]*"|'[^']*')*>"#)
|
||||
let alt = try! NSRegularExpression(pattern: #"(?is)^<img\b.*?\s+alt\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))"#)
|
||||
var result = source
|
||||
for tag in tags.matches(in: source, range: NSRange(source.startIndex..., in: source)).reversed() {
|
||||
let range = Range(tag.range, in: source)!
|
||||
let value = String(source[range])
|
||||
let match = alt.firstMatch(in: value, range: NSRange(value.startIndex..., in: value))
|
||||
let label = match.flatMap { match in
|
||||
(1...3).compactMap { Range(match.range(at: $0), in: value).map { String(value[$0]) } }.first
|
||||
} ?? ""
|
||||
result.replaceSubrange(Range(tag.range, in: result)!, with: label)
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -46,7 +46,10 @@ actor SybilAPIClient: SybilAPIClienting, AgentAPIClienting, AgentPushAPIClientin
|
||||
guard value.type == "persistent-agent", value.protocol == "pi-room.v1" else {
|
||||
throw AgentFailure(message: "Unsupported Agent room protocol.")
|
||||
}
|
||||
if configuration.authToken?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty != false { value.capabilities?.attachments = nil }
|
||||
if configuration.authToken?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty != false {
|
||||
value.capabilities?.attachments = nil
|
||||
value.capabilities?.assistantImages = nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
|
||||
@@ -174,9 +174,15 @@ struct AgentAttachmentCard: View {
|
||||
if attachment.kind == .image {
|
||||
ZStack {
|
||||
SybilTheme.surface
|
||||
if let image { Image(uiImage: image).resizable().scaledToFit() }
|
||||
if let image {
|
||||
Image(uiImage: image).resizable().scaledToFit()
|
||||
.accessibilityLabel("Image: \(attachment.filename)")
|
||||
.accessibilityIdentifier("agent-image-preview")
|
||||
}
|
||||
else { Image(systemName: "photo").foregroundStyle(SybilTheme.textMuted) }
|
||||
}.frame(height: 180).clipped()
|
||||
.accessibilityElement(children: .contain)
|
||||
.accessibilityIdentifier("agent-image-frame")
|
||||
}
|
||||
if let text {
|
||||
Text(text).font(.system(.caption, design: .monospaced)).lineLimit(6)
|
||||
|
||||
@@ -44,14 +44,24 @@ struct AgentAttachmentCapabilities: Codable, Equatable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
struct AgentAssistantImageCapabilities: Decodable, Equatable, Sendable {
|
||||
var version: Int
|
||||
var supported: Bool { version == 1 }
|
||||
}
|
||||
|
||||
struct AgentCapabilities: Decodable, Sendable {
|
||||
var attachments: AgentAttachmentCapabilities?
|
||||
init(attachments: AgentAttachmentCapabilities? = nil) { self.attachments = attachments }
|
||||
private enum CodingKeys: String, CodingKey { case attachments }
|
||||
var assistantImages: AgentAssistantImageCapabilities?
|
||||
init(attachments: AgentAttachmentCapabilities? = nil, assistantImages: AgentAssistantImageCapabilities? = nil) {
|
||||
self.attachments = attachments
|
||||
self.assistantImages = assistantImages
|
||||
}
|
||||
private enum CodingKeys: String, CodingKey { case attachments, assistantImages }
|
||||
init(from decoder: Decoder) throws {
|
||||
let values = try decoder.container(keyedBy: CodingKeys.self)
|
||||
// Unknown versions or malformed future capabilities must not break text chat.
|
||||
attachments = try? values.decode(AgentAttachmentCapabilities.self, forKey: .attachments)
|
||||
assistantImages = try? values.decode(AgentAssistantImageCapabilities.self, forKey: .assistantImages)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,9 +101,21 @@ struct AgentAttachment: Codable, Hashable, Identifiable, Sendable {
|
||||
|
||||
extension Message {
|
||||
var agentAttachments: [AgentAttachment] {
|
||||
guard let entries = metadata?.objectValue?["agentAttachments"]?.arrayValue else { return [] }
|
||||
return entries.compactMap {
|
||||
guard let data = try? JSONEncoder().encode($0),
|
||||
guard role == .user || role == .assistant,
|
||||
let info = metadata?.objectValue,
|
||||
let entries = info["agentAttachments"]?.arrayValue else { return [] }
|
||||
if role == .assistant {
|
||||
guard info["kind"]?.stringValue == "message", info["status"]?.stringValue == "complete" else { return [] }
|
||||
}
|
||||
return entries.compactMap { entry in
|
||||
// Inspect the original JSON before Codable can discard unknown fields. Output
|
||||
// descriptors are deliberately path/URL-free; legacy user uploads stay compatible.
|
||||
if role == .assistant {
|
||||
let safeFields: Set<String> = ["id", "kind", "filename", "mimeType", "sizeBytes", "sha256"]
|
||||
guard let fields = entry.objectValue, Set(fields.keys) == safeFields,
|
||||
fields["kind"]?.stringValue == "image" else { return nil }
|
||||
}
|
||||
guard let data = try? JSONEncoder().encode(entry),
|
||||
let attachment = try? JSONDecoder().decode(AgentAttachment.self, from: data),
|
||||
(try? attachment.validate()) != nil else { return nil }
|
||||
return attachment
|
||||
|
||||
@@ -249,7 +249,7 @@ struct AgentPresentation {
|
||||
.sorted { AgentCursor.less($0.agentEntryCursor ?? "0", $1.agentEntryCursor ?? "0") }
|
||||
if !replay {
|
||||
for message in fresh {
|
||||
if message.isAgentPublicReply && !message.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { suppressed = true }
|
||||
if message.isAgentPublicReply && (!message.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !message.agentAttachments.isEmpty) { suppressed = true }
|
||||
else if message.role == .user { suppressed = false }
|
||||
else if message.toolCallMetadata?.status == "initiated",
|
||||
!Self.quietTools.contains(message.toolCallMetadata?.toolName ?? ""), !seenTools.contains(message.agentToolID) { suppressed = false }
|
||||
|
||||
@@ -275,7 +275,9 @@ private struct AgentMessageRow: View {
|
||||
ForEach(message.agentAttachments) { attachment in
|
||||
AgentAttachmentCard(attachment: attachment, room: room)
|
||||
}
|
||||
MessageBubble(message: message, isSending: false, streamingAssistantPresentation: nil, isForking: false, onFork: nil, agentStyle: true)
|
||||
if !message.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || message.agentAttachments.isEmpty {
|
||||
MessageBubble(message: message, isSending: false, streamingAssistantPresentation: nil, isForking: false, onFork: nil, agentStyle: true)
|
||||
}
|
||||
HStack(spacing: 6) {
|
||||
Text(own ? "You" : "Sybil")
|
||||
Text(message.createdAt, style: .time)
|
||||
|
||||
@@ -223,14 +223,14 @@ struct MessageBubble: View {
|
||||
)
|
||||
}
|
||||
|
||||
if isStreamingAssistant {
|
||||
if isStreamingAssistant && !agentStyle {
|
||||
StreamingAssistantContentView(
|
||||
content: message.content,
|
||||
isSending: isSending,
|
||||
presentation: matchingStreamingAssistantPresentation
|
||||
)
|
||||
} else if !message.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
||||
SybilMessageText(content: message.content, format: isUser ? .literal : .markdown)
|
||||
SybilMessageText(content: message.content, format: isUser ? .literal : .markdown, agentStyle: agentStyle)
|
||||
}
|
||||
}
|
||||
.padding(.horizontal, isUser || agentStyle ? 14 : 2)
|
||||
|
||||
@@ -7,6 +7,7 @@ struct SybilMessageText: View {
|
||||
enum Format { case literal, markdown }
|
||||
var content: String
|
||||
var format: Format
|
||||
var agentStyle = false
|
||||
|
||||
static func literalContent(_ text: String) -> MarkdownContent {
|
||||
// Pasted CRLF/CR and blank lines containing spaces or tabs have the same
|
||||
@@ -31,7 +32,7 @@ struct SybilMessageText: View {
|
||||
#if DEBUG
|
||||
let _ = AgentRenderDiagnostics.record("messageTextBuild")
|
||||
#endif
|
||||
Markdown(format == .literal ? Self.literalContent(content) : MarkdownContent(content))
|
||||
Markdown(format == .literal ? Self.literalContent(content) : agentStyle ? AgentMessageMarkdown.content(content) : MarkdownContent(content))
|
||||
.tint(SybilTheme.primary)
|
||||
.foregroundStyle(format == .literal ? SybilTheme.text : SybilTheme.text.opacity(0.95))
|
||||
.markdownTheme(.sybilReadable)
|
||||
|
||||
Reference in New Issue
Block a user