301 lines
18 KiB
Swift
301 lines
18 KiB
Swift
import CryptoKit
|
||
import Foundation
|
||
import ImageIO
|
||
import UniformTypeIdentifiers
|
||
import UIKit
|
||
|
||
struct AgentAttachmentCapabilities: Codable, Equatable, Sendable {
|
||
var version: Int
|
||
var maxCount: Int
|
||
var maxTotalBytes: Int
|
||
var maxImageBytes: Int
|
||
var maxFileBytes: Int
|
||
var imageMimeTypes: [String]
|
||
var maxImagePixels: Int
|
||
var maxImageDimension: Int
|
||
var quotaBytes: Int64? = nil
|
||
var usedBytes: Int64? = nil
|
||
var remainingBytes: Int64? = nil
|
||
var maxConcurrentUploads: Int? = nil
|
||
var maxConcurrentDecodes: Int? = nil
|
||
|
||
var supported: Bool {
|
||
version == 1 && maxCount > 0 && maxTotalBytes > 0 && maxImageBytes > 0 && maxFileBytes > 0
|
||
&& maxImagePixels > 0 && maxImageDimension > 0
|
||
}
|
||
var countLimit: Int { min(maxCount, 8) }
|
||
var totalLimit: Int { min(maxTotalBytes, 32 * 1024 * 1024) }
|
||
var imageLimit: Int { min(maxImageBytes, 6 * 1024 * 1024) }
|
||
var fileLimit: Int { min(maxFileBytes, 8 * 1024 * 1024) }
|
||
var pixelLimit: Int { min(maxImagePixels, 50_000_000) }
|
||
var dimensionLimit: Int { min(maxImageDimension, 16_384) }
|
||
func validate(_ attachments: [AgentAttachment]) throws {
|
||
guard supported else { throw AgentAttachmentIssue.unavailable }
|
||
for attachment in attachments { try attachment.validate() }
|
||
guard attachments.count <= countLimit, attachments.reduce(0, { $0 + $1.sizeBytes }) <= totalLimit else {
|
||
throw AgentAttachmentIssue.local("Attachments exceed the advertised message limit.", code: "attachment_limit_exceeded")
|
||
}
|
||
for attachment in attachments {
|
||
guard attachment.sizeBytes <= (attachment.kind == .image ? imageLimit : fileLimit),
|
||
attachment.kind != .image || imageMimeTypes.contains(attachment.mimeType) else {
|
||
throw AgentAttachmentIssue.local("An attachment exceeds the current Agent limits or uses an unavailable image format.", code: "attachment_limit_exceeded")
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
struct AgentAssistantImageCapabilities: Decodable, Equatable, Sendable {
|
||
var version: Int
|
||
var supported: Bool { version == 1 }
|
||
}
|
||
|
||
struct AgentCapabilities: Decodable, Sendable {
|
||
var attachments: AgentAttachmentCapabilities?
|
||
var assistantImages: AgentAssistantImageCapabilities?
|
||
init(attachments: AgentAttachmentCapabilities? = nil, assistantImages: AgentAssistantImageCapabilities? = nil) {
|
||
self.attachments = attachments
|
||
self.assistantImages = assistantImages
|
||
}
|
||
private enum CodingKeys: String, CodingKey { case attachments, assistantImages }
|
||
init(from decoder: Decoder) throws {
|
||
let values = try decoder.container(keyedBy: CodingKeys.self)
|
||
// Unknown versions or malformed future capabilities must not break text chat.
|
||
attachments = try? values.decode(AgentAttachmentCapabilities.self, forKey: .attachments)
|
||
assistantImages = try? values.decode(AgentAssistantImageCapabilities.self, forKey: .assistantImages)
|
||
}
|
||
}
|
||
|
||
struct AgentAttachment: Codable, Hashable, Identifiable, Sendable {
|
||
enum Kind: String, Codable, Sendable { case image, text, file }
|
||
var id: String
|
||
var kind: Kind
|
||
var filename: String
|
||
var mimeType: String
|
||
var sizeBytes: Int
|
||
var sha256: String
|
||
var workspacePath: String? = nil
|
||
|
||
func validate() throws {
|
||
guard !id.isEmpty, id.utf8.count <= 1024, !id.unicodeScalars.contains(where: CharacterSet.controlCharacters.contains),
|
||
try AgentAttachmentPreparation.filename(filename) == filename,
|
||
sizeBytes >= 0, sizeBytes <= 8 * 1024 * 1024,
|
||
sha256.count == 64, sha256.utf8.allSatisfy({ (48...57).contains($0) || (97...102).contains($0) }),
|
||
try AgentAttachmentPreparation.mime(mimeType) == mimeType else {
|
||
throw AgentAttachmentIssue.local("The Agent returned an invalid attachment descriptor.", code: "invalid_upstream_response")
|
||
}
|
||
guard (kind == .image) == AgentAttachmentPreparation.nativeMIMEs.contains(mimeType),
|
||
kind != .image || sizeBytes <= 6 * 1024 * 1024 else {
|
||
throw AgentAttachmentIssue.local("Invalid image attachment descriptor.", code: "invalid_upstream_response")
|
||
}
|
||
if let workspacePath {
|
||
guard workspacePath.hasPrefix("/"), !workspacePath.unicodeScalars.contains(where: CharacterSet.controlCharacters.contains) else {
|
||
throw AgentAttachmentIssue.local("The Agent returned an invalid workspace path.", code: "invalid_upstream_response")
|
||
}
|
||
}
|
||
}
|
||
var metadataValue: JSONValue {
|
||
// All stored fields are JSON-compatible; this also keeps optional future fields separate from ChatAttachment.
|
||
(try? JSONDecoder().decode(JSONValue.self, from: JSONEncoder().encode(self))) ?? .null
|
||
}
|
||
}
|
||
|
||
extension Message {
|
||
var agentAttachments: [AgentAttachment] {
|
||
guard role == .user || role == .assistant,
|
||
let info = metadata?.objectValue,
|
||
let entries = info["agentAttachments"]?.arrayValue else { return [] }
|
||
if role == .assistant {
|
||
guard info["kind"]?.stringValue == "message", info["status"]?.stringValue == "complete",
|
||
entries.count <= 8 else { return [] }
|
||
}
|
||
let attachments: [AgentAttachment] = entries.compactMap { entry in
|
||
// Inspect the original JSON before Codable can discard unknown fields. Output
|
||
// descriptors are deliberately path/URL-free; legacy user uploads stay compatible.
|
||
if role == .assistant {
|
||
let safeFields: Set<String> = ["id", "kind", "filename", "mimeType", "sizeBytes", "sha256"]
|
||
guard let fields = entry.objectValue, Set(fields.keys) == safeFields,
|
||
fields["kind"]?.stringValue == "image" else { return nil }
|
||
}
|
||
guard let data = try? JSONEncoder().encode(entry),
|
||
let attachment = try? JSONDecoder().decode(AgentAttachment.self, from: data),
|
||
(try? attachment.validate()) != nil else { return nil }
|
||
return attachment
|
||
}
|
||
if role == .assistant {
|
||
// Reject the entire output set, including partially invalid arrays. The
|
||
// count and per-image bounds above also bound this aggregate calculation.
|
||
guard attachments.count == entries.count,
|
||
Set(attachments.map(\.id)).count == attachments.count,
|
||
attachments.reduce(0, { $0 + $1.sizeBytes }) <= 32 * 1024 * 1024 else { return [] }
|
||
}
|
||
return attachments
|
||
}
|
||
}
|
||
|
||
struct AgentSubmission: Encodable, Sendable {
|
||
struct Reference: Encodable, Sendable { let id: String }
|
||
let requestId: String
|
||
let content: String
|
||
let attachments: [Reference]?
|
||
init(content: String, requestID: String, attachments: [AgentAttachment] = []) throws {
|
||
guard !requestID.isEmpty, requestID.utf16.count <= 256, content.utf16.count <= 40_000,
|
||
!content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !attachments.isEmpty,
|
||
attachments.count <= 8, Set(attachments.map(\.id)).count == attachments.count else {
|
||
throw AgentAttachmentIssue.local("Enter a message of at most 40,000 characters or attach files.", code: "invalid_attachment")
|
||
}
|
||
for attachment in attachments { try attachment.validate() }
|
||
guard attachments.reduce(0, { $0 + $1.sizeBytes }) <= 32 * 1024 * 1024 else {
|
||
throw AgentAttachmentIssue.local("Attachments exceed the total message size limit.", code: "attachment_limit_exceeded")
|
||
}
|
||
self.content = content; requestId = requestID
|
||
self.attachments = attachments.isEmpty ? nil : attachments.map { Reference(id: $0.id) }
|
||
}
|
||
var attachmentIDs: [String] { attachments?.map(\.id) ?? [] }
|
||
}
|
||
|
||
struct AgentAttachmentIssue: Error, LocalizedError, Codable, Equatable, Sendable {
|
||
var message: String
|
||
var code: String
|
||
var statusCode: Int?
|
||
// Preserve additive quota information without inventing field names.
|
||
var details: [String: JSONValue] = [:]
|
||
var retryAfter: String? = nil
|
||
var errorDescription: String? {
|
||
var result = "\(message) [\(code)]"
|
||
if code == "attachment_quota_exceeded" {
|
||
let numbers = ["requiredBytes", "remainingBytes", "quotaBytes"].compactMap { key -> String? in
|
||
guard let number = details[key]?.numberValue, number.isFinite, number >= 0, number < Double(Int64.max) else { return nil }
|
||
return "\(key): \(ByteCountFormatter.string(fromByteCount: Int64(number), countStyle: .file))"
|
||
}
|
||
if !numbers.isEmpty { result += " " + numbers.joined(separator: ", ") }
|
||
}
|
||
if let retryAfter { result += " Retry-After: \(retryAfter)." }
|
||
return result
|
||
}
|
||
static func local(_ message: String, code: String = "invalid_attachment") -> Self { Self(message: message, code: code) }
|
||
static let unavailable = Self.local("Attachments are unavailable on this Agent. Reconnect to check support, or remove the files to send text.", code: "attachments_unavailable")
|
||
static func wrapping(_ error: any Error) -> Self {
|
||
error as? Self ?? .local(error.localizedDescription, code: "attachment_transfer_failed")
|
||
}
|
||
static func response(_ data: Data, status: Int, retryAfter: String? = nil) -> Self {
|
||
let fields = (try? JSONDecoder().decode(JSONValue.self, from: data))?.objectValue ?? [:]
|
||
return Self(message: fields["message"]?.stringValue ?? fields["error"]?.stringValue ?? "Attachment request failed (HTTP \(status)).",
|
||
code: fields["code"]?.stringValue ?? "attachment_request_failed", statusCode: status, details: fields, retryAfter: retryAfter)
|
||
}
|
||
}
|
||
|
||
struct AgentPreparedAttachment: Sendable {
|
||
var filename: String
|
||
var mimeType: String
|
||
var data: Data
|
||
var sha256: String { SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() }
|
||
}
|
||
|
||
enum AgentAttachmentPreparation {
|
||
static func offMain<T: Sendable>(_ operation: @escaping @Sendable () throws -> T) async throws -> T {
|
||
let task = Task.detached(operation: operation)
|
||
return try await withTaskCancellationHandler { try await task.value } onCancel: { task.cancel() }
|
||
}
|
||
static let nativeMIMEs: Set<String> = ["image/png", "image/jpeg", "image/gif", "image/webp"]
|
||
static func filename(_ value: String) throws -> String {
|
||
let value = value.precomposedStringWithCanonicalMapping
|
||
guard !value.isEmpty, value.utf8.count <= 255, value != ".", value != "..",
|
||
!value.contains("/"), !value.contains("\\"),
|
||
!value.unicodeScalars.contains(where: CharacterSet.controlCharacters.contains) else {
|
||
throw AgentAttachmentIssue.local("The filename must be a safe basename of 1–255 UTF-8 bytes.")
|
||
}
|
||
return value
|
||
}
|
||
static func mime(_ value: String) throws -> String {
|
||
let value = value.lowercased().trimmingCharacters(in: .whitespacesAndNewlines)
|
||
let parts = value.split(separator: "/", omittingEmptySubsequences: false)
|
||
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyz0123456789!#$%&'*+-.^_`|~")
|
||
guard parts.count == 2, parts.allSatisfy({ !$0.isEmpty && $0.unicodeScalars.allSatisfy(allowed.contains) }), value.utf8.count <= 255 else {
|
||
throw AgentAttachmentIssue.local("The selected file has an invalid media type.")
|
||
}
|
||
return value
|
||
}
|
||
static func file(_ url: URL, limits: AgentAttachmentCapabilities) throws -> AgentPreparedAttachment {
|
||
let accessed = url.startAccessingSecurityScopedResource()
|
||
defer { if accessed { url.stopAccessingSecurityScopedResource() } }
|
||
let name = try filename(url.lastPathComponent)
|
||
let type = (try? url.resourceValues(forKeys: [.contentTypeKey]))?.contentType ?? UTType(filenameExtension: url.pathExtension)
|
||
let media = try mime(type?.preferredMIMEType ?? "application/octet-stream")
|
||
let handle = try FileHandle(forReadingFrom: url)
|
||
defer { try? handle.close() }
|
||
var data = Data()
|
||
while let chunk = try handle.read(upToCount: 64 * 1024), !chunk.isEmpty {
|
||
try Task.checkCancellation()
|
||
guard data.count + chunk.count <= (nativeMIMEs.contains(media) ? limits.imageLimit : limits.fileLimit) else {
|
||
throw AgentAttachmentIssue.local("\(name) exceeds the file size limit.", code: "attachment_limit_exceeded")
|
||
}
|
||
data.append(chunk)
|
||
}
|
||
return try prepare(data, filename: name, mimeType: media, limits: limits)
|
||
}
|
||
static func prepare(_ data: Data, filename name: String, mimeType: String, limits: AgentAttachmentCapabilities) throws -> AgentPreparedAttachment {
|
||
guard limits.supported else { throw AgentAttachmentIssue.unavailable }
|
||
let name = try filename(name), media = try mime(mimeType)
|
||
guard data.count <= (nativeMIMEs.contains(media) ? limits.imageLimit : limits.fileLimit) else { throw AgentAttachmentIssue.local("\(name) exceeds the file size limit.", code: "attachment_limit_exceeded") }
|
||
if nativeMIMEs.contains(media) {
|
||
guard limits.imageMimeTypes.contains(media), data.count <= limits.imageLimit else {
|
||
throw AgentAttachmentIssue.local("\(name) exceeds the image limit or uses an unavailable image format.", code: "attachment_limit_exceeded")
|
||
}
|
||
_ = try raster(data, mime: media, pixels: limits.pixelLimit, dimension: limits.dimensionLimit)
|
||
}
|
||
return AgentPreparedAttachment(filename: name, mimeType: media, data: data)
|
||
}
|
||
static func raster(_ data: Data, mime: String, pixels: Int = 50_000_000, dimension: Int = 16_384) throws -> CGImage {
|
||
guard nativeMIMEs.contains(mime), let source = CGImageSourceCreateWithData(data as CFData, nil),
|
||
let type = CGImageSourceGetType(source), UTType(type as String)?.preferredMIMEType == mime,
|
||
CGImageSourceGetStatus(source) == .statusComplete,
|
||
let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any],
|
||
let width = properties[kCGImagePropertyPixelWidth] as? Int, let height = properties[kCGImagePropertyPixelHeight] as? Int,
|
||
width > 0, height > 0, width <= dimension, height <= dimension, width <= pixels / height,
|
||
let image = CGImageSourceCreateThumbnailAtIndex(source, 0, [kCGImageSourceCreateThumbnailFromImageAlways: true,
|
||
kCGImageSourceThumbnailMaxPixelSize: 600, kCGImageSourceCreateThumbnailWithTransform: true] as CFDictionary) else {
|
||
throw AgentAttachmentIssue.local("The image is invalid, has a mismatched media type, or exceeds the dimension/pixel limit.", code: "invalid_image")
|
||
}
|
||
return image
|
||
}
|
||
static func photo(_ data: Data, filename originalName: String?, mimeType: String?, limits: AgentAttachmentCapabilities) throws -> AgentPreparedAttachment {
|
||
let media = mimeType ?? "application/octet-stream"
|
||
if nativeMIMEs.contains(media) {
|
||
return try prepare(data, filename: originalName ?? "photo.\(UTType(mimeType: media)?.preferredFilenameExtension ?? "jpg")", mimeType: media, limits: limits)
|
||
}
|
||
// Photo Library is an explicit visual-input selection. Convert HEIC/HEIF with bounded decoding.
|
||
guard let source = CGImageSourceCreateWithData(data as CFData, nil), let sourceType = CGImageSourceGetType(source),
|
||
["public.heic", "public.heif"].contains(sourceType as String), limits.imageMimeTypes.contains("image/jpeg"),
|
||
let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any],
|
||
let width = properties[kCGImagePropertyPixelWidth] as? Int, let height = properties[kCGImagePropertyPixelHeight] as? Int,
|
||
width > 0, height > 0, width <= limits.dimensionLimit, height <= limits.dimensionLimit, width <= limits.pixelLimit / height,
|
||
let image = CGImageSourceCreateThumbnailAtIndex(source, 0, [kCGImageSourceCreateThumbnailFromImageAlways: true,
|
||
kCGImageSourceThumbnailMaxPixelSize: min(2000, limits.dimensionLimit), kCGImageSourceCreateThumbnailWithTransform: true] as CFDictionary),
|
||
let jpeg = UIImage(cgImage: image).jpegData(compressionQuality: 0.88) else {
|
||
throw AgentAttachmentIssue.local("This photo could not be prepared as a supported image. Select its original in Files to upload it as a file.", code: "invalid_image")
|
||
}
|
||
let safe = try filename(originalName ?? "photo.heic")
|
||
let stem = (safe as NSString).deletingPathExtension
|
||
return try prepare(jpeg, filename: stem + ".jpg", mimeType: "image/jpeg", limits: limits)
|
||
}
|
||
}
|
||
|
||
struct AgentMultipart: Sendable {
|
||
let boundary: String
|
||
let body: Data
|
||
init(file: AgentPreparedAttachment, uploadID: String) throws {
|
||
guard UUID(uuidString: uploadID) != nil else { throw AgentAttachmentIssue.local("Invalid upload identity.") }
|
||
let name = try AgentAttachmentPreparation.filename(file.filename)
|
||
let media = try AgentAttachmentPreparation.mime(file.mimeType)
|
||
var boundary: String
|
||
repeat { boundary = "Sybil-" + UUID().uuidString } while file.data.range(of: Data(boundary.utf8)) != nil
|
||
self.boundary = boundary
|
||
let quoted = name.replacingOccurrences(of: "\"", with: "\\\"")
|
||
var bytes = Data("--\(boundary)\r\nContent-Disposition: form-data; name=\"uploadId\"\r\n\r\n\(uploadID)\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"file\"; filename=\"\(quoted)\"\r\nContent-Type: \(media)\r\n\r\n".utf8)
|
||
bytes.append(file.data)
|
||
bytes.append(Data("\r\n--\(boundary)--\r\n".utf8))
|
||
guard bytes.count <= 8 * 1024 * 1024 + 64 * 1024 else { throw AgentAttachmentIssue.local("The upload exceeds the request size limit.", code: "attachment_limit_exceeded") }
|
||
body = bytes
|
||
}
|
||
}
|