import Accelerate import CoreImage import Foundation import Vision import simd /// How a dropped image is matched to a latent. `quick` is the instant /// nearest-neighbour; `search` adds the live latent refinement. Encoder-based /// inversion (e4e/pSp/ReStyle) will be added here later. enum FindStrategy: String, CaseIterable, Identifiable, Sendable { case quick case search case encoder var id: String { rawValue } var name: String { switch self { case .quick: "Quick" case .search: "Search" case .encoder: "Encoder" } } } /// Turns a dropped face photo into one of a model's latents, fully offline: /// aligns the face (Vision), embeds it (ArcFace `w600k_r50`), and returns the /// z of the closest face in a per-model bank of generated samples. The bank is /// built once per model and cached to Application Support. actor FaceInversion { static let shared = FaceInversion() private let ciContext = CIContext(options: [.cacheIntermediates: false]) private var recognizer: ORTSession? private var e4eEncoder: ORTSession? /// In-memory bank cache: model id → [(z, arcface embedding)]. private var banks: [String: [(z: [Float], emb: [Float])]] = [:] /// Alignment points for a model's generated faces (all framed alike), so /// the bank build runs Vision once, not once per sample. private var alignPoints: [String: [SIMD2]] = [:] private static let arcTemplate: [SIMD2] = [ SIMD2(0.34191607, 1 - 0.46157411), SIMD2(0.65653393, 1 - 0.45983393), SIMD2(0.50022500, 1 - 0.64050536), SIMD2(0.37097589, 1 - 0.82469196), SIMD2(0.63151696, 1 - 0.82325089), ] private static let embedSize = 112 /// FaceFusion's `ffhq_512` 5-point template (lower-left origin), used to /// align a dropped face for the e4e encoder. private static let ffhqTemplate: [SIMD2] = [ SIMD2(0.37691676, 1 - 0.46864664), SIMD2(0.62285697, 1 - 0.46912813), SIMD2(0.50123859, 1 - 0.61331904), SIMD2(0.39308822, 1 - 0.72541100), SIMD2(0.61150205, 1 - 0.72490465), ] private static let encoderSize = 256 enum InversionError: LocalizedError { case noFace var errorDescription: String? { switch self { case .noFace: "No face was found in the dropped image." } } } /// The L2-normalized ArcFace embedding of the dropped face, or nil if /// Vision finds no face. func embedDropped(_ image: CIImage, directory: URL) throws -> [Float]? { try detectAndEmbed(image, directory: directory)?.emb } /// Encode a dropped face to a W+ latent [1,18,512] (9216 floats) via the /// e4e encoder — the high-fidelity inversion. Nil if no face is found. func encodeWPlus(_ image: CIImage, directory: URL) throws -> [Float]? { guard let points = Self.fivePoints(in: image) else { return nil } guard let warp = Self.similarityTransform( from: points, template: Self.ffhqTemplate, cropSize: CGFloat(Self.encoderSize)) else { return nil } let rgba = renderCrop(image: image, transform: warp, size: Self.encoderSize) // e4e: (rgb/255 − 0.5) / 0.5 → [−1, 1], NCHW. let blob = Self.nchwBlob( rgba: rgba, size: Self.encoderSize, mean: SIMD3(repeating: 0.5), std: SIMD3(repeating: 0.5)) let session = try encoderSession(directory: directory) let output = try session.run([ session.inputNames[0]: ORTTensor( shape: [1, 3, Int64(Self.encoderSize), Int64(Self.encoderSize)], data: blob) ]) return output.first?.data } private func encoderSession(directory: URL) throws -> ORTSession { if let e4eEncoder { return e4eEncoder } let url = directory.appendingPathComponent("e4e-ffhq-encoder.onnx") guard FileManager.default.fileExists(atPath: url.path) else { throw FaceLocalError.modelsMissing } let session = try ORTSession(modelPath: url.path, useCoreML: false) e4eEncoder = session return session } /// The z of the bank face closest (cosine) to `query`. Builds/extends the /// bank to `target` if needed. func nearest( to query: [Float], model: LocalFaceModel, directory: URL, target: Int, progress: @Sendable @escaping (Int, Int) -> Void ) async throws -> [Float] { var bank = banks[model.id] ?? loadBank(model, directory: directory) if bank.count < target { try await buildBank( model: model, directory: directory, to: target, bank: &bank, progress: progress) banks[model.id] = bank } else { banks[model.id] = bank } var bestZ = bank.first?.z ?? FaceLocalEngine.gaussianLatent(seed: 1) var bestSim = -Float.greatestFiniteMagnitude for entry in bank { let sim = Self.cosine(query, entry.emb) if sim > bestSim { bestSim = sim; bestZ = entry.z } } return bestZ } /// Convenience used by tests: embed the image, then nearest. func nearestLatent( to image: CIImage, model: LocalFaceModel, directory: URL, target: Int, progress: @Sendable @escaping (Int, Int) -> Void ) async throws -> [Float] { guard let query = try embedDropped(image, directory: directory) else { throw InversionError.noFace } return try await nearest( to: query, model: model, directory: directory, target: target, progress: progress) } /// Gradient-free refinement: starting from `start`, sample small latent /// perturbations, generate + embed each, and keep any that raise the cosine /// similarity to `query` — a (1+λ) evolution strategy with adaptive step. /// Drives the face much closer to the dropped identity than the bank alone. /// Reports (iteration, total, bestSimilarity). Returns the best z found. func refine( query: [Float], start: [Float], model: LocalFaceModel, directory: URL, iterations: Int, lambda: Int, progress: @Sendable @escaping (Int, Int, Float, [Float]) -> Void ) async throws -> [Float] { var best = start var bestSim = try await embedGenerated(best, model: model, directory: directory) .map { Self.cosine($0, query) } ?? -1 var sigma: Float = 0.35 for iteration in 0.. bestSim { bestSim = sim; best = candidate; improved = true } } // Grow the step while we're making progress, shrink when stuck. sigma = improved ? min(sigma * 1.2, 0.8) : max(sigma * 0.7, 0.05) progress(iteration + 1, iterations, bestSim, best) } return best } /// Cosine similarity between `query` and the face `z` generates. Used by /// tests to check that refinement raises the match. func similarity( of z: [Float], to query: [Float], model: LocalFaceModel, directory: URL ) async throws -> Float? { guard let emb = try await embedGenerated(z, model: model, directory: directory) else { return nil } return Self.cosine(emb, query) } private func embedGenerated( _ z: [Float], model: LocalFaceModel, directory: URL ) async throws -> [Float]? { let png = try await FaceLocalEngine.shared.png( model: model, directory: directory, latent: z) guard let face = CIImage(data: png) else { return nil } return try detectAndEmbed(face, directory: directory)?.emb } private static func cosine(_ a: [Float], _ b: [Float]) -> Float { var sim: Float = 0 vDSP_dotpr(a, 1, b, 1, &sim, vDSP_Length(min(a.count, b.count))) return sim } // MARK: - Bank private func buildBank( model: LocalFaceModel, directory: URL, to target: Int, bank: inout [(z: [Float], emb: [Float])], progress: @Sendable @escaping (Int, Int) -> Void ) async throws { // Deterministic seeds so a cached bank is reproducible; a safety cap // above `target` covers the rare generated face Vision can't read. var points = alignPoints[model.id] var index = bank.count let maxAttempts = target * 2 while bank.count < target && index < maxAttempts { if Task.isCancelled { return } let z = FaceLocalEngine.gaussianLatent(seed: UInt64(index + 1)) index += 1 let png = try await FaceLocalEngine.shared.png( model: model, directory: directory, latent: z) guard let face = CIImage(data: png) else { continue } if let points { // Fast path: reuse the model's alignment, skip Vision. guard let emb = try embedding(for: face, points: points, directory: directory) else { continue } bank.append((z, emb)) } else { // Calibrate alignment once, from the first readable face. guard let detected = try detectAndEmbed(face, directory: directory) else { continue } points = detected.points alignPoints[model.id] = detected.points bank.append((z, detected.emb)) } progress(bank.count, target) } } private func bankFileName(_ model: LocalFaceModel) -> String { "bank-\(model.id.replacingOccurrences(of: "/", with: "_")).bin" } /// Where runtime-built banks are cached (Application Support). private func cacheBankURL(_ model: LocalFaceModel) -> URL? { let fm = FileManager.default guard let base = fm.urls(for: .applicationSupportDirectory, in: .userDomainMask).first else { return nil } let dir = base.appendingPathComponent("FaceMaker", isDirectory: true) try? fm.createDirectory(at: dir, withIntermediateDirectories: true) return dir.appendingPathComponent(bankFileName(model)) } /// Prefer a bank shipped in the app bundle (instant, precomputed), then /// fall back to a runtime-built cache. func loadBank(_ model: LocalFaceModel, directory: URL) -> [(z: [Float], emb: [Float])] { let bundled = directory.appendingPathComponent(bankFileName(model)) if let bank = load(from: bundled), !bank.isEmpty { return bank } if let url = cacheBankURL(model), let bank = load(from: url) { return bank } return [] } /// Binary layout: Int32 count, then per entry 512 float z + 512 float emb. private func load(from url: URL) -> [(z: [Float], emb: [Float])]? { guard let data = try? Data(contentsOf: url), data.count >= 4 else { return nil } return data.withUnsafeBytes { raw -> [(z: [Float], emb: [Float])] in let count = Int(raw.load(fromByteOffset: 0, as: Int32.self)) let floats = raw.baseAddress!.advanced(by: 4) .assumingMemoryBound(to: Float.self) let stride = 1024 // 512 z + 512 emb guard 4 + count * stride * 4 <= data.count else { return [] } var result: [(z: [Float], emb: [Float])] = [] result.reserveCapacity(count) for i in 0.. Void ) async throws { var bank: [(z: [Float], emb: [Float])] = [] try await buildBank( model: model, directory: directory, to: target, bank: &bank, progress: progress) save(bank, to: url) } // MARK: - ArcFace embedding /// Detect the face and embed it — used for the dropped photo and to /// calibrate a model's bank alignment once. Nil if Vision finds no face. private func detectAndEmbed( _ image: CIImage, directory: URL ) throws -> (points: [SIMD2], emb: [Float])? { guard let points = Self.fivePoints(in: image) else { return nil } guard let emb = try embedding(for: image, points: points, directory: directory) else { return nil } return (points, emb) } /// L2-normalized ArcFace embedding using pre-supplied alignment points. private func embedding( for image: CIImage, points: [SIMD2], directory: URL ) throws -> [Float]? { guard let warp = Self.similarityTransform( from: points, template: Self.arcTemplate, cropSize: CGFloat(Self.embedSize)) else { return nil } let rgba = renderCrop(image: image, transform: warp, size: Self.embedSize) let blob = Self.nchwBlob( rgba: rgba, size: Self.embedSize, mean: SIMD3(repeating: 0.5), std: SIMD3(repeating: 0.5)) let session = try recognizerSession(directory: directory) let output = try session.run([ session.inputNames[0]: ORTTensor( shape: [1, 3, Int64(Self.embedSize), Int64(Self.embedSize)], data: blob) ]) guard let raw = output.first?.data else { return nil } return Self.normalized(raw) } private func recognizerSession(directory: URL) throws -> ORTSession { if let recognizer { return recognizer } let url = directory.appendingPathComponent("w600k_r50.onnx") guard FileManager.default.fileExists(atPath: url.path) else { throw FaceLocalError.modelsMissing } // ArcFace on the Neural Engine — far faster than CPU for the 174 MB // r50, and the bank build embeds hundreds of faces. let session = try ORTSession(modelPath: url.path, useCoreML: true) recognizer = session return session } private func renderCrop(image: CIImage, transform: CGAffineTransform, size: Int) -> [UInt8] { var buffer = [UInt8](repeating: 0, count: size * size * 4) let bounds = CGRect(x: 0, y: 0, width: size, height: size) ciContext.render( image.transformed(by: transform).cropped(to: bounds), toBitmap: &buffer, rowBytes: size * 4, bounds: bounds, format: .RGBA8, colorSpace: CGColorSpace(name: CGColorSpace.sRGB)) return buffer } // MARK: - Vision landmarks → 5 points /// The InsightFace 5 points (left eye, right eye, nose, left mouth, right /// mouth) in Core Image coordinates (origin lower-left). private static func fivePoints(in image: CIImage) -> [SIMD2]? { let handler = VNImageRequestHandler(ciImage: image, options: [:]) let request = VNDetectFaceLandmarksRequest() try? handler.perform([request]) guard let face = (request.results)? .max(by: { $0.boundingBox.width * $0.boundingBox.height < $1.boundingBox.width * $1.boundingBox.height }), let landmarks = face.landmarks, let leftEye = landmarks.leftEye, let rightEye = landmarks.rightEye, let lips = landmarks.outerLips else { return nil } let size = image.extent.size func imgPoints(_ region: VNFaceLandmarkRegion2D) -> [CGPoint] { region.pointsInImage(imageSize: size) } func centroid(_ points: [CGPoint]) -> CGPoint { guard !points.isEmpty else { return .zero } let sum = points.reduce(CGPoint.zero) { CGPoint(x: $0.x + $1.x, y: $0.y + $1.y) } return CGPoint(x: sum.x / CGFloat(points.count), y: sum.y / CGFloat(points.count)) } let eyeA = centroid(imgPoints(leftEye)) let eyeB = centroid(imgPoints(rightEye)) let (eyeLeft, eyeRight) = eyeA.x <= eyeB.x ? (eyeA, eyeB) : (eyeB, eyeA) let mouth = imgPoints(lips) guard let mouthLeft = mouth.min(by: { $0.x < $1.x }), let mouthRight = mouth.max(by: { $0.x < $1.x }) else { return nil } let eyeMid = CGPoint(x: (eyeLeft.x + eyeRight.x) / 2, y: (eyeLeft.y + eyeRight.y) / 2) let mouthMid = CGPoint(x: (mouthLeft.x + mouthRight.x) / 2, y: (mouthLeft.y + mouthRight.y) / 2) let axis = CGPoint(x: mouthMid.x - eyeMid.x, y: mouthMid.y - eyeMid.y) let crest = (landmarks.noseCrest ?? landmarks.nose).map(imgPoints) ?? [] guard let noseTip = crest.max(by: { ($0.x * axis.x + $0.y * axis.y) < ($1.x * axis.x + $1.y * axis.y) }) else { return nil } return [eyeLeft, eyeRight, noseTip, mouthLeft, mouthRight].map { SIMD2(Float($0.x), Float($0.y)) } } /// Least-squares similarity transform mapping `points` onto /// `template × cropSize` (rotation + uniform scale + translation). private static func similarityTransform( from points: [SIMD2], template: [SIMD2], cropSize: CGFloat ) -> CGAffineTransform? { guard points.count == template.count, points.count >= 2 else { return nil } let targets = template.map { CGPoint(x: CGFloat($0.x) * cropSize, y: CGFloat($0.y) * cropSize) } let src = points.map { CGPoint(x: CGFloat($0.x), y: CGFloat($0.y)) } func centroid(_ p: [CGPoint]) -> CGPoint { let s = p.reduce(CGPoint.zero) { CGPoint(x: $0.x + $1.x, y: $0.y + $1.y) } return CGPoint(x: s.x / CGFloat(p.count), y: s.y / CGFloat(p.count)) } let zMean = centroid(src), wMean = centroid(targets) var numRe: CGFloat = 0, numIm: CGFloat = 0, denom: CGFloat = 0 for (z, w) in zip(src, targets) { let zx = z.x - zMean.x, zy = z.y - zMean.y let wx = w.x - wMean.x, wy = w.y - wMean.y numRe += zx * wx + zy * wy numIm += zx * wy - zy * wx denom += zx * zx + zy * zy } guard denom > 1e-6 else { return nil } let aRe = numRe / denom, aIm = numIm / denom let tx = wMean.x - (aRe * zMean.x - aIm * zMean.y) let ty = wMean.y - (aIm * zMean.x + aRe * zMean.y) return CGAffineTransform(a: aRe, b: aIm, c: -aIm, d: aRe, tx: tx, ty: ty) } // MARK: - Blob helpers private static func nchwBlob( rgba: [UInt8], size: Int, mean: SIMD3, std: SIMD3 ) -> [Float] { let pixels = size * size var blob = [Float](repeating: 0, count: 3 * pixels) rgba.withUnsafeBufferPointer { source in blob.withUnsafeMutableBufferPointer { dest in for channel in 0..<3 { let plane = dest.baseAddress! + channel * pixels vDSP_vfltu8(source.baseAddress! + channel, 4, plane, 1, vDSP_Length(pixels)) var scale = 1 / (255 * std[channel]) var bias = -mean[channel] / std[channel] vDSP_vsmsa(plane, 1, &scale, &bias, plane, 1, vDSP_Length(pixels)) } } } return blob } private static func normalized(_ vector: [Float]) -> [Float] { var result = vector var norm: Float = 0 vDSP_svesq(result, 1, &norm, vDSP_Length(result.count)) norm = max(sqrt(norm), 1e-6) vDSP_vsdiv(result, 1, &norm, &result, 1, vDSP_Length(result.count)) return result } }