Sources/FaceMaker/FaceInversion.swift

Swift source for Making faces, 482 lines. Raw file

import Accelerate
import CoreImage
import Foundation
import Vision
import simd

/// How a dropped image is matched to a latent. `quick` is the instant
/// nearest-neighbour; `search` adds the live latent refinement. Encoder-based
/// inversion (e4e/pSp/ReStyle) will be added here later.
enum FindStrategy: String, CaseIterable, Identifiable, Sendable {
    case quick
    case search
    case encoder
    var id: String { rawValue }
    var name: String {
        switch self {
        case .quick: "Quick"
        case .search: "Search"
        case .encoder: "Encoder"
        }
    }
}

/// Turns a dropped face photo into one of a model's latents, fully offline:
/// aligns the face (Vision), embeds it (ArcFace `w600k_r50`), and returns the
/// z of the closest face in a per-model bank of generated samples. The bank is
/// built once per model and cached to Application Support.
actor FaceInversion {
    static let shared = FaceInversion()

    private let ciContext = CIContext(options: [.cacheIntermediates: false])
    private var recognizer: ORTSession?
    private var e4eEncoder: ORTSession?

    /// In-memory bank cache: model id → [(z, arcface embedding)].
    private var banks: [String: [(z: [Float], emb: [Float])]] = [:]
    /// Alignment points for a model's generated faces (all framed alike), so
    /// the bank build runs Vision once, not once per sample.
    private var alignPoints: [String: [SIMD2<Float>]] = [:]

    private static let arcTemplate: [SIMD2<Float>] = [
        SIMD2(0.34191607, 1 - 0.46157411),
        SIMD2(0.65653393, 1 - 0.45983393),
        SIMD2(0.50022500, 1 - 0.64050536),
        SIMD2(0.37097589, 1 - 0.82469196),
        SIMD2(0.63151696, 1 - 0.82325089),
    ]
    private static let embedSize = 112

    /// FaceFusion's `ffhq_512` 5-point template (lower-left origin), used to
    /// align a dropped face for the e4e encoder.
    private static let ffhqTemplate: [SIMD2<Float>] = [
        SIMD2(0.37691676, 1 - 0.46864664),
        SIMD2(0.62285697, 1 - 0.46912813),
        SIMD2(0.50123859, 1 - 0.61331904),
        SIMD2(0.39308822, 1 - 0.72541100),
        SIMD2(0.61150205, 1 - 0.72490465),
    ]
    private static let encoderSize = 256

    enum InversionError: LocalizedError {
        case noFace
        var errorDescription: String? {
            switch self {
            case .noFace: "No face was found in the dropped image."
            }
        }
    }

    /// The L2-normalized ArcFace embedding of the dropped face, or nil if
    /// Vision finds no face.
    func embedDropped(_ image: CIImage, directory: URL) throws -> [Float]? {
        try detectAndEmbed(image, directory: directory)?.emb
    }

    /// Encode a dropped face to a W+ latent [1,18,512] (9216 floats) via the
    /// e4e encoder — the high-fidelity inversion. Nil if no face is found.
    func encodeWPlus(_ image: CIImage, directory: URL) throws -> [Float]? {
        guard let points = Self.fivePoints(in: image) else { return nil }
        guard let warp = Self.similarityTransform(
            from: points, template: Self.ffhqTemplate,
            cropSize: CGFloat(Self.encoderSize)) else { return nil }
        let rgba = renderCrop(image: image, transform: warp, size: Self.encoderSize)
        // e4e: (rgb/255 − 0.5) / 0.5  →  [−1, 1], NCHW.
        let blob = Self.nchwBlob(
            rgba: rgba, size: Self.encoderSize,
            mean: SIMD3(repeating: 0.5), std: SIMD3(repeating: 0.5))
        let session = try encoderSession(directory: directory)
        let output = try session.run([
            session.inputNames[0]: ORTTensor(
                shape: [1, 3, Int64(Self.encoderSize), Int64(Self.encoderSize)], data: blob)
        ])
        return output.first?.data
    }

    private func encoderSession(directory: URL) throws -> ORTSession {
        if let e4eEncoder { return e4eEncoder }
        let url = directory.appendingPathComponent("e4e-ffhq-encoder.onnx")
        guard FileManager.default.fileExists(atPath: url.path) else {
            throw FaceLocalError.modelsMissing
        }
        let session = try ORTSession(modelPath: url.path, useCoreML: false)
        e4eEncoder = session
        return session
    }

    /// The z of the bank face closest (cosine) to `query`. Builds/extends the
    /// bank to `target` if needed.
    func nearest(
        to query: [Float],
        model: LocalFaceModel,
        directory: URL,
        target: Int,
        progress: @Sendable @escaping (Int, Int) -> Void
    ) async throws -> [Float] {
        var bank = banks[model.id] ?? loadBank(model, directory: directory)
        if bank.count < target {
            try await buildBank(
                model: model, directory: directory, to: target,
                bank: &bank, progress: progress)
            banks[model.id] = bank
        } else {
            banks[model.id] = bank
        }

        var bestZ = bank.first?.z ?? FaceLocalEngine.gaussianLatent(seed: 1)
        var bestSim = -Float.greatestFiniteMagnitude
        for entry in bank {
            let sim = Self.cosine(query, entry.emb)
            if sim > bestSim { bestSim = sim; bestZ = entry.z }
        }
        return bestZ
    }

    /// Convenience used by tests: embed the image, then nearest.
    func nearestLatent(
        to image: CIImage, model: LocalFaceModel, directory: URL,
        target: Int, progress: @Sendable @escaping (Int, Int) -> Void
    ) async throws -> [Float] {
        guard let query = try embedDropped(image, directory: directory) else {
            throw InversionError.noFace
        }
        return try await nearest(
            to: query, model: model, directory: directory, target: target, progress: progress)
    }

    /// Gradient-free refinement: starting from `start`, sample small latent
    /// perturbations, generate + embed each, and keep any that raise the cosine
    /// similarity to `query` — a (1+λ) evolution strategy with adaptive step.
    /// Drives the face much closer to the dropped identity than the bank alone.
    /// Reports (iteration, total, bestSimilarity). Returns the best z found.
    func refine(
        query: [Float], start: [Float],
        model: LocalFaceModel, directory: URL,
        iterations: Int, lambda: Int,
        progress: @Sendable @escaping (Int, Int, Float, [Float]) -> Void
    ) async throws -> [Float] {
        var best = start
        var bestSim = try await embedGenerated(best, model: model, directory: directory)
            .map { Self.cosine($0, query) } ?? -1
        var sigma: Float = 0.35

        for iteration in 0..<iterations {
            if Task.isCancelled { break }
            var improved = false
            for k in 0..<lambda {
                if Task.isCancelled { break }
                let seed = UInt64(iteration * lambda + k + 1) &* 0x9E37_79B9_7F4A_7C15
                let noise = FaceLocalEngine.gaussianLatent(seed: seed, count: best.count)
                var candidate = best
                for i in 0..<min(candidate.count, noise.count) {
                    candidate[i] += sigma * noise[i]
                }
                guard let emb = try await embedGenerated(
                    candidate, model: model, directory: directory) else { continue }
                let sim = Self.cosine(emb, query)
                if sim > bestSim { bestSim = sim; best = candidate; improved = true }
            }
            // Grow the step while we're making progress, shrink when stuck.
            sigma = improved ? min(sigma * 1.2, 0.8) : max(sigma * 0.7, 0.05)
            progress(iteration + 1, iterations, bestSim, best)
        }
        return best
    }

    /// Cosine similarity between `query` and the face `z` generates. Used by
    /// tests to check that refinement raises the match.
    func similarity(
        of z: [Float], to query: [Float], model: LocalFaceModel, directory: URL
    ) async throws -> Float? {
        guard let emb = try await embedGenerated(z, model: model, directory: directory)
        else { return nil }
        return Self.cosine(emb, query)
    }

    private func embedGenerated(
        _ z: [Float], model: LocalFaceModel, directory: URL
    ) async throws -> [Float]? {
        let png = try await FaceLocalEngine.shared.png(
            model: model, directory: directory, latent: z)
        guard let face = CIImage(data: png) else { return nil }
        return try detectAndEmbed(face, directory: directory)?.emb
    }

    private static func cosine(_ a: [Float], _ b: [Float]) -> Float {
        var sim: Float = 0
        vDSP_dotpr(a, 1, b, 1, &sim, vDSP_Length(min(a.count, b.count)))
        return sim
    }

    // MARK: - Bank

    private func buildBank(
        model: LocalFaceModel,
        directory: URL,
        to target: Int,
        bank: inout [(z: [Float], emb: [Float])],
        progress: @Sendable @escaping (Int, Int) -> Void
    ) async throws {
        // Deterministic seeds so a cached bank is reproducible; a safety cap
        // above `target` covers the rare generated face Vision can't read.
        var points = alignPoints[model.id]
        var index = bank.count
        let maxAttempts = target * 2
        while bank.count < target && index < maxAttempts {
            if Task.isCancelled { return }
            let z = FaceLocalEngine.gaussianLatent(seed: UInt64(index + 1))
            index += 1
            let png = try await FaceLocalEngine.shared.png(
                model: model, directory: directory, latent: z)
            guard let face = CIImage(data: png) else { continue }

            if let points {
                // Fast path: reuse the model's alignment, skip Vision.
                guard let emb = try embedding(for: face, points: points, directory: directory)
                else { continue }
                bank.append((z, emb))
            } else {
                // Calibrate alignment once, from the first readable face.
                guard let detected = try detectAndEmbed(face, directory: directory)
                else { continue }
                points = detected.points
                alignPoints[model.id] = detected.points
                bank.append((z, detected.emb))
            }
            progress(bank.count, target)
        }
    }

    private func bankFileName(_ model: LocalFaceModel) -> String {
        "bank-\(model.id.replacingOccurrences(of: "/", with: "_")).bin"
    }

    /// Where runtime-built banks are cached (Application Support).
    private func cacheBankURL(_ model: LocalFaceModel) -> URL? {
        let fm = FileManager.default
        guard let base = fm.urls(for: .applicationSupportDirectory, in: .userDomainMask).first
        else { return nil }
        let dir = base.appendingPathComponent("FaceMaker", isDirectory: true)
        try? fm.createDirectory(at: dir, withIntermediateDirectories: true)
        return dir.appendingPathComponent(bankFileName(model))
    }

    /// Prefer a bank shipped in the app bundle (instant, precomputed), then
    /// fall back to a runtime-built cache.
    func loadBank(_ model: LocalFaceModel, directory: URL) -> [(z: [Float], emb: [Float])] {
        let bundled = directory.appendingPathComponent(bankFileName(model))
        if let bank = load(from: bundled), !bank.isEmpty { return bank }
        if let url = cacheBankURL(model), let bank = load(from: url) { return bank }
        return []
    }

    /// Binary layout: Int32 count, then per entry 512 float z + 512 float emb.
    private func load(from url: URL) -> [(z: [Float], emb: [Float])]? {
        guard let data = try? Data(contentsOf: url), data.count >= 4 else { return nil }
        return data.withUnsafeBytes { raw -> [(z: [Float], emb: [Float])] in
            let count = Int(raw.load(fromByteOffset: 0, as: Int32.self))
            let floats = raw.baseAddress!.advanced(by: 4)
                .assumingMemoryBound(to: Float.self)
            let stride = 1024  // 512 z + 512 emb
            guard 4 + count * stride * 4 <= data.count else { return [] }
            var result: [(z: [Float], emb: [Float])] = []
            result.reserveCapacity(count)
            for i in 0..<count {
                let base = i * stride
                let z = Array(UnsafeBufferPointer(start: floats + base, count: 512))
                let emb = Array(UnsafeBufferPointer(start: floats + base + 512, count: 512))
                result.append((z, emb))
            }
            return result
        }
    }

    private func save(_ bank: [(z: [Float], emb: [Float])], to url: URL) {
        var data = Data()
        var count = Int32(bank.count)
        withUnsafeBytes(of: &count) { data.append(contentsOf: $0) }
        for entry in bank {
            entry.z.withUnsafeBytes { data.append(contentsOf: $0) }
            entry.emb.withUnsafeBytes { data.append(contentsOf: $0) }
        }
        try? data.write(to: url)
    }

    /// Build a bank from scratch and write it to `url` — used offline to
    /// generate the precomputed banks that ship in the app bundle.
    func exportBank(
        model: LocalFaceModel, directory: URL, target: Int, to url: URL,
        progress: @Sendable @escaping (Int, Int) -> Void
    ) async throws {
        var bank: [(z: [Float], emb: [Float])] = []
        try await buildBank(
            model: model, directory: directory, to: target,
            bank: &bank, progress: progress)
        save(bank, to: url)
    }

    // MARK: - ArcFace embedding

    /// Detect the face and embed it — used for the dropped photo and to
    /// calibrate a model's bank alignment once. Nil if Vision finds no face.
    private func detectAndEmbed(
        _ image: CIImage, directory: URL
    ) throws -> (points: [SIMD2<Float>], emb: [Float])? {
        guard let points = Self.fivePoints(in: image) else { return nil }
        guard let emb = try embedding(for: image, points: points, directory: directory)
        else { return nil }
        return (points, emb)
    }

    /// L2-normalized ArcFace embedding using pre-supplied alignment points.
    private func embedding(
        for image: CIImage, points: [SIMD2<Float>], directory: URL
    ) throws -> [Float]? {
        guard let warp = Self.similarityTransform(
            from: points, template: Self.arcTemplate, cropSize: CGFloat(Self.embedSize))
        else { return nil }

        let rgba = renderCrop(image: image, transform: warp, size: Self.embedSize)
        let blob = Self.nchwBlob(
            rgba: rgba, size: Self.embedSize,
            mean: SIMD3(repeating: 0.5), std: SIMD3(repeating: 0.5))

        let session = try recognizerSession(directory: directory)
        let output = try session.run([
            session.inputNames[0]: ORTTensor(
                shape: [1, 3, Int64(Self.embedSize), Int64(Self.embedSize)], data: blob)
        ])
        guard let raw = output.first?.data else { return nil }
        return Self.normalized(raw)
    }

    private func recognizerSession(directory: URL) throws -> ORTSession {
        if let recognizer { return recognizer }
        let url = directory.appendingPathComponent("w600k_r50.onnx")
        guard FileManager.default.fileExists(atPath: url.path) else {
            throw FaceLocalError.modelsMissing
        }
        // ArcFace on the Neural Engine — far faster than CPU for the 174 MB
        // r50, and the bank build embeds hundreds of faces.
        let session = try ORTSession(modelPath: url.path, useCoreML: true)
        recognizer = session
        return session
    }

    private func renderCrop(image: CIImage, transform: CGAffineTransform, size: Int) -> [UInt8] {
        var buffer = [UInt8](repeating: 0, count: size * size * 4)
        let bounds = CGRect(x: 0, y: 0, width: size, height: size)
        ciContext.render(
            image.transformed(by: transform).cropped(to: bounds),
            toBitmap: &buffer, rowBytes: size * 4, bounds: bounds,
            format: .RGBA8, colorSpace: CGColorSpace(name: CGColorSpace.sRGB))
        return buffer
    }

    // MARK: - Vision landmarks → 5 points

    /// The InsightFace 5 points (left eye, right eye, nose, left mouth, right
    /// mouth) in Core Image coordinates (origin lower-left).
    private static func fivePoints(in image: CIImage) -> [SIMD2<Float>]? {
        let handler = VNImageRequestHandler(ciImage: image, options: [:])
        let request = VNDetectFaceLandmarksRequest()
        try? handler.perform([request])
        guard let face = (request.results)?
            .max(by: { $0.boundingBox.width * $0.boundingBox.height
                     < $1.boundingBox.width * $1.boundingBox.height }),
            let landmarks = face.landmarks,
            let leftEye = landmarks.leftEye,
            let rightEye = landmarks.rightEye,
            let lips = landmarks.outerLips
        else { return nil }

        let size = image.extent.size
        func imgPoints(_ region: VNFaceLandmarkRegion2D) -> [CGPoint] {
            region.pointsInImage(imageSize: size)
        }
        func centroid(_ points: [CGPoint]) -> CGPoint {
            guard !points.isEmpty else { return .zero }
            let sum = points.reduce(CGPoint.zero) { CGPoint(x: $0.x + $1.x, y: $0.y + $1.y) }
            return CGPoint(x: sum.x / CGFloat(points.count), y: sum.y / CGFloat(points.count))
        }

        let eyeA = centroid(imgPoints(leftEye))
        let eyeB = centroid(imgPoints(rightEye))
        let (eyeLeft, eyeRight) = eyeA.x <= eyeB.x ? (eyeA, eyeB) : (eyeB, eyeA)

        let mouth = imgPoints(lips)
        guard let mouthLeft = mouth.min(by: { $0.x < $1.x }),
              let mouthRight = mouth.max(by: { $0.x < $1.x }) else { return nil }

        let eyeMid = CGPoint(x: (eyeLeft.x + eyeRight.x) / 2, y: (eyeLeft.y + eyeRight.y) / 2)
        let mouthMid = CGPoint(x: (mouthLeft.x + mouthRight.x) / 2, y: (mouthLeft.y + mouthRight.y) / 2)
        let axis = CGPoint(x: mouthMid.x - eyeMid.x, y: mouthMid.y - eyeMid.y)
        let crest = (landmarks.noseCrest ?? landmarks.nose).map(imgPoints) ?? []
        guard let noseTip = crest.max(by: {
            ($0.x * axis.x + $0.y * axis.y) < ($1.x * axis.x + $1.y * axis.y)
        }) else { return nil }

        return [eyeLeft, eyeRight, noseTip, mouthLeft, mouthRight].map {
            SIMD2(Float($0.x), Float($0.y))
        }
    }

    /// Least-squares similarity transform mapping `points` onto
    /// `template × cropSize` (rotation + uniform scale + translation).
    private static func similarityTransform(
        from points: [SIMD2<Float>], template: [SIMD2<Float>], cropSize: CGFloat
    ) -> CGAffineTransform? {
        guard points.count == template.count, points.count >= 2 else { return nil }
        let targets = template.map { CGPoint(x: CGFloat($0.x) * cropSize, y: CGFloat($0.y) * cropSize) }
        let src = points.map { CGPoint(x: CGFloat($0.x), y: CGFloat($0.y)) }

        func centroid(_ p: [CGPoint]) -> CGPoint {
            let s = p.reduce(CGPoint.zero) { CGPoint(x: $0.x + $1.x, y: $0.y + $1.y) }
            return CGPoint(x: s.x / CGFloat(p.count), y: s.y / CGFloat(p.count))
        }
        let zMean = centroid(src), wMean = centroid(targets)
        var numRe: CGFloat = 0, numIm: CGFloat = 0, denom: CGFloat = 0
        for (z, w) in zip(src, targets) {
            let zx = z.x - zMean.x, zy = z.y - zMean.y
            let wx = w.x - wMean.x, wy = w.y - wMean.y
            numRe += zx * wx + zy * wy
            numIm += zx * wy - zy * wx
            denom += zx * zx + zy * zy
        }
        guard denom > 1e-6 else { return nil }
        let aRe = numRe / denom, aIm = numIm / denom
        let tx = wMean.x - (aRe * zMean.x - aIm * zMean.y)
        let ty = wMean.y - (aIm * zMean.x + aRe * zMean.y)
        return CGAffineTransform(a: aRe, b: aIm, c: -aIm, d: aRe, tx: tx, ty: ty)
    }

    // MARK: - Blob helpers

    private static func nchwBlob(
        rgba: [UInt8], size: Int, mean: SIMD3<Float>, std: SIMD3<Float>
    ) -> [Float] {
        let pixels = size * size
        var blob = [Float](repeating: 0, count: 3 * pixels)
        rgba.withUnsafeBufferPointer { source in
            blob.withUnsafeMutableBufferPointer { dest in
                for channel in 0..<3 {
                    let plane = dest.baseAddress! + channel * pixels
                    vDSP_vfltu8(source.baseAddress! + channel, 4, plane, 1, vDSP_Length(pixels))
                    var scale = 1 / (255 * std[channel])
                    var bias = -mean[channel] / std[channel]
                    vDSP_vsmsa(plane, 1, &scale, &bias, plane, 1, vDSP_Length(pixels))
                }
            }
        }
        return blob
    }

    private static func normalized(_ vector: [Float]) -> [Float] {
        var result = vector
        var norm: Float = 0
        vDSP_svesq(result, 1, &norm, vDSP_Length(result.count))
        norm = max(sqrt(norm), 1e-6)
        vDSP_vsdiv(result, 1, &norm, &result, 1, vDSP_Length(result.count))
        return result
    }
}