Sources/FaceMaker/FaceInversion.swift
Swift source for Making faces, 482 lines. Raw file
import Accelerate
import CoreImage
import Foundation
import Vision
import simd
/// How a dropped image is matched to a latent. `quick` is the instant
/// nearest-neighbour; `search` adds the live latent refinement. Encoder-based
/// inversion (e4e/pSp/ReStyle) will be added here later.
enum FindStrategy: String, CaseIterable, Identifiable, Sendable {
case quick
case search
case encoder
var id: String { rawValue }
var name: String {
switch self {
case .quick: "Quick"
case .search: "Search"
case .encoder: "Encoder"
}
}
}
/// Turns a dropped face photo into one of a model's latents, fully offline:
/// aligns the face (Vision), embeds it (ArcFace `w600k_r50`), and returns the
/// z of the closest face in a per-model bank of generated samples. The bank is
/// built once per model and cached to Application Support.
actor FaceInversion {
static let shared = FaceInversion()
private let ciContext = CIContext(options: [.cacheIntermediates: false])
private var recognizer: ORTSession?
private var e4eEncoder: ORTSession?
/// In-memory bank cache: model id → [(z, arcface embedding)].
private var banks: [String: [(z: [Float], emb: [Float])]] = [:]
/// Alignment points for a model's generated faces (all framed alike), so
/// the bank build runs Vision once, not once per sample.
private var alignPoints: [String: [SIMD2<Float>]] = [:]
private static let arcTemplate: [SIMD2<Float>] = [
SIMD2(0.34191607, 1 - 0.46157411),
SIMD2(0.65653393, 1 - 0.45983393),
SIMD2(0.50022500, 1 - 0.64050536),
SIMD2(0.37097589, 1 - 0.82469196),
SIMD2(0.63151696, 1 - 0.82325089),
]
private static let embedSize = 112
/// FaceFusion's `ffhq_512` 5-point template (lower-left origin), used to
/// align a dropped face for the e4e encoder.
private static let ffhqTemplate: [SIMD2<Float>] = [
SIMD2(0.37691676, 1 - 0.46864664),
SIMD2(0.62285697, 1 - 0.46912813),
SIMD2(0.50123859, 1 - 0.61331904),
SIMD2(0.39308822, 1 - 0.72541100),
SIMD2(0.61150205, 1 - 0.72490465),
]
private static let encoderSize = 256
enum InversionError: LocalizedError {
case noFace
var errorDescription: String? {
switch self {
case .noFace: "No face was found in the dropped image."
}
}
}
/// The L2-normalized ArcFace embedding of the dropped face, or nil if
/// Vision finds no face.
func embedDropped(_ image: CIImage, directory: URL) throws -> [Float]? {
try detectAndEmbed(image, directory: directory)?.emb
}
/// Encode a dropped face to a W+ latent [1,18,512] (9216 floats) via the
/// e4e encoder — the high-fidelity inversion. Nil if no face is found.
func encodeWPlus(_ image: CIImage, directory: URL) throws -> [Float]? {
guard let points = Self.fivePoints(in: image) else { return nil }
guard let warp = Self.similarityTransform(
from: points, template: Self.ffhqTemplate,
cropSize: CGFloat(Self.encoderSize)) else { return nil }
let rgba = renderCrop(image: image, transform: warp, size: Self.encoderSize)
// e4e: (rgb/255 − 0.5) / 0.5 → [−1, 1], NCHW.
let blob = Self.nchwBlob(
rgba: rgba, size: Self.encoderSize,
mean: SIMD3(repeating: 0.5), std: SIMD3(repeating: 0.5))
let session = try encoderSession(directory: directory)
let output = try session.run([
session.inputNames[0]: ORTTensor(
shape: [1, 3, Int64(Self.encoderSize), Int64(Self.encoderSize)], data: blob)
])
return output.first?.data
}
private func encoderSession(directory: URL) throws -> ORTSession {
if let e4eEncoder { return e4eEncoder }
let url = directory.appendingPathComponent("e4e-ffhq-encoder.onnx")
guard FileManager.default.fileExists(atPath: url.path) else {
throw FaceLocalError.modelsMissing
}
let session = try ORTSession(modelPath: url.path, useCoreML: false)
e4eEncoder = session
return session
}
/// The z of the bank face closest (cosine) to `query`. Builds/extends the
/// bank to `target` if needed.
func nearest(
to query: [Float],
model: LocalFaceModel,
directory: URL,
target: Int,
progress: @Sendable @escaping (Int, Int) -> Void
) async throws -> [Float] {
var bank = banks[model.id] ?? loadBank(model, directory: directory)
if bank.count < target {
try await buildBank(
model: model, directory: directory, to: target,
bank: &bank, progress: progress)
banks[model.id] = bank
} else {
banks[model.id] = bank
}
var bestZ = bank.first?.z ?? FaceLocalEngine.gaussianLatent(seed: 1)
var bestSim = -Float.greatestFiniteMagnitude
for entry in bank {
let sim = Self.cosine(query, entry.emb)
if sim > bestSim { bestSim = sim; bestZ = entry.z }
}
return bestZ
}
/// Convenience used by tests: embed the image, then nearest.
func nearestLatent(
to image: CIImage, model: LocalFaceModel, directory: URL,
target: Int, progress: @Sendable @escaping (Int, Int) -> Void
) async throws -> [Float] {
guard let query = try embedDropped(image, directory: directory) else {
throw InversionError.noFace
}
return try await nearest(
to: query, model: model, directory: directory, target: target, progress: progress)
}
/// Gradient-free refinement: starting from `start`, sample small latent
/// perturbations, generate + embed each, and keep any that raise the cosine
/// similarity to `query` — a (1+λ) evolution strategy with adaptive step.
/// Drives the face much closer to the dropped identity than the bank alone.
/// Reports (iteration, total, bestSimilarity). Returns the best z found.
func refine(
query: [Float], start: [Float],
model: LocalFaceModel, directory: URL,
iterations: Int, lambda: Int,
progress: @Sendable @escaping (Int, Int, Float, [Float]) -> Void
) async throws -> [Float] {
var best = start
var bestSim = try await embedGenerated(best, model: model, directory: directory)
.map { Self.cosine($0, query) } ?? -1
var sigma: Float = 0.35
for iteration in 0..<iterations {
if Task.isCancelled { break }
var improved = false
for k in 0..<lambda {
if Task.isCancelled { break }
let seed = UInt64(iteration * lambda + k + 1) &* 0x9E37_79B9_7F4A_7C15
let noise = FaceLocalEngine.gaussianLatent(seed: seed, count: best.count)
var candidate = best
for i in 0..<min(candidate.count, noise.count) {
candidate[i] += sigma * noise[i]
}
guard let emb = try await embedGenerated(
candidate, model: model, directory: directory) else { continue }
let sim = Self.cosine(emb, query)
if sim > bestSim { bestSim = sim; best = candidate; improved = true }
}
// Grow the step while we're making progress, shrink when stuck.
sigma = improved ? min(sigma * 1.2, 0.8) : max(sigma * 0.7, 0.05)
progress(iteration + 1, iterations, bestSim, best)
}
return best
}
/// Cosine similarity between `query` and the face `z` generates. Used by
/// tests to check that refinement raises the match.
func similarity(
of z: [Float], to query: [Float], model: LocalFaceModel, directory: URL
) async throws -> Float? {
guard let emb = try await embedGenerated(z, model: model, directory: directory)
else { return nil }
return Self.cosine(emb, query)
}
private func embedGenerated(
_ z: [Float], model: LocalFaceModel, directory: URL
) async throws -> [Float]? {
let png = try await FaceLocalEngine.shared.png(
model: model, directory: directory, latent: z)
guard let face = CIImage(data: png) else { return nil }
return try detectAndEmbed(face, directory: directory)?.emb
}
private static func cosine(_ a: [Float], _ b: [Float]) -> Float {
var sim: Float = 0
vDSP_dotpr(a, 1, b, 1, &sim, vDSP_Length(min(a.count, b.count)))
return sim
}
// MARK: - Bank
private func buildBank(
model: LocalFaceModel,
directory: URL,
to target: Int,
bank: inout [(z: [Float], emb: [Float])],
progress: @Sendable @escaping (Int, Int) -> Void
) async throws {
// Deterministic seeds so a cached bank is reproducible; a safety cap
// above `target` covers the rare generated face Vision can't read.
var points = alignPoints[model.id]
var index = bank.count
let maxAttempts = target * 2
while bank.count < target && index < maxAttempts {
if Task.isCancelled { return }
let z = FaceLocalEngine.gaussianLatent(seed: UInt64(index + 1))
index += 1
let png = try await FaceLocalEngine.shared.png(
model: model, directory: directory, latent: z)
guard let face = CIImage(data: png) else { continue }
if let points {
// Fast path: reuse the model's alignment, skip Vision.
guard let emb = try embedding(for: face, points: points, directory: directory)
else { continue }
bank.append((z, emb))
} else {
// Calibrate alignment once, from the first readable face.
guard let detected = try detectAndEmbed(face, directory: directory)
else { continue }
points = detected.points
alignPoints[model.id] = detected.points
bank.append((z, detected.emb))
}
progress(bank.count, target)
}
}
private func bankFileName(_ model: LocalFaceModel) -> String {
"bank-\(model.id.replacingOccurrences(of: "/", with: "_")).bin"
}
/// Where runtime-built banks are cached (Application Support).
private func cacheBankURL(_ model: LocalFaceModel) -> URL? {
let fm = FileManager.default
guard let base = fm.urls(for: .applicationSupportDirectory, in: .userDomainMask).first
else { return nil }
let dir = base.appendingPathComponent("FaceMaker", isDirectory: true)
try? fm.createDirectory(at: dir, withIntermediateDirectories: true)
return dir.appendingPathComponent(bankFileName(model))
}
/// Prefer a bank shipped in the app bundle (instant, precomputed), then
/// fall back to a runtime-built cache.
func loadBank(_ model: LocalFaceModel, directory: URL) -> [(z: [Float], emb: [Float])] {
let bundled = directory.appendingPathComponent(bankFileName(model))
if let bank = load(from: bundled), !bank.isEmpty { return bank }
if let url = cacheBankURL(model), let bank = load(from: url) { return bank }
return []
}
/// Binary layout: Int32 count, then per entry 512 float z + 512 float emb.
private func load(from url: URL) -> [(z: [Float], emb: [Float])]? {
guard let data = try? Data(contentsOf: url), data.count >= 4 else { return nil }
return data.withUnsafeBytes { raw -> [(z: [Float], emb: [Float])] in
let count = Int(raw.load(fromByteOffset: 0, as: Int32.self))
let floats = raw.baseAddress!.advanced(by: 4)
.assumingMemoryBound(to: Float.self)
let stride = 1024 // 512 z + 512 emb
guard 4 + count * stride * 4 <= data.count else { return [] }
var result: [(z: [Float], emb: [Float])] = []
result.reserveCapacity(count)
for i in 0..<count {
let base = i * stride
let z = Array(UnsafeBufferPointer(start: floats + base, count: 512))
let emb = Array(UnsafeBufferPointer(start: floats + base + 512, count: 512))
result.append((z, emb))
}
return result
}
}
private func save(_ bank: [(z: [Float], emb: [Float])], to url: URL) {
var data = Data()
var count = Int32(bank.count)
withUnsafeBytes(of: &count) { data.append(contentsOf: $0) }
for entry in bank {
entry.z.withUnsafeBytes { data.append(contentsOf: $0) }
entry.emb.withUnsafeBytes { data.append(contentsOf: $0) }
}
try? data.write(to: url)
}
/// Build a bank from scratch and write it to `url` — used offline to
/// generate the precomputed banks that ship in the app bundle.
func exportBank(
model: LocalFaceModel, directory: URL, target: Int, to url: URL,
progress: @Sendable @escaping (Int, Int) -> Void
) async throws {
var bank: [(z: [Float], emb: [Float])] = []
try await buildBank(
model: model, directory: directory, to: target,
bank: &bank, progress: progress)
save(bank, to: url)
}
// MARK: - ArcFace embedding
/// Detect the face and embed it — used for the dropped photo and to
/// calibrate a model's bank alignment once. Nil if Vision finds no face.
private func detectAndEmbed(
_ image: CIImage, directory: URL
) throws -> (points: [SIMD2<Float>], emb: [Float])? {
guard let points = Self.fivePoints(in: image) else { return nil }
guard let emb = try embedding(for: image, points: points, directory: directory)
else { return nil }
return (points, emb)
}
/// L2-normalized ArcFace embedding using pre-supplied alignment points.
private func embedding(
for image: CIImage, points: [SIMD2<Float>], directory: URL
) throws -> [Float]? {
guard let warp = Self.similarityTransform(
from: points, template: Self.arcTemplate, cropSize: CGFloat(Self.embedSize))
else { return nil }
let rgba = renderCrop(image: image, transform: warp, size: Self.embedSize)
let blob = Self.nchwBlob(
rgba: rgba, size: Self.embedSize,
mean: SIMD3(repeating: 0.5), std: SIMD3(repeating: 0.5))
let session = try recognizerSession(directory: directory)
let output = try session.run([
session.inputNames[0]: ORTTensor(
shape: [1, 3, Int64(Self.embedSize), Int64(Self.embedSize)], data: blob)
])
guard let raw = output.first?.data else { return nil }
return Self.normalized(raw)
}
private func recognizerSession(directory: URL) throws -> ORTSession {
if let recognizer { return recognizer }
let url = directory.appendingPathComponent("w600k_r50.onnx")
guard FileManager.default.fileExists(atPath: url.path) else {
throw FaceLocalError.modelsMissing
}
// ArcFace on the Neural Engine — far faster than CPU for the 174 MB
// r50, and the bank build embeds hundreds of faces.
let session = try ORTSession(modelPath: url.path, useCoreML: true)
recognizer = session
return session
}
private func renderCrop(image: CIImage, transform: CGAffineTransform, size: Int) -> [UInt8] {
var buffer = [UInt8](repeating: 0, count: size * size * 4)
let bounds = CGRect(x: 0, y: 0, width: size, height: size)
ciContext.render(
image.transformed(by: transform).cropped(to: bounds),
toBitmap: &buffer, rowBytes: size * 4, bounds: bounds,
format: .RGBA8, colorSpace: CGColorSpace(name: CGColorSpace.sRGB))
return buffer
}
// MARK: - Vision landmarks → 5 points
/// The InsightFace 5 points (left eye, right eye, nose, left mouth, right
/// mouth) in Core Image coordinates (origin lower-left).
private static func fivePoints(in image: CIImage) -> [SIMD2<Float>]? {
let handler = VNImageRequestHandler(ciImage: image, options: [:])
let request = VNDetectFaceLandmarksRequest()
try? handler.perform([request])
guard let face = (request.results)?
.max(by: { $0.boundingBox.width * $0.boundingBox.height
< $1.boundingBox.width * $1.boundingBox.height }),
let landmarks = face.landmarks,
let leftEye = landmarks.leftEye,
let rightEye = landmarks.rightEye,
let lips = landmarks.outerLips
else { return nil }
let size = image.extent.size
func imgPoints(_ region: VNFaceLandmarkRegion2D) -> [CGPoint] {
region.pointsInImage(imageSize: size)
}
func centroid(_ points: [CGPoint]) -> CGPoint {
guard !points.isEmpty else { return .zero }
let sum = points.reduce(CGPoint.zero) { CGPoint(x: $0.x + $1.x, y: $0.y + $1.y) }
return CGPoint(x: sum.x / CGFloat(points.count), y: sum.y / CGFloat(points.count))
}
let eyeA = centroid(imgPoints(leftEye))
let eyeB = centroid(imgPoints(rightEye))
let (eyeLeft, eyeRight) = eyeA.x <= eyeB.x ? (eyeA, eyeB) : (eyeB, eyeA)
let mouth = imgPoints(lips)
guard let mouthLeft = mouth.min(by: { $0.x < $1.x }),
let mouthRight = mouth.max(by: { $0.x < $1.x }) else { return nil }
let eyeMid = CGPoint(x: (eyeLeft.x + eyeRight.x) / 2, y: (eyeLeft.y + eyeRight.y) / 2)
let mouthMid = CGPoint(x: (mouthLeft.x + mouthRight.x) / 2, y: (mouthLeft.y + mouthRight.y) / 2)
let axis = CGPoint(x: mouthMid.x - eyeMid.x, y: mouthMid.y - eyeMid.y)
let crest = (landmarks.noseCrest ?? landmarks.nose).map(imgPoints) ?? []
guard let noseTip = crest.max(by: {
($0.x * axis.x + $0.y * axis.y) < ($1.x * axis.x + $1.y * axis.y)
}) else { return nil }
return [eyeLeft, eyeRight, noseTip, mouthLeft, mouthRight].map {
SIMD2(Float($0.x), Float($0.y))
}
}
/// Least-squares similarity transform mapping `points` onto
/// `template × cropSize` (rotation + uniform scale + translation).
private static func similarityTransform(
from points: [SIMD2<Float>], template: [SIMD2<Float>], cropSize: CGFloat
) -> CGAffineTransform? {
guard points.count == template.count, points.count >= 2 else { return nil }
let targets = template.map { CGPoint(x: CGFloat($0.x) * cropSize, y: CGFloat($0.y) * cropSize) }
let src = points.map { CGPoint(x: CGFloat($0.x), y: CGFloat($0.y)) }
func centroid(_ p: [CGPoint]) -> CGPoint {
let s = p.reduce(CGPoint.zero) { CGPoint(x: $0.x + $1.x, y: $0.y + $1.y) }
return CGPoint(x: s.x / CGFloat(p.count), y: s.y / CGFloat(p.count))
}
let zMean = centroid(src), wMean = centroid(targets)
var numRe: CGFloat = 0, numIm: CGFloat = 0, denom: CGFloat = 0
for (z, w) in zip(src, targets) {
let zx = z.x - zMean.x, zy = z.y - zMean.y
let wx = w.x - wMean.x, wy = w.y - wMean.y
numRe += zx * wx + zy * wy
numIm += zx * wy - zy * wx
denom += zx * zx + zy * zy
}
guard denom > 1e-6 else { return nil }
let aRe = numRe / denom, aIm = numIm / denom
let tx = wMean.x - (aRe * zMean.x - aIm * zMean.y)
let ty = wMean.y - (aIm * zMean.x + aRe * zMean.y)
return CGAffineTransform(a: aRe, b: aIm, c: -aIm, d: aRe, tx: tx, ty: ty)
}
// MARK: - Blob helpers
private static func nchwBlob(
rgba: [UInt8], size: Int, mean: SIMD3<Float>, std: SIMD3<Float>
) -> [Float] {
let pixels = size * size
var blob = [Float](repeating: 0, count: 3 * pixels)
rgba.withUnsafeBufferPointer { source in
blob.withUnsafeMutableBufferPointer { dest in
for channel in 0..<3 {
let plane = dest.baseAddress! + channel * pixels
vDSP_vfltu8(source.baseAddress! + channel, 4, plane, 1, vDSP_Length(pixels))
var scale = 1 / (255 * std[channel])
var bias = -mean[channel] / std[channel]
vDSP_vsmsa(plane, 1, &scale, &bias, plane, 1, vDSP_Length(pixels))
}
}
}
return blob
}
private static func normalized(_ vector: [Float]) -> [Float] {
var result = vector
var norm: Float = 0
vDSP_svesq(result, 1, &norm, vDSP_Length(result.count))
norm = max(sqrt(norm), 1e-6)
vDSP_vsdiv(result, 1, &norm, &result, 1, vDSP_Length(result.count))
return result
}
}