Sources/FaceMaker/FaceLocal.swift
Swift source for Making faces, 658 lines. Raw file
import CoreGraphics
import Foundation
import ImageIO
import UniformTypeIdentifiers
/// Which face fields a model actually accepts. A hosted model takes them as
/// words in one prompt. MobileStyleGAN samples a face and accepts none of them.
struct FaceFields: Equatable, Hashable, Sendable {
var age = false
var gender = false
var expression = false
var glasses = false
var pose = false
var details = false
static let prompt = FaceFields(
age: true, gender: true, expression: true, glasses: true, pose: true, details: true)
}
/// How one slider changes the latent.
enum FaceEdit: Hashable, Sendable {
/// Add the coefficient times this 512-vector to every W row.
case direction(file: String)
/// One input of the group's controller. The controller output replaces a slice of W.
case component
/// Scale one SemanticStyleGAN part code. Zero leaves that part alone.
case part(index: Int)
}
/// One slider for a model.
struct FaceSlider: Identifiable, Hashable, Sendable {
var id: String
var name: String
var minimum: Double
var maximum: Double
/// The value that means "no extra edit" for a direction or a part scale.
var neutral: Double
var edit: FaceEdit
}
/// A block of sliders that belongs to one model.
struct FaceControlGroup: Identifiable, Hashable, Sendable {
var id: String
var name: String
/// When set, the group stays off until the user turns it on. Off leaves
/// the sampled latent alone instead of forcing the slider values.
var optional: Bool
var sliders: [FaceSlider]
/// Controller ONNX. Its input is the slider values in order.
var controllerFile: String? = nil
/// Where this group is written into the 512-d W vector.
var latentStart: Int? = nil
/// Add the slider values onto W instead of replacing the slice with a controller.
var offsetsLatent: Bool = false
}
enum LocalFaceKind: Hashable, Sendable {
/// Mapping input `var` [1, 512], synthesis input `style`.
case mobile
/// Mapping input `z` [1, 512], synthesis input `ws`.
case stylegan2
/// Mapping input `z` [1, 512] to W. Controllers replace slices, then W is repeated for synthesis.
case ganControl
/// One network. Input `z` and per-part `scales`.
case semanticParts
}
struct LocalFaceModel: Identifiable, Hashable, Sendable {
var id: String
var name: String
var licence: String
var mapFile: String
var synthFile: String
var kind: LocalFaceKind
/// Smaller ranks are faster and sort first.
var speedRank: Int
var fields: FaceFields
var controls: [FaceControlGroup]
var note: String
var menuTitle: String { "\(name) (\(licence))" }
var sliders: [FaceSlider] { controls.flatMap(\.sliders) }
static let all: [LocalFaceModel] = [
LocalFaceModel(
id: "local/mobilestylegan-ffhq",
name: "MobileStyleGAN FFHQ",
licence: "Apache 2.0, FFHQ non-commercial",
mapFile: "mobilestylegan_ffhq_v2-map.onnx",
synthFile: "mobilestylegan_ffhq_v2-synth.onnx",
kind: .mobile,
speedRank: 0,
fields: FaceFields(),
controls: [],
note: "Runs on this Mac. This model samples a random face and has no edit directions."),
LocalFaceModel(
id: "local/stylegan2-ffhq",
name: "StyleGAN2 FFHQ",
licence: "NVIDIA non-commercial",
mapFile: "stylegan2-ffhq-map.onnx",
synthFile: "stylegan2-ffhq-synth.onnx",
kind: .stylegan2,
speedRank: 1,
fields: FaceFields(),
controls: [Self.ffhqDirections],
note: "Runs on this Mac. Each slider adds one published direction. Zero leaves that direction alone."),
LocalFaceModel(
id: "local/stylegan2-metfaces",
name: "StyleGAN2 MetFaces",
licence: "NVIDIA non-commercial",
mapFile: "stylegan2-metfaces-map.onnx",
synthFile: "stylegan2-metfaces-synth.onnx",
kind: .stylegan2,
speedRank: 2,
fields: FaceFields(),
controls: [],
note: "Runs on this Mac. This model samples a random painted face and has no edit directions."),
LocalFaceModel(
id: "local/gan-control-ffhq",
name: "GAN-Control FFHQ",
licence: "Apache 2.0, FFHQ non-commercial",
mapFile: "gancontrol-ffhq-map.onnx",
synthFile: "gancontrol-ffhq-synth.onnx",
kind: .ganControl,
speedRank: 3,
fields: FaceFields(),
controls: Self.ganControls,
note: "Runs on this Mac. Tick a group to edit it. Its sliders appear when it is on. Off leaves the random sample alone."),
LocalFaceModel(
id: "local/semantic-celeba",
name: "SemanticStyleGAN CelebAMask",
licence: "CC BY-NC-SA 4.0",
mapFile: "semantic-celeba-synth.onnx",
synthFile: "semantic-celeba-synth.onnx",
kind: .semanticParts,
speedRank: 4,
fields: FaceFields(),
controls: [Self.semanticParts],
note: "Runs on this Mac. Each slider scales one face part. Zero leaves that part alone."),
]
/// InterFaceGAN W boundaries for StyleGAN FFHQ. Each file is one unit vector.
private static let ffhqDirections = FaceControlGroup(
id: "directions",
name: "Directions",
optional: false,
sliders: [
FaceSlider(id: "stylegan2-ffhq/age", name: "Age", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_age_w_boundary.fp32.f32")),
FaceSlider(id: "stylegan2-ffhq/gender", name: "Gender", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_gender_w_boundary.fp32.f32")),
FaceSlider(id: "stylegan2-ffhq/smile", name: "Smile", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_smile_w_boundary.fp32.f32")),
FaceSlider(id: "stylegan2-ffhq/glasses", name: "Glasses", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_eyeglasses_w_boundary.fp32.f32")),
FaceSlider(id: "stylegan2-ffhq/pose", name: "Pose", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_pose_w_boundary.fp32.f32")),
])
/// Latent order: identity, expression, pose, light, age, hair, then the residual slice.
private static let ganControls: [FaceControlGroup] = [
numberedGroup(
id: "gan/identity", name: "Identity", count: 128, minimum: -5, maximum: 5,
latentStart: 0, offsetsLatent: true),
numberedGroup(
id: "gan/expression", name: "Expression", count: 64, minimum: -2, maximum: 2,
controller: "gancontrol-ffhq-expression.onnx", latentStart: 128),
FaceControlGroup(
id: "gan/pose",
name: "Pose",
optional: true,
sliders: [
FaceSlider(id: "gan/yaw", name: "Yaw", minimum: -60, maximum: 60, neutral: 0, edit: .component),
FaceSlider(id: "gan/pitch", name: "Pitch", minimum: -30, maximum: 30, neutral: 0, edit: .component),
FaceSlider(id: "gan/roll", name: "Roll", minimum: -30, maximum: 30, neutral: 0, edit: .component),
],
controllerFile: "gancontrol-ffhq-orientation.onnx",
latentStart: 192),
numberedGroup(
id: "gan/light", name: "Light", count: 27, minimum: -2, maximum: 2,
controller: "gancontrol-ffhq-gamma.onnx", latentStart: 256),
FaceControlGroup(
id: "gan/age",
name: "Age",
optional: true,
sliders: [
FaceSlider(id: "gan/age", name: "Years", minimum: 0, maximum: 100, neutral: 30, edit: .component),
],
controllerFile: "gancontrol-ffhq-age.onnx",
latentStart: 320),
FaceControlGroup(
id: "gan/hair",
name: "Hair color",
optional: true,
sliders: [
FaceSlider(id: "gan/hair-r", name: "Red", minimum: 0, maximum: 1, neutral: 0.4, edit: .component),
FaceSlider(id: "gan/hair-g", name: "Green", minimum: 0, maximum: 1, neutral: 0.25, edit: .component),
FaceSlider(id: "gan/hair-b", name: "Blue", minimum: 0, maximum: 1, neutral: 0.15, edit: .component),
],
controllerFile: "gancontrol-ffhq-hair.onnx",
latentStart: 384),
numberedGroup(
id: "gan/other", name: "Other", count: 64, minimum: -5, maximum: 5,
latentStart: 448, offsetsLatent: true),
]
/// CelebAMask part codes, in the order the generator stores them.
private static let semanticParts = FaceControlGroup(
id: "semantic/parts",
name: "Parts",
optional: false,
sliders: [
("coarse-1", "Coarse 1", 0), ("coarse-2", "Coarse 2", 1),
("background-1", "Background 1", 2), ("background-2", "Background 2", 3),
("face-shape", "Face shape", 4), ("face-texture", "Face texture", 5),
("eye-shape", "Eye shape", 6), ("eye-texture", "Eye texture", 7),
("brow-shape", "Brow shape", 8), ("brow-texture", "Brow texture", 9),
("mouth-shape", "Mouth shape", 10), ("mouth-texture", "Mouth texture", 11),
("nose-shape", "Nose shape", 12), ("nose-texture", "Nose texture", 13),
("ear-shape", "Ear shape", 14), ("ear-texture", "Ear texture", 15),
("hair-shape", "Hair shape", 16), ("hair-texture", "Hair texture", 17),
("neck-shape", "Neck shape", 18), ("neck-texture", "Neck texture", 19),
("clothes-shape", "Clothes shape", 20), ("clothes-texture", "Clothes texture", 21),
("glasses", "Glasses", 22), ("glasses-2", "Glasses 2", 23),
("hat", "Hat", 24), ("hat-2", "Hat 2", 25),
("earring", "Earring", 26), ("earring-2", "Earring 2", 27),
].map { suffix, name, index in
FaceSlider(
id: "semantic/\(suffix)",
name: name,
minimum: -1,
maximum: 1,
neutral: 0,
edit: .part(index: index))
})
private static func numberedGroup(
id: String,
name: String,
count: Int,
minimum: Double,
maximum: Double,
controller: String? = nil,
latentStart: Int,
offsetsLatent: Bool = false
) -> FaceControlGroup {
FaceControlGroup(
id: id,
name: name,
optional: true,
sliders: (0..<count).map { index in
FaceSlider(
id: "\(id)/\(index)",
name: "\(name) \(index + 1)",
minimum: minimum,
maximum: maximum,
neutral: 0,
edit: .component)
},
controllerFile: controller,
latentStart: latentStart,
offsetsLatent: offsetsLatent)
}
static var fastest: LocalFaceModel {
all.min { $0.speedRank < $1.speedRank } ?? all[0]
}
static func find(_ id: String) -> LocalFaceModel? {
all.first { $0.id == id }
}
}
enum FaceLocalError: LocalizedError {
case modelsMissing
case badOutput
var errorDescription: String? {
switch self {
case .modelsMissing:
"The local face model is not in the app."
case .badOutput:
"The local face model returned an unreadable image."
}
}
}
enum FaceNumbers {
/// Blank, or any text that is not a single number, means no age.
static func age(_ text: String) -> Double? {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let normalized = trimmed.replacingOccurrences(of: ",", with: ".")
guard let value = Double(normalized), value.isFinite, value >= 0 else { return nil }
return value
}
static func display(_ value: Double) -> String {
if value.rounded() == value, abs(value) < 10_000 {
return String(Int(value))
}
return String(format: "%g", value)
}
}
enum FacePrompt {
static func api(
age: String,
gender: String,
expression: String,
glasses: String,
pose: String,
details: String,
variation: String
) -> String {
var parts = [
"Photorealistic headshot of one person, facing the camera, shoulders visible, plain background."
]
if let years = FaceNumbers.age(age) {
parts.append("Age \(FaceNumbers.display(years)).")
}
append(&parts, "Gender", gender)
append(&parts, "Expression", expression)
append(&parts, "Glasses", glasses)
append(&parts, "Pose", pose)
if let words = clause(details) {
parts.append("\(words).")
}
parts.append("\(variation). A different individual from the other portraits.")
return parts.joined(separator: " ")
}
static func brief(
age: String,
gender: String,
expression: String,
glasses: String,
pose: String,
details: String,
fields: FaceFields
) -> String {
var parts: [String] = []
if fields.age, let years = FaceNumbers.age(age) {
parts.append("age \(FaceNumbers.display(years))")
}
if fields.gender, let value = clause(gender) { parts.append(value) }
if fields.expression, let value = clause(expression) { parts.append(value) }
if fields.glasses, let value = clause(glasses) { parts.append(value) }
if fields.pose, let value = clause(pose) { parts.append(value) }
if fields.details, let value = clause(details) { parts.append(value) }
if parts.isEmpty { return "Face." }
return "Face, \(parts.joined(separator: ", "))."
}
/// Non-zero direction sliders, for the media card.
static func localBrief(_ edits: [(name: String, value: Double)]) -> String {
let parts = edits.map { name, value in
"\(name) \(String(format: "%+.1f", value))"
}
if parts.isEmpty { return "Face." }
return "Face, \(parts.joined(separator: ", "))."
}
private static func append(_ parts: inout [String], _ label: String, _ text: String) {
guard let value = clause(text) else { return }
parts.append("\(label): \(value).")
}
private static func clause(_ text: String) -> String? {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? nil : trimmed
}
}
/// Runs a bundled face generator through ONNX Runtime. No download and no
/// helper process: the weights ship inside the app.
actor FaceLocalEngine {
static let shared = FaceLocalEngine()
private var loaded: [String: (map: ORTSession, synth: ORTSession)] = [:]
private var controllers: [String: ORTSession] = [:]
private var directions: [String: [Float]] = [:]
func png(
model: LocalFaceModel,
directory: URL,
latent: [Float],
edits: [String: Float] = [:]
) throws -> Data {
let pair = try sessions(for: model, directory: directory)
let z = ORTTensor(shape: [1, 512], data: latent)
let mapped: [ORTTensor]
let image: [ORTTensor]
switch model.kind {
case .mobile:
mapped = try pair.map.run(["var": z])
guard let style = mapped.first else { throw FaceLocalError.badOutput }
image = try pair.synth.run(["style": style])
case .stylegan2:
mapped = try pair.map.run(["z": z])
guard let styles = mapped.first else { throw FaceLocalError.badOutput }
let edited = try applyDirections(styles, model: model, edits: edits, directory: directory)
image = try pair.synth.run(["ws": edited])
case .ganControl:
mapped = try pair.map.run(["z": z])
guard let styles = mapped.first else { throw FaceLocalError.badOutput }
let edited = try applyControllers(styles, model: model, edits: edits, directory: directory)
image = try pair.synth.run(["ws": Self.repeatStyle(edited, rows: 16)])
case .semanticParts:
mapped = []
let scales = partScales(model: model, edits: edits)
image = try pair.synth.run([
"z": z,
"scales": ORTTensor(shape: [1, 28], data: scales),
])
}
guard let tensor = image.first, let png = Self.png(from: tensor) else {
throw FaceLocalError.badOutput
}
return png
}
/// Generate directly from a W+ latent [1, 18, 512], bypassing the mapping
/// network — used by encoder inversion. Direction edits still apply, so the
/// reconstructed face can be aged / made to smile, etc.
func png(
model: LocalFaceModel,
directory: URL,
wplus: [Float],
edits: [String: Float] = [:]
) throws -> Data {
guard model.kind == .stylegan2 else { throw FaceLocalError.badOutput }
let pair = try sessions(for: model, directory: directory)
let ws = ORTTensor(shape: [1, 18, 512], data: wplus)
let edited = try applyDirections(ws, model: model, edits: edits, directory: directory)
let image = try pair.synth.run(["ws": edited])
guard let tensor = image.first, let png = Self.png(from: tensor) else {
throw FaceLocalError.badOutput
}
return png
}
/// Loads a model's networks ahead of time, so the first face isn't slowed by it.
func preload(model: LocalFaceModel, directory: URL) throws {
_ = try sessions(for: model, directory: directory)
}
private func sessions(for model: LocalFaceModel, directory: URL) throws -> (map: ORTSession, synth: ORTSession) {
if let cached = loaded[model.id] { return cached }
let mapURL = directory.appendingPathComponent(model.mapFile)
let synthURL = directory.appendingPathComponent(model.synthFile)
guard FileManager.default.fileExists(atPath: mapURL.path),
FileManager.default.fileExists(atPath: synthURL.path)
else { throw FaceLocalError.modelsMissing }
// This distilled network is faster on the CPU than waiting for a
// Core ML compile. The runtime is still the vendored ONNX Runtime.
let synth = try ORTSession(modelPath: synthURL.path, useCoreML: false)
let map = model.mapFile == model.synthFile
? synth
: try ORTSession(modelPath: mapURL.path, useCoreML: false)
let pair = (map, synth)
loaded[model.id] = pair
return pair
}
/// Adds each non-zero slider to every row of W+. A zero coefficient is a no-op.
private func applyDirections(
_ styles: ORTTensor,
model: LocalFaceModel,
edits: [String: Float],
directory: URL
) throws -> ORTTensor {
guard styles.shape.count == 3 else { return styles }
let width = Int(styles.shape[2])
let layers = Int(styles.shape[1])
guard width == 512, styles.data.count >= layers * width else { return styles }
var data = styles.data
var changed = false
for slider in model.sliders {
guard case .direction(let file) = slider.edit else { continue }
let coeff = edits[slider.id] ?? 0
if abs(coeff) < 0.0001 { continue }
let vector = try direction(file: file, directory: directory)
guard vector.count == width else { continue }
changed = true
for layer in 0..<layers {
let base = layer * width
for index in 0..<width {
data[base + index] += coeff * vector[index]
}
}
}
if !changed { return styles }
return ORTTensor(shape: styles.shape, data: data)
}
private func direction(file: String, directory: URL) throws -> [Float] {
if let cached = directions[file] { return cached }
let url = directory.appendingPathComponent(file)
guard let data = try? Data(contentsOf: url), data.count == 512 * MemoryLayout<Float>.size else {
throw FaceLocalError.modelsMissing
}
var values = [Float](repeating: 0, count: 512)
_ = values.withUnsafeMutableBytes { raw in
data.copyBytes(to: raw)
}
directions[file] = values
return values
}
/// Writes every enabled group into W. Controllers replace their slice. Offset groups add to it.
private func applyControllers(
_ styles: ORTTensor,
model: LocalFaceModel,
edits: [String: Float],
directory: URL
) throws -> ORTTensor {
guard styles.data.count == 512 else { return styles }
var data = styles.data
for group in model.controls {
guard let start = group.latentStart else { continue }
guard group.sliders.contains(where: { edits[$0.id] != nil }) else { continue }
if group.offsetsLatent {
for (offset, slider) in group.sliders.enumerated() {
let index = start + offset
guard data.indices.contains(index) else { continue }
data[index] += edits[slider.id] ?? Float(slider.neutral)
}
continue
}
guard let file = group.controllerFile else { continue }
let input = group.sliders.map { edits[$0.id] ?? Float($0.neutral) }
let session = try controller(file: file, directory: directory)
let output = try session.run([
"control": ORTTensor(shape: [1, Int64(input.count)], data: input),
])
guard let values = output.first?.data, start + values.count <= data.count else {
throw FaceLocalError.badOutput
}
for index in values.indices {
data[start + index] = values[index]
}
}
return ORTTensor(shape: styles.shape, data: data)
}
private func controller(file: String, directory: URL) throws -> ORTSession {
if let cached = controllers[file] { return cached }
let url = directory.appendingPathComponent(file)
guard FileManager.default.fileExists(atPath: url.path) else { throw FaceLocalError.modelsMissing }
let session = try ORTSession(modelPath: url.path, useCoreML: false)
controllers[file] = session
return session
}
/// Slider 0 keeps the truncated part. −1 pulls it to the average, +1 exaggerates it.
private func partScales(model: LocalFaceModel, edits: [String: Float]) -> [Float] {
var scales = [Float](repeating: 1, count: 28)
for slider in model.sliders {
guard case .part(let index) = slider.edit, scales.indices.contains(index) else { continue }
let value = edits[slider.id] ?? Float(slider.neutral)
scales[index] = 1 + value
}
return scales
}
private static func repeatStyle(_ styles: ORTTensor, rows: Int) -> ORTTensor {
var data: [Float] = []
data.reserveCapacity(rows * styles.data.count)
for _ in 0..<rows {
data.append(contentsOf: styles.data)
}
return ORTTensor(shape: [1, Int64(rows), Int64(styles.data.count)], data: data)
}
static func bundledDirectory() throws -> URL {
guard let folder = Bundle.main.resourceURL?.appendingPathComponent("Models", isDirectory: true),
FileManager.default.fileExists(atPath: folder.path)
else { throw FaceLocalError.modelsMissing }
return folder
}
/// The deterministic 512-d Gaussian z a seed maps to. Exposed so the app
/// can build a face's latent itself (for vector mode, perturbation, and
/// "Use Vector"), then hand it to `png(latent:)`.
static func gaussianLatent(seed: UInt64, count: Int = 512) -> [Float] {
var rng = SplitMix64(state: seed == 0 ? 0x1234_5678_9ABC_DEF : seed)
var values: [Float] = []
values.reserveCapacity(count)
while values.count < count {
var u1 = 0.0
repeat { u1 = unit(&rng) } while u1 <= .leastNonzeroMagnitude
let u2 = unit(&rng)
let radius = sqrt(-2 * log(u1))
let angle = 2 * Double.pi * u2
values.append(Float(radius * cos(angle)))
if values.count < count {
values.append(Float(radius * sin(angle)))
}
}
return values
}
private static func unit(_ rng: inout SplitMix64) -> Double {
Double(rng.next() >> 11) / Double(1 << 53)
}
private static func png(from tensor: ORTTensor) -> Data? {
guard tensor.shape.count == 4, tensor.shape[0] == 1, tensor.shape[1] == 3 else { return nil }
let height = Int(tensor.shape[2])
let width = Int(tensor.shape[3])
let plane = width * height
guard plane > 0, tensor.data.count >= plane * 3 else { return nil }
var pixels = [UInt8](repeating: 255, count: plane * 4)
let source = tensor.data
for index in 0..<plane {
pixels[index * 4] = byte(source[index])
pixels[index * 4 + 1] = byte(source[plane + index])
pixels[index * 4 + 2] = byte(source[plane * 2 + index])
}
let data = Data(pixels) as CFData
guard let provider = CGDataProvider(data: data) else { return nil }
let bitmap = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue)
guard let image = CGImage(
width: width,
height: height,
bitsPerComponent: 8,
bitsPerPixel: 32,
bytesPerRow: width * 4,
space: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: bitmap,
provider: provider,
decode: nil,
shouldInterpolate: false,
intent: .defaultIntent)
else { return nil }
let encoded = NSMutableData()
guard let destination = CGImageDestinationCreateWithData(
encoded, UTType.png.identifier as CFString, 1, nil)
else { return nil }
CGImageDestinationAddImage(destination, image, nil)
guard CGImageDestinationFinalize(destination) else { return nil }
return encoded as Data
}
private static func byte(_ value: Float) -> UInt8 {
let unit = min(max((value + 1) * 0.5, 0), 1)
return UInt8((unit * 255).rounded())
}
}
private struct SplitMix64: RandomNumberGenerator {
var state: UInt64
mutating func next() -> UInt64 {
state &+= 0x9E37_79B9_7F4A_7C15
var mixed = state
mixed = (mixed ^ (mixed >> 30)) &* 0xBF58_476D_1CE4_E5B9
mixed = (mixed ^ (mixed >> 27)) &* 0x94D0_49BB_1331_11EB
return mixed ^ (mixed >> 31)
}
}