import CoreGraphics import Foundation import ImageIO import UniformTypeIdentifiers /// Which face fields a model actually accepts. A hosted model takes them as /// words in one prompt. MobileStyleGAN samples a face and accepts none of them. struct FaceFields: Equatable, Hashable, Sendable { var age = false var gender = false var expression = false var glasses = false var pose = false var details = false static let prompt = FaceFields( age: true, gender: true, expression: true, glasses: true, pose: true, details: true) } /// How one slider changes the latent. enum FaceEdit: Hashable, Sendable { /// Add the coefficient times this 512-vector to every W row. case direction(file: String) /// One input of the group's controller. The controller output replaces a slice of W. case component /// Scale one SemanticStyleGAN part code. Zero leaves that part alone. case part(index: Int) } /// One slider for a model. struct FaceSlider: Identifiable, Hashable, Sendable { var id: String var name: String var minimum: Double var maximum: Double /// The value that means "no extra edit" for a direction or a part scale. var neutral: Double var edit: FaceEdit } /// A block of sliders that belongs to one model. struct FaceControlGroup: Identifiable, Hashable, Sendable { var id: String var name: String /// When set, the group stays off until the user turns it on. Off leaves /// the sampled latent alone instead of forcing the slider values. var optional: Bool var sliders: [FaceSlider] /// Controller ONNX. Its input is the slider values in order. var controllerFile: String? = nil /// Where this group is written into the 512-d W vector. var latentStart: Int? = nil /// Add the slider values onto W instead of replacing the slice with a controller. var offsetsLatent: Bool = false } enum LocalFaceKind: Hashable, Sendable { /// Mapping input `var` [1, 512], synthesis input `style`. case mobile /// Mapping input `z` [1, 512], synthesis input `ws`. case stylegan2 /// Mapping input `z` [1, 512] to W. Controllers replace slices, then W is repeated for synthesis. case ganControl /// One network. Input `z` and per-part `scales`. case semanticParts } struct LocalFaceModel: Identifiable, Hashable, Sendable { var id: String var name: String var licence: String var mapFile: String var synthFile: String var kind: LocalFaceKind /// Smaller ranks are faster and sort first. var speedRank: Int var fields: FaceFields var controls: [FaceControlGroup] var note: String var menuTitle: String { "\(name) (\(licence))" } var sliders: [FaceSlider] { controls.flatMap(\.sliders) } static let all: [LocalFaceModel] = [ LocalFaceModel( id: "local/mobilestylegan-ffhq", name: "MobileStyleGAN FFHQ", licence: "Apache 2.0, FFHQ non-commercial", mapFile: "mobilestylegan_ffhq_v2-map.onnx", synthFile: "mobilestylegan_ffhq_v2-synth.onnx", kind: .mobile, speedRank: 0, fields: FaceFields(), controls: [], note: "Runs on this Mac. This model samples a random face and has no edit directions."), LocalFaceModel( id: "local/stylegan2-ffhq", name: "StyleGAN2 FFHQ", licence: "NVIDIA non-commercial", mapFile: "stylegan2-ffhq-map.onnx", synthFile: "stylegan2-ffhq-synth.onnx", kind: .stylegan2, speedRank: 1, fields: FaceFields(), controls: [Self.ffhqDirections], note: "Runs on this Mac. Each slider adds one published direction. Zero leaves that direction alone."), LocalFaceModel( id: "local/stylegan2-metfaces", name: "StyleGAN2 MetFaces", licence: "NVIDIA non-commercial", mapFile: "stylegan2-metfaces-map.onnx", synthFile: "stylegan2-metfaces-synth.onnx", kind: .stylegan2, speedRank: 2, fields: FaceFields(), controls: [], note: "Runs on this Mac. This model samples a random painted face and has no edit directions."), LocalFaceModel( id: "local/gan-control-ffhq", name: "GAN-Control FFHQ", licence: "Apache 2.0, FFHQ non-commercial", mapFile: "gancontrol-ffhq-map.onnx", synthFile: "gancontrol-ffhq-synth.onnx", kind: .ganControl, speedRank: 3, fields: FaceFields(), controls: Self.ganControls, note: "Runs on this Mac. Tick a group to edit it. Its sliders appear when it is on. Off leaves the random sample alone."), LocalFaceModel( id: "local/semantic-celeba", name: "SemanticStyleGAN CelebAMask", licence: "CC BY-NC-SA 4.0", mapFile: "semantic-celeba-synth.onnx", synthFile: "semantic-celeba-synth.onnx", kind: .semanticParts, speedRank: 4, fields: FaceFields(), controls: [Self.semanticParts], note: "Runs on this Mac. Each slider scales one face part. Zero leaves that part alone."), ] /// InterFaceGAN W boundaries for StyleGAN FFHQ. Each file is one unit vector. private static let ffhqDirections = FaceControlGroup( id: "directions", name: "Directions", optional: false, sliders: [ FaceSlider(id: "stylegan2-ffhq/age", name: "Age", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_age_w_boundary.fp32.f32")), FaceSlider(id: "stylegan2-ffhq/gender", name: "Gender", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_gender_w_boundary.fp32.f32")), FaceSlider(id: "stylegan2-ffhq/smile", name: "Smile", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_smile_w_boundary.fp32.f32")), FaceSlider(id: "stylegan2-ffhq/glasses", name: "Glasses", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_eyeglasses_w_boundary.fp32.f32")), FaceSlider(id: "stylegan2-ffhq/pose", name: "Pose", minimum: -5, maximum: 5, neutral: 0, edit: .direction(file: "stylegan_ffhq_pose_w_boundary.fp32.f32")), ]) /// Latent order: identity, expression, pose, light, age, hair, then the residual slice. private static let ganControls: [FaceControlGroup] = [ numberedGroup( id: "gan/identity", name: "Identity", count: 128, minimum: -5, maximum: 5, latentStart: 0, offsetsLatent: true), numberedGroup( id: "gan/expression", name: "Expression", count: 64, minimum: -2, maximum: 2, controller: "gancontrol-ffhq-expression.onnx", latentStart: 128), FaceControlGroup( id: "gan/pose", name: "Pose", optional: true, sliders: [ FaceSlider(id: "gan/yaw", name: "Yaw", minimum: -60, maximum: 60, neutral: 0, edit: .component), FaceSlider(id: "gan/pitch", name: "Pitch", minimum: -30, maximum: 30, neutral: 0, edit: .component), FaceSlider(id: "gan/roll", name: "Roll", minimum: -30, maximum: 30, neutral: 0, edit: .component), ], controllerFile: "gancontrol-ffhq-orientation.onnx", latentStart: 192), numberedGroup( id: "gan/light", name: "Light", count: 27, minimum: -2, maximum: 2, controller: "gancontrol-ffhq-gamma.onnx", latentStart: 256), FaceControlGroup( id: "gan/age", name: "Age", optional: true, sliders: [ FaceSlider(id: "gan/age", name: "Years", minimum: 0, maximum: 100, neutral: 30, edit: .component), ], controllerFile: "gancontrol-ffhq-age.onnx", latentStart: 320), FaceControlGroup( id: "gan/hair", name: "Hair color", optional: true, sliders: [ FaceSlider(id: "gan/hair-r", name: "Red", minimum: 0, maximum: 1, neutral: 0.4, edit: .component), FaceSlider(id: "gan/hair-g", name: "Green", minimum: 0, maximum: 1, neutral: 0.25, edit: .component), FaceSlider(id: "gan/hair-b", name: "Blue", minimum: 0, maximum: 1, neutral: 0.15, edit: .component), ], controllerFile: "gancontrol-ffhq-hair.onnx", latentStart: 384), numberedGroup( id: "gan/other", name: "Other", count: 64, minimum: -5, maximum: 5, latentStart: 448, offsetsLatent: true), ] /// CelebAMask part codes, in the order the generator stores them. private static let semanticParts = FaceControlGroup( id: "semantic/parts", name: "Parts", optional: false, sliders: [ ("coarse-1", "Coarse 1", 0), ("coarse-2", "Coarse 2", 1), ("background-1", "Background 1", 2), ("background-2", "Background 2", 3), ("face-shape", "Face shape", 4), ("face-texture", "Face texture", 5), ("eye-shape", "Eye shape", 6), ("eye-texture", "Eye texture", 7), ("brow-shape", "Brow shape", 8), ("brow-texture", "Brow texture", 9), ("mouth-shape", "Mouth shape", 10), ("mouth-texture", "Mouth texture", 11), ("nose-shape", "Nose shape", 12), ("nose-texture", "Nose texture", 13), ("ear-shape", "Ear shape", 14), ("ear-texture", "Ear texture", 15), ("hair-shape", "Hair shape", 16), ("hair-texture", "Hair texture", 17), ("neck-shape", "Neck shape", 18), ("neck-texture", "Neck texture", 19), ("clothes-shape", "Clothes shape", 20), ("clothes-texture", "Clothes texture", 21), ("glasses", "Glasses", 22), ("glasses-2", "Glasses 2", 23), ("hat", "Hat", 24), ("hat-2", "Hat 2", 25), ("earring", "Earring", 26), ("earring-2", "Earring 2", 27), ].map { suffix, name, index in FaceSlider( id: "semantic/\(suffix)", name: name, minimum: -1, maximum: 1, neutral: 0, edit: .part(index: index)) }) private static func numberedGroup( id: String, name: String, count: Int, minimum: Double, maximum: Double, controller: String? = nil, latentStart: Int, offsetsLatent: Bool = false ) -> FaceControlGroup { FaceControlGroup( id: id, name: name, optional: true, sliders: (0.. LocalFaceModel? { all.first { $0.id == id } } } enum FaceLocalError: LocalizedError { case modelsMissing case badOutput var errorDescription: String? { switch self { case .modelsMissing: "The local face model is not in the app." case .badOutput: "The local face model returned an unreadable image." } } } enum FaceNumbers { /// Blank, or any text that is not a single number, means no age. static func age(_ text: String) -> Double? { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return nil } let normalized = trimmed.replacingOccurrences(of: ",", with: ".") guard let value = Double(normalized), value.isFinite, value >= 0 else { return nil } return value } static func display(_ value: Double) -> String { if value.rounded() == value, abs(value) < 10_000 { return String(Int(value)) } return String(format: "%g", value) } } enum FacePrompt { static func api( age: String, gender: String, expression: String, glasses: String, pose: String, details: String, variation: String ) -> String { var parts = [ "Photorealistic headshot of one person, facing the camera, shoulders visible, plain background." ] if let years = FaceNumbers.age(age) { parts.append("Age \(FaceNumbers.display(years)).") } append(&parts, "Gender", gender) append(&parts, "Expression", expression) append(&parts, "Glasses", glasses) append(&parts, "Pose", pose) if let words = clause(details) { parts.append("\(words).") } parts.append("\(variation). A different individual from the other portraits.") return parts.joined(separator: " ") } static func brief( age: String, gender: String, expression: String, glasses: String, pose: String, details: String, fields: FaceFields ) -> String { var parts: [String] = [] if fields.age, let years = FaceNumbers.age(age) { parts.append("age \(FaceNumbers.display(years))") } if fields.gender, let value = clause(gender) { parts.append(value) } if fields.expression, let value = clause(expression) { parts.append(value) } if fields.glasses, let value = clause(glasses) { parts.append(value) } if fields.pose, let value = clause(pose) { parts.append(value) } if fields.details, let value = clause(details) { parts.append(value) } if parts.isEmpty { return "Face." } return "Face, \(parts.joined(separator: ", "))." } /// Non-zero direction sliders, for the media card. static func localBrief(_ edits: [(name: String, value: Double)]) -> String { let parts = edits.map { name, value in "\(name) \(String(format: "%+.1f", value))" } if parts.isEmpty { return "Face." } return "Face, \(parts.joined(separator: ", "))." } private static func append(_ parts: inout [String], _ label: String, _ text: String) { guard let value = clause(text) else { return } parts.append("\(label): \(value).") } private static func clause(_ text: String) -> String? { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) return trimmed.isEmpty ? nil : trimmed } } /// Runs a bundled face generator through ONNX Runtime. No download and no /// helper process: the weights ship inside the app. actor FaceLocalEngine { static let shared = FaceLocalEngine() private var loaded: [String: (map: ORTSession, synth: ORTSession)] = [:] private var controllers: [String: ORTSession] = [:] private var directions: [String: [Float]] = [:] func png( model: LocalFaceModel, directory: URL, latent: [Float], edits: [String: Float] = [:] ) throws -> Data { let pair = try sessions(for: model, directory: directory) let z = ORTTensor(shape: [1, 512], data: latent) let mapped: [ORTTensor] let image: [ORTTensor] switch model.kind { case .mobile: mapped = try pair.map.run(["var": z]) guard let style = mapped.first else { throw FaceLocalError.badOutput } image = try pair.synth.run(["style": style]) case .stylegan2: mapped = try pair.map.run(["z": z]) guard let styles = mapped.first else { throw FaceLocalError.badOutput } let edited = try applyDirections(styles, model: model, edits: edits, directory: directory) image = try pair.synth.run(["ws": edited]) case .ganControl: mapped = try pair.map.run(["z": z]) guard let styles = mapped.first else { throw FaceLocalError.badOutput } let edited = try applyControllers(styles, model: model, edits: edits, directory: directory) image = try pair.synth.run(["ws": Self.repeatStyle(edited, rows: 16)]) case .semanticParts: mapped = [] let scales = partScales(model: model, edits: edits) image = try pair.synth.run([ "z": z, "scales": ORTTensor(shape: [1, 28], data: scales), ]) } guard let tensor = image.first, let png = Self.png(from: tensor) else { throw FaceLocalError.badOutput } return png } /// Generate directly from a W+ latent [1, 18, 512], bypassing the mapping /// network — used by encoder inversion. Direction edits still apply, so the /// reconstructed face can be aged / made to smile, etc. func png( model: LocalFaceModel, directory: URL, wplus: [Float], edits: [String: Float] = [:] ) throws -> Data { guard model.kind == .stylegan2 else { throw FaceLocalError.badOutput } let pair = try sessions(for: model, directory: directory) let ws = ORTTensor(shape: [1, 18, 512], data: wplus) let edited = try applyDirections(ws, model: model, edits: edits, directory: directory) let image = try pair.synth.run(["ws": edited]) guard let tensor = image.first, let png = Self.png(from: tensor) else { throw FaceLocalError.badOutput } return png } /// Loads a model's networks ahead of time, so the first face isn't slowed by it. func preload(model: LocalFaceModel, directory: URL) throws { _ = try sessions(for: model, directory: directory) } private func sessions(for model: LocalFaceModel, directory: URL) throws -> (map: ORTSession, synth: ORTSession) { if let cached = loaded[model.id] { return cached } let mapURL = directory.appendingPathComponent(model.mapFile) let synthURL = directory.appendingPathComponent(model.synthFile) guard FileManager.default.fileExists(atPath: mapURL.path), FileManager.default.fileExists(atPath: synthURL.path) else { throw FaceLocalError.modelsMissing } // This distilled network is faster on the CPU than waiting for a // Core ML compile. The runtime is still the vendored ONNX Runtime. let synth = try ORTSession(modelPath: synthURL.path, useCoreML: false) let map = model.mapFile == model.synthFile ? synth : try ORTSession(modelPath: mapURL.path, useCoreML: false) let pair = (map, synth) loaded[model.id] = pair return pair } /// Adds each non-zero slider to every row of W+. A zero coefficient is a no-op. private func applyDirections( _ styles: ORTTensor, model: LocalFaceModel, edits: [String: Float], directory: URL ) throws -> ORTTensor { guard styles.shape.count == 3 else { return styles } let width = Int(styles.shape[2]) let layers = Int(styles.shape[1]) guard width == 512, styles.data.count >= layers * width else { return styles } var data = styles.data var changed = false for slider in model.sliders { guard case .direction(let file) = slider.edit else { continue } let coeff = edits[slider.id] ?? 0 if abs(coeff) < 0.0001 { continue } let vector = try direction(file: file, directory: directory) guard vector.count == width else { continue } changed = true for layer in 0.. [Float] { if let cached = directions[file] { return cached } let url = directory.appendingPathComponent(file) guard let data = try? Data(contentsOf: url), data.count == 512 * MemoryLayout.size else { throw FaceLocalError.modelsMissing } var values = [Float](repeating: 0, count: 512) _ = values.withUnsafeMutableBytes { raw in data.copyBytes(to: raw) } directions[file] = values return values } /// Writes every enabled group into W. Controllers replace their slice. Offset groups add to it. private func applyControllers( _ styles: ORTTensor, model: LocalFaceModel, edits: [String: Float], directory: URL ) throws -> ORTTensor { guard styles.data.count == 512 else { return styles } var data = styles.data for group in model.controls { guard let start = group.latentStart else { continue } guard group.sliders.contains(where: { edits[$0.id] != nil }) else { continue } if group.offsetsLatent { for (offset, slider) in group.sliders.enumerated() { let index = start + offset guard data.indices.contains(index) else { continue } data[index] += edits[slider.id] ?? Float(slider.neutral) } continue } guard let file = group.controllerFile else { continue } let input = group.sliders.map { edits[$0.id] ?? Float($0.neutral) } let session = try controller(file: file, directory: directory) let output = try session.run([ "control": ORTTensor(shape: [1, Int64(input.count)], data: input), ]) guard let values = output.first?.data, start + values.count <= data.count else { throw FaceLocalError.badOutput } for index in values.indices { data[start + index] = values[index] } } return ORTTensor(shape: styles.shape, data: data) } private func controller(file: String, directory: URL) throws -> ORTSession { if let cached = controllers[file] { return cached } let url = directory.appendingPathComponent(file) guard FileManager.default.fileExists(atPath: url.path) else { throw FaceLocalError.modelsMissing } let session = try ORTSession(modelPath: url.path, useCoreML: false) controllers[file] = session return session } /// Slider 0 keeps the truncated part. −1 pulls it to the average, +1 exaggerates it. private func partScales(model: LocalFaceModel, edits: [String: Float]) -> [Float] { var scales = [Float](repeating: 1, count: 28) for slider in model.sliders { guard case .part(let index) = slider.edit, scales.indices.contains(index) else { continue } let value = edits[slider.id] ?? Float(slider.neutral) scales[index] = 1 + value } return scales } private static func repeatStyle(_ styles: ORTTensor, rows: Int) -> ORTTensor { var data: [Float] = [] data.reserveCapacity(rows * styles.data.count) for _ in 0.. URL { guard let folder = Bundle.main.resourceURL?.appendingPathComponent("Models", isDirectory: true), FileManager.default.fileExists(atPath: folder.path) else { throw FaceLocalError.modelsMissing } return folder } /// The deterministic 512-d Gaussian z a seed maps to. Exposed so the app /// can build a face's latent itself (for vector mode, perturbation, and /// "Use Vector"), then hand it to `png(latent:)`. static func gaussianLatent(seed: UInt64, count: Int = 512) -> [Float] { var rng = SplitMix64(state: seed == 0 ? 0x1234_5678_9ABC_DEF : seed) var values: [Float] = [] values.reserveCapacity(count) while values.count < count { var u1 = 0.0 repeat { u1 = unit(&rng) } while u1 <= .leastNonzeroMagnitude let u2 = unit(&rng) let radius = sqrt(-2 * log(u1)) let angle = 2 * Double.pi * u2 values.append(Float(radius * cos(angle))) if values.count < count { values.append(Float(radius * sin(angle))) } } return values } private static func unit(_ rng: inout SplitMix64) -> Double { Double(rng.next() >> 11) / Double(1 << 53) } private static func png(from tensor: ORTTensor) -> Data? { guard tensor.shape.count == 4, tensor.shape[0] == 1, tensor.shape[1] == 3 else { return nil } let height = Int(tensor.shape[2]) let width = Int(tensor.shape[3]) let plane = width * height guard plane > 0, tensor.data.count >= plane * 3 else { return nil } var pixels = [UInt8](repeating: 255, count: plane * 4) let source = tensor.data for index in 0.. UInt8 { let unit = min(max((value + 1) * 0.5, 0), 1) return UInt8((unit * 255).rounded()) } } private struct SplitMix64: RandomNumberGenerator { var state: UInt64 mutating func next() -> UInt64 { state &+= 0x9E37_79B9_7F4A_7C15 var mixed = state mixed = (mixed ^ (mixed >> 30)) &* 0xBF58_476D_1CE4_E5B9 mixed = (mixed ^ (mixed >> 27)) &* 0x94D0_49BB_1331_11EB return mixed ^ (mixed >> 31) } }