models/convert/gancontrol.py
Python source for Making faces, 124 lines. Raw file
"""GAN-Control FFHQ: Amazon's generator and controllers to ONNX.
python gancontrol.py <model-folder> <repo> <output-prefix>
GAN-Control is a StyleGAN2 trained so that W splits into named slices:
identity, expression, pose, light, age, hair and the rest, 64 or 128 numbers
each. A small controller network for each attribute turns meaningful values
(yaw, pitch and roll in degrees, an age in years, a hair colour) into that
attribute's slice of W. The networks are built with the authors' classes (the
`repo` argument, gan-control) and exported separately:
<prefix>-map.onnx z [1, 512] -> w [1, 512]
<prefix>-synth.onnx ws [1, 16, 512] -> image [1, 3, 512, 512] in -1..1
<prefix>-orientation.onnx control [1, 3] -> [1, 64] yaw, pitch, roll
<prefix>-age.onnx control [1, 1] -> [1, 64] age in years
<prefix>-hair.onnx control [1, 3] -> [1, 64] hair colour, RGB 0..1
<prefix>-gamma.onnx control [1, 27] -> [1, 64] spherical-harmonic lighting
<prefix>-expression.onnx control [1, 64] -> [1, 64] expression coefficients
The app writes each controller's output into its slice of W, then repeats W
for the 16 rows of the synthesis network. The noise is fixed, so a seed always
gives the same face.
"""
import json
import os
import sys
from types import SimpleNamespace
import torch
import cpu_ops
from export import export, verify
folder, repo, prefix = sys.argv[1:4]
sys.path.insert(0, os.path.join(repo, "src"))
from gan_control.models import gan_model # noqa: E402
from gan_control.models.controller_model import FcStack # noqa: E402
from gan_control.models.gan_model import Generator # noqa: E402
from gan_control.utils.mini_batch_multi_split_utils import MiniBatchUtils # noqa: E402
# The same convolution with weights of a fixed shape, which ONNX can follow
gan_model.ModulatedConv2d.forward = cpu_ops.unfused_modulated_conv2d
def newest_checkpoint(path):
checkpoints = sorted(os.listdir(os.path.join(path, "checkpoint")))
return torch.load(os.path.join(path, "checkpoint", checkpoints[-1]), map_location="cpu", weights_only=False)
def settings(path):
with open(os.path.join(path, "args.json")) as f:
return SimpleNamespace(**json.load(f))
# The generator
generator_dir = os.path.join(folder, "generator")
config = settings(generator_dir)
model_config, training = config.model_config, config.training_config
batch_utils = MiniBatchUtils(training["mini_batch"], training["sub_groups_dict"], total_batch=training["batch"])
G = Generator(
model_config["size"],
model_config["latent_size"],
model_config["n_mlp"],
channel_multiplier=model_config["channel_multiplier"],
out_channels=model_config["img_channels"],
split_fc=model_config["split_fc"],
fc_config=batch_utils.get_fc_config(),
conv_transpose=model_config["conv_transpose"],
noise_mode=model_config["g_noise_mode"],
)
G.load_state_dict(newest_checkpoint(generator_dir)["g_ema"])
G = G.float().eval()
print(f" generator: {model_config['size']}px, {G.n_latent} ws rows")
class Mapping(torch.nn.Module):
def __init__(self, G):
super().__init__()
self.G = G
def forward(self, z):
return self.G.style(z)
class Synthesis(torch.nn.Module):
def __init__(self, G):
super().__init__()
self.G = G
torch.manual_seed(0)
self.noise = torch.nn.ParameterList(
[torch.nn.Parameter(n, requires_grad=False) for n in G.make_noise(device="cpu")])
def forward(self, ws):
image, _ = self.G([ws], input_is_latent=True, noise=list(self.noise))
return image
z = torch.randn(1, 512, generator=torch.Generator().manual_seed(1))
with torch.no_grad():
w = Mapping(G)(z)
ws = w[:, None, :].repeat(1, G.n_latent, 1)
export(Mapping(G), (z,), ["z"], ["w"], f"{prefix}-map.onnx")
export(Synthesis(G), (ws,), ["ws"], ["image"], f"{prefix}-synth.onnx")
verify(f"{prefix}-map.onnx", {"z": z}, Mapping(G)(z))
verify(f"{prefix}-synth.onnx", {"ws": ws}, Synthesis(G)(ws))
# The controllers, one per attribute
for name in ["orientation", "age", "hair", "gamma", "expression"]:
path = next(
os.path.join(folder, entry) for entry in sorted(os.listdir(folder))
if entry.startswith(name + "_") and not entry.startswith("expression_q"))
control_config = settings(path).model_config
start, end = batch_utils.place_in_latent_dict[name]
controller = FcStack(
control_config["lr_mlp"], control_config["n_mlp"], control_config["in_dim"],
control_config["mid_dim"], end - start)
controller.load_state_dict(newest_checkpoint(path)["controller"])
controller = controller.float().eval()
control = torch.rand(1, control_config["in_dim"], generator=torch.Generator().manual_seed(1))
output = f"{prefix}-{name}.onnx"
export(controller, (control,), ["control"], ["latent"], output)
verify(output, {"control": control}, controller(control))