"""Training of the models: Adam on mini-batches, early stopping on a validation set.
The parameters follow Adam (Kingma & Ba 2015) on the gradients of mini-batches of the
training set (points, or whole paths for a recurrent model), shuffled at each epoch;
the global norm of a gradient is clipped to ``clip``, the learning rate falls
geometrically from ``learning_rate`` to ``final`` times it over the epochs. The
parameters kept are those of the least loss of the validation set (of the training set
without one); the training stops after ``patience`` epochs without a better one. The
seeds of the initial parameters (the model's) and of the shuffles (the training's)
make it reproducible. The mini-batches may be turned at each epoch by the symmetries of
the behaviour: uniform rotations (isotropic behaviours) or the 24 rotations of the cube (a
cubic crystal in its axes), data that the behaviour gives without more responses.
"""
from __future__ import annotations
import copy
import itertools
import time
from collections.abc import Callable
from dataclasses import dataclass, field
import numpy as np
from morphogen.learning.data import Dataset, _tensor, _voigt
from morphogen.learning.networks import Model
[docs]
@dataclass
class Training:
"""A training: the model (its parameters those of the best epoch), the losses of the
training and validation sets at each epoch (mean squares of the gaps of the
normalized stresses), the best epoch, whether it stopped early, its time."""
model: Model
train_loss: list[float] = field(default_factory=list)
validation_loss: list[float] = field(default_factory=list)
best_epoch: int = -1
stopped: bool = False
time_s: float = 0.0
@property
def epochs(self) -> int:
return len(self.train_loss)
@property
def best_loss(self) -> float:
return float(self.validation_loss[self.best_epoch])
[docs]
def summary(self) -> dict:
return {"epochs": self.epochs, "best_epoch": self.best_epoch,
"train_loss": float(self.train_loss[self.best_epoch]),
"validation_loss": self.best_loss, "stopped": self.stopped,
"time_s": round(self.time_s, 3)} # fmt: skip
[docs]
def rotations(n: int, rng) -> np.ndarray:
"""``n`` rotation matrices drawn uniformly (n, 3, 3): the QR decompositions of Gaussian
matrices, the signs of the diagonals of R taken out, the determinants made positive."""
q, r = np.linalg.qr(rng.normal(size=(n, 3, 3)))
q = q * np.sign(np.diagonal(r, axis1=1, axis2=2))[:, None, :]
q[:, :, 0] *= np.sign(np.linalg.det(q))[:, None]
return q
[docs]
def cubic_rotations() -> np.ndarray:
"""The 24 rotations of the cube (24, 3, 3): the signed permutation matrices of
determinant 1, the proper symmetries of a cubic crystal in its axes."""
out = []
for perm in itertools.permutations(range(3)):
for signs in itertools.product((1.0, -1.0), repeat=3):
m = np.zeros((3, 3))
m[range(3), perm] = signs
if np.linalg.det(m) > 0:
out.append(m)
return np.array(out)
#: The symmetries the mini-batches may be turned by (``rotate`` of :func:`train`)
SYMMETRIES = ("isotropic", "cubic")
def _symmetry(rotate):
"""The rotations of ``rotate``: None (none), "isotropic" (uniform: True) or the group
of the cube (``"cubic"``)."""
if rotate in (False, None, "none"):
return None
if rotate is True or rotate == "isotropic":
return "isotropic"
if rotate == "cubic":
return cubic_rotations()
raise ValueError(f"rotate: false, true or one of {', '.join(SYMMETRIES)}, got {rotate!r}")
[docs]
def voigt_rotations(rot) -> tuple[np.ndarray, np.ndarray]:
"""The 6 x 6 matrices (n, 6, 6) that turn the Voigt components of strains (engineering
shears) and of stresses by rotations ``rot`` (n, 3, 3): ``R t R^T``."""
eye = np.eye(6)
out = []
for engineering in (True, False):
basis = _tensor(eye, engineering) # (6, 3, 3): the tensors of the components
turned = np.einsum("nij,kjl,nml->nkim", rot, basis, rot)
out.append(np.swapaxes(_voigt(turned, engineering), 1, 2))
return out[0], out[1]
[docs]
def rotated(data: Dataset, rng, points: bool = False) -> Dataset:
"""The paths of ``data`` each turned by a random rotation (each point, with
``points``): the responses of an isotropic behaviour along the turned paths."""
n = data.n_paths * (data.n_points if points else 1)
te, ts = voigt_rotations(rotations(n, rng))
shape = (data.n_paths, data.n_points if points else 1, 6, 6)
te, ts = te.reshape(shape), ts.reshape(shape)
strain = np.einsum("ptij,ptj->pti", te, data.strain)
stress = np.einsum("ptij,ptj->pti", ts, data.stress)
return Dataset(data.times, strain, stress, data.measure, data.info)
def _turned(model: Model, arrays, rng, group="isotropic") -> tuple:
"""A mini-batch of normalized arrays (:meth:`~morphogen.learning.networks.Model.
prepare`) turned by a random rotation per sample (a point, or a path): uniform, or of
the rotations ``group`` (n, 3, 3); the scales are scalars, the rotations act on the
normalized strains, increments and stresses."""
n = arrays[0].shape[0]
rot = rotations(n, rng) if isinstance(group, str) else group[rng.integers(len(group), size=n)]
te, ts = voigt_rotations(rot)
if not model.recurrent:
x, y = arrays
return np.einsum("nij,nj->ni", te, x), np.einsum("nij,nj->ni", ts, y)
if model.kind == "standard": # the strains, the time steps, the stresses
x, dt, y = arrays
return np.einsum("nij,ntj->nti", te, x), dt, np.einsum("nij,ntj->nti", ts, y)
u, x, y = arrays
u = u.copy()
u[..., :6] = np.einsum("nij,ntj->nti", te, u[..., :6])
u[..., 6:12] = np.einsum("nij,ntj->nti", te, u[..., 6:12])
return u, np.einsum("nij,ntj->nti", te, x), np.einsum("nij,ntj->nti", ts, y)
[docs]
def train(
model: Model,
data: Dataset,
validation: Dataset | None = None,
*,
epochs: int = 200,
batch: int = 32,
learning_rate: float = 0.01,
final: float = 0.1,
patience: int | None = None,
clip: float = 1.0,
rotate: bool | str = False,
seed: int = 0,
callback: Callable[[int, float, float], None] | None = None,
) -> Training:
"""Train ``model`` on ``data`` (see the module), set up on them first when it has no
parameters; ``validation``: the paths of the early stopping. With ``rotate`` (true or
``"isotropic"``: an isotropic behaviour), each mini-batch of paths is turned by random
rotations at each epoch (:func:`rotated`): the data of every direction, without more
responses; with ``"cubic"`` (a cubic crystal, its data in its axes), by random
rotations of the cube (:func:`cubic_rotations`). ``callback(epoch, train_loss,
validation_loss)`` after each epoch (it may raise :class:`~morphogen.stopping.
Stopped`)."""
if int(epochs) < 1 or int(batch) < 1 or not learning_rate > 0 or not 0 < final <= 1:
raise ValueError("epochs, batch: at least one; learning_rate positive; final in (0, 1]")
group = _symmetry(rotate)
if data.measure != model.measure and model.ready:
raise ValueError(f"data in {data.measure} strains, the model in {model.measure}")
t0 = time.perf_counter()
if not model.ready:
model.setup(data)
arrays = model.prepare(data)
checks = model.prepare(validation) if validation is not None else None
n = arrays[0].shape[0]
rng = np.random.default_rng(seed)
first = {k: np.zeros_like(v) for k, v in model.params.items()}
second = {k: np.zeros_like(v) for k, v in model.params.items()}
beta1, beta2, eps = 0.9, 0.999, 1e-8
out = Training(model)
best, kept, wait, count = np.inf, copy.deepcopy(model.params), 0, 0
for epoch in range(int(epochs)):
rate = learning_rate * final ** (epoch / max(int(epochs) - 1, 1))
order = rng.permutation(n)
total = 0.0
for start in range(0, n, int(batch)):
idx = order[start : start + int(batch)]
sub = tuple(a[idx] for a in arrays)
if group is not None:
sub = _turned(model, sub, rng, group)
loss, grads = model.objective(sub)
total += loss * idx.size
norm = np.sqrt(sum(float(np.sum(g * g)) for g in grads.values()))
factor = min(1.0, clip / norm) if clip and norm > 0 else 1.0
count += 1
c1, c2 = 1.0 - beta1**count, 1.0 - beta2**count
for k, g in grads.items():
g = g * factor
first[k] = beta1 * first[k] + (1.0 - beta1) * g
second[k] = beta2 * second[k] + (1.0 - beta2) * g * g
model.params[k] = model.params[k] - rate * (first[k] / c1) / (
np.sqrt(second[k] / c2) + eps
)
out.train_loss.append(total / n)
check = model.objective(checks, grad=False)[0] if checks is not None else total / n
out.validation_loss.append(float(check))
if callback is not None:
callback(epoch, out.train_loss[-1], float(check))
if check < best:
best, kept, wait, out.best_epoch = check, copy.deepcopy(model.params), 0, epoch
else:
wait += 1
if patience is not None and wait >= int(patience):
out.stopped = True
break
model.params = kept
out.time_s = time.perf_counter() - t0
model.metadata["training"] = out.summary()
return out
[docs]
def errors(model: Model, data: Dataset) -> dict:
"""The gaps of the stresses of ``model`` to those of ``data``: relative to the root
mean square of the stresses of each path (``paths``: one per path; their ``mean`` and
``max``) and of all (``relative``), and the root mean square of the gaps (``rmse``,
in the units of the stresses). A recurrent model runs each path from its natural
state."""
if model.recurrent:
predicted = model.predict(data.strain, data.times)
else:
predicted = model.predict(data.strain)
gap = np.sum((predicted - data.stress) ** 2, axis=-1)
norm = np.sum(data.stress**2, axis=-1)
paths = np.sqrt(gap.sum(axis=1) / np.maximum(norm.sum(axis=1), 1e-300))
return {"relative": float(np.sqrt(gap.sum() / max(norm.sum(), 1e-300))),
"mean": float(paths.mean()), "max": float(paths.max()),
"rmse": float(np.sqrt(gap.mean() / 6.0)), "paths": paths.tolist()} # fmt: skip