"""Database of the configurations tested and of their results, and Ashby charts.
A :class:`ResultsDatabase` is one SQLite file (Python's standard library). Every run
directory added (:meth:`ResultsDatabase.add_run`: the outputs of ``morphogen run``, or of
a run with ``outputs.database``) contributes one record per realization: its
configuration flattened into dotted keys (``model.type``, ``model.radius``,
``domain.size``, ``homogenization.materials.1.young``...) and its row of results
(measures, apparent properties, solid fraction, relative, normalized and specific values:
:mod:`morphogen.homogenization.postprocess`). Records are selected by conditions on any
key (:func:`parse_condition`): ``model.type == gyroid``, ``solid_fraction < 0.3``,
``model.type in gyroid,octet``, ``run_name ~ lattice*``.
Ashby charts (:func:`ashby_chart`) place records in a space of two properties, by default
normalized: a relative property (``E / E_s``) against the relative density (the solid
fraction), on logarithmic axes, one family per value of a key (the model by default),
with the scaling laws of cellular solids ``P / P_s = phi^n`` (Gibson & Ashby 1997:
``n = 1`` for stretching-dominated architectures, the rule of mixtures; ``n = 2`` for
bending-dominated open-cell foams).
"""
from __future__ import annotations
import csv
import fnmatch
import json
import math
import sqlite3
import time
from pathlib import Path
import numpy as np
#: Runs and their realizations; a record is the JSON of the flattened configuration
#: merged with the row of results of one realization.
SCHEMA = """
CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY,
path TEXT UNIQUE,
name TEXT,
added TEXT,
config TEXT,
version TEXT
);
CREATE TABLE IF NOT EXISTS samples (
run INTEGER REFERENCES runs(id) ON DELETE CASCADE,
realization INTEGER,
data TEXT,
PRIMARY KEY (run, realization)
);
"""
#: Parts of a configuration that do not describe the microstructure or its computation.
_SKIPPED = ("outputs", "rve")
OPERATORS = ("==", "!=", "<=", ">=", "<", ">", " not in ", " in ", "~")
[docs]
def flatten(data: dict, prefix: str = "") -> dict:
"""Nested mappings as one level of dotted keys; lists become JSON text."""
out: dict = {}
for key, value in data.items():
name = f"{prefix}{key}"
if isinstance(value, dict):
out.update(flatten(value, name + "."))
elif isinstance(value, (list, tuple)):
out[name] = json.dumps(value)
else:
out[name] = value
return out
def _value(text: str):
"""A condition's value: a number, a boolean, or a string."""
t = text.strip().strip("'\"")
low = t.lower()
if low in ("true", "false"):
return low == "true"
try:
return float(t)
except ValueError:
return t
[docs]
def parse_condition(text: str) -> tuple[str, str, object]:
"""``"key OP value"`` with OP one of ``== != < <= > >= in, not in`` (comma-separated
values) or ``~`` (shell pattern, e.g. ``lattice*``)."""
for op in OPERATORS:
key, sep, value = text.partition(op)
if sep and key.strip():
if op.strip() in ("in", "not in"):
return key.strip(), op.strip(), [_value(v) for v in value.split(",")]
return key.strip(), op, _value(value)
raise ValueError(f"condition {text!r}: expected 'key OP value' with OP in == != < <= > >= in ~")
def _equal(a, b) -> bool:
if isinstance(a, bool) or isinstance(b, bool):
return bool(a) == bool(b) and isinstance(a, bool) == isinstance(b, bool)
if isinstance(a, (int, float)) and isinstance(b, (int, float)):
return float(a) == float(b)
return str(a) == str(b)
[docs]
def matches(record: dict, conditions) -> bool:
"""Whether a record satisfies every condition (strings or parsed tuples); a missing
key fails every condition but ``!=`` and ``not in``."""
for cond in conditions:
key, op, value = parse_condition(cond) if isinstance(cond, str) else cond
if key not in record:
if op in ("!=", "not in"):
continue
return False
v = record[key]
if op == "==":
ok = _equal(v, value)
elif op == "!=":
ok = not _equal(v, value)
elif op == "in":
ok = any(_equal(v, w) for w in value)
elif op == "not in":
ok = not any(_equal(v, w) for w in value)
elif op == "~":
ok = fnmatch.fnmatchcase(str(v), str(value))
else:
try:
a, b = float(v), float(value)
except (TypeError, ValueError):
return False
ok = {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op]
if not ok:
return False
return True
[docs]
class ResultsDatabase:
"""SQLite database of runs and of the results of their realizations."""
def __init__(self, path):
self.path = Path(path)
self.connection = sqlite3.connect(str(self.path))
self.connection.execute("PRAGMA foreign_keys = ON")
self.connection.executescript(SCHEMA)
def __enter__(self) -> ResultsDatabase:
return self
def __exit__(self, *exc) -> None:
self.close()
[docs]
def close(self) -> None:
self.connection.close()
[docs]
def add_run(self, directory, *, name: str | None = None) -> int:
"""Add (or replace) the realizations of a run directory: its configuration
(``run.json``) and results (``results.csv``, or the files of split runs). Returns
the number of records."""
from morphogen import pipeline
directory = Path(directory).resolve()
manifests = [directory / "run.json", *sorted(directory.glob("run_task*of*.json"))]
manifest = next((m for m in manifests if m.exists()), None)
if manifest is None:
raise ValueError(f"{directory}: not the output directory of a run (no run.json)")
info = json.loads(manifest.read_text(encoding="utf-8"))
config = info.get("config", {})
rows = pipeline.collect_results(directory)
name = name or config.get("outputs", {}).get("name") or directory.name
version = info.get("versions", {}).get("morphogen", "")
return self.add_records(
rows, name=name, config=config, path=str(directory), version=version
)
[docs]
def add_records(
self,
rows: list[dict],
*,
name: str,
config: dict | None = None,
path: str | None = None,
version: str = "",
) -> int:
"""Add the rows of a run (one per realization) with its configuration; a run of
the same ``path`` (default: the name) is replaced."""
config = config or {}
base = flatten({k: v for k, v in config.items() if k not in _SKIPPED})
key = path or f"name:{name}"
with self.connection:
self.connection.execute("DELETE FROM runs WHERE path = ?", (key,))
cur = self.connection.execute(
"INSERT INTO runs (path, name, added, config, version) VALUES (?, ?, ?, ?, ?)",
(key, name, time.strftime("%Y-%m-%d %H:%M:%S"), json.dumps(config), version),
)
run = cur.lastrowid
data = []
for i, row in enumerate(rows):
record = {"run": run, "run_name": name, **base, **row}
realization = int(row.get("realization", i))
data.append((run, realization, json.dumps(record)))
self.connection.executemany(
"INSERT OR REPLACE INTO samples (run, realization, data) VALUES (?, ?, ?)", data
)
return len(rows)
[docs]
def remove(self, run) -> int:
"""Remove a run (its id, path or name); returns the number of runs removed."""
with self.connection:
if isinstance(run, int) or str(run).isdigit():
cur = self.connection.execute("DELETE FROM runs WHERE id = ?", (int(run),))
else:
text = str(run)
resolved = str(Path(text).resolve())
cur = self.connection.execute(
"DELETE FROM runs WHERE path IN (?, ?) OR name = ?", (text, resolved, text)
)
return cur.rowcount
[docs]
def runs(self) -> list[dict]:
"""Runs: id, name, path, date added, version and number of records."""
rows = self.connection.execute(
"SELECT r.id, r.name, r.path, r.added, r.version, COUNT(s.run) FROM runs r "
"LEFT JOIN samples s ON s.run = r.id GROUP BY r.id ORDER BY r.id"
).fetchall()
keys = ("id", "name", "path", "added", "version", "records")
return [dict(zip(keys, r, strict=True)) for r in rows]
[docs]
def records(self, where=()) -> list[dict]:
"""Records satisfying every condition of ``where`` (see :func:`matches`)."""
conditions = [parse_condition(c) if isinstance(c, str) else c for c in where]
out = []
for (data,) in self.connection.execute(
"SELECT data FROM samples ORDER BY run, realization"
):
record = json.loads(data)
if matches(record, conditions):
out.append(record)
return out
[docs]
def keys(self, where=()) -> dict[str, dict]:
"""Keys of the records: how many records hold each, and whether its values are all
numbers."""
out: dict[str, dict] = {}
for record in self.records(where):
for k, v in record.items():
entry = out.setdefault(k, {"count": 0, "numeric": True})
entry["count"] += 1
if isinstance(v, bool) or not isinstance(v, (int, float)):
entry["numeric"] = False
return dict(sorted(out.items()))
[docs]
def record_run(directory, database, *, name: str | None = None) -> int:
"""Add a run directory to a database file (created if needed)."""
with ResultsDatabase(database) as db:
return db.add_run(directory, name=name)
[docs]
def write_records(records: list[dict], path, keys=None) -> Path:
"""CSV of records (all their keys, or ``keys``): the table view of a chart."""
if keys is None:
keys = []
for r in records:
keys.extend(k for k in r if k not in keys)
path = Path(path)
with open(path, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(keys), lineterminator="\n")
writer.writeheader()
for r in records:
writer.writerow({k: r.get(k, "") for k in keys})
return path
# -- Ashby charts ----------------------------------------------------------------------------
#: Categorical slots of scatter charts (any two families can be neighbours: three hues pass
#: the colour-vision checks together; more families fold into "Other"), light and dark.
SERIES = {"light": ("#2a78d6", "#eb6834", "#1baf7a"), "dark": ("#3987e5", "#d95926", "#199e70")}
MARKERS = ("o", "s", "^")
THEMES = {
"light": {
"surface": "#fcfcfb",
"ink": "#0b0b0b",
"secondary": "#52514e",
"muted": "#898781",
"grid": "#e1e0d9",
"axis": "#c3c2b7",
},
"dark": {
"surface": "#1a1a19",
"ink": "#f0efec",
"secondary": "#c3c2b7",
"muted": "#898781",
"grid": "#2c2c2a",
"axis": "#383835",
},
}
OTHER = "Other"
[docs]
def default_axes(records: list[dict]) -> tuple[str, str]:
"""The relative density and the first relative property found (Young's modulus first,
then the bulk and shear moduli, then any)."""
keys = {k for r in records for k in r}
relative = sorted(k for k in keys if k.endswith(".relative"))
for prop in ("E1", "bulk_modulus", "shear_modulus", "conductivity"):
hits = [k for k in relative if prop in k.split(".")]
if hits:
return "solid_fraction", hits[0]
if relative:
return "solid_fraction", relative[0]
raise ValueError("no relative property in the records: give the keys of the axes")
[docs]
def ashby_points(
records: list[dict],
x: str | None = None,
y: str | None = None,
by: str | None = "model.type",
*,
log: bool = True,
groups: list | None = None,
max_groups: int = 3,
) -> dict:
"""Points of an Ashby chart: the families (values of ``by``) with their points, the
first ``max_groups`` in their own colour (by size, or the order of ``groups``), the
others folded into "Other". Records without finite values (positive ones on
logarithmic axes) are left out."""
if x is None or y is None:
dx, dy = default_axes(records)
x, y = x or dx, y or dy
families: dict[str, list] = {}
for r in records:
try:
vx, vy = float(r[x]), float(r[y])
except (KeyError, TypeError, ValueError):
continue
if not (math.isfinite(vx) and math.isfinite(vy)) or (log and (vx <= 0 or vy <= 0)):
continue
name = str(r.get(by, "")) if by else ""
families.setdefault(name, []).append(
{"x": vx, "y": vy, "run": r.get("run_name", ""), "realization": r.get("realization")}
)
order = [str(g) for g in groups if str(g) in families] if groups else []
order += sorted((g for g in families if g not in order), key=lambda g: -len(families[g]))
shown, folded = order[:max_groups], order[max_groups:]
out = [{"name": g, "slot": i, "points": families[g]} for i, g in enumerate(shown)]
if folded:
pts = [p for g in folded for p in families[g]]
out.append({"name": OTHER, "slot": None, "points": pts, "members": folded})
return {"x": x, "y": y, "by": by, "log": log, "groups": out}
#: Names and symbols of the properties in the keys of the records.
_NAMES = {
"bulk_modulus": ("bulk modulus", "K"),
"shear_modulus": ("shear modulus", "G"),
"conductivity": ("conductivity", "k"),
"mean": ("conductivity", "k"),
"E1": ("Young's modulus E1", "E"),
"E2": ("Young's modulus E2", "E"),
"E3": ("Young's modulus E3", "E"),
"G12": ("shear modulus G12", "G"),
"G23": ("shear modulus G23", "G"),
"G31": ("shear modulus G31", "G"),
}
[docs]
def axis_label(key: str) -> str:
"""Readable name of a key: the relative density, a relative property..."""
if key == "solid_fraction":
return "Relative density ρ / ρs (solid fraction)"
if key == "density":
return "Density ρ"
parts = key.split(".")
kind = parts[-1] if parts[-1] in ("relative", "normalized", "specific") else None
core = parts[:-1] if kind else parts
bcs = ("kubc", "periodic", "subc", "mixed")
bc = next((p for p in core if p in bcs), None)
core = [p for p in core if p not in bcs] or [key]
name, symbol = _NAMES.get(core[-1], (core[-1].replace("_", " "), "P"))
entry, notes = core[-1], [bc] if bc else []
if core[0] == "stiffness" and len(entry) == 3 and entry[0] in "CS" and entry[1:].isdigit():
# an entry of the stiffness or of its compliance, in morphoGen's Voigt notation
name = f"{'stiffness' if entry[0] == 'C' else 'compliance'} {entry}"
symbol, notes = entry, [*notes, "Voigt 11 22 33 23 31 12, engineering shears"]
suffix = f", {bc}" if bc else ""
if kind == "relative":
return f"Relative {name} ({symbol} / {symbol}s{suffix})"
if kind == "normalized":
return f"Normalized {name} ({symbol} / (ρ/ρs {symbol}s){suffix})"
if kind == "specific":
return f"Specific {name} ({symbol} / ρ{suffix})"
return f"{name[:1].upper()}{name[1:]}" + (f" ({'; '.join(notes)})" if notes else "")
def _hull(points: np.ndarray) -> np.ndarray | None:
"""Convex hull (in order) of 2D points, or None for fewer than three points or a
degenerate set."""
if len(points) < 3:
return None
from scipy.spatial import ConvexHull, QhullError
try:
return points[ConvexHull(points).vertices]
except QhullError:
return None
[docs]
def ashby_chart(
records: list[dict],
path,
x: str | None = None,
y: str | None = None,
by: str | None = "model.type",
*,
log: bool = True,
guides=None,
groups: list | None = None,
theme: str = "light",
title: str | None = None,
) -> Path:
"""Ashby chart of records as a PNG, SVG or PDF file (needs matplotlib): one family per
value of ``by`` (three in colour, with their markers, envelopes and direct labels;
others in "Other"), on logarithmic axes by default, with the lines ``y = x^n`` for
``guides`` (default: n = 1 and 2 when the axes are relative)."""
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
except ImportError: # pragma: no cover - optional dependency
raise ImportError("Ashby charts need matplotlib: pip install 'morphogen[plot]'") from None
if theme not in THEMES:
raise ValueError(f"theme: one of {list(THEMES)}")
data = ashby_points(records, x, y, by, log=log, groups=groups)
if not any(g["points"] for g in data["groups"]):
raise ValueError(f"no record has values of {data['x']!r} and {data['y']!r}")
colours, ink = SERIES[theme], THEMES[theme]
relative = data["x"] == "solid_fraction" and data["y"].endswith(".relative")
if guides is None:
guides = (1, 2) if relative and log else ()
fig, ax = plt.subplots(figsize=(6.4, 4.8), dpi=150)
fig.patch.set_facecolor(ink["surface"])
ax.set_facecolor(ink["surface"])
for side in ("top", "right"):
ax.spines[side].set_visible(False)
for side in ("left", "bottom"):
ax.spines[side].set_color(ink["axis"])
ax.spines[side].set_linewidth(0.8)
ax.tick_params(colors=ink["axis"], labelcolor=ink["secondary"], labelsize=8, width=0.8)
ax.grid(True, which="major", color=ink["grid"], linewidth=0.6)
ax.set_axisbelow(True)
if log:
ax.set_xscale("log")
ax.set_yscale("log")
count = sum(len(g["points"]) for g in data["groups"])
placed: list = [] # boxes of the labels drawn, in display coordinates
for g in data["groups"]:
pts = np.array([[p["x"], p["y"]] for p in g["points"]])
colour = ink["muted"] if g["slot"] is None else colours[g["slot"]]
marker = "o" if g["slot"] is None else MARKERS[g["slot"]]
hull = None if g["slot"] is None else _hull(np.log10(pts) if log else pts)
if hull is not None: # the envelope of a family (not of the others folded)
ax.fill(*(10.0**hull.T if log else hull.T), color=colour, alpha=0.1, linewidth=0)
if g["slot"] is not None:
label = g["name"]
elif len(g["members"]) == 1:
label = f"{OTHER}: {g['members'][0]}"
else:
label = f"{OTHER} ({len(g['members'])} families)"
ax.scatter(
pts[:, 0],
pts[:, 1],
s=40,
marker=marker,
color=colour,
edgecolors=ink["surface"],
linewidths=1.2,
zorder=3,
label=label or "all",
)
lo, hi = ax.get_xlim(), ax.get_ylim()
guide_marks = []
for n in guides:
xs = np.geomspace(max(lo[0], 1e-12), lo[1], 64) if log else np.linspace(*lo, 64)
ys = xs ** float(n)
keep = (ys >= hi[0]) & (ys <= hi[1])
if keep.sum() < 2:
continue
ax.plot(xs[keep], ys[keep], color=ink["muted"], linewidth=0.8, zorder=1)
guide_marks.append((f"n = {n:g}", np.column_stack([xs[keep], ys[keep]])))
ax.set_xlim(lo)
ax.set_ylim(hi)
fig.canvas.draw()
renderer = fig.canvas.get_renderer()
from matplotlib.transforms import Bbox
half = 4.0 * fig.dpi / 72.0 # the markers (s = 40 points^2), that labels must not hide
for g in data["groups"]:
for p in g["points"]:
cx, cy = ax.transData.transform((p["x"], p["y"]))
placed.append(Bbox.from_extents(cx - half, cy - half, cx + half, cy + half))
def place(text, anchors, offsets, **kw) -> None:
"""Draw a label at the first anchor and offset where it overlaps no other label
and no marker (``anchors``: one point or a list of them)."""
points = [anchors] if np.ndim(anchors) == 1 else list(anchors)
frame = ax.get_window_extent(renderer)
for xy in points:
for dx, dy, ha in offsets:
t = ax.annotate(text, xy, xytext=(dx, dy), textcoords="offset points", ha=ha, **kw)
box = t.get_window_extent(renderer).expanded(1.05, 1.15)
inside = frame.contains(box.x0, box.y0) and frame.contains(box.x1, box.y1)
if inside and not any(box.overlaps(b) for b in placed):
placed.append(box)
return
t.remove()
if len(data["groups"]) > 1: # beside the plot: it hides no point
legend = ax.legend(
frameon=False,
fontsize=8,
loc="upper left",
bbox_to_anchor=(1.02, 1.0),
title=by,
scatterpoints=1,
)
plt.setp(legend.get_texts(), color=ink["secondary"])
legend.get_title().set_color(ink["secondary"])
legend.get_title().set_fontsize(8)
for text, line in guide_marks: # the scaling laws, from their left end along them,
# under or over the rising line rather than across it
anchors = [line[int(f * (len(line) - 1))] for f in (0.0, 0.2, 0.4, 0.6)]
offsets = [(6, -12, "left"), (-6, 4, "right")]
place(text, anchors, offsets, fontsize=7, color=ink["muted"])
if len(data["groups"]) > 1: # direct labels of the families, where they fit
for g in data["groups"]:
if g["slot"] is None:
continue
pts = np.array([[p["x"], p["y"]] for p in g["points"]])
k = int(np.argmin(pts[:, 0]))
place(
g["name"],
pts[k],
[(-7, -3, "right"), (0, -13, "center"), (0, 7, "center"), (7, -3, "left")],
fontsize=8,
color=ink["secondary"],
)
if log:
from matplotlib.ticker import FuncFormatter, LogLocator
plain = FuncFormatter(lambda v, _: f"{v:g}")
for axis, (a, b) in ((ax.xaxis, lo), (ax.yaxis, hi)):
if np.log10(b / a) < 2.0: # less than two decades: ticks at 1, 2, 5
axis.set_major_locator(LogLocator(subs=(1.0, 2.0, 5.0)))
axis.set_major_formatter(plain)
axis.set_minor_formatter(FuncFormatter(lambda v, _: ""))
ax.set_xlabel(axis_label(data["x"]), color=ink["secondary"], fontsize=9)
ax.set_ylabel(axis_label(data["y"]), color=ink["secondary"], fontsize=9)
head = title or f"{axis_label(data['y'])} — {count} realization{'s' * (count != 1)}"
ax.set_title(head, color=ink["ink"], fontsize=10, loc="left")
path = Path(path)
fig.savefig(path, facecolor=ink["surface"], bbox_inches="tight")
plt.close(fig)
return path