"""Parallel runs: processes on one machine, job arrays on clusters, MPI.
A run is a list of realizations (items), each with its own seed derived from the seed
of the run and its indices (:mod:`morphogen.seeding`): the results do not depend on how
the items are spread over processes, tasks or nodes.
* Every realization leaves a *record* (``records/<name>.json``: its row of results and
its rows of apparent properties) as soon as it is done. A run started again with
``resume`` skips the realizations that have a record, so that an interrupted task (a
time limit on a cluster) loses at most the realizations in progress.
* ``jobs`` processes run the realizations of a task on one machine (``spawn`` start
method, on every platform); the threads (``threads``, by default all the cores) are
shared between them, so that the compiled kernels and the linear algebra do not
oversubscribe the cores.
* A run split in ``n_tasks`` tasks (``task_id`` = 0 ... n - 1) gives task ``i`` the items
``i, i + n, i + 2n...``: a job array of a scheduler, or the ranks of an MPI program
(:func:`run_mpi`). :func:`merge` gathers the records of all the tasks into the results
tables and writes the RVE report; :func:`status` counts them.
* :func:`job_script` writes the script of a job array for SLURM, PBS Pro, Torque, SGE,
OAR or LSF, and the script of the merging job.
"""
from __future__ import annotations
import json
import logging
import os
import re
import shlex
import subprocess
import sys
import time
from collections.abc import Iterator
from concurrent.futures import ProcessPoolExecutor, as_completed
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
import numpy as np
from morphogen._utils import to_jsonable
from morphogen.stopping import Stopped, checkpoint
log = logging.getLogger("morphogen")
#: Environment variables bounding the threads of the linear algebra libraries.
THREAD_VARIABLES = (
"OMP_NUM_THREADS",
"OPENBLAS_NUM_THREADS",
"MKL_NUM_THREADS",
"VECLIB_MAXIMUM_THREADS",
"NUMEXPR_NUM_THREADS",
)
[docs]
@dataclass(frozen=True)
class Item:
"""One realization of a run: its position, seed key, file stem and domain size
(``None``: the domain of the configuration), and the columns identifying its rows."""
index: int
key: tuple[int, ...]
stem: str
size: float | None
ident: dict = field(default_factory=dict, hash=False, compare=False)
[docs]
def items(cfg) -> list[Item]:
"""Realizations of a run (a batch, or the sizes and realizations of an ensemble
RVE study)."""
if cfg.rve is None:
width = max(4, len(str(cfg.realizations - 1)))
return [
Item(r, (r,), f"{cfg.outputs.name}_{r:0{width}d}", None, {"realization": r})
for r in range(cfg.realizations)
]
if cfg.rve.method != "ensemble":
raise ValueError("the window method of RVE studies runs one realization")
out: list[Item] = []
for s, n in enumerate(cfg.rve.realizations):
size = float(cfg.rve.sizes[s])
ident = {"size_index": s, "size": size, "volume": size**3}
for r in range(n):
stem = f"{cfg.outputs.name}_L{size:g}_{r:04d}"
out.append(Item(len(out), (s, r), stem, size, {**ident, "realization": r}))
return out
[docs]
def task_items(all_items: list[Item], task_id: int, n_tasks: int) -> list[Item]:
"""The items of a task among ``n_tasks`` (``task_id`` from 0): one item in ``n_tasks``, so that
the tasks share them evenly."""
if n_tasks < 1 or not 0 <= task_id < n_tasks:
raise ValueError(f"task id {task_id} must lie in [0, {n_tasks})")
return all_items[task_id::n_tasks]
# -- records -----------------------------------------------------------------------------
[docs]
def records_dir(outdir) -> Path:
"""The folder of the records of the items of a run, in its output folder."""
return Path(outdir) / "records"
[docs]
def record_path(outdir, item: Item) -> Path:
"""The record of an item of a run (``records/<item>.json`` in the output folder)."""
return records_dir(outdir) / f"{item.stem}.json"
[docs]
def write_record(outdir, item: Item, record: dict) -> Path:
"""Write a record atomically (a partial file never looks complete)."""
path = record_path(outdir, item)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(f".{os.getpid()}.tmp")
tmp.write_text(json.dumps(to_jsonable(record)), encoding="utf-8")
os.replace(tmp, path)
return path
[docs]
def read_records(outdir) -> list[dict]:
"""All the records of a run directory, in the order of the run."""
folder = records_dir(outdir)
out = []
for path in sorted(folder.glob("*.json")) if folder.is_dir() else []:
try:
out.append(json.loads(path.read_text(encoding="utf-8")))
except (OSError, ValueError):
log.warning("unreadable record %s (skipped)", path)
return sorted(out, key=lambda r: r["index"])
# -- execution -----------------------------------------------------------------------------
[docs]
def realize(cfg, item: Item, outdir) -> dict:
"""Generate and process one realization; returns its record."""
from morphogen.domain import Domain
from morphogen.pipeline import _process
from morphogen.seeding import derive_seed
t0 = time.time()
domain = cfg.domain
if item.size is not None:
domain = Domain(
size=(item.size,) * 3,
origin=cfg.domain.origin,
periodic=cfg.domain.periodic,
unit=cfg.domain.unit,
)
micro = cfg.model.generate(domain, derive_seed(cfg.seed, *item.key))
measures = cfg.analysis if cfg.rve is None else cfg.rve.properties
values, fe = _process(micro, cfg, item.stem, Path(outdir), measures, item.index)
if cfg.rve is None:
row = {**item.ident, "seed": cfg.seed if cfg.seed is not None else "", **micro.summary()}
else:
row = {**item.ident, **micro.summary()}
row.update(values)
row["time_s"] = round(time.time() - t0, 4)
record = {
"index": item.index,
"stem": item.stem,
"row": row,
"fe_rows": [{**item.ident, **f} for f in fe],
}
return json.loads(json.dumps(to_jsonable(record))) # as read back from a record
def _worker_init(threads: int, level: int) -> None:
from morphogen import settings
from morphogen._platform import full_speed
logging.basicConfig(level=level, format="%(levelname)s: %(message)s")
log.setLevel(level)
settings.set_num_threads(threads)
full_speed() # Windows: not inherited from the parent process
def _worker(cfg, item: Item, outdir) -> dict:
record = realize(cfg, item, outdir)
write_record(outdir, item, record)
return record
[docs]
def threads_per_job(jobs: int, threads: int | None = None) -> int:
"""Threads of each of ``jobs`` processes sharing ``threads`` (default: the cores)."""
total = threads if threads is not None else (os.cpu_count() or 1)
return max(1, int(total) // max(1, int(jobs)))
[docs]
@contextmanager
def process_pool(jobs: int, threads: int) -> Iterator[ProcessPoolExecutor]:
"""Pool of ``jobs`` processes (``spawn``) with ``threads`` threads each."""
import multiprocessing
env = {name: str(threads) for name in THREAD_VARIABLES}
saved = {name: os.environ.get(name) for name in env}
os.environ.update(env) # inherited by the processes: read when they import NumPy
try:
with ProcessPoolExecutor(
max_workers=jobs,
mp_context=multiprocessing.get_context("spawn"),
initializer=_worker_init,
initargs=(threads, log.getEffectiveLevel()),
) as pool:
yield pool
finally:
for name, value in saved.items():
if value is None:
os.environ.pop(name, None)
else:
os.environ[name] = value
[docs]
def execute(cfg, todo: list[Item], outdir, *, jobs: int = 1, resume: bool = False) -> list[dict]:
"""Records of the items ``todo`` (in their order), computed by ``jobs`` processes;
with ``resume``, the items that have a record are not computed again. Each
realization is a checkpoint of :mod:`morphogen.stopping` (a stop cancels those the
processes have not started)."""
outdir = Path(outdir)
records: dict[int, dict] = {}
pending = []
for item in todo:
path = record_path(outdir, item)
if resume and path.exists():
try:
records[item.index] = json.loads(path.read_text(encoding="utf-8"))
continue
except ValueError:
pass
pending.append(item)
if resume and records:
log.info("resuming: %d of %d realizations already done", len(records), len(todo))
if jobs <= 1 or len(pending) <= 1:
for item in pending:
checkpoint()
t0 = time.time()
records[item.index] = _worker(cfg, item, outdir)
log.info("%s done in %.2f s", item.stem, time.time() - t0)
else:
threads = threads_per_job(jobs, cfg.threads)
log.info("%d processes of %d threads", jobs, threads)
with process_pool(jobs, threads) as pool:
futures = {pool.submit(_worker, cfg, item, outdir): item for item in pending}
try:
for future in as_completed(futures):
item = futures[future]
records[item.index] = future.result()
log.info("%s done (%d/%d)", item.stem, len(records), len(todo))
checkpoint()
except Stopped:
pool.shutdown(wait=False, cancel_futures=True)
raise
return [records[item.index] for item in todo]
[docs]
def rows_of(records: list[dict]) -> tuple[list[dict], list[dict]]:
"""Results rows and homogenization rows of records."""
rows = [r["row"] for r in records]
fe_rows = [f for r in records for f in r["fe_rows"]]
return rows, fe_rows
# -- MPI -------------------------------------------------------------------------------------
[docs]
def run_mpi(cfg, *, jobs: int = 1, resume: bool = False) -> Path:
"""Run with the ranks of an MPI program (``mpirun -n N morphogen run config --mpi``):
rank ``i`` of ``N`` is task ``i``; rank 0 merges the results when all are done."""
try:
from mpi4py import MPI
except ImportError: # pragma: no cover - depends on the environment
raise ImportError("MPI runs need the optional 'mpi4py' package") from None
from morphogen import pipeline
comm = MPI.COMM_WORLD
rank, size = comm.Get_rank(), comm.Get_size()
if cfg.seed is None: # one seed for all the ranks
cfg.seed = comm.bcast(np.random.SeedSequence().entropy if rank == 0 else None, root=0)
outdir = pipeline.run(cfg, task_id=rank, n_tasks=size, jobs=jobs, resume=resume)
comm.Barrier()
if rank == 0 and size > 1:
merge(outdir)
comm.Barrier()
return outdir
# -- merging and status ------------------------------------------------------------------
def _manifest_value(outdir: Path, key: str):
"""An entry of the manifest of a run (``run.json``, or that of one of its tasks)."""
for path in [outdir / "run.json", *sorted(outdir.glob("run_task*of*.json"))]:
if path.exists():
return json.loads(path.read_text(encoding="utf-8")).get(key)
return None
def _run_config(outdir: Path):
"""Configuration of a run directory (from its run.json or run_task*.json)."""
from morphogen.config import load_config
files = [outdir / "run.json", *sorted(outdir.glob("run_task*of*.json"))]
for path in files:
if path.exists():
data = json.loads(path.read_text(encoding="utf-8"))
source = dict(data["config"])
if data.get("seed") is not None:
source["seed"] = data["seed"]
cfg = load_config(source, base_dir=outdir)
cfg.outputs.directory = outdir
return cfg
raise ValueError(f"{outdir}: no run.json (not a morphoGen run directory)")
[docs]
def merge(outdir, *, report: bool = True) -> Path:
"""Gather the records of all the tasks of a run into ``results.csv`` and
``homogenization.csv``, and write the RVE report of an ensemble study."""
from morphogen import pipeline
outdir = Path(outdir)
cfg = _run_config(outdir)
expected = items(cfg)
records = read_records(outdir)
done = {r["stem"] for r in records}
missing = [it.stem for it in expected if it.stem not in done]
if missing:
raise ValueError(
f"{len(missing)} of {len(expected)} realizations have no record yet (e.g. "
f"{missing[0]}); run the missing tasks again with --resume, see 'morphogen status'"
)
rows, fe_rows = rows_of(records)
pipeline.write_rows(outdir / "results.csv", rows)
if fe_rows:
pipeline.write_rows(outdir / "homogenization.csv", fe_rows)
if report and cfg.rve is not None and rows:
pipeline.write_report(rows, outdir, cfg.rve, cfg, fe_rows=fe_rows)
database = _manifest_value(outdir, "database")
if database is not None:
from morphogen.results import record_run
record_run(outdir, database)
log.info("merged %d realizations in %s", len(records), outdir)
return outdir
[docs]
def status(outdir) -> dict:
"""Realizations done and missing in a run directory."""
outdir = Path(outdir)
cfg = _run_config(outdir)
expected = items(cfg)
records = read_records(outdir)
done = {r["stem"] for r in records}
times = [r["row"].get("time_s", 0.0) for r in records]
missing = [it.stem for it in expected if it.stem not in done]
return {
"directory": str(outdir),
"expected": len(expected),
"done": len(expected) - len(missing),
"missing": missing,
"mean_time_s": float(np.mean(times)) if times else float("nan"),
"merged": (outdir / "results.csv").exists(),
}
# -- job scripts -------------------------------------------------------------------------------
#: Array directive, task index (0-based) and submission command of each scheduler.
SCHEDULERS = {
"slurm": {
"prefix": "#SBATCH",
"array": "--array=0-{last}",
"index": "${SLURM_ARRAY_TASK_ID}",
"name": "--job-name={name}",
"cores": "--cpus-per-task={cores}",
"time": "--time={time}",
"memory": "--mem={memory}",
"queue": "--partition={queue}",
"account": "--account={account}",
"log": ["--output={logs}/%x_%A_%a.log"],
"submit": ["sbatch", "--parsable"],
"after": ["--dependency=afterok:{job}"],
},
"pbs": {
"prefix": "#PBS",
"array": "-J 0-{last}",
"index": "${PBS_ARRAY_INDEX}",
"name": "-N {name}",
"cores": "-l select=1:ncpus={cores}",
"time": "-l walltime={time}",
"memory": "-l mem={memory}",
"queue": "-q {queue}",
"account": "-A {account}",
"log": ["-o {logs}/", "-j oe"],
"submit": ["qsub"],
"after": ["-W", "depend=afterok:{job}"],
},
"torque": {
"prefix": "#PBS",
"array": "-t 0-{last}",
"index": "${PBS_ARRAYID}",
"name": "-N {name}",
"cores": "-l nodes=1:ppn={cores}",
"time": "-l walltime={time}",
"memory": "-l mem={memory}",
"queue": "-q {queue}",
"account": "-A {account}",
"log": ["-o {logs}/", "-j oe"],
"submit": ["qsub"],
"after": ["-W", "depend=afterokarray:{job}"],
},
"sge": {
"prefix": "#$",
"array": "-t 1-{count}",
"index": "$((SGE_TASK_ID - 1))",
"name": "-N {name}",
"cores": "-pe smp {cores}",
"time": "-l h_rt={time}",
"memory": "-l h_vmem={memory}",
"queue": "-q {queue}",
"account": "-A {account}",
"log": ["-o {logs}/", "-j y"],
"submit": ["qsub", "-terse"],
"after": ["-hold_jid", "{job}"],
},
"oar": {
"prefix": "#OAR",
"array": "--array {count}",
"index": "$((OAR_ARRAY_INDEX - 1))",
"name": "-n {name}",
"cores": "-l /nodes=1/core={cores},walltime={time}",
"time": None, # with the cores
"memory": None,
"queue": "-q {queue}",
"account": "--project {account}",
"log": ["-O {logs}/%jobid%.log", "-E {logs}/%jobid%.log"],
"submit": ["oarsub", "-S"],
"after": ["--anterior={job}"],
},
"lsf": {
"prefix": "#BSUB",
"array": None, # in the job name
"index": "$((LSB_JOBINDEX - 1))",
"name": "-J {name}[1-{count}]",
"cores": "-n {cores}",
"time": "-W {time}",
"memory": "-M {memory}",
"queue": "-q {queue}",
"account": "-P {account}",
"log": ["-o {logs}/%J_%I.log"],
"submit": ["bsub"],
"after": ["-w", "done({job})"],
},
}
[docs]
@dataclass
class JobOptions:
"""Resources of the tasks of a job array."""
tasks: int
jobs: int = 1
threads: int = 1
time: str = "01:00:00"
memory: str | None = None
queue: str | None = None
account: str | None = None
name: str = "morphogen"
setup: list[str] = field(default_factory=list) # shell lines (modules, environments)
@property
def cores(self) -> int:
return self.jobs * self.threads
def _directives(scheduler: str, opts: JobOptions, logs: Path, *, array: bool) -> list[str]:
spec = SCHEDULERS[scheduler]
time_limit = opts.time
if scheduler == "lsf": # LSF: [hours:]minutes
parts = [int(x) for x in time_limit.split(":")]
minutes = parts[0] * 60 + parts[1] + (1 if len(parts) > 2 and parts[2] else 0)
time_limit = f"{minutes // 60}:{minutes % 60:02d}" if len(parts) > 2 else time_limit
values = {
"name": opts.name if array else f"{opts.name}-merge",
"last": opts.tasks - 1,
"count": opts.tasks,
"cores": opts.cores if array else 1,
"time": time_limit,
"memory": opts.memory,
"queue": opts.queue,
"account": opts.account,
"logs": logs,
}
array = array and opts.tasks > 1
name = spec["name"] if (array or scheduler != "lsf") else "-J {name}"
lines = [name]
if array and spec["array"]:
lines.append(spec["array"])
lines += [spec["cores"], *spec["log"]]
for key in ("time", "memory", "queue", "account"):
if spec[key] and values[key] is not None:
lines.append(spec[key])
return [f"{spec['prefix']} {line.format(**values)}" for line in lines]
[docs]
def job_script(
scheduler: str,
config,
outdir,
opts: JobOptions,
*,
seed: int | None = None,
executable: str | None = None,
) -> tuple[str, str]:
"""Scripts of the job array running a configuration in ``opts.tasks`` tasks (each
with ``opts.jobs`` processes of ``opts.threads`` threads, resumable), and of the job
merging the results."""
if scheduler not in SCHEDULERS:
raise ValueError(f"scheduler: one of {sorted(SCHEDULERS)}")
if opts.tasks < 1 or opts.jobs < 1 or opts.threads < 1:
raise ValueError("tasks, jobs and threads: positive integers")
outdir = Path(outdir).resolve()
logs = outdir / "logs"
exe = executable or f"{shlex.quote(sys.executable)} -m morphogen"
index = SCHEDULERS[scheduler]["index"] if opts.tasks > 1 else "0"
command = [
exe,
"run",
shlex.quote(str(Path(config).resolve())),
"-o",
shlex.quote(str(outdir)),
"--task-id",
f'"{index}"',
"--n-tasks",
str(opts.tasks),
"--jobs",
str(opts.jobs),
"--threads",
str(opts.cores),
"--resume",
"--quiet",
]
if seed is not None:
command += ["--seed", str(seed)]
header = ["#!/bin/bash"]
body = [
"set -euo pipefail",
*opts.setup,
f"mkdir -p {shlex.quote(str(logs))}",
]
array = "\n".join(
[*header, *_directives(scheduler, opts, logs, array=True), *body, " ".join(command)]
)
merge_cmd = f"{exe} merge {shlex.quote(str(outdir))}"
merging = "\n".join(
[*header, *_directives(scheduler, opts, logs, array=False), *body, merge_cmd]
)
return array + "\n", merging + "\n"
def _job_id(scheduler: str, output: str) -> str:
"""Identifier of a submitted job in the output of the submission command."""
text = output.strip()
last = text.splitlines()[-1].strip() if text else ""
if scheduler == "slurm":
return last.split(";")[0]
if scheduler in ("pbs", "torque"):
return last
if scheduler == "lsf":
match = re.search(r"<(\d+)>", text)
elif scheduler == "oar":
match = re.search(r"OAR_ARRAY_ID=(\d+)", text) or re.search(r"OAR_JOB_ID=(\d+)", text)
else:
match = re.match(r"\d+", last)
if match is None:
raise RuntimeError(f"no job id in the output of the submission: {text!r}")
return match.group(1) if match.groups() else match.group(0)
[docs]
def submit(scheduler: str, array_script: Path, merge_script: Path) -> list[str]:
"""Submit the job array, then the merging job after it; returns their ids."""
spec = SCHEDULERS[scheduler]
def call(extra, script):
if scheduler == "lsf": # bsub reads the directives of a script from its input
with open(script, encoding="utf-8") as f:
res = subprocess.run(
[*spec["submit"], *extra], stdin=f, capture_output=True, text=True, check=True
)
else:
res = subprocess.run(
[*spec["submit"], *extra, str(script)], capture_output=True, text=True, check=True
)
return _job_id(scheduler, res.stdout)
for script in (array_script, merge_script):
Path(script).chmod(0o755)
first = call([], array_script)
after = [part.format(job=first) for part in spec["after"]]
return [first, call(after, merge_script)]