Source code for morphogen.dynamics.openradioss

"""OpenRadioss on this machine: where it is installed (:func:`find_openradioss`), the
environment its programs need, its runs (:func:`run_openradioss`: the Starter, then the
Engine, their progress followed in their listings, stopped at a checkpoint of
:mod:`morphogen.stopping`), the conversion of its outputs (``anim_to_vtk`` for the
animations, ``th_to_csv`` for the time histories) and the installation of a release that
the OpenRadioss project publishes on GitHub (:func:`install_openradioss`).

OpenRadioss (https://github.com/OpenRadioss/OpenRadioss, AGPL-3.0) is a separate
program: morphoGen writes its inputs, runs its executables and reads their outputs, and
neither ships nor links it. An installation is the folder of a release (its ``exec``,
``hm_cfg_files`` and ``extlib`` folders), found from, in order: the path given, the
environment variables ``MORPHOGEN_OPENRADIOSS`` and ``OPENRADIOSS_PATH`` (that of
OpenRadioss's documentation), the folder where :func:`install_openradioss` puts it
(``~/morphogen/openradioss``), and a Starter on the ``PATH``."""

from __future__ import annotations

import json
import os
import platform
import shutil
import stat
import subprocess
import sys
import tempfile
import time
import urllib.request
import zipfile
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path

from morphogen._version import __version__
from morphogen.stopping import Stopped, checkpoint

#: The environment variables of the folder of an installation, in order
ENVIRONMENT = ("MORPHOGEN_OPENRADIOSS", "OPENRADIOSS_PATH")
#: The repository of OpenRadioss, whose releases :func:`install_openradioss` installs
REPOSITORY = "OpenRadioss/OpenRadioss"
GITHUB_API = "https://api.github.com"
#: The file that records the release installed in its folder
RELEASE_FILE = "morphogen_release.json"
#: The words of the names of the archives of the releases, by system and machine
ASSETS = {
    ("Linux", "x86_64"): ("linux64",),
    ("Linux", "aarch64"): ("linuxa64", "linux_arm64", "aarch64", "arm64"),
    ("Linux", "arm64"): ("linuxa64", "linux_arm64", "aarch64", "arm64"),
    ("Windows", "AMD64"): ("win64",),
    ("Windows", "x86_64"): ("win64",),
    ("Darwin", "arm64"): ("macos", "mac", "darwin"),
    ("Darwin", "x86_64"): ("macos", "mac", "darwin"),
}


[docs] def default_home() -> Path: """The folder where :func:`install_openradioss` installs OpenRadioss by default.""" return Path.home() / "morphogen" / "openradioss"
[docs] @dataclass class OpenRadioss: """An installation of OpenRadioss: its folder (``root``), the Starter, the Engine (and its MPI version), the converters of its outputs when it has them, the suffix of the names of its executables (``linux64_gf``, ``win64``...) and the release installed (when :func:`install_openradioss` installed it).""" root: Path starter: Path engine: Path engine_mpi: Path | None = None anim_to_vtk: Path | None = None th_to_csv: Path | None = None suffix: str = "" release: dict = field(default_factory=dict) @property def version(self) -> str | None: return self.release.get("tag")
[docs] def library_folders(self) -> list[Path]: """The folders of the shared libraries of its programs (the reader of the inputs, H3D, the runtimes of the compilers), for the platform of the executables.""" extlib = self.root / "extlib" if not extlib.is_dir(): return [] names = {self.suffix.split("_")[0], self.suffix} found = [] for folder in sorted(extlib.rglob("*")): if folder.is_dir() and folder.name in names: found.append(folder) return found
[docs] def environment(self, threads: int | None = None, base: dict | None = None) -> dict: """The environment of its programs: ``OPENRADIOSS_PATH``, ``RAD_CFG_PATH`` (the configuration of the reader of the inputs), ``RAD_H3D_PATH``, the stack of the OpenMP threads, their number, and the folders of its libraries on the search path of the system.""" env = dict(os.environ if base is None else base) env["OPENRADIOSS_PATH"] = str(self.root) env["RAD_CFG_PATH"] = str(self.root / "hm_cfg_files") folders = self.library_folders() h3d = [f for f in folders if "h3d" in f.parts] if h3d: env["RAD_H3D_PATH"] = str(h3d[0]) env.setdefault("OMP_STACKSIZE", "400m") env.setdefault("KMP_STACKSIZE", "400m") if threads: env["OMP_NUM_THREADS"] = str(int(threads)) if folders: name = {"win32": "PATH", "darwin": "DYLD_LIBRARY_PATH"}.get( sys.platform, "LD_LIBRARY_PATH" ) paths = [str(f) for f in folders] if env.get(name): paths.append(env[name]) env[name] = os.pathsep.join(paths) return env
[docs] def describe(self) -> dict: """Its folder, executables and release, for the page and the command line.""" out = { "root": str(self.root), "starter": str(self.starter), "engine": str(self.engine), "engine (MPI)": str(self.engine_mpi) if self.engine_mpi else "none", "anim_to_vtk": str(self.anim_to_vtk) if self.anim_to_vtk else "none", "th_to_csv": str(self.th_to_csv) if self.th_to_csv else "none", } if self.version: out["release"] = self.version return out
def _executable(path: Path) -> bool: return path.is_file() and (sys.platform == "win32" or os.access(path, os.X_OK)) def _make_executable(folder: Path) -> None: """Restore the permission to run the programs of a folder (an archive unpacked without its permission bits).""" if sys.platform == "win32" or not folder.is_dir(): return for path in folder.iterdir(): if path.is_file() and not os.access(path, os.X_OK): try: path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) except OSError: pass def _exec_folder(path: Path) -> Path | None: """The folder of the executables of the installation at ``path``: ``path/exec``, a folder holding a Starter, the folder of a Starter given, or that of the only installation in ``path`` (an archive unpacked into it).""" path = Path(path).expanduser() if path.is_file(): return path.parent if path.name.lower().startswith("starter_") else None if not path.is_dir(): return None for candidate in (path / "exec", path): if candidate.is_dir() and any( p.name.lower().startswith("starter_") for p in candidate.iterdir() ): return candidate inner = [p for p in path.iterdir() if p.is_dir() and (p / "exec").is_dir()] return inner[0] / "exec" if len(inner) == 1 else None def _pick(folder: Path, prefix: str, suffix: str, variants=("",)) -> Path | None: """The program ``<prefix><suffix><variant>`` of a folder (with ``.exe`` on Windows), the first variant found.""" for variant in variants: for ext in ("", ".exe"): path = folder / f"{prefix}{suffix}{variant}{ext}" if path.is_file(): return path return None def _installation(folder: Path) -> OpenRadioss: folder = Path(folder).resolve() _make_executable(folder) starters = sorted( p for p in folder.iterdir() if p.name.lower().startswith("starter_") and p.suffix.lower() in ("", ".exe") and not p.stem.endswith("_sp") ) if not starters: raise FileNotFoundError(f"no Starter of OpenRadioss in {folder}") # the double precision Starter of the machine: the shortest name (no _sp, no variant) starter = min(starters, key=lambda p: len(p.stem)) suffix = starter.stem[len("starter_") :] engine = _pick(folder, "engine_", suffix) if engine is None: raise FileNotFoundError(f"no Engine of OpenRadioss in {folder} (engine_{suffix})") root = folder.parent if folder.name == "exec" else folder release = {} for place in (root, root.parent): record = place / RELEASE_FILE if record.is_file(): try: release = json.loads(record.read_text(encoding="utf-8")) except (OSError, ValueError): release = {} break return OpenRadioss( root=root, starter=starter, engine=engine, engine_mpi=_pick(folder, "engine_", suffix, ("_ompi", "_impi", "_mpi")), anim_to_vtk=_pick(folder, "anim_to_vtk_", suffix), th_to_csv=_pick(folder, "th_to_csv_", suffix), suffix=suffix, release=release, )
[docs] def find_openradioss(path=None) -> OpenRadioss: """The installation of OpenRadioss at ``path`` (its folder, its ``exec`` folder or its Starter), else the first found in the places of the module; raises FileNotFoundError, saying how to install it, when there is none.""" tried = [] places = [(str(path), "the path given")] if path not in (None, "") else [] places += [(os.environ[v], f"${v}") for v in ENVIRONMENT if os.environ.get(v)] places.append((str(default_home()), "the folder of morphogen radioss install")) for place, origin in places: folder = _exec_folder(Path(place)) if folder is not None: return _installation(folder) tried.append(f"{origin} ({place})") if origin == "the path given": raise FileNotFoundError(f"no installation of OpenRadioss at {place}") for name in ("starter_linux64_gf", "starter_win64", "starter_linuxa64", "starter_macos64"): exe = shutil.which(name) if exe: return _installation(Path(exe).resolve().parent) raise FileNotFoundError( "OpenRadioss not found (tried " + ", ".join(tried) + " and the PATH): install a " "release with `morphogen radioss install` (or the tool OpenRadioss installation of the " "interface), or set OPENRADIOSS_PATH to the folder of one (https://github.com/OpenRadioss/" "OpenRadioss/releases)" )
[docs] def has_openradioss(path=None) -> bool: """Whether an installation of OpenRadioss is found.""" try: find_openradioss(path) except FileNotFoundError: return False return True
# -- runs ----------------------------------------------------------------------------------
[docs] @dataclass class RadiossRun: """A run of OpenRadioss in its ``directory``: its animations converted to legacy VTK files (``frames``, in their order), its time history converted to CSV (``history``, None without ``th_to_csv``), its listings and the time it took (s).""" directory: Path name: str frames: list[Path] history: Path | None listings: dict[str, Path] elapsed: float = 0.0 warnings: list[str] = field(default_factory=list)
def _wait( process: subprocess.Popen, listing: Path | None, end_time: float | None, progress: Callable | None, span: tuple[float, float], label: str, timeout: float | None, ) -> int: """Wait for a program, its progress read in its listing (the time of its last cycle over ``end_time``, mapped on ``span``), stopped at a checkpoint (it is killed).""" from morphogen.io.radioss import read_radioss_listing started = time.monotonic() last = -1.0 while True: try: code = process.wait(timeout=0.5) except subprocess.TimeoutExpired: code = None try: checkpoint() except Stopped: _kill(process) raise if timeout is not None and time.monotonic() - started > timeout: _kill(process) raise RuntimeError(f"{label}: stopped after {timeout:g} s (timeout)") if progress is not None and listing is not None and end_time and listing.is_file(): try: now = read_radioss_listing(listing).time except OSError: now = last if now > last: last = now share = min(max(now / end_time, 0.0), 1.0) progress( span[0] + share * (span[1] - span[0]), f"{label}: t = {now:.4g} of {end_time:.4g}", ) if code is not None: return int(code) def _kill(process: subprocess.Popen) -> None: try: process.kill() process.wait(timeout=10) except (OSError, subprocess.TimeoutExpired): pass def _check(listing: Path, code: int, label: str, log: Path) -> None: """Raise RuntimeError when a program failed: its exit code, its listing's errors or its abnormal termination (the end of its log otherwise).""" from morphogen.io.radioss import read_radioss_listing state = read_radioss_listing(listing) if listing.is_file() else None errors = state.errors if state is not None else [] if code == 0 and state is not None and state.normal is not False and not errors: return if code == 0 and state is None: return detail = "; ".join(errors[:3]) if not detail and log.is_file(): tail = log.read_text(encoding="utf-8", errors="replace").strip().splitlines()[-8:] detail = " | ".join(s.strip() for s in tail if s.strip()) where = listing.name if listing.is_file() else log.name raise RuntimeError(f"{label} failed (exit code {code}; see {where}): {detail or 'no detail'}")
[docs] def run_openradioss( directory, name: str | None = None, *, threads: int | None = None, mpi: int | None = None, installation=None, progress: Callable | None = None, convert: bool = True, timeout: float | None = None, ) -> RadiossRun: """Run the inputs written in ``directory`` (:func:`morphogen.io.write_radioss`): the Starter, then the Engine with ``threads`` OpenMP threads (all the cores by default) or on ``mpi`` processes (``mpiexec`` and the Engine built with MPI), then the conversion of the animations and of the time history (``convert``). ``progress(fraction, message)`` is called as the run advances (the time of the last cycle printed in the listing of the Engine); a stop asked by :mod:`morphogen.stopping` kills the programs. Raises RuntimeError when a program fails (the errors of its listing); its logs stay in the directory.""" from morphogen.io.radioss import read_manifest directory = Path(directory).resolve() manifest = read_manifest(directory) name = name or manifest["name"] install = ( installation if isinstance(installation, OpenRadioss) else find_openradioss(installation) ) threads = int(threads or os.cpu_count() or 1) env = install.environment(threads) started = time.monotonic() end_time = float(manifest.get("end_time") or 0.0) or None domains = int(mpi or 1) listings = {"starter": directory / f"{name}_0000.out", "engine": directory / f"{name}_0001.out"} for stale in listings.values(): stale.unlink(missing_ok=True) if progress: progress(0.0, "OpenRadioss Starter") log = directory / "starter.log" with open(log, "w", encoding="utf-8") as out: process = subprocess.Popen( [str(install.starter), "-i", f"{name}_0000.rad", "-np", str(domains)], cwd=directory, env=env, stdout=out, stderr=subprocess.STDOUT, ) code = _wait(process, None, None, None, (0.0, 0.05), "Starter", timeout) _check(listings["starter"], code, "the Starter of OpenRadioss", log) if domains > 1: mpiexec = shutil.which("mpiexec") or shutil.which("mpirun") if install.engine_mpi is None or mpiexec is None: raise RuntimeError( f"{domains} MPI processes: an Engine built with MPI " f"({'found' if install.engine_mpi else 'none in ' + str(install.root)}) and " f"mpiexec ({'found' if mpiexec else 'not on the PATH'})" ) command = [mpiexec, "-n", str(domains), str(install.engine_mpi)] env["OMP_NUM_THREADS"] = str(max(1, threads // domains)) else: command = [str(install.engine)] command += ["-i", f"{name}_0001.rad", "-nt", env.get("OMP_NUM_THREADS", str(threads))] log = directory / "engine.log" with open(log, "w", encoding="utf-8") as out: process = subprocess.Popen( command, cwd=directory, env=env, stdout=out, stderr=subprocess.STDOUT ) code = _wait( process, listings["engine"], end_time, progress, (0.05, 0.9), "OpenRadioss", timeout ) _check(listings["engine"], code, "the Engine of OpenRadioss", log) run = RadiossRun(directory, name, [], None, listings, time.monotonic() - started) if convert: convert_outputs(run, install, env=env, progress=progress) run.elapsed = time.monotonic() - started return run
[docs] def animation_files(directory, name: str) -> list[Path]: """The animation files of a run (``<name>A001``, ``<name>A002``...), in order.""" found = [] for path in Path(directory).glob(f"{name}A*"): tail = path.name[len(name) + 1 :] if tail.isdigit(): found.append(path) return sorted(found, key=lambda p: int(p.name[len(name) + 1 :]))
[docs] def convert_outputs(run: RadiossRun, installation=None, *, env=None, progress=None) -> RadiossRun: """Convert the outputs of a run: each animation to a legacy VTK file (``anim_to_vtk``, ``<name>A001.vtk``...) and the time history to CSV (``th_to_csv``); the outputs already converted are kept. A converter missing from the installation is skipped with a warning of the run.""" install = ( installation if isinstance(installation, OpenRadioss) else find_openradioss(installation) ) env = env or install.environment() directory, name = Path(run.directory), run.name animations = animation_files(directory, name) frames = [] if animations and install.anim_to_vtk is None: run.warnings.append("no anim_to_vtk in the installation: the animations are not read") for k, path in enumerate(animations): target = path.with_name(path.name + ".vtk") if install.anim_to_vtk is not None and not target.is_file(): checkpoint() if progress: progress( 0.9 + 0.08 * k / len(animations), f"animation {k + 1} of {len(animations)}" ) with open(target, "wb") as out: done = subprocess.run( [str(install.anim_to_vtk), path.name], cwd=directory, env=env, stdout=out, stderr=subprocess.PIPE, ) if done.returncode != 0 or target.stat().st_size == 0: target.unlink(missing_ok=True) run.warnings.append( f"anim_to_vtk failed on {path.name}: " f"{done.stderr.decode(errors='replace').strip()[:200]}" ) continue if target.is_file(): frames.append(target) run.frames = frames history = directory / f"{name}T01" if history.is_file(): found = sorted(directory.glob(f"{name}T01*.csv")) if not found and install.th_to_csv is not None: checkpoint() done = subprocess.run( [str(install.th_to_csv), history.name], cwd=directory, env=env, capture_output=True ) found = sorted(directory.glob(f"{name}T01*.csv")) if not found and done.returncode == 0 and done.stdout.strip(): target = directory / f"{name}T01.csv" # a converter writing to its output target.write_bytes(done.stdout) found = [target] if not found: run.warnings.append( f"th_to_csv gave no CSV file: " f"{done.stderr.decode(errors='replace').strip()[:200]}" ) elif not found: run.warnings.append("no th_to_csv in the installation: the time history is not read") run.history = found[0] if found else None return run
# -- installation of a release ------------------------------------------------------------- def _request(url: str, timeout: float): request = urllib.request.Request( url, headers={ "Accept": "application/vnd.github+json, application/octet-stream;q=0.9", "User-Agent": f"morphoGen/{__version__}", }, ) return urllib.request.urlopen(request, timeout=timeout)
[docs] def release_info(tag: str | None = None, *, api: str = GITHUB_API, timeout: float = 60) -> dict: """The release ``tag`` of OpenRadioss on GitHub (the latest by default): its name, tag and archives (``assets``: name, size, URL).""" path = f"tags/{tag}" if tag else "latest" with _request(f"{api.rstrip('/')}/repos/{REPOSITORY}/releases/{path}", timeout) as res: data = json.loads(res.read().decode("utf-8")) assets = [ { "name": a.get("name", ""), "size": int(a.get("size") or 0), "url": a.get("browser_download_url", ""), } for a in data.get("assets", []) ] return { "tag": data.get("tag_name", tag or ""), "name": data.get("name", ""), "published": data.get("published_at", ""), "assets": assets, }
[docs] def release_asset(release: dict, system: str | None = None, machine: str | None = None) -> dict: """The archive of a release (:func:`release_info`) for this system and machine.""" system = system or platform.system() machine = machine or platform.machine() words = ASSETS.get((system, machine)) if words is None: raise RuntimeError( f"no release of OpenRadioss is published for {system} on {machine}: build it from " "its sources (https://github.com/OpenRadioss/OpenRadioss) and set OPENRADIOSS_PATH" ) archives = [ a for a in release.get("assets", []) if a["name"].lower().endswith((".zip", ".tar.gz", ".tgz")) ] for word in words: for asset in archives: name = asset["name"].lower() if word in name and "src" not in name and "source" not in name: if word == "linux64" and ("linuxa64" in name or "arm" in name): continue return asset raise RuntimeError( f"the release {release.get('tag')} of OpenRadioss has no archive for {system} on " f"{machine} (archives: {', '.join(a['name'] for a in archives) or 'none'})" )
def _extract(archive: Path, target: Path) -> None: """Unpack a zip or tar archive into ``target``, refusing the members that would land outside it, with the permission bits of the zip members kept.""" target = target.resolve() if archive.name.lower().endswith(".zip"): with zipfile.ZipFile(archive) as zf: for member in zf.infolist(): path = (target / member.filename).resolve() if target != path and target not in path.parents: raise RuntimeError( f"{archive.name}: the member {member.filename} leaves the folder" ) zf.extract(member, target) mode = (member.external_attr >> 16) & 0o777 if mode and not member.is_dir(): path.chmod(mode | stat.S_IRUSR | stat.S_IWUSR) return import tarfile with tarfile.open(archive) as tf: tf.extractall(target, filter="data")
[docs] def install_openradioss( target=None, *, tag: str | None = None, api: str = GITHUB_API, progress: Callable | None = None, timeout: float = 120, system: str | None = None, machine: str | None = None, ) -> OpenRadioss: """Download the release ``tag`` of OpenRadioss (the latest by default) for this system from GitHub and unpack it into ``target`` (``~/morphogen/openradioss`` by default), which replaces an earlier installation there; returns the installation. Releases are published for Linux (x86-64, ARM) and Windows; elsewhere OpenRadioss is built from its sources. ``progress(fraction, message)`` follows the download.""" target = Path(target).expanduser() if target else default_home() if progress: progress(0.0, "the release of OpenRadioss") release = release_info(tag, api=api, timeout=timeout) asset = release_asset(release, system, machine) target.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix=".openradioss-", dir=target.parent) as tmp: archive = Path(tmp) / Path(asset["name"]).name with _request(asset["url"], timeout) as res, open(archive, "wb") as out: total = int(res.headers.get("Content-Length") or asset.get("size") or 0) done = 0 while True: checkpoint() block = res.read(1 << 20) if not block: break out.write(block) done += len(block) if progress and total: progress( 0.9 * done / total, f"{asset['name']}: {done / 2**20:.0f} of {total / 2**20:.0f} MB", ) if progress: progress(0.92, f"unpacking {asset['name']}") unpacked = Path(tmp) / "unpacked" unpacked.mkdir() _extract(archive, unpacked) folder = _exec_folder(unpacked) if folder is None: raise RuntimeError(f"{asset['name']}: no Starter of OpenRadioss in the archive") record = { "tag": release["tag"], "name": release["name"], "asset": asset["name"], "url": asset["url"], "installed": time.strftime("%Y-%m-%d %H:%M:%S"), } (unpacked / RELEASE_FILE).write_text(json.dumps(record, indent=1), encoding="utf-8") if target.exists(): shutil.rmtree(target) shutil.move(str(unpacked), str(target)) if progress: progress(1.0, f"OpenRadioss {release['tag']} in {target}") return find_openradioss(target)