"""
Debug metadata subsystem.
rainbow's normal read path surfaces a lean set of metadata to keep overhead low.
A vendor run directory, however, ships many sidecar files (registers, INIs,
audit trails, method dumps, ...) carrying far more context: instrument module
serials, firmware/software versions, the acquisition operator, method-save
details, injection volume, signal optics, and so on. This subsystem decodes
those sidecars into structured fields on demand.
Two entry points:
from rainbow import debug
debug.inspect(path) # {relpath: structured metadata} - faithful per file
debug.fields(path) # one merged canonical run-metadata record
Each format has a parser module exposing ``matches(name)``, ``parse(path)``
(full, lossless structure), and ``canonical(parsed)`` (the well-known keys
promoted to shared canonical names). The canonical field vocabulary grows as
formats are added; see ``docs/source/debug/`` for the per-format catalog and the
depth/output-shape decisions behind each parser.
Read-only; nothing is ever written back.
"""
import os
import tempfile
import zipfile
__all__ = ["inspect", "fields"]
from rainbow.debug import ini
from rainbow.debug import xml
from rainbow.debug import chemstation_text
from rainbow.debug import chemstation_header
from rainbow.debug import dotnet
from rainbow.debug import reg
from rainbow.debug import method
from rainbow.debug import waters_header
from rainbow.debug import waters_inf
from rainbow.debug import mzxml
# Parser registry. Each module: matches(name)->bool, parse(path)->dict,
# canonical(parsed)->dict. Order matters only if patterns overlap.
_PARSERS = [ini, xml, chemstation_text, chemstation_header, dotnet, reg, method,
waters_header, waters_inf, mzxml]
# Canonical fields that accumulate across files instead of taking the first
# value (a run legitimately has several serials / devices / users / signals).
_LIST_FIELDS = ("serials", "signal_optics", "devices", "computers", "users")
# Container archives the walker transparently steps into, treating each member
# like a file in a sub-directory. An Agilent OpenLab ``.dx`` is an OPC (zip)
# package; rb.read decodes its signal *data*, while debug mines the per-signal
# ChemStation *headers* (operator, full method path, signal optics) that rb.read
# drops. The manifest (injection.acmd) is left to rb.read to avoid duplicating
# its job; debug only reaches the sidecar identity inside the archive.
_ARCHIVE_EXTS = (".dx",)
def _validate(path):
"""Raises if ``path`` is not a readable file or directory.
Mirrors ``rb.read``'s contract: a debug entry point accepts a run directory
or a single sidecar file, and anything else is an error rather than a
silently empty result.
"""
if not isinstance(path, str) or not os.path.exists(path):
raise Exception(f"{path} is not a file or directory.")
def _iter_files(path):
"""Yields ``(abspath, relpath)`` for every file under ``path`` (or the file
itself), sorted for stable output. A ``.dx`` archive (file or nested) is
transparently descended into, yielding its identity-bearing members."""
if os.path.isfile(path):
if path.lower().endswith(_ARCHIVE_EXTS):
yield from _iter_archive(path, os.path.basename(path))
else:
yield path, os.path.basename(path)
return
# Sorted by relpath across the whole tree, not per directory: os.walk lists
# a directory's own files before descending, and hands back sibling
# directories in readdir order, which is the creation order on APFS and a
# hash order on ext4. fields() keeps the first value it sees for a scalar,
# so an unsorted walk makes the reported instrument and method depend on the
# filesystem the run happens to sit on.
entries = []
for dirpath, dirnames, names in os.walk(path):
dirnames.sort()
for name in names:
full = os.path.join(dirpath, name)
entries.append((os.path.relpath(full, path), full, name))
for rel, full, name in sorted(entries):
if name.lower().endswith(_ARCHIVE_EXTS):
yield from _iter_archive(full, rel)
else:
yield full, rel
def _iter_archive(archive_path, relbase):
"""Yields ``(temp_path, relpath)`` for each recognized member of a ``.dx``
(OPC zip) package.
Only members a parser actually claims are extracted - the GUID-named binary
signal payloads (``.CH``/``.UV``) carry the ChemStation identity headers, so
they are kept. The OPC packaging parts (the ``[Content_Types].xml`` map and
the ``_rels`` relationship files), the numeric ``.IT`` traces, and the
manifest (``injection.acmd``, rb.read's territory) carry no instrument
identity and are skipped without extraction. Member relpaths are reported as
``<archive>!<member>`` to mark the boundary.
The temp directory lives for the life of this generator: the ``with`` stays
open while suspended at each ``yield``, so a member file is still on disk
when the caller parses it, and is cleaned up once iteration completes.
"""
try:
archive = zipfile.ZipFile(archive_path)
members = archive.namelist()
except Exception:
# An archive rainbow cannot open contributes nothing, and must not stop
# the walk reaching the sidecars beside it. namelist() is inside the
# guard because reading the central directory is where a truncated or
# otherwise malformed package fails.
return
with archive, tempfile.TemporaryDirectory() as tmp:
for member in members:
base = os.path.basename(member)
# OPC packaging parts are pure container plumbing, no identity.
low = member.lower()
if base.lower() == "[content_types].xml" or "_rels/" in low:
continue
if not base or _dispatch(base) is None:
continue
dest = os.path.join(tmp, base)
try:
with open(dest, "wb") as out:
out.write(archive.read(member))
except Exception:
# One member is skipped, never the archive and never the walk.
# zipfile.read raises more than a bad-zip error: an unsupported
# compression method (deflate64) is a NotImplementedError and an
# encrypted member a RuntimeError, neither of which is an
# OSError. This generator is advanced by the caller's `for`
# header, outside its per-file try, so anything escaping here
# takes down the whole inspection rather than one sidecar.
continue
yield dest, "{}!{}".format(relbase, member)
def _dispatch(name):
"""Returns the parser module that handles ``name``, or None."""
for module in _PARSERS:
if module.matches(name):
return module
return None
[docs]
def inspect(path):
"""
Decodes every recognized metadata sidecar under ``path`` into its full
structure.
Args:
path (str): Path of a run directory (or a single sidecar file).
Returns:
Dict ``{relpath: parsed}`` where ``parsed`` is the parser's faithful,
lossless structure. A file that fails to parse maps to
``{'parser': ..., 'error': ...}`` rather than being dropped.
"""
_validate(path)
out = {}
for full, rel in _iter_files(path):
module = _dispatch(os.path.basename(full))
if module is None:
continue
try:
out[rel] = module.parse(full)
except ImportError:
# A missing dependency is the caller's environment, not a bad file:
# reporting it per-file would bury it in every entry at once.
raise
except Exception as e: # never let one bad sidecar abort the inspection
out[rel] = {"parser": module.NAME, "error": repr(e)}
return out
[docs]
def fields(path):
"""
Merges every recognized sidecar under ``path`` into one canonical
run-metadata record.
List-valued fields (e.g. ``serials``) accumulate unique values across
files; scalar fields take the first non-empty value seen. Each merged value
records which file(s) it came from under the ``_sources`` key.
Args:
path (str): Path of a run directory (or a single sidecar file).
Returns:
Dict of canonical fields, plus ``_sources`` mapping each field to the
relpath(s) that supplied it.
"""
_validate(path)
merged = {}
sources = {}
for full, rel in _iter_files(path):
module = _dispatch(os.path.basename(full))
if module is None:
continue
try:
contributed = module.canonical(module.parse(full))
except ImportError:
# Otherwise a missing dependency looks like a run with no metadata.
raise
except Exception:
continue
for key, value in contributed.items():
if key in _LIST_FIELDS:
bucket = merged.setdefault(key, [])
for item in (value if isinstance(value, list) else [value]):
if item not in bucket:
bucket.append(item)
sources.setdefault(key, []).append(rel)
elif key not in merged and value not in (None, ""):
merged[key] = value
sources[key] = rel
if merged:
merged["_sources"] = sources
return merged