How energy is measured on the ET-SoC-1

etpower.py

#!/usr/bin/env python3
"""Read an ET-SoC-1 card's power meter in plain Python: no libDM, nothing to build.

It asks the card's service processor (SP) three questions over the management node /dev/et<N>_mgmt,
using the device-management API's own message format (et-platform device-api headers):

  DM_CMD_GET_MODULE_POWER (33)                board input power, 10 mW steps (the PMIC's reading)
  DM_CMD_GET_SP_STATS (60)                    the three metered rails (minion cores, SRAM, mesh) as
                                              [average, min, max]; the average is the PMIC's own
                                              running average (time constant about 1 s)
  DM_CMD_GET_MODULE_CURRENT_TEMPERATURE (27)  die temperature (average and hottest minion shire)

One JSON line per sample on stdout. The node has one opener, so hold the card's lock around it:

  flock -n /run/lock/etsoc-shire0.lock python3 etpower.py --seconds 10 > idle.jsonl

Stop it early with Ctrl-C or a plain `kill`: the request in flight finishes first. Never `kill -9`
it: a reply left in the management queue breaks the next program that opens the card.

How a message travels (the same two ioctls libDM's device layer uses): an 8-byte header (message
size, tag, command code, flags) is pushed with ETSOC1_IOCTL_PUSH_SQ, and the reply, a 24-byte
header (the same 8 bytes, the SP's latency in µs, a status) plus the command's struct, is popped
with ETSOC1_IOCTL_POP_CQ.
"""
import argparse
import ctypes
import errno
import fcntl
import json
import os
import select
import signal
import struct
import sys
import time

MAGIC = 0xE7  # ESPERANTO_PCIE_IOCTL_MAGIC (et-driver/et_ioctl.h)


def _ioc(direction, nr, size):
    return (direction << 30) | (size << 16) | (MAGIC << 8) | nr


class CmdDesc(ctypes.Structure):  # struct cmd_desc (et_ioctl.h)
    _fields_ = [("cmd", ctypes.c_void_p), ("size", ctypes.c_uint16),
                ("sq_index", ctypes.c_uint16), ("flags", ctypes.c_uint8)]


class RspDesc(ctypes.Structure):  # struct rsp_desc (et_ioctl.h)
    _fields_ = [("rsp", ctypes.c_void_p), ("size", ctypes.c_uint16), ("cq_index", ctypes.c_uint16)]


PUSH_SQ = _ioc(1, 6, ctypes.sizeof(CmdDesc))  # _IOW(MAGIC, 6, struct cmd_desc)
POP_CQ = _ioc(3, 7, ctypes.sizeof(RspDesc))   # _IOWR(MAGIC, 7, struct rsp_desc)
MAX_MSG = 1760

GET_MODULE_CURRENT_TEMPERATURE = 27
GET_MODULE_POWER = 33
GET_SP_STATS = 60

# struct get_sp_stats_t (device_mgmt_api_rpc_types.h): 42 uint16 values, in this order.
SP_FIELDS = ["system_power", "system_temperature"] + [
    "%s_%s" % (part, q) for part in ("minion", "sram", "noc")
    for q in ("power", "temperature", "voltage", "freq")]

_stop = False


def _ask_to_stop(_sig, _frame):
    global _stop
    _stop = True


def drain(fd):
    """Pop and drop any reply a killed program left behind (nobody else can have the node open)."""
    rbuf = ctypes.create_string_buffer(MAX_MSG)
    desc = RspDesc(ctypes.cast(rbuf, ctypes.c_void_p), MAX_MSG, 0)
    dropped = 0
    while True:
        try:
            fcntl.ioctl(fd, POP_CQ, desc)
            dropped += 1
        except OSError as e:
            if e.errno == errno.EAGAIN:
                return dropped
            raise


def request(fd, cmd, timeout_s=2.0):
    """Send one command (header only, as libDM does for these) and return its reply bytes."""
    tag = (os.getpid() ^ int(time.time() * 1000)) & 0xFFFF or 1
    msg = struct.pack("<HHHH", 8, tag, cmd, 0)
    sbuf = ctypes.create_string_buffer(msg, len(msg))
    fcntl.ioctl(fd, PUSH_SQ, CmdDesc(ctypes.cast(sbuf, ctypes.c_void_p), len(msg), 0, 0))
    rbuf = ctypes.create_string_buffer(MAX_MSG)
    desc = RspDesc(ctypes.cast(rbuf, ctypes.c_void_p), MAX_MSG, 0)
    poller = select.poll()
    poller.register(fd, select.POLLIN)
    end = time.time() + timeout_s
    while time.time() < end:
        poller.poll(20)
        try:
            n = fcntl.ioctl(fd, POP_CQ, desc)
        except OSError as e:
            if e.errno == errno.EAGAIN:
                continue
            raise
        data = rbuf.raw[:n]
        _size, rtag, rcmd, _flags = struct.unpack_from("<HHHH", data, 0)
        if rtag == tag and rcmd == cmd:
            status, = struct.unpack_from("<i", data, 16)
            if status != 0:
                raise RuntimeError("command %d returned status %d" % (cmd, status))
            return data
    raise TimeoutError("no reply to command %d within %.1f s" % (cmd, timeout_s))


def read_once(fd):
    """One sample: board power, the three rails and the die temperature."""
    t = time.time()
    board_w = struct.unpack_from("<H", request(fd, GET_MODULE_POWER), 24)[0] / 100.0  # 10 mW units
    v = struct.unpack_from("<42H", request(fd, GET_SP_STATS), 24)
    sp = {name: list(v[3 * i:3 * i + 3]) for i, name in enumerate(SP_FIELDS)}  # [avg, min, max]
    temp = struct.unpack_from("<6hB", request(fd, GET_MODULE_CURRENT_TEMPERATURE), 24)
    return {
        "t": round(t, 3),
        "took_ms": round((time.time() - t) * 1e3, 1),
        "board_w": board_w,
        # the rails' running averages (index 0 of each triple), mW -> W
        "minion_w": sp["minion_power"][0] / 1000.0,
        "sram_w": sp["sram_power"][0] / 1000.0,
        "noc_w": sp["noc_power"][0] / 1000.0,
        "minion_mhz": sp["minion_freq"][0],
        "die_c": temp[3],       # minion shires, average
        "die_max_c": temp[5],   # hottest minion shire
    }


def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--card", type=int, default=int(os.environ.get("ET_DEVICES", "0") or 0),
                    help="card number N in /dev/etN_mgmt (default: $ET_DEVICES or 0)")
    ap.add_argument("--seconds", type=float, default=10.0, help="how long to sample")
    ap.add_argument("--every-ms", type=int, default=100, help="sampling period (default 100 ms = 10 Hz)")
    a = ap.parse_args()

    signal.signal(signal.SIGINT, _ask_to_stop)
    signal.signal(signal.SIGTERM, _ask_to_stop)
    # The node has one opener. Another reader (the lab's live dashboard takes one reading a second on a
    # free card) holds it for a few tens of ms, so wait up to 5 s for it before giving up.
    end = time.time() + 5
    while True:
        try:
            fd = os.open("/dev/et%d_mgmt" % a.card, os.O_RDWR | os.O_NONBLOCK)
            break
        except OSError as e:
            if e.errno not in (errno.EBUSY, errno.EPERM) or time.time() > end:
                raise
            time.sleep(0.05)
    try:
        stale = drain(fd)
        if stale:
            print("dropped %d stale repl%s left in the queue" % (stale, "y" if stale == 1 else "ies"),
                  file=sys.stderr)
        start = time.time()
        while not _stop and time.time() - start < a.seconds:
            tick = time.time()
            print(json.dumps(read_once(fd)), flush=True)
            time.sleep(max(0.0, a.every_ms / 1000.0 - (time.time() - tick)))
    finally:
        os.close(fd)


if __name__ == "__main__":
    main()

measure.sh

#!/usr/bin/env bash
# measure.sh: one energy measurement on an ET-SoC-1 card, made the way the lab's agents make them.
#
#   idle (6 s)  ->  the workload (one process, capped at 10 s)  ->  idle (8 s)
#
# with the card's power meter read 10 times a second the whole time. The idle stretches on both
# sides are the baseline: energy = (power during the work - idle power) x time.
#
# Hold the card's lock for the whole measurement, so nobody else starts on the card meanwhile:
#   flock -n /run/lock/etsoc-shire0.lock ./measure.sh OUT_DIR -- <workload command ...>
#   python3 energy.py OUT_DIR
#
# ETTELEM points at the sampler (default build/ettelem/ettelem); IDLE_BEFORE and IDLE_AFTER are seconds.
set -u
OUT=$1; shift; [ "${1:-}" = "--" ] && shift
ETTELEM=${ETTELEM:-build/ettelem/ettelem}
IDLE_BEFORE=${IDLE_BEFORE:-6}
IDLE_AFTER=${IDLE_AFTER:-8}
mkdir -p "$OUT"

# 1. Start the sampler. About one start in three fails right after another sampler exits, so retry.
SAMPLER=
for attempt in 1 2 3 4; do
  "$ETTELEM" sample --seconds 120 --every-ms 100 > "$OUT/power.jsonl" 2> "$OUT/sampler.err" < /dev/null &
  SAMPLER=$!
  for i in $(seq 1 40); do
    sleep 0.25
    grep -q '^{' "$OUT/power.jsonl" && break 2
  done
  kill -TERM "$SAMPLER"; wait "$SAMPLER"; SAMPLER=; sleep 2
done
[ -n "$SAMPLER" ] || { echo "the sampler did not start (is the card free? et-who)" >&2; exit 1; }
# Stop it with a plain kill (SIGTERM): the request in flight completes. Never kill -9 a card tool.
trap 'kill -TERM "$SAMPLER" 2>/dev/null; wait "$SAMPLER"' EXIT

# 2. Idle, the work, idle.
sleep "$IDLE_BEFORE"
date +%s.%N > "$OUT/t_start"
timeout 10 "$@" > "$OUT/workload.out" 2>&1 < /dev/null
echo $? > "$OUT/exit_code"
date +%s.%N > "$OUT/t_end"
sleep "$IDLE_AFTER"
echo "done: $OUT (workload exit code $(cat "$OUT/exit_code"))"

energy.py

#!/usr/bin/env python3
"""energy.py OUT_DIR: the energy of the workload that measure.sh ran, from the card's power meter.

  idle power    = mean board power over the idle stretches before and after the work
                  (their mean removes most of idle power's drift with die temperature)
  energy        = sum over samples of (board power - idle power) x sample interval,
                  over the work plus a short margin either side (the meter lags by up to one
                  service-processor pass, about 0.13-0.27 s)
  per operation = energy / operations completed (the workload's own count, from the device)

The published tables add one refinement, also computed here: the die runs a little warmer during
the work than in the idle stretches, and the warmer die leaks more, so that extra leakage (the idle
law's slope times the temperature difference) is taken out (energy manual §9; the constants are
workloads/enercat/analyze_catalogue.py's).

Prints a summary and writes OUT_DIR/summary.json and OUT_DIR/trace.csv (time from the start of the work).
"""
import json
import math
import os
import re
import statistics
import sys


def load_samples(path):
    rows = []
    with open(path) as f:
        for line in f:
            if not line.startswith("{"):
                continue
            d = json.loads(line)
            if "board_w" not in d:
                continue
            t = d["t_ms"] / 1000.0 if "t_ms" in d else d["t"]          # ettelem / etpower.py
            sp = d.get("sp", {})
            rail = lambda k: sp[k][0] if k in sp else d.get(k)           # the PMIC's running average
            die = d.get("temp_c", {}).get("minshire", [d.get("die_c")])[0]
            mhz = d.get("mhz", {}).get("minion", d.get("minion_mhz"))
            rows.append({"t": t, "board_w": d["board_w"], "minion_w": rail("minion_w"), "sram_w": rail("sram_w"),
                         "noc_w": rail("noc_w"), "die_c": die, "mhz": mhz, "took_ms": d.get("took_ms")})
    return rows


def load_launches(path):
    """The workload's own records: sparsity_host prints one 'SPARSITY {json}' line per launch."""
    out = []
    if os.path.exists(path):
        for line in open(path):
            m = re.match(r"SPARSITY (\{.*\})", line)
            if m:
                r = json.loads(m.group(1))
                if r.get("launch", 0) >= 0:      # launch -1 is a calibration run
                    out.append(r)
    return out


def mean(xs):
    xs = [x for x in xs if x is not None]
    return statistics.mean(xs) if xs else None


A_LEAK_80, T_L = 23.257, 36.0  # idle law's leakage: 23.3 W x e^((T-80)/36)


def leak_slope(t_c):
    """Watts of extra leakage per degree C of die temperature, at t_c."""
    return A_LEAK_80 / T_L * math.exp((t_c - 80.0) / T_L)


def main():
    out = sys.argv[1]
    idle_before = float(os.environ.get("IDLE_BEFORE", 6))
    idle_after = float(os.environ.get("IDLE_AFTER", 8))
    s = load_samples(os.path.join(out, "power.jsonl"))
    t_start = float(open(os.path.join(out, "t_start")).read())
    t_end = float(open(os.path.join(out, "t_end")).read())
    launches = load_launches(os.path.join(out, "workload.out"))

    # The work itself: from the first launch to the end of the last (device-side records), else the process.
    w0 = min(r["t_start_ms"] for r in launches) / 1000.0 if launches else t_start
    w1 = max(r["t_end_ms"] for r in launches) / 1000.0 if launches else t_end
    busy_s = w1 - w0

    before = [x for x in s if t_start - idle_before + 1.0 <= x["t"] <= t_start - 0.1]
    after = [x for x in s if t_end + 3.0 <= x["t"] <= t_end + idle_after - 0.3]
    idle_b, idle_a = mean(x["board_w"] for x in before), mean(x["board_w"] for x in after)
    idle_w = (idle_b + idle_a) / 2

    # Energy above idle, integrated over the work plus a margin (sample interval from the timestamps).
    win = [x for x in s if w0 - 0.5 <= x["t"] <= w1 + 1.0]
    energy_j = 0.0
    for a, b in zip(win, win[1:]):
        energy_j += (a["board_w"] - idle_w) * (b["t"] - a["t"])
    inside = [x for x in s if w0 + 0.3 <= x["t"] <= w1]
    t_idle = mean([x["die_c"] for x in before + after])
    t_busy = mean([x["die_c"] for x in inside])
    leak_w = leak_slope((t_busy + t_idle) / 2) * (t_busy - t_idle) if None not in (t_idle, t_busy) else 0.0
    energy_corr_j = energy_j - leak_w * busy_s
    res = {
        "idle_before_w": round(idle_b, 3), "idle_after_w": round(idle_a, 3), "idle_w": round(idle_w, 3),
        "busy_s": round(busy_s, 3), "energy_above_idle_j": round(energy_j, 3),
        "mean_above_idle_w": round(energy_j / busy_s, 3) if busy_s > 0 else None,
        "leakage_correction_w": round(leak_w, 3), "energy_corrected_j": round(energy_corr_j, 3),
        "busy_mean_board_w": round(mean(x["board_w"] for x in inside), 3) if inside else None,
        "die_c_before": mean(x["die_c"] for x in before), "die_c_busy": mean(x["die_c"] for x in inside),
        "die_c_after": mean(x["die_c"] for x in after),
        "minion_mhz_seen": sorted({x["mhz"] for x in s if x["mhz"] is not None and w0 <= x["t"] <= w1}),
        "samples": len(s), "sample_period_ms": round(1000 * (s[-1]["t"] - s[0]["t"]) / (len(s) - 1), 1),
        "distinct_board_values_per_s": round(sum(1 for a, b in zip(s, s[1:]) if a["board_w"] != b["board_w"])
                                             / (s[-1]["t"] - s[0]["t"]), 2),
        "launches": len(launches),
    }
    if launches and launches[0].get("test") == "fma":
        ops = sum(r["iters"] * r["minions"] for r in launches)          # TensorFMA instructions completed
        macs = ops * 16 * launches[0]["a_elems"]                         # 16 x 16 x 16 = 4,096 per fp32 op
        res.update({"values": launches[0]["values"], "type": launches[0]["type"], "tensorfma_ops": ops,
                    "multiply_adds": macs, "pj_per_multiply_add": round(energy_j / macs * 1e12, 3),
                    "pj_per_multiply_add_corrected": round(energy_corr_j / macs * 1e12, 3),
                    "ghz_device": launches[0].get("ghz"),
                    "all_checked_ok": all(r.get("ok") for r in launches)})
    json.dump(res, open(os.path.join(out, "summary.json"), "w"), indent=1)

    with open(os.path.join(out, "trace.csv"), "w") as f:
        f.write("t_s,board_w,minion_w,sram_w,noc_w,die_c,mhz\n")
        for x in s:
            f.write("%.3f,%s,%s,%s,%s,%s,%s\n" % (x["t"] - w0, x["board_w"], x["minion_w"], x["sram_w"],
                                                 x["noc_w"], x["die_c"], x["mhz"]))

    print("idle: %.2f W before, %.2f W after, baseline %.2f W" % (idle_b, idle_a, idle_w))
    print("work: %.2f s, %.2f W above idle on average, %.1f J above idle" % (busy_s, res["mean_above_idle_w"], energy_j))
    print("      leakage correction %.2f W (die %.1f C busy vs %.1f C idle) -> %.1f J"
          % (leak_w, t_busy or 0, t_idle or 0, energy_corr_j))
    if "pj_per_multiply_add" in res:
        print("      %s data, %d TensorFMA ops = %.3g multiply-adds -> %.3f pJ per multiply-add (%.3f corrected)"
              % (res["values"], res["tensorfma_ops"], res["multiply_adds"], res["pj_per_multiply_add"],
                 res["pj_per_multiply_add_corrected"]))
    print("die: %s C before, %s C during, %s C after; minion clock %s MHz"
          % tuple([None if v is None else round(v, 1) for v in (res["die_c_before"], res["die_c_busy"],
                                                                 res["die_c_after"])] + [res["minion_mhz_seen"]]))


if __name__ == "__main__":
    main()

• Runs: aifoundry3 card 0, 10:26 PDT, by an agent on the box. The two passes ran in opposite orders, with 20 s between runs. Raw data is on the box in ~/nekko/build/energy-tutorial/runs/.
• The sensors and the service-processor loop: docs/research/power-telemetry.md. The method: energy manual §9.
• Published per-multiply-add values for this loop, energy manual §3.2: 0.273, 2.131 and 5.782 pJ on zeros, ones and random data, the mean of three cards. For aifoundry3 alone, 0.163, 1.925 and 5.369 pJ as registered. The manual's launch-temperature note puts aifoundry3 at 0.37–0.48 pJ on zeros and 5.60–5.72 on random data; the ones band applies the same note's 0.91–1.43 W offset.
• The Python reader's message layout: et-platform's device-api headers (device_mgmt_api_rpc_types.h) and the driver's ioctls (et_ioctl.h).