Organize the project by domain and name analyses by their rule

Energy code moves to src/energy/ and its docs to docs/energy/; shared code
(CSV loading, venv switch, browser, zoom script) goes to src/common/, and the
AIMsight originals to docs/aimsight/. Analyses are no longer numbered: two
Energy scripts remain, daily.py (delivered state and trend) and
all_analyses.py (comparison of the four). Calculations and pages unchanged (D-026).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maximeandClaude Opus 5.5 committed 2026-10-12 17:12:37 +02:00
1 parent 77c8f24d2f
commit 8a32f4d418
44 files changed
+1757 -771

No files matched your search

-882
View File
@@ -1,882 +0,0 @@
#!/usr/bin/env python3
import os, threading, signal, json, time, argparse, socket, re, datetime, collections
import serial
from .. import __version__ as _AIMBEDDED_VERSION
from ..libs.logging import init_logger, set_logging_level, log_levels
import subprocess
from ina219 import INA219, SmbusI2cDevice
import psutil
try:
import redis
except ImportError:
redis = None
logger = init_logger(level="info")
def send_at_command(ser, command, timeout=1):
"""
Sends an AT command to the modem and reads the response.
"""
ser.write((command + "\r\n").encode())
time.sleep(timeout)
response = ser.read_all().decode(errors="ignore").strip()
return response
def map_csq_rssi(rssi):
"""
Maps RSSI value to dBm.
"""
if rssi == 0:
return -113 # -113 dBm or less
elif rssi == 31:
return -51 # -51 dBm or greater
elif 1 <= rssi <= 30:
return -113 + (rssi * 2) # dBm
elif rssi == 99:
return None
return None
def map_csq_ber(ber):
"""
Maps BER value to percentage range.
"""
ber_mapping = {
0: 0.1,
1: 0.3,
2: 0.6,
3: 1.2,
4: 2.4,
5: 5.0,
6: 9.0,
7: 13.0,
99: None,
}
return ber_mapping.get(ber, None)
def parse_csq(response):
"""
Parses AT+CSQ response for signal quality.
"""
match = re.search(r'\+CSQ: (\d+),(\d+)', response)
if match:
rssi = int(match.group(1))
ber = int(match.group(2))
rssi_dbm = map_csq_rssi(rssi)
ber_percent = map_csq_ber(ber)
return rssi_dbm, ber_percent
return None, None
def parse_cesq(response):
"""
Parses AT+CESQ response and interprets rsrq and rsrp.
"""
match = re.search(r'\+CESQ: (\d+),(\d+),(\d+),(\d+),(\d+),(\d+)', response)
if match:
# Extract rsrq and rsrp values (ignore first 4 values)
rsrq = int(match.group(5))
rsrp = int(match.group(6))
# Map rsrq to dB ranges
rsrq_db = (
-19.5 + (rsrq * 0.5) if 0 <= rsrq <= 34
else None if rsrq == 255
else None
)
# Map rsrp to dBm ranges
rsrp_dbm = (
-140 + rsrp if 0 <= rsrp <= 97
else None if rsrp == 255
else None
)
return rsrq_db, rsrp_dbm
return None, None
def parse_cereg(response):
"""
Parses AT+CEREG? response into a registration status string.
"""
mapping = {0: "not_registered", 1: "home", 2: "searching",
3: "denied", 4: "unknown", 5: "roaming"}
match = re.search(r'\+CEREG: \d+,(\d+)', response)
if match:
return mapping.get(int(match.group(1)), "unknown")
return None
def parse_cgact(response):
"""
Parses AT+CGACT? response. Returns the list of active PDP context ids.
"""
return [int(m.group(1)) for m in re.finditer(r'\+CGACT: (\d+),1', response)]
def parse_cgpaddr(response):
"""
Parses AT+CGPADDR=<cid> response. Returns the PDP address or None.
"""
match = re.search(r'\+CGPADDR: \d+,"([^"]+)"', response)
if match and match.group(1) != "0.0.0.0":
return match.group(1)
return None
def get_network_info(response):
"""
Extracts network operator and technology from the response.
"""
match = re.search(r'\+COPS: \d,(\d),"([^"]+)",(\d+)', response)
if match:
mode = int(match.group(1))
operator = match.group(2)
technology = int(match.group(3))
tech_mapping = {0: "GSM", 2: "UMTS", 3: "LTE", 7: "E-UTRAN"}
return operator, tech_mapping.get(technology, "Unknown")
return None, None
class systemInfo(threading.Thread):
# BusVoltage trend analysis: sliding window and classification deadband.
# INA219 bus voltage LSB is 4 mV, so slopes below a few mV/min are noise.
TREND_WINDOW_S = 60
TREND_MIN_SAMPLES = 10
TREND_MIN_SPAN_S = 30
TREND_THRESHOLD_MV_MIN = 2.5
# Rest voltage vs state-of-charge for a 4S LiFePO4 pack (12.8 V nominal).
# The curve is very flat between 30% and 80%, so mid-range estimates are
# coarse and only reliable when the battery is not under charge.
LIFEPO4_OCV_SOC = [
(10.0, 0), (12.0, 10), (12.5, 20), (12.8, 30), (12.9, 40),
(13.0, 50), (13.1, 60), (13.2, 70), (13.3, 80), (13.4, 90), (13.6, 100),
]
RUNTIME_MIN_CURRENT_MA = 50
# Effective series resistance [Ohm] between battery and INA219 bus
# measurement: pack internal resistance + wiring + 100 mOhm shunt.
# Measured 0.34 Ohm on a deployed unit via a load-step V/I regression
# (R^2 = 0.93). Without IR compensation, load changes masquerade as
# charge/discharge swings on LiFePO4's flat voltage curve.
BATTERY_R_OHM_DEFAULT = 0.34
def __init__(
self,
system_name=None,
device=1,
enable_logging=False,
no_modem=False,
refresh_rate=1,
battery_capacity_ah=None,
battery_r_ohm=BATTERY_R_OHM_DEFAULT,
config_folder="/etc/aimsight",
redis_socket=None,
redis_host=None,
redis_port=None,
):
self.config_folder = config_folder
self.system_name = system_name
self.device_index = device
self.stop_service = threading.Event()
self.event_notification = threading.Condition()
self.refresh_rate = refresh_rate
self.enable_logging = enable_logging
self.modem = not no_modem
self.power_monitor = self.init_ina(i2cBus=device)
self.voltage_history = collections.deque()
self.battery_state = "unknown"
self.battery_capacity_ah = battery_capacity_ah
self.battery_r_ohm = battery_r_ohm
self.accumulator = None
self.last_sync = time.time()
self.modem_count = 0
self.modem_resp = None
self.network_count = 0
self.network_resp = None
# Redis init (graceful degradation)
self._redis = None
if redis is not None:
try:
if redis_socket:
self._redis = redis.Redis(unix_socket_path=redis_socket, decode_responses=False)
else:
self._redis = redis.Redis(
host=redis_host or "localhost",
port=redis_port or 6379,
decode_responses=False,
)
self._redis.ping()
logger.info("Redis connection established")
except Exception as e:
logger.warning("Redis not available: %s", e)
self._redis = None
else:
logger.warning("redis-py not installed")
super().__init__()
def stop(self):
logger.info("Stop instruction received. notifing acquisition thread.")
self.stop_service.set()
with self.event_notification:
self.event_notification.notify()
def init_ina(self, shunt_ohms=0.1, i2cBus=1):
i2c_addr = INA219.I2C_ADDR_DEFAULT
i2c_device = SmbusI2cDevice(interface=1, address=i2c_addr)
ina_device = INA219(i2c_device, shunt_ohms=shunt_ohms) # Using the BBB I2C bus nb.2
try:
ina_device.configure(INA219.RANGE_32V, INA219.GAIN_8_320MV)
except OSError:
logger.fatal("INA219: Cannot communicate with peripheral %d", i2cBus)
return ina_device
def _store_in_redis(self, key, data, ttl):
"""Store data as a Redis hash with TTL. No-op if Redis is unavailable."""
if self._redis is None:
return
try:
mapping = {"_updated_at": str(time.time())}
for k, v in data.items():
if v is None:
mapping[k] = ""
elif isinstance(v, (list, tuple)):
mapping[k] = json.dumps(v)
else:
mapping[k] = str(v)
self._redis.hset(key, mapping=mapping)
self._redis.expire(key, ttl)
except Exception as e:
logger.warning("Redis store failed for %s: %s", key, e)
def _get_gateways(self):
"""Parse 'ip route' to build a {iface: gateway} map."""
gateways = {}
try:
output = subprocess.check_output(["ip", "route"], text=True)
for line in output.splitlines():
# e.g. "default via 192.168.1.1 dev eth0 ..."
parts = line.split()
if "via" in parts and "dev" in parts:
gw = parts[parts.index("via") + 1]
iface = parts[parts.index("dev") + 1]
if parts[0] == "default":
gateways[iface] = gw
except Exception:
pass
return gateways
def get_interface_info(self):
"""Get network interface info (IP, netmask, gateway). Returns flat dict or None."""
self.network_count = (self.network_count + 1) % 5
if self.network_count > 0 and self.network_resp is not None:
return self.network_resp
try:
result = {}
addrs = psutil.net_if_addrs()
gateways = self._get_gateways()
skip_prefixes = ("lo", "docker", "br-")
for iface, addr_list in addrs.items():
if any(iface.startswith(p) for p in skip_prefixes):
continue
for addr in addr_list:
if addr.family == socket.AF_INET:
result[f"{iface}_ip"] = addr.address
result[f"{iface}_netmask"] = addr.netmask
result[f"{iface}_gateway"] = gateways.get(iface)
break
self.network_resp = result if result else None
except Exception as e:
logger.warning("Failed to get network info: %s", e)
return self.network_resp
def publish_new_system_info(self):
power_info = self.get_power_info()
proc_info = self.get_proc_info()
thermal_info = self.get_thermal_info()
self._store_in_redis("system_info:power", power_info, 60)
self._store_in_redis("system_info:proc", proc_info, 60)
self._store_in_redis("system_info:temperature", thermal_info, 60)
modem_info = None
if self.modem:
modem_info = self.get_modem_info()
if modem_info is not None:
self._store_in_redis("system_info:modem", modem_info, 120)
network_info = self.get_interface_info()
if network_info is not None:
self._store_in_redis("system_info:network", network_info, 120)
if self.enable_logging:
logger.info("Power: %s", power_info)
logger.info("Proc : %s", proc_info)
logger.info("Temp : %s", thermal_info)
if self.modem:
logger.info("Modem: %s", modem_info)
logger.info("Net : %s", network_info)
def update_voltage_trend(self, voltage):
"""
Tracks BusVoltage over a sliding time window and classifies the
battery state from its slope.
Returns (slope_mv_per_min, state) with state one of "charging",
"discharging", "stable" or "unknown".
"""
now = time.time()
self.voltage_history.append((now, voltage))
while now - self.voltage_history[0][0] > self.TREND_WINDOW_S:
self.voltage_history.popleft()
n = len(self.voltage_history)
span = self.voltage_history[-1][0] - self.voltage_history[0][0]
if n < self.TREND_MIN_SAMPLES or span < self.TREND_MIN_SPAN_S:
return None, "unknown"
# Least-squares slope of voltage vs time, converted to mV/min
t_mean = sum(t for t, _ in self.voltage_history) / n
v_mean = sum(v for _, v in self.voltage_history) / n
num = sum((t - t_mean) * (v - v_mean) for t, v in self.voltage_history)
den = sum((t - t_mean) ** 2 for t, _ in self.voltage_history)
if den == 0:
return None, "unknown"
slope_mv_min = num / den * 1000 * 60
if slope_mv_min > self.TREND_THRESHOLD_MV_MIN:
self.battery_state = "charging"
elif slope_mv_min < -self.TREND_THRESHOLD_MV_MIN:
self.battery_state = "discharging"
else:
self.battery_state = "stable"
return slope_mv_min, self.battery_state
def battery_percent(self, voltage):
"""
Estimates state-of-charge [%] from pack voltage by linear
interpolation of the LiFePO4 OCV table. While charging, the bus
follows the charger voltage and the estimate saturates at 100%.
"""
table = self.LIFEPO4_OCV_SOC
if voltage <= table[0][0]:
return 0.0
if voltage >= table[-1][0]:
return 100.0
for (v0, p0), (v1, p1) in zip(table, table[1:]):
if voltage <= v1:
return p0 + (p1 - p0) * (voltage - v0) / (v1 - v0)
return 100.0
def get_power_info(self):
bus_voltage = self.power_monitor.voltage()
power_info = {"BusVoltage": bus_voltage}
try:
power_info.update({
"BusCurrent": self.power_monitor.current(),
"Power": self.power_monitor.power(),
"ShuntVoltage": self.power_monitor.shunt_voltage(),
})
except Exception as exception:
# Current out of device range with specified shunt resistor
logger.error("INA219: %s", exception.__repr__())
# Compensate the IR drop (V_ocv ~ V_bus + I*R) so that trend and SoC
# track the battery, not the load: a load step sags the bus voltage
# and would otherwise read as a discharge, and the relaxation after
# load release as a charge.
current_ma = power_info.get("BusCurrent")
if current_ma is not None:
ocv = bus_voltage + (current_ma / 1000) * self.battery_r_ohm
else:
ocv = bus_voltage
slope_mv_min, battery_state = self.update_voltage_trend(ocv)
# SoC from the windowed mean of the compensated voltage: filters
# INA219 quantization noise and residual load transients.
mean_ocv = sum(v for _, v in self.voltage_history) / len(self.voltage_history)
soc = self.battery_percent(mean_ocv)
power_info.update({
"BusVoltageOCV": round(ocv, 3),
"BusVoltageTrend": round(slope_mv_min, 2) if slope_mv_min is not None else None,
"BatteryState": battery_state,
"BatteryPercent": round(soc, 1),
})
if self.battery_capacity_ah is not None:
remaining_ah = self.battery_capacity_ah * soc / 100
power_info["BatteryRemainingAh"] = round(remaining_ah, 2)
current_ma = power_info.get("BusCurrent")
if (battery_state == "discharging" and current_ma is not None
and current_ma > self.RUNTIME_MIN_CURRENT_MA):
power_info["BatteryRuntime_h"] = round(remaining_ah / (current_ma / 1000), 1)
return power_info
def get_thermal_info(self):
folder_skeleton = "/sys/devices/virtual/thermal/thermal_zone{}/temp"
thermal_info = {}
mapping = {"A0": 0, "CPU": 1, "GPU": 2, "PLL": 3, "thermal": 5}
for k, i in mapping.items():
with open(folder_skeleton.format(i), "r") as fp:
value = fp.read()
thermal_info[k] = int(value) / 1000
return thermal_info
def get_proc_info(self):
gpu_load_file = "/sys/devices/gpu.0/load"
with open(gpu_load_file, "r") as fp:
value = fp.read()
gpu_load = int(value)/10
data_storage = psutil.disk_usage('/data')
proc_info = {
"gpu_load_percent": gpu_load,
"cpu_load_percent": psutil.cpu_percent(),
"percpu_load_percent": psutil.cpu_percent(percpu=True),
"cpu_load_info": psutil.getloadavg(),
"data_storage_percent": data_storage.percent,
"data_storage_free_gb": data_storage.free / 1024**3,
"data_storage_used_gb": data_storage.used / 1024**3,
"data_storage_total_gb": data_storage.total / 1024**3,
}
return proc_info
def get_modem_info(self):
self.modem_count = (self.modem_count + 1) % 10
if self.modem_count > 0 and self.modem_resp is not None:
return self.modem_resp
try:
ser = serial.Serial(
port="/dev/ttyTHS1",
baudrate=115200,
timeout=1,
write_timeout=1
)
if not ser.is_open:
ser.open()
response = send_at_command(ser, "AT+CSQ")
rssi_dbm, ber_percent = parse_csq(response)
response = send_at_command(ser, "AT+CESQ")
rsrq_db, rsrp_dbm = parse_cesq(response)
response = send_at_command(ser, "AT+COPS?")
operator, technology = get_network_info(response)
response = send_at_command(ser, "AT#CCID")
match = re.search(r'#CCID: (\d+)', response)
ccid = ""
if match:
ccid = match.group(1)
# Link status and IP from the PDP context, mode-agnostic: works
# in both PPP and ECM since it asks the modem about the cellular
# session, not the host interface (in ECM the host only sees the
# modem's NAT LAN address, e.g. 192.168.225.x on wwan0).
response = send_at_command(ser, "AT+CEREG?", timeout=0.5)
registration = parse_cereg(response)
response = send_at_command(ser, "AT+CGACT?", timeout=0.5)
active_cids = parse_cgact(response)
ip = None
if active_cids:
response = send_at_command(ser, "AT+CGPADDR=%d" % active_cids[0], timeout=0.5)
ip = parse_cgpaddr(response)
link = "up" if ip is not None else "down"
ser.close()
except:
# Report an explicit "down" default instead of returning the
# cached response: stale good values would keep telling the
# cloud the link is fine while the modem is unreachable.
logger.exception("Error when reading modem, reporting link down")
self.modem_resp = {
"SIM_CCID": None,
"rsrq_dB": None,
"rsrp_dBm": None,
"rssi_dbm": None,
"ber_percent": None,
"network_operator": None,
"network_technology": None,
"registration": None,
"link": "down",
"ip": None,
}
else:
self.modem_resp = {
"SIM_CCID": ccid,
"rsrq_dB": rsrq_db,
"rsrp_dBm": rsrp_dbm,
"rssi_dbm": rssi_dbm,
"ber_percent": ber_percent,
"network_operator": operator,
"network_technology": technology,
"registration": registration,
"link": link,
"ip": ip,
}
return self.modem_resp
def run(self):
jiter_time = 0
while "Running Indefinitely":
if self.refresh_rate is None:
timeout = None
else:
refresh_interval = 1 / self.refresh_rate
timeout = refresh_interval - jiter_time - 0.002
if timeout < 0:
timeout = 0
with self.event_notification:
self.event_notification.wait(timeout=timeout)
if self.stop_service.is_set():
break
t0 = time.time()
self.publish_new_system_info()
jiter_time = time.time() - t0
if self.accumulator is not None:
self.accumulator.commit()
if self._redis is not None:
try:
self._redis.close()
except Exception:
pass
def _add_redis_args(parser):
"""Add common Redis connection arguments to a parser/subparser."""
redis_group = parser.add_argument_group("Redis", "Redis connection options")
redis_group.add_argument(
"--redis-socket",
default=os.environ.get("SYSTEM_INFO_REDIS_SOCKET"),
help="Unix socket path (takes precedence over host:port)",
)
redis_group.add_argument(
"--redis-host",
default=os.environ.get("SYSTEM_INFO_REDIS_HOST", "localhost"),
help="Redis host (default: localhost)",
)
redis_group.add_argument(
"--redis-port",
type=int,
default=int(os.environ.get("SYSTEM_INFO_REDIS_PORT", 6379)),
help="Redis port (default: 6379)",
)
redis_group.add_argument(
"--redis-password",
default=os.environ.get("SYSTEM_INFO_REDIS_PASSWORD"),
help="Redis password",
)
redis_group.add_argument(
"--redis-db",
type=int,
default=int(os.environ.get("SYSTEM_INFO_REDIS_DB", 0)),
help="Redis database number (default: 0)",
)
redis_group.add_argument(
"--redis-stream",
default=os.environ.get("SYSTEM_INFO_REDIS_STREAM", "aimsight:sync"),
help="Redis stream name (default: aimsight:sync)",
)
return redis_group
def _redis_kwargs(args):
"""Extract Redis connection kwargs from parsed args."""
return {
"socket": args.redis_socket,
"host": args.redis_host,
"port": args.redis_port,
"password": args.redis_password,
"db": args.redis_db,
}
def cmd_monitor(args):
"""Run the system info monitor (default command)."""
ENABLE_LOGGING = json.loads(os.environ.get("SYSTEM_INFO_ENABLE_LOGGING", "false"))
DISABLE_MODEM = json.loads(os.environ.get("SYSTEM_INFO_DISABLE_MODEM", "false"))
pw = systemInfo(
device=args.device,
system_name=args.system_name,
enable_logging=args.enable_logging or ENABLE_LOGGING,
refresh_rate=args.refresh,
no_modem=args.modem or DISABLE_MODEM,
battery_capacity_ah=args.battery_capacity,
battery_r_ohm=args.battery_resistance,
redis_socket=args.redis_socket,
redis_host=args.redis_host,
redis_port=args.redis_port,
)
signal.signal(signal.SIGTERM, lambda x, y: pw.stop())
signal.signal(signal.SIGINT, lambda x, y: pw.stop())
pw.start()
while pw.is_alive():
pw.join(1)
# Per-Redis-topic sync spec: each topic maps to a list of directives, applied
# in order against the topic's decoded hash. Directive shapes:
#
# {"fields": [key, ...]}
# Heterogeneous quantities: publish each key as its own metric whose
# data_type is the key itself.
#
# {"metric": name, "label": key, "fields": {redis_key: label_value, ...}}
# Same-unit scalar series that only differ along one dimension: publish
# every field under a single metric ``name``, distinguished by a
# Prometheus-style ``label=label_value`` label instead of baking the
# dimension into the metric name.
#
# {"metric": name, "label": key, "list": redis_key, "list_labels": [...]}
# A list-valued field (one element per series member): publish one metric
# ``name`` per element, labelled by ``list_labels[i]`` (or the index).
SYNC_SPEC = {
"power": [
{"fields": [
"Power", "BusCurrent", "BusVoltage", "BusVoltageOCV", "ShuntVoltage",
"BusVoltageTrend", "BatteryState", "BatteryPercent",
"BatteryRemainingAh", "BatteryRuntime_h",
]},
],
"modem": [
# ``ip`` is published as ip_address{interface="modem"}, see below.
{"fields": [
"rsrq_dB", "SIM_CCID", "rsrp_dBm", "rssi_dbm", "ber_percent",
"network_operator", "network_technology", "registration", "link",
]},
],
"temperature": [
{"metric": "temperature_deg_c", "label": "sensor",
"fields": {"A0": "A0", "CPU": "CPU", "GPU": "GPU", "PLL": "PLL", "thermal": "thermal"}},
],
"proc": [
{"fields": ["data_storage_percent"]},
{"metric": "data_storage_gb", "label": "state",
"fields": {
"data_storage_free_gb": "free",
"data_storage_used_gb": "used",
"data_storage_total_gb": "total",
}},
{"metric": "load_percent", "label": "unit",
"fields": {"cpu_load_percent": "cpu", "gpu_load_percent": "gpu"}},
{"metric": "cpu_load_percent", "label": "cpu", "list": "percpu_load_percent"},
{"metric": "load_average", "label": "window", "list": "cpu_load_info",
"list_labels": ["1m", "5m", "15m"]},
],
}
# Interface IPs published as ip_address{interface=...}, one series per link.
# nebula1 is the Nebula overlay tun (config ``tun.dev``), only present while the
# VPN is up; the modem IP is the cellular PDP address from the modem topic.
IP_SOURCES = [
("eth0", "network", "eth0_ip"),
("nebula", "network", "nebula1_ip"),
("modem", "modem", "ip"),
]
def _emit_directive(publisher, data, directive):
"""Publish one directive's metrics from a decoded Redis hash.
Returns the number of metrics published. None-valued fields/elements and a
missing (or non-list) ``list`` source are skipped.
"""
metric = directive.get("metric")
label = directive.get("label")
count = 0
list_key = directive.get("list")
if list_key is not None:
values = data.get(list_key)
if not isinstance(values, list):
return 0
list_labels = directive.get("list_labels")
for i, value in enumerate(values):
if value is None:
continue
label_value = list_labels[i] if list_labels and i < len(list_labels) else str(i)
publisher.metric(json.dumps(value), data_type=metric, labels={label: label_value})
count += 1
return count
if label is not None:
for redis_key, label_value in directive["fields"].items():
value = data.get(redis_key)
if value is None:
continue
publisher.metric(json.dumps(value), data_type=metric, labels={label: label_value})
count += 1
return count
for key in directive["fields"]:
value = data.get(key)
if value is None:
continue
publisher.metric(json.dumps(value), data_type=key)
count += 1
return count
def _module_version(*dist_names):
"""Best-effort version of an installed distribution, or None.
Reads distribution metadata (does not import the package, so no heavy
optional deps are pulled in); tries each candidate name in turn.
"""
try:
from importlib.metadata import version, PackageNotFoundError
except ImportError:
return None
for dist in dist_names:
try:
return version(dist)
except PackageNotFoundError:
continue
except Exception:
return None
return None
def _distro_version(path="/etc/build"):
"""Parse ``DISTRO_VERSION`` from the image build file, or None if absent."""
try:
with open(path) as fp:
for line in fp:
if line.startswith("DISTRO_VERSION"):
_, _, value = line.partition("=")
return value.strip().strip('"').strip("'") or None
except OSError:
return None
return None
def _software_versions():
"""Component versions for the ``software_version`` info-metric labels.
Missing components report "unknown" so the series keeps a stable set of
label keys.
"""
return {
"aimbedded": _AIMBEDDED_VERSION or "unknown",
"aimsight_optical": _module_version("aimsight-optical", "aimsight_optical") or "unknown",
"distro": _distro_version() or "unknown",
}
def _ntp_synchronized():
"""Whether the system clock is NTP-synchronized, or None if undetermined.
Read live from ``timedatectl`` (an unprivileged query): a drifted clock
breaks TLS and makes every reported timestamp suspect, so it is worth
surfacing on its own.
"""
try:
out = subprocess.check_output(
["timedatectl", "show", "-p", "NTPSynchronized", "--value"],
text=True, stderr=subprocess.DEVNULL, timeout=5,
).strip().lower()
except Exception:
return None
if out in ("yes", "true", "1"):
return True
if out in ("no", "false", "0"):
return False
return None
def cmd_sync(args):
"""Sync system_info snapshot from Redis to cloud via MQTT."""
from .client import SystemInfoClient
from ..mqtt_sync.publisher import SyncPublisher
redis_kw = _redis_kwargs(args)
with SystemInfoClient(**redis_kw) as client, \
SyncPublisher(**redis_kw, stream=args.redis_stream) as publisher:
snap = client.snapshot()
if not snap:
logger.warning("No system_info data in Redis, skipping sync")
return
count = 0
for topic, directives in SYNC_SPEC.items():
data = snap.get(topic)
if not data:
continue
for directive in directives:
count += _emit_directive(publisher, data, directive)
# Interface IPs, one series per interface via an ``interface`` label.
for label, topic, key in IP_SOURCES:
ip = (snap.get(topic) or {}).get(key)
if ip:
publisher.metric(json.dumps(ip), data_type="ip_address", labels={"interface": label})
count += 1
# Clock health: 1 synchronized, 0 drifted. Reported even when 0 (a
# de-synced clock must be visible, not merely absent); omitted only
# when it cannot be determined.
ntp = _ntp_synchronized()
if ntp is not None:
publisher.metric(json.dumps(1 if ntp else 0), data_type="ntp_synchronized")
count += 1
# Component versions as a Prometheus-style info metric (value 1, the
# versions carried as labels).
publisher.metric(json.dumps(1), data_type="software_version", labels=_software_versions())
count += 1
publisher.metric(json.dumps(datetime.datetime.now().isoformat()), data_type="last_sync")
logger.info("Synced %d system_info metrics to cloud", count + 1)
def main():
SYSTEM_NAME = os.environ.get("SYSTEM_NAME", None)
REFRESH_RATE = os.environ.get("SYSTEM_INFO_REFRESH_RATE", "1")
parser = argparse.ArgumentParser(description="System info collector")
subparsers = parser.add_subparsers(dest="command")
# --- monitor (default) ---
monitor_parser = subparsers.add_parser("monitor", help="Run system info monitor")
monitor_parser.add_argument("-d", "--device", type=int, help="device index", default=1)
monitor_parser.add_argument("-s", "--system_name", help="System name", default=SYSTEM_NAME)
monitor_parser.add_argument("-r", "--refresh", type=float, help="Refresh rate in [Hz]", default=REFRESH_RATE)
monitor_parser.add_argument("-l", "--enable_logging", action="store_true", help="print values")
monitor_parser.add_argument("-m", "--modem", action="store_true", help="disable modem")
BATTERY_CAPACITY = os.environ.get("SYSTEM_INFO_BATTERY_CAPACITY_AH")
monitor_parser.add_argument(
"-c", "--battery-capacity", type=float,
default=float(BATTERY_CAPACITY) if BATTERY_CAPACITY else None,
help="Battery capacity in Ah (7.3 or 19); enables remaining-Ah and runtime estimates",
)
BATTERY_R = os.environ.get("SYSTEM_INFO_BATTERY_R_OHM")
monitor_parser.add_argument(
"--battery-resistance", type=float,
default=float(BATTERY_R) if BATTERY_R else systemInfo.BATTERY_R_OHM_DEFAULT,
help="Effective series resistance battery-to-INA219 in Ohm, for IR compensation "
"(default: %(default)s, measured via load-step V/I regression)",
)
_add_redis_args(monitor_parser)
# --- sync ---
sync_parser = subparsers.add_parser("sync", help="Sync system_info snapshot to cloud")
_add_redis_args(sync_parser)
args = parser.parse_args()
if args.command == "sync":
cmd_sync(args)
else:
# Default to monitor (handles both explicit "monitor" and no subcommand)
if args.command is None:
args = monitor_parser.parse_args()
cmd_monitor(args)
if __name__ == "__main__":
main()
-33
View File
@@ -1,33 +0,0 @@
"""Analyse 01 : MOVING_AVERAGE_7_DAYS_DROP (tendance de BusVoltage en moyenne glissante), graphiques HTML ouverts dans le navigateur.
La page montre aussi, dessous, les analyses 02 (MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP), 03 (DAILY_BATTERY_STATE) et 04 (DAILY_BATTERY_TREND).
Usage : python3 src/analyse_01.py [CSV ou dossiers...] (défaut : data/, sortie dans out/)
Documentation : docs/analyses/analyse_01.md
"""
import importlib.util
import os
import shutil
import subprocess
import sys
import webbrowser
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
VENV = ROOT / ".venv" / "bin" / "python"
if importlib.util.find_spec("plotly") is None and VENV.exists():
os.execv(VENV, [str(VENV), *sys.argv]) # plotly n'est installé que dans .venv
import energy_plot # noqa: E402
def open_html(path):
if shutil.which("explorer.exe") and shutil.which("wslpath"): # WSL : navigateur Windows
subprocess.run(["explorer.exe", subprocess.check_output(["wslpath", "-w", path], text=True).strip()])
else:
webbrowser.open(Path(path).resolve().as_uri())
if __name__ == "__main__":
args = sys.argv[1:] or [str(ROOT / "data")]
for p in energy_plot.main([*args, "--out", str(ROOT / "out")]):
open_html(p)
-10
View File
@@ -1,10 +0,0 @@
"""Analyse 02 : MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP (MOVING_AVERAGE_7_DAYS_DROP sur la seule mesure de 00:00 UTC).
Son graphique est le panneau placé sous MOVING_AVERAGE_7_DAYS_DROP, dans la page commune : ce script lance analyse_01.py.
Usage : python3 src/analyse_02.py [CSV ou dossiers...] (défaut : data/, sortie dans out/)
Documentation : docs/analyses/analyse_02.md
"""
import runpy
from pathlib import Path
runpy.run_path(str(Path(__file__).with_name("analyse_01.py")), run_name="__main__")
-10
View File
@@ -1,10 +0,0 @@
"""Analyse 03 : DAILY_BATTERY_STATE (état de la batterie jour par jour, phase C).
Son graphique est le troisième panneau d'analyse de la page commune : ce script lance analyse_01.py.
Usage : python3 src/analyse_03.py [CSV ou dossiers...] (défaut : data/, sortie dans out/)
Documentation : docs/analyses/analyse_03.md
"""
import runpy
from pathlib import Path
runpy.run_path(str(Path(__file__).with_name("analyse_01.py")), run_name="__main__")
-10
View File
@@ -1,10 +0,0 @@
"""Analyse 04 : DAILY_BATTERY_TREND (tendance de la batterie d'un jour à l'autre, J face à J−1).
Son graphique est le quatrième panneau d'analyse de la page commune : ce script lance analyse_01.py.
Usage : python3 src/analyse_04.py [CSV ou dossiers...] (défaut : data/, sortie dans out/)
Documentation : docs/analyses/analyse_04.md
"""
import runpy
from pathlib import Path
runpy.run_path(str(Path(__file__).with_name("analyse_01.py")), run_name="__main__")
View File
Whitespace-only changes.
+59
View File
@@ -0,0 +1,59 @@
"""Pages HTML du poste de dev : bascule sur .venv si plotly manque, ouverture dans le navigateur, zoom (D-013).
À importer avant plotly : si plotly n'est pas installé, l'import relance le script sous .venv/bin/python.
"""
import importlib.util
import os
import shutil
import subprocess
import sys
import webbrowser
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
VENV = ROOT / ".venv" / "bin" / "python"
if importlib.util.find_spec("plotly") is None and VENV.exists():
os.execv(VENV, [str(VENV), *sys.argv]) # plotly n'est installé que dans .venv
# Plotly ne recale pas l'ordonnée sur la plage visible : après chaque zoom ou déplacement horizontal,
# ce script recalcule l'axe vertical de chaque graphique sur les points visibles (5 % de marge),
# seuil compris (c'est une courbe). Double-clic : retour à la vue complète.
AUTOSCALE_Y_JS = """
const gd = document.getElementById('{plot_id}');
const name = (t, a) => a + 'axis' + (t[a + 'axis'] || a).slice(1);
const xs = gd.data.map(t => t.x.map(v => gd._fullLayout[name(t, 'x')].d2l(v)));
gd.on('plotly_relayout', ev => {
const keys = Object.keys(ev);
if (!keys.some(k => k.startsWith('xaxis'))) return;
const upd = {};
if (keys.some(k => k.endsWith('autorange'))) {
gd.data.forEach(t => { upd[name(t, 'y') + '.autorange'] = true; });
return Plotly.relayout(gd, upd);
}
const box = {};
gd.data.forEach((t, i) => {
if (t.visible === 'legendonly' || t.visible === false) return;
const xa = gd._fullLayout[name(t, 'x')];
const [a, b] = xa.range.map(r => xa.r2l(r));
const r = box[name(t, 'y')] || (box[name(t, 'y')] = [Infinity, -Infinity]);
t.y.forEach((y, j) => {
if (y === null || !isFinite(y) || xs[i][j] < a || xs[i][j] > b) return;
r[0] = Math.min(r[0], y);
r[1] = Math.max(r[1], y);
});
});
for (const [ya, [lo, hi]] of Object.entries(box)) {
if (lo > hi) continue;
const pad = (hi - lo) * 0.05 || Math.abs(hi) * 0.01 || 1;
upd[ya + '.range'] = [lo - pad, hi + pad];
}
Plotly.relayout(gd, upd);
});
"""
def open_html(path):
if shutil.which("explorer.exe") and shutil.which("wslpath"): # WSL : navigateur Windows
subprocess.run(["explorer.exe", subprocess.check_output(["wslpath", "-w", path], text=True).strip()])
else:
webbrowser.open(Path(path).resolve().as_uri())
+67
View File
@@ -0,0 +1,67 @@
"""Chargement des exports CSV des stations, commun à tous les domaines.
Format des CSV : docs/domaine/ce-que-mesure-la-station.md. Bibliothèque standard uniquement.
"""
import csv
import glob
import os
import re
import sys
from datetime import datetime
NAME_RE = re.compile(r"^(?P<station>[^_]+)_(?P<metric>.+?)_\d{4}_\d{2}_\d{2}_\d{2}_\d{2} to .+\.csv$")
def parse_name(path):
"""'<STATION>_<Metric>_<début> to <fin>.csv' -> (station, metric), ou None."""
m = NAME_RE.match(os.path.basename(path))
return (m["station"], m["metric"]) if m else None
def load_csv(path):
"""CSV ',time_utc,metric' -> {datetime UTC: float ou None (pas de mesure)}."""
try:
with open(path, newline="") as f:
rows = csv.reader(f)
next(rows)
return {datetime.fromisoformat(r[1]): float(r[2]) if r[2] else None for r in rows}
except (StopIteration, IndexError, ValueError) as e:
raise ValueError(f"{path} : CSV illisible ({e!r})") from e
def load_stations(paths):
"""Fichiers et/ou dossiers -> {station: {metric: série}}."""
files = []
for p in paths:
files += sorted(glob.glob(os.path.join(p, "*.csv"))) if os.path.isdir(p) else [p]
stations, seen = {}, {}
for path in files:
real = os.path.realpath(path)
if real in seen.values():
continue
key = parse_name(path)
if key is None:
print(f"ignoré (nom non reconnu) : {path}", file=sys.stderr)
continue
if key in seen:
raise ValueError(f"doublon {key[0]}/{key[1]} : {seen[key]} et {path}")
seen[key] = real
try:
stations.setdefault(key[0], {})[key[1]] = load_csv(path)
except ValueError as e:
print(f"ignoré : {e}", file=sys.stderr)
return stations
def gaps(v):
"""Plages de créneaux sans mesure : [(premier créneau vide, première valeur suivante)]."""
out, start = [], None
for t in sorted(v):
if v[t] is None and start is None:
start = t
elif v[t] is not None and start is not None:
out.append((start, t))
start = None
if start is not None:
out.append((start, max(v)))
return out
View File
Whitespace-only changes.
+95
View File
@@ -0,0 +1,95 @@
"""Les quatre analyses Energy ensemble, pour les comparer (D-016) : un panneau chacune, puis un graphique par série.
Usage : python3 src/energy/all_analyses.py [CSV ou dossiers...] (défaut : data/)
Écrit out/<STATION>_energy.html et l'ouvre. Documentation : docs/energy/analyses/ (une fiche par analyse)
"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # src/ : paquets common et energy
from common.page import AUTOSCALE_Y_JS, ROOT, open_html # noqa: E402 (bascule seule sur .venv si plotly manque)
from common.stations import gaps, load_stations # noqa: E402
from plotly.subplots import make_subplots # noqa: E402
from energy import battery_state as bs, battery_trend as bt, moving_average as ma # noqa: E402
from energy.daily import daily_state_panel, daily_trend_panel # noqa: E402
# Zones de BusVoltage selon l'état de la batterie (bandes de fond) : docs/energy/domaine/tensions-selon-l-etat-de-la-batterie.md.
# [V] = VERMONT, 5e-95e centiles horaires ; [M] = manuel ; [E] = experts. Les zones « en charge » et « décharge »
# se recouvrent de 13,1 à 13,2 V : sur le plateau, la tension seule ne sépare pas la charge de la décharge.
ZONES = [
("décharge", 12.0, 13.2, "steelblue", "bottom left"), # 12,0 : coupure de la station [M] ; 13,2 : haut du plateau [V]
("en charge", 13.1, 13.8, "gold", "top left"), # bulk [V]
("pleine charge", 14.0, 14.4, "green", "top left"), # absorption 14,0-14,3 [V] ; secteur 14,2-14,4 [E]
]
def figure(station, series, results):
"""Haut : une analyse par graphique, courbes superposées. Dessous : une série temporelle par graphique."""
metrics = ["BusVoltage"] + sorted(m for m in series if m != "BusVoltage")
n = len(results)
fig = make_subplots(rows=n + len(metrics), cols=1, shared_xaxes=True, vertical_spacing=0.03,
row_heights=[1.5] * n + [1] * len(metrics),
subplot_titles=[f"Analyse {r['analysis']}" for r in results] + metrics)
for row, result in enumerate(results, start=1):
if result["analysis"] == bs.RULE["id"]:
daily_state_panel(fig, result, row)
continue
if result["analysis"] == bt.RULE["id"]:
daily_trend_panel(fig, result, row)
continue
analyse = dict(legendgroup=result["analysis"], legendgrouptitle_text=f"Analyse {result['analysis']}")
thr = result["rule"]["threshold_v"]
pts = result["points"]
x = [p["t"] for p in pts]
ref = [p["facts"].get("ref_v") for p in pts]
hover = [f"écart={p['value'] if p['value'] is None else round(p['value'], 3)}<br>anomalie={p['anomaly']}<br>"
+ "<br>".join(f"{k}={val}" for k, val in p["facts"].items()) for p in pts]
fig.add_scatter(x=x, y=[p["facts"].get("ma_v") for p in pts], name="moyenne 7 j", text=hover,
hovertemplate="%{y:.3f} V<br>%{text}", line=dict(color="royalblue"), row=row, col=1,
**analyse)
fig.add_scatter(x=x, y=[None if r is None else r + thr for r in ref], name=f"seuil (max 30 j {thr:+} V)",
line=dict(color="red", dash="dash"), row=row, col=1, **analyse)
for p in ma.periods(pts):
fig.add_vrect(x0=p["start"], x1=p["end"] or x[-1], fillcolor="red", opacity=0.15,
line_width=0, row=row, col=1)
fig.update_yaxes(title_text="BusVoltage moyennée [V]", row=row, col=1)
mesures = dict(legendgroup="mesures", legendgrouptitle_text="Mesures")
for row, m in enumerate(metrics, start=n + 1):
s = series[m]
fig.add_scatter(x=sorted(s), y=[s[t] for t in sorted(s)], name=m, line=dict(width=1),
row=row, col=1, **mesures)
for a, b in gaps(series["BusVoltage"]):
fig.add_vrect(x0=a, x1=b, fillcolor="gray", opacity=0.3, line_width=0, row=n + 1, col=1)
for name, lo, hi, color, pos in ZONES:
fig.add_hrect(y0=lo, y1=hi, fillcolor=color, opacity=0.15, line_width=0, layer="below",
annotation_text=f"{name} {lo}-{hi} V", annotation_position=pos, row=n + 1, col=1)
fig.update_xaxes(showticklabels=True) # dates sous chaque graphique, pas seulement le dernier
fig.update_yaxes(fixedrange=True) # zoom horizontal seulement : l'ordonnée est recalée par AUTOSCALE_Y_JS
fig.update_layout(title=f"{station} : analyses " + ", ".join(r["analysis"] for r in results),
height=300 * (n + len(metrics)),
barmode="overlay", legend=dict(groupclick="toggleitem"))
return fig
def main(paths):
"""Pour chaque station qui a BusVoltage : out/<STATION>_energy.html. -> chemins écrits."""
(ROOT / "out").mkdir(exist_ok=True)
written = []
for station, series in sorted(load_stations(paths).items()):
if "BusVoltage" not in series:
print(f"{station} : ignorée, pas de BusVoltage")
continue
path = ROOT / "out" / f"{station}_energy.html"
results = [ma.run(series), ma.run_midnight(series), bs.run(series), bt.run(series)]
figure(station, series, results).write_html(path, include_plotlyjs="cdn", post_script=AUTOSCALE_Y_JS,
config={"scrollZoom": True})
print(path)
written.append(path)
return written
if __name__ == "__main__":
for p in main(sys.argv[1:] or [str(ROOT / "data")]):
open_html(p)
@@ -1,19 +1,19 @@
"""Analyse 03 DAILY_BATTERY_STATE : état de la batterie jour par jour (phase C : état seul, sans verdict).
"""DAILY_BATTERY_STATE : état de la batterie jour par jour (phase C : état seul, sans verdict).
Documentation : docs/analyses/analyse_03.md
Documentation : docs/energy/analyses/daily_battery_state.md
Bibliothèque standard uniquement (cible : système embarqué de la station).
"""
import math
from collections import defaultdict
from datetime import timedelta
HOUR = timedelta(hours=1) # grille des CSV : moyennes horaires (sans import d'energy_check : 03 et 04 se livrent seules)
HOUR = timedelta(hours=1) # grille des CSV : moyennes horaires (aucun import du projet : DAILY_BATTERY_STATE et DAILY_BATTERY_TREND se livrent seules)
# Constantes (fiche analyse_03, « Constantes et sources »).
# Constantes (fiche daily_battery_state, « Constantes et sources »).
FULL_CHARGE_V = 14.0 # VERMONT : maxima journaliers < 13,55 V (35 j) ou 14,02-14,33 V (135 j) ; absorption ~14,2 V
PLATEAU_FLOOR_V = 12.8 # coude LiFePO4 : table OCV du collecteur (12,8 V = 30 %) ; pente nocturne doublée sous 12,8 V
MIN_RISE_V = 0.05 # > variations horaires sur le plateau (<= 15 mV/h), < plus faible vraie montée (+0,18 V, VERMONT 04-11)
MIN_COVERAGE = 0.8 # repris de l'analyse 01 : 20 h mesurées sur 24
MIN_COVERAGE = 0.8 # repris de MOVING_AVERAGE_7_DAYS_DROP : 20 h mesurées sur 24
RULE = {
"id": "DAILY_BATTERY_STATE",
@@ -22,12 +22,12 @@ RULE = {
"plateau_floor_v": PLATEAU_FLOOR_V,
"min_rise_v": MIN_RISE_V,
"min_coverage": MIN_COVERAGE,
"doc": "docs/analyses/analyse_03.md",
"doc": "docs/energy/analyses/daily_battery_state.md",
}
def day_facts(day):
"""{heure UTC: tension mesurée} d'un jour -> faits et état (fiche analyse_03, « Principe »)."""
"""{heure UTC: tension mesurée} d'un jour -> faits et état (fiche daily_battery_state, « Principe »)."""
if not day:
return {"state": None, "coverage": 0.0}
max_v, min_v = max(day.values()), min(day.values())
@@ -1,22 +1,22 @@
"""Analyse 04 DAILY_BATTERY_TREND : tendance de la batterie d'un jour à l'autre (modèle markovien : J face à J−1).
"""DAILY_BATTERY_TREND : tendance de la batterie d'un jour à l'autre (modèle markovien : J face à J−1).
Documentation : docs/analyses/analyse_04.md
Documentation : docs/energy/analyses/daily_battery_trend.md
Bibliothèque standard uniquement (cible : système embarqué de la station).
"""
from datetime import timedelta
import battery_state as bs
from energy import battery_state as bs
# Constantes (fiche analyse_04, « Constantes et sources »).
# Constantes (fiche daily_battery_trend, « Constantes et sources »).
NIGHT_END_DEADBAND_V = 0.05 # VERMONT : FULL_CHARGE enchaînés, écart de fin de nuit -0,026 à +0,043 V (5e-95e centiles, 125 paires)
STATE_RANK = {"FULL_CHARGE": 3, "PARTIAL_CHARGE": 2, "NO_CHARGE": 1, "BELOW_PLATEAU": 0} # choix de conception (fiche 04)
STATE_RANK = {"FULL_CHARGE": 3, "PARTIAL_CHARGE": 2, "NO_CHARGE": 1, "BELOW_PLATEAU": 0} # choix de conception (fiche daily_battery_trend)
RULE = {
"id": "DAILY_BATTERY_TREND",
"verdict": "aucun : anomaly = None ; la tendance est dans facts.trend",
"night_end_deadband_v": NIGHT_END_DEADBAND_V,
"state_rank": STATE_RANK,
"doc": "docs/analyses/analyse_04.md",
"doc": "docs/energy/analyses/daily_battery_trend.md",
}
+121
View File
@@ -0,0 +1,121 @@
"""Analyses livrées (D-020) : DAILY_BATTERY_STATE et DAILY_BATTERY_TREND, en une figure pour le site (D-021).
Usage : python3 src/energy/daily.py [CSV ou dossiers...] (défaut : data/)
Écrit out/<STATION>_daily.html (page de démo) et .json (figure Plotly pour le site, Plotly.newPlot), puis les ouvre.
Documentation : docs/energy/analyses/daily_battery_state.md et daily_battery_trend.md
"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # src/ : paquets common et energy
from datetime import datetime, time, timezone # noqa: E402
from common.page import AUTOSCALE_Y_JS, ROOT, open_html # noqa: E402 (bascule seule sur .venv si plotly manque)
from common.stations import load_stations # noqa: E402
from plotly.subplots import make_subplots # noqa: E402
from energy import battery_state as bs, battery_trend as bt # noqa: E402
# États de DAILY_BATTERY_STATE : couleurs validées par le skill dataviz (toutes paires, fond blanc).
# Le jaune contraste peu avec le fond : chaque état a aussi son symbole, sa légende et son survol.
STATE_STYLE = {
"FULL_CHARGE": ("full charge", "#008300", "circle"),
"PARTIAL_CHARGE": ("partial charge (uncertain)", "#eda100", "diamond"),
"BELOW_PLATEAU": ("below plateau", "#2a78d6", "triangle-down"),
"NO_CHARGE": ("no charge", "#4a3aa7", "x"),
None: ("unknown state", "#8c8c8c", "circle-open"),
}
def noon(day):
"""Abscisse d'un point journalier (DAILY_BATTERY_STATE et DAILY_BATTERY_TREND) : midi UTC de son jour, au milieu de la journée."""
return datetime.combine(day, time(12), timezone.utc)
def daily_state_panel(fig, result, row):
"""Panneau DAILY_BATTERY_STATE : maximum et fin de nuit par jour, seuils, état en marqueurs (fiche daily_battery_state)."""
group = dict(legendgroup=result["analysis"], legendgrouptitle_text=f"Analysis {result['analysis']}")
rule, pts = result["rule"], result["points"]
x = [noon(p["t"].date()) for p in pts]
fig.add_scatter(x=x, y=[p["facts"].get("max_v") for p in pts], name="daily maximum", mode="lines",
line=dict(color="#52514e", width=1.5), hoverinfo="skip", row=row, col=1, **group)
fig.add_scatter(x=x, y=[p["facts"].get("night_end_v") for p in pts], name="night end", mode="lines",
line=dict(color="#9a9893", width=1.5), hovertemplate="night end %{y:.2f} V<extra></extra>",
row=row, col=1, **group)
# Seuils tracés comme des courbes (un point par jour) : AUTOSCALE_Y_JS les garde visibles au zoom (D-013).
for v, name, state in ((rule["full_charge_v"], "full charge", "FULL_CHARGE"),
(rule["plateau_floor_v"], "plateau floor", "BELOW_PLATEAU")):
fig.add_scatter(x=x, y=[v] * len(x), name=f"{name} {v} V", mode="lines", hoverinfo="skip",
line=dict(color=STATE_STYLE[state][1], dash="dash", width=1), row=row, col=1, **group)
for state, (label, color, symbol) in STATE_STYLE.items():
sel = [p for p in pts if p["facts"]["state"] == state and p["facts"].get("max_v") is not None]
if not sel:
continue
hover = [f"{p['t']:%Y-%m-%d}: {label}<br>" + "<br>".join(f"{k}={val}" for k, val in p["facts"].items())
for p in sel]
fig.add_scatter(x=[noon(p["t"].date()) for p in sel], y=[p["facts"]["max_v"] for p in sel], name=label,
mode="markers",
marker=dict(color=color, symbol=symbol, size=8, line=dict(color="white", width=1)),
text=hover, hovertemplate="%{text}<extra></extra>", row=row, col=1, **group)
fig.update_yaxes(title_text="BusVoltage [V]", row=row, col=1)
# Tendances de DAILY_BATTERY_TREND : paire divergente bleu/rouge et gris neutre du skill dataviz
# (vert/rouge refusé par son validateur : deutéranopie, ΔE 5,9). Chaque tendance a aussi sa légende et son survol.
TREND_STYLE = {"BETTER": ("better", "#2a78d6"), "STABLE": ("stable", "#8c8c8c"), "WORSE": ("worse", "#d03b3b")}
def daily_trend_panel(fig, result, row):
"""Panneau DAILY_BATTERY_TREND : écart de fin de nuit J − J−1 en barres, couleur = tendance (fiche daily_battery_trend)."""
group = dict(legendgroup=result["analysis"], legendgrouptitle_text=f"Analysis {result['analysis']}")
pts, db = result["points"], result["rule"]["night_end_deadband_v"]
x = [noon(p["t"].date()) for p in pts]
# Zone morte tracée comme une courbe (un point par jour, None entre +db et −db) : visible au zoom (D-013).
fig.add_scatter(x=x + [None] + x, y=[db] * len(x) + [None] + [-db] * len(x), name=f"dead band ±{db} V",
mode="lines", hoverinfo="skip", line=dict(color="#52514e", dash="dash", width=1),
row=row, col=1, **group)
for trend, (label, color) in TREND_STYLE.items():
sel = [p for p in pts if p["facts"]["trend"] == trend and p["value"] is not None]
if not sel:
continue
hover = [f"{p['t']:%Y-%m-%d}: {label} (decided by {p['facts']['decided_by']})<br>"
f"{p['facts']['state_prev']} → {p['facts']['state']}<br>"
+ "<br>".join(f"{k}={v}" for k, v in p["facts"].items() if k.startswith("delta")) for p in sel]
fig.add_bar(x=[noon(p["t"].date()) for p in sel], y=[p["value"] for p in sel], name=label,
marker_color=color, width=0.8 * 86_400_000, hovertext=hover, hovertemplate="%{hovertext}<extra></extra>",
row=row, col=1, **group)
fig.update_yaxes(title_text="Δ night end [V]", row=row, col=1)
def daily_figure(station, series):
"""Graphique pour le site de monitoring (D-021) : DAILY_BATTERY_STATE en haut, DAILY_BATTERY_TREND dessous, même axe du temps."""
fig = make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.08,
subplot_titles=[f"Analysis {bs.RULE['id']}", f"Analysis {bt.RULE['id']}"])
daily_state_panel(fig, bs.run(series), 1)
daily_trend_panel(fig, bt.run(series), 2)
fig.update_xaxes(showticklabels=True)
fig.update_yaxes(fixedrange=True) # zoom horizontal seulement : l'ordonnée est recalée par AUTOSCALE_Y_JS
fig.update_layout(title=f"{station}: battery state and trend", height=700, barmode="overlay",
legend=dict(groupclick="toggleitem"))
return fig
def main(paths):
"""Pour chaque station qui a BusVoltage : out/<STATION>_daily.html et .json. -> chemins écrits."""
(ROOT / "out").mkdir(exist_ok=True)
written = []
for station, series in sorted(load_stations(paths).items()):
if "BusVoltage" not in series:
print(f"{station} : ignorée, pas de BusVoltage")
continue
path = ROOT / "out" / f"{station}_daily.html"
fig = daily_figure(station, series)
fig.write_html(path, include_plotlyjs="cdn", post_script=AUTOSCALE_Y_JS, config={"scrollZoom": True})
fig.write_json(path.with_suffix(".json"))
print(path)
written.append(path)
return written
if __name__ == "__main__":
for p in main(sys.argv[1:] or [str(ROOT / "data")]):
open_html(p)
+6 -7
View File
@@ -1,14 +1,13 @@
"""Couleur du jour : matrice état (analyse 03) × tendance (analyse 04) -> vert / orange / rouge (D-020).
"""Couleur du jour : matrice état (DAILY_BATTERY_STATE) × tendance (DAILY_BATTERY_TREND) -> vert / orange / rouge (D-020).
Documentation : docs/guide/diagnostic-energetique.md (§ 7, la matrice)
Documentation : docs/energy/guide/diagnostic-energetique.md (§ 7, la matrice)
Bibliothèque standard uniquement (cible : système embarqué de la station).
"""
from datetime import timedelta
import battery_state as bs
import battery_trend as bt
from energy import battery_state as bs, battery_trend as bt
# Matrice tenue par l'utilisateur dans cas-d-usage-03-04.drawio : MATRIX[état][tendance].
# Matrice tenue par l'utilisateur dans docs/energy/matrice-etat-tendance.drawio : MATRIX[état][tendance].
MATRIX = {
"FULL_CHARGE": {"BETTER": "GREEN", "STABLE": "GREEN", "WORSE": "GREEN"},
"PARTIAL_CHARGE": {"BETTER": "GREEN", "STABLE": "ORANGE", "WORSE": "ORANGE"},
@@ -20,12 +19,12 @@ RULE = {
"id": "DAY_COLOR",
"verdict": "aucun : anomaly = None ; la couleur est dans facts.color (None = inconnu, état ou tendance absent)",
"matrix": MATRIX,
"doc": "docs/guide/diagnostic-energetique.md",
"doc": "docs/energy/guide/diagnostic-energetique.md",
}
def run(series):
"""Couleur du jour, causale comme les analyses 03 et 04 (J et J−1). Un point par jour UTC."""
"""Couleur du jour, causale comme DAILY_BATTERY_STATE et DAILY_BATTERY_TREND (J et J−1). Un point par jour UTC."""
points = []
for s, t in zip(bs.run(series)["points"], bt.run(series)["points"]): # mêmes jours, même ordre (bt part de bs)
facts = {**s["facts"], **t["facts"]}
+21 -21
View File
@@ -1,24 +1,24 @@
"""Focus sur un jour : sa couleur (matrice 03 × 04) et ce qui l'explique, sur le jour J et la veille seulement (D-024).
"""Focus sur un jour : sa couleur (matrice état × tendance) et ce qui l'explique, sur le jour J et la veille seulement (D-024).
Usage : python3 src/focus_jour.py JJ.MM[.AAAA] [CSV ou dossiers...] (défaut : data/ ; sans année, celle des données)
Usage : python3 src/energy/focus_jour.py JJ.MM[.AAAA] [CSV ou dossiers...] (défaut : data/ ; sans année, celle des données)
Affiche le résumé des faits, écrit out/<STATION>_focus_<AAAA-MM-JJ>.html (+ .json, la figure Plotly pour le site)
et l'ouvre dans le navigateur. Textes du résumé et du graphique en anglais : destinés au site (D-023).
"""
import sys
from datetime import date, timedelta
from pathlib import Path
from analyse_01 import ROOT, open_html # bascule seule sur .venv si plotly manque
import plotly.graph_objects as go
import battery_state as bs
import battery_trend as bt
import day_color as dc
import energy_check as ec
import energy_plot as ep
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # src/ : paquets common et energy
from common.page import ROOT, open_html # noqa: E402 (bascule seule sur .venv si plotly manque)
from common.stations import gaps, load_stations # noqa: E402
import plotly.graph_objects as go # noqa: E402
from energy import battery_state as bs, battery_trend as bt, day_color as dc # noqa: E402
from energy.daily import STATE_STYLE, TREND_STYLE, noon # noqa: E402
# Statuts : noms et couleurs de la maquette System Summary (maquettes/user/energy_template.html).
COLOR = {"GREEN": ("Green", "#008300"), "ORANGE": ("Orange", "#eda100"), "RED": ("Red", "#d03b3b"),
None: ("Unknown", "#8c8c8c")}
# Condition qui a donné l'état (fiche analyse_03, « Principe », dans l'ordre d'évaluation).
# Condition qui a donné l'état (fiche daily_battery_state, « Principe », dans l'ordre d'évaluation).
STATE_RULE = {
"FULL_CHARGE": f"at least one hour ≥ {bs.FULL_CHARGE_V} V",
"BELOW_PLATEAU": f"one hour < {bs.PLATEAU_FLOOR_V} V",
@@ -66,24 +66,24 @@ def provisional(f):
def figure(station, f):
"""BusVoltage horaire sur J−1 et J, seuils de l'analyse 03, maximum et fin de nuit de chaque jour,
flèche de fin de nuit J−1 -> J (comparaison de l'analyse 04)."""
"""BusVoltage horaire sur J−1 et J, seuils de DAILY_BATTERY_STATE, maximum et fin de nuit de chaque jour,
flèche de fin de nuit J−1 -> J (comparaison de DAILY_BATTERY_TREND)."""
d, h = f["day"], f["hours"]
name, color = COLOR[f["color"]]
fig = go.Figure()
fig.add_scatter(x=list(h), y=list(h.values()), mode="lines+markers", line=dict(color="#636efa", width=2),
marker_size=4, hovertemplate="%{x|%d/%m %H:%M} UTC: %{y:.3f} V<extra></extra>")
for a, b in ep.gaps(h):
for a, b in gaps(h):
fig.add_vrect(x0=a, x1=b, fillcolor="gray", opacity=0.3, line_width=0)
for v, label, state in ((bs.FULL_CHARGE_V, "full charge", "FULL_CHARGE"),
(bs.PLATEAU_FLOOR_V, "plateau floor", "BELOW_PLATEAU")):
fig.add_hline(y=v, line_dash="dash", line_width=1, line_color=ep.STATE_STYLE[state][1],
fig.add_hline(y=v, line_dash="dash", line_width=1, line_color=STATE_STYLE[state][1],
annotation_text=f"{label} {v} V", annotation_position="top right")
midnight = ep.noon(d) - timedelta(hours=12) # début de J (UTC) : l'axe couvre toujours J−1 et J, 48 h
midnight = noon(d) - timedelta(hours=12) # début de J (UTC) : l'axe couvre toujours J−1 et J, 48 h
fig.add_vline(x=midnight, line_width=1, line_color="#52514e")
night_ends = []
for tag, x, facts in (("D-1", d - timedelta(days=1), f["prev"]), ("D", d, f["cur"])):
fig.add_annotation(x=ep.noon(x), y=1.07, yref="paper", showarrow=False,
fig.add_annotation(x=noon(x), y=1.07, yref="paper", showarrow=False,
text=f"<b>{tag} {x:%d/%m}</b>: {facts['state'] or 'unknown state' if facts else 'no data'}")
if not facts or facts.get("max_v") is None:
continue
@@ -92,7 +92,7 @@ def figure(station, f):
t_end = next(t for t in day_h if t.hour == facts["night_end_h"])
night_ends.append((t_end, facts["night_end_v"]))
hover = "<br>".join(f"{k}={v}" for k, v in facts.items())
label, c, symbol = ep.STATE_STYLE[facts["state"]]
label, c, symbol = STATE_STYLE[facts["state"]]
fig.add_scatter(x=[t_max], y=[facts["max_v"]], mode="markers+text", text=[f"max {facts['max_v']} V"], cliponaxis=False,
textposition="top center", marker=dict(color=c, symbol=symbol, size=12),
hovertext=[f"{x:%d/%m}: {label}<br>{hover}"], hovertemplate="%{hovertext}<extra></extra>")
@@ -103,7 +103,7 @@ def figure(station, f):
trend = f["cur"]["trend"]
if len(night_ends) == 2 and trend:
(t0, v0), (t1, v1) = night_ends
label, c = ep.TREND_STYLE[trend]
label, c = TREND_STYLE[trend]
fig.add_annotation(x=t1, y=v1, ax=t0, ay=v0, axref="x", ayref="y", text="", showarrow=True,
arrowhead=2, arrowwidth=2, arrowcolor=c)
fig.add_annotation(x=t0 + (t1 - t0) / 2, y=(v0 + v1) / 2, showarrow=False, yshift=14, font_color=c,
@@ -115,8 +115,8 @@ def figure(station, f):
fig.update_layout(title=f"{station}, {d:%d %b %Y}: <span style='color:{color}'>■ {name}</span>"
f" · {f['cur']['state']} × {trend or 'unknown trend'}"
+ (" · provisional (day not over)" if provisional(f) else "")
+ "<br><sup>gray band: no measurement · dashed: thresholds of the state (analysis 03)"
" · arrow: night end D-1 → D (trend, analysis 04)</sup>",
+ "<br><sup>gray band: no measurement · dashed: thresholds of the daily state"
" · arrow: night end D-1 → D (trend)</sup>",
height=500, showlegend=False, margin=dict(t=120))
return fig
@@ -125,7 +125,7 @@ if __name__ == "__main__":
if len(sys.argv) < 2:
sys.exit(__doc__)
(ROOT / "out").mkdir(exist_ok=True)
for station, series in sorted(ec.load_stations(sys.argv[2:] or [str(ROOT / "data")]).items()):
for station, series in sorted(load_stations(sys.argv[2:] or [str(ROOT / "data")]).items()):
if "BusVoltage" not in series:
continue
try:
+126
View File
@@ -0,0 +1,126 @@
"""Tendance de BusVoltage en moyenne glissante sur 7 jours (comparaison seulement, non livrée : D-020).
- MOVING_AVERAGE_7_DAYS_DROP (docs/energy/analyses/moving_average_7_days_drop.md) : sur toutes les heures ;
- MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP (docs/energy/analyses/moving_average_7_days_midnight_utc_drop.md) :
le même calcul sur la seule mesure de 00:00 UTC.
Bibliothèque standard uniquement (cible : système embarqué de la station).
"""
import math
from datetime import timedelta
HOUR = timedelta(hours=1)
DAY = timedelta(days=1)
# Constantes de calibration (fiche moving_average_7_days_drop, « Constantes et sources »).
WINDOW_H = 168 # multiple de 24 h : annule le cycle jour/nuit ; 72 h ne lisse pas la météo
REF_H = 720 # 30 jours : contient le dernier régime sain avant un déficit (choix de conception)
MIN_COVERAGE = 0.8 # à 0,5, JUNGFRAU (sain) tombe à -0,19 V en début d'historique ; à 0,8 : -0,13 V
MIN_REF_POINTS = 24 # au moins un jour de référence (choix de conception)
THRESHOLD_V = -0.2 # cas de régression : pire écart sain -0,16 V (VERMONT 05-11)
RULE = {
"id": "MOVING_AVERAGE_7_DAYS_DROP",
"window_h": WINDOW_H,
"ref_h": REF_H,
"min_coverage": MIN_COVERAGE,
"min_ref_points": MIN_REF_POINTS,
"threshold_v": THRESHOLD_V,
"threshold_source": "cas de régression FLORIDA, JUNGFRAU, VERMONT, WILDSTRUBEL, MISSISSIPPI (fiche moving_average_7_days_drop, « Calibration »)",
"doc": "docs/energy/analyses/moving_average_7_days_drop.md",
}
# MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP : les mêmes constantes, converties en jours (un point par jour, fiche moving_average_7_days_midnight_utc_drop).
RULE_MIDNIGHT = {
"id": "MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP",
"sample": "BusVoltage à 00:00 UTC",
"window_d": WINDOW_H // 24,
"ref_d": REF_H // 24,
"min_coverage": MIN_COVERAGE,
"min_ref_points": MIN_REF_POINTS // 24,
"threshold_v": THRESHOLD_V,
"threshold_source": "repris de MOVING_AVERAGE_7_DAYS_DROP pour comparer les méthodes, non recalibré (fiche moving_average_7_days_midnight_utc_drop)",
"doc": "docs/energy/analyses/moving_average_7_days_midnight_utc_drop.md",
}
def run(series):
"""Analyse MOVING_AVERAGE_7_DAYS_DROP, causale : le point t n'utilise que les données <= t (fiche moving_average_7_days_drop, « Principe »)."""
return _ma_drop(series["BusVoltage"], HOUR, WINDOW_H, REF_H, MIN_REF_POINTS, RULE)
def run_midnight(series):
"""Analyse MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP : le même calcul sur la seule mesure de 00:00 UTC (fiche moving_average_7_days_midnight_utc_drop)."""
v = {t: x for t, x in series["BusVoltage"].items() if t.hour == t.minute == 0}
return _ma_drop(v, DAY, WINDOW_H // 24, REF_H // 24, MIN_REF_POINTS // 24, RULE_MIDNIGHT)
def _ma_drop(v, step, window, ref_n, min_ref, rule):
"""Moyenne sur `window` pas de `step`, comparée à son maximum des `ref_n` pas précédents (causal)."""
if not v:
return {"analysis": rule["id"], "rule": rule, "points": []}
t0 = min(v)
if any((t - t0) % step for t in v):
raise ValueError(f"BusVoltage : horodatages hors de la grille (pas de {step})")
grid = [t0 + i * step for i in range((max(v) - t0) // step + 1)]
# NaN/inf (ex. export pandas) = pas de mesure : sinon un NaN pollue la somme courante.
vals = [x if x is not None and math.isfinite(x) else None for x in (v.get(t) for t in grid)]
# Moyenne glissante sur `window` créneaux (somme courante), définie si couverture suffisante.
ma, cov = [None] * len(grid), [0.0] * len(grid)
total = count = 0
for i, x in enumerate(vals):
if x is not None:
total += x
count += 1
if i >= window and vals[i - window] is not None:
total -= vals[i - window]
count -= 1
cov[i] = count / window
if count >= MIN_COVERAGE * window:
ma[i] = total / count
points = []
for i, t in enumerate(grid):
if t not in v:
continue
facts = {"coverage": round(cov[i], 3)}
value = anomaly = None
if ma[i] is not None:
facts["ma_v"] = round(ma[i], 3)
lo = max(0, i - ref_n)
ref_vals = [m for m in ma[lo:i] if m is not None]
if len(ref_vals) >= min_ref:
ref = max(ref_vals)
value = ma[i] - ref
anomaly = value < rule["threshold_v"]
facts["ref_v"] = round(ref, 3)
facts["ref_t"] = grid[ma.index(ref, lo, i)]
points.append({"t": t, "value": value, "anomaly": anomaly, "facts": facts})
return {"analysis": rule["id"], "rule": rule, "points": points}
def periods(points):
"""Séquences maximales True/None commençant et finissant par True."""
spans, cur = [], None
for k, p in enumerate(points):
if p["anomaly"] is True:
if cur is None:
cur = {"first": k, "last": k, "worst": k}
cur["last"] = k
if p["value"] < points[cur["worst"]]["value"]:
cur["worst"] = k
elif p["anomaly"] is False and cur is not None:
spans.append(cur)
cur = None
if cur is not None:
spans.append(cur)
out = []
for s in spans:
ongoing = all(p["anomaly"] is None for p in points[s["last"] + 1:])
out.append({
"start": points[s["first"]]["t"],
"end": None if ongoing else points[s["last"]]["t"],
"worst_value": points[s["worst"]]["value"],
"worst_t": points[s["worst"]]["t"],
})
return out
-253
View File
@@ -1,253 +0,0 @@
"""Analyses énergétiques : tendance de BusVoltage en moyenne glissante.
- MOVING_AVERAGE_7_DAYS_DROP (docs/analyses/analyse_01.md) : sur toutes les heures ;
- MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP (docs/analyses/analyse_02.md) : le même calcul sur la seule mesure de 00:00 UTC.
Bibliothèque standard uniquement (cible : système embarqué de la station).
"""
import argparse
import csv
import glob
import json
import math
import os
import re
import sys
from datetime import datetime, timedelta
HOUR = timedelta(hours=1)
DAY = timedelta(days=1)
# Constantes de calibration (fiche analyse_01, « Constantes et sources »).
WINDOW_H = 168 # multiple de 24 h : annule le cycle jour/nuit ; 72 h ne lisse pas la météo
REF_H = 720 # 30 jours : contient le dernier régime sain avant un déficit (choix de conception)
MIN_COVERAGE = 0.8 # à 0,5, JUNGFRAU (sain) tombe à -0,19 V en début d'historique ; à 0,8 : -0,13 V
MIN_REF_POINTS = 24 # au moins un jour de référence (choix de conception)
THRESHOLD_V = -0.2 # cas de régression : pire écart sain -0,16 V (VERMONT 05-11)
RULE = {
"id": "MOVING_AVERAGE_7_DAYS_DROP",
"window_h": WINDOW_H,
"ref_h": REF_H,
"min_coverage": MIN_COVERAGE,
"min_ref_points": MIN_REF_POINTS,
"threshold_v": THRESHOLD_V,
"threshold_source": "cas de régression FLORIDA, JUNGFRAU, VERMONT, WILDSTRUBEL, MISSISSIPPI (fiche analyse_01, « Calibration »)",
"doc": "docs/analyses/analyse_01.md",
}
# Analyse 02 : les mêmes constantes, converties en jours (un point par jour, fiche analyse_02).
RULE_MIDNIGHT = {
"id": "MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP",
"sample": "BusVoltage à 00:00 UTC",
"window_d": WINDOW_H // 24,
"ref_d": REF_H // 24,
"min_coverage": MIN_COVERAGE,
"min_ref_points": MIN_REF_POINTS // 24,
"threshold_v": THRESHOLD_V,
"threshold_source": "repris de MOVING_AVERAGE_7_DAYS_DROP pour comparer les méthodes, non recalibré (fiche analyse_02)",
"doc": "docs/analyses/analyse_02.md",
}
NAME_RE = re.compile(r"^(?P<station>[^_]+)_(?P<metric>.+?)_\d{4}_\d{2}_\d{2}_\d{2}_\d{2} to .+\.csv$")
def parse_name(path):
"""'<STATION>_<Metric>_<début> to <fin>.csv' -> (station, metric), ou None."""
m = NAME_RE.match(os.path.basename(path))
return (m["station"], m["metric"]) if m else None
def load_csv(path):
"""CSV ',time_utc,metric' -> {datetime UTC: float ou None (pas de mesure)}."""
try:
with open(path, newline="") as f:
rows = csv.reader(f)
next(rows)
return {datetime.fromisoformat(r[1]): float(r[2]) if r[2] else None for r in rows}
except (StopIteration, IndexError, ValueError) as e:
raise ValueError(f"{path} : CSV illisible ({e!r})") from e
def load_stations(paths):
"""Fichiers et/ou dossiers -> {station: {metric: série}}."""
files = []
for p in paths:
files += sorted(glob.glob(os.path.join(p, "*.csv"))) if os.path.isdir(p) else [p]
stations, seen = {}, {}
for path in files:
real = os.path.realpath(path)
if real in seen.values():
continue
key = parse_name(path)
if key is None:
print(f"ignoré (nom non reconnu) : {path}", file=sys.stderr)
continue
if key in seen:
raise ValueError(f"doublon {key[0]}/{key[1]} : {seen[key]} et {path}")
seen[key] = real
try:
stations.setdefault(key[0], {})[key[1]] = load_csv(path)
except ValueError as e:
print(f"ignoré : {e}", file=sys.stderr)
return stations
def run(series):
"""Analyse MOVING_AVERAGE_7_DAYS_DROP, causale : le point t n'utilise que les données <= t (fiche analyse_01, « Principe »)."""
return _ma_drop(series["BusVoltage"], HOUR, WINDOW_H, REF_H, MIN_REF_POINTS, RULE)
def run_midnight(series):
"""Analyse MOVING_AVERAGE_7_DAYS_MIDNIGHT_UTC_DROP : le même calcul sur la seule mesure de 00:00 UTC (fiche analyse_02)."""
v = {t: x for t, x in series["BusVoltage"].items() if t.hour == t.minute == 0}
return _ma_drop(v, DAY, WINDOW_H // 24, REF_H // 24, MIN_REF_POINTS // 24, RULE_MIDNIGHT)
def _ma_drop(v, step, window, ref_n, min_ref, rule):
"""Moyenne sur `window` pas de `step`, comparée à son maximum des `ref_n` pas précédents (causal)."""
if not v:
return {"analysis": rule["id"], "rule": rule, "points": []}
t0 = min(v)
if any((t - t0) % step for t in v):
raise ValueError(f"BusVoltage : horodatages hors de la grille (pas de {step})")
grid = [t0 + i * step for i in range((max(v) - t0) // step + 1)]
# NaN/inf (ex. export pandas) = pas de mesure : sinon un NaN pollue la somme courante.
vals = [x if x is not None and math.isfinite(x) else None for x in (v.get(t) for t in grid)]
# Moyenne glissante sur `window` créneaux (somme courante), définie si couverture suffisante.
ma, cov = [None] * len(grid), [0.0] * len(grid)
total = count = 0
for i, x in enumerate(vals):
if x is not None:
total += x
count += 1
if i >= window and vals[i - window] is not None:
total -= vals[i - window]
count -= 1
cov[i] = count / window
if count >= MIN_COVERAGE * window:
ma[i] = total / count
points = []
for i, t in enumerate(grid):
if t not in v:
continue
facts = {"coverage": round(cov[i], 3)}
value = anomaly = None
if ma[i] is not None:
facts["ma_v"] = round(ma[i], 3)
lo = max(0, i - ref_n)
ref_vals = [m for m in ma[lo:i] if m is not None]
if len(ref_vals) >= min_ref:
ref = max(ref_vals)
value = ma[i] - ref
anomaly = value < rule["threshold_v"]
facts["ref_v"] = round(ref, 3)
facts["ref_t"] = grid[ma.index(ref, lo, i)]
points.append({"t": t, "value": value, "anomaly": anomaly, "facts": facts})
return {"analysis": rule["id"], "rule": rule, "points": points}
def periods(points):
"""Séquences maximales True/None commençant et finissant par True."""
spans, cur = [], None
for k, p in enumerate(points):
if p["anomaly"] is True:
if cur is None:
cur = {"first": k, "last": k, "worst": k}
cur["last"] = k
if p["value"] < points[cur["worst"]]["value"]:
cur["worst"] = k
elif p["anomaly"] is False and cur is not None:
spans.append(cur)
cur = None
if cur is not None:
spans.append(cur)
out = []
for s in spans:
ongoing = all(p["anomaly"] is None for p in points[s["last"] + 1:])
out.append({
"start": points[s["first"]]["t"],
"end": None if ongoing else points[s["last"]]["t"],
"worst_value": points[s["worst"]]["value"],
"worst_t": points[s["worst"]]["t"],
})
return out
def _d(t):
return t.strftime("%Y-%m-%d %H:%M")
def _v(x):
return f"{x:.2f}".replace(".", ",")
def _unknown_line(station, points):
"""Heures sans verdict et plus longue plage : l'absence de verdict n'est ni une anomalie ni un état sain."""
best = cur = None
for p in points:
if p["value"] is None:
cur = (cur[0], p["t"], cur[2] + 1) if cur else (p["t"], p["t"], 1)
if best is None or cur[2] > best[2]:
best = cur
else:
cur = None
n = sum(p["value"] is None for p in points)
line = f"{station} : sans verdict (UNKNOWN) {n} h sur {len(points)}"
if best:
line += f" ; plus longue plage du {_d(best[0])} au {_d(best[1])}"
return line + "."
def summary(station, result, per):
"""Résumé texte généré à partir des faits calculés (une ligne par période + heures sans verdict)."""
rule = f"Règle {RULE['id']} (seuil {_v(THRESHOLD_V)} V, source : cas de régression)."
points = result["points"]
defined = [p for p in points if p["value"] is not None]
if not defined:
lines = [f"{station} : analyse impossible (insufficient evidence) : aucun point avec verdict "
f"(historique trop court, vide ou trop lacunaire). {rule}"]
elif not per:
w = min(defined, key=lambda p: p["value"])
lines = [f"{station} : aucune anomalie énergétique (not observed) sur les points avec verdict. "
f"Pire écart {_v(w['value'])} V le {_d(w['t'])}. {rule}"]
else:
lines = []
for p in per:
w = next(q for q in points if q["t"] == p["worst_t"])
f = w["facts"]
until = "la fin de l'export" if p["end"] is None else _d(p["end"])
lines.append(
f"{station} : anomalie énergétique (consistent with) du {_d(p['start'])} à {until}. "
f"Pire écart le {_d(w['t'])} : moyenne sur 7 jours {_v(f['ma_v'])} V, soit {_v(-w['value'])} V "
f"sous son maximum du {_d(f['ref_t'])} ({_v(f['ref_v'])} V). {rule}")
if points:
lines.append(_unknown_line(station, points))
return "\n".join(lines)
def main(argv=None):
ap = argparse.ArgumentParser(description="Analyse énergétique MOVING_AVERAGE_7_DAYS_DROP")
ap.add_argument("paths", nargs="+", help="fichiers CSV et/ou dossiers")
ap.add_argument("--out", default="out", help="dossier de sortie (défaut : out/)")
a = ap.parse_args(argv)
os.makedirs(a.out, exist_ok=True)
for station, series in sorted(load_stations(a.paths).items()):
if "BusVoltage" not in series:
print(f"{station} : ignorée, pas de BusVoltage")
continue
try:
result = run(series)
except ValueError as e:
print(f"{station} : ignorée, {e}", file=sys.stderr)
continue
per = periods(result["points"])
with open(os.path.join(a.out, f"{station}_energy.json"), "w") as f:
json.dump({"station": station, **result, "periods": per}, f, indent=1,
default=lambda o: o.isoformat())
print(summary(station, result, per))
if __name__ == "__main__":
main()
-236
View File
@@ -1,236 +0,0 @@
"""Graphique HTML des analyses énergétiques, un panneau par analyse (poste de dev, plotly dans .venv).
Documentation : docs/analyses/analyse_01.md à analyse_04.md
"""
import argparse
import os
from datetime import datetime, time, timezone
from plotly.subplots import make_subplots
import energy_check as ec
import battery_state as bs
import battery_trend as bt
# Plotly ne recale pas l'ordonnée sur la plage visible : après chaque zoom ou déplacement horizontal,
# ce script recalcule l'axe vertical de chaque graphique sur les points visibles (5 % de marge),
# seuil compris (c'est une courbe). Double-clic : retour à la vue complète.
AUTOSCALE_Y_JS = """
const gd = document.getElementById('{plot_id}');
const name = (t, a) => a + 'axis' + (t[a + 'axis'] || a).slice(1);
const xs = gd.data.map(t => t.x.map(v => gd._fullLayout[name(t, 'x')].d2l(v)));
gd.on('plotly_relayout', ev => {
const keys = Object.keys(ev);
if (!keys.some(k => k.startsWith('xaxis'))) return;
const upd = {};
if (keys.some(k => k.endsWith('autorange'))) {
gd.data.forEach(t => { upd[name(t, 'y') + '.autorange'] = true; });
return Plotly.relayout(gd, upd);
}
const box = {};
gd.data.forEach((t, i) => {
if (t.visible === 'legendonly' || t.visible === false) return;
const xa = gd._fullLayout[name(t, 'x')];
const [a, b] = xa.range.map(r => xa.r2l(r));
const r = box[name(t, 'y')] || (box[name(t, 'y')] = [Infinity, -Infinity]);
t.y.forEach((y, j) => {
if (y === null || !isFinite(y) || xs[i][j] < a || xs[i][j] > b) return;
r[0] = Math.min(r[0], y);
r[1] = Math.max(r[1], y);
});
});
for (const [ya, [lo, hi]] of Object.entries(box)) {
if (lo > hi) continue;
const pad = (hi - lo) * 0.05 || Math.abs(hi) * 0.01 || 1;
upd[ya + '.range'] = [lo - pad, hi + pad];
}
Plotly.relayout(gd, upd);
});
"""
# Zones de BusVoltage selon l'état de la batterie (bandes de fond) : docs/domaine/tensions-selon-l-etat-de-la-batterie.md.
# [V] = VERMONT, 5e-95e centiles horaires ; [M] = manuel ; [E] = experts. Les zones « en charge » et « décharge »
# se recouvrent de 13,1 à 13,2 V : sur le plateau, la tension seule ne sépare pas la charge de la décharge.
ZONES = [
("décharge", 12.0, 13.2, "steelblue", "bottom left"), # 12,0 : coupure de la station [M] ; 13,2 : haut du plateau [V]
("en charge", 13.1, 13.8, "gold", "top left"), # bulk [V]
("pleine charge", 14.0, 14.4, "green", "top left"), # absorption 14,0-14,3 [V] ; secteur 14,2-14,4 [E]
]
# États de DAILY_BATTERY_STATE : couleurs validées par le skill dataviz (toutes paires, fond blanc).
# Le jaune contraste peu avec le fond : chaque état a aussi son symbole, sa légende et son survol.
STATE_STYLE = {
"FULL_CHARGE": ("full charge", "#008300", "circle"),
"PARTIAL_CHARGE": ("partial charge (uncertain)", "#eda100", "diamond"),
"BELOW_PLATEAU": ("below plateau", "#2a78d6", "triangle-down"),
"NO_CHARGE": ("no charge", "#4a3aa7", "x"),
None: ("unknown state", "#8c8c8c", "circle-open"),
}
def noon(day):
"""Abscisse d'un point journalier (analyses 03 et 04) : midi UTC de son jour, au milieu de la journée."""
return datetime.combine(day, time(12), timezone.utc)
def daily_state_panel(fig, result, row):
"""Panneau DAILY_BATTERY_STATE : maximum et fin de nuit par jour, seuils, état en marqueurs (fiche analyse_03)."""
group = dict(legendgroup=result["analysis"], legendgrouptitle_text=f"Analysis {result['analysis']}")
rule, pts = result["rule"], result["points"]
x = [noon(p["t"].date()) for p in pts]
fig.add_scatter(x=x, y=[p["facts"].get("max_v") for p in pts], name="daily maximum", mode="lines",
line=dict(color="#52514e", width=1.5), hoverinfo="skip", row=row, col=1, **group)
fig.add_scatter(x=x, y=[p["facts"].get("night_end_v") for p in pts], name="night end", mode="lines",
line=dict(color="#9a9893", width=1.5), hovertemplate="night end %{y:.2f} V<extra></extra>",
row=row, col=1, **group)
# Seuils tracés comme des courbes (un point par jour) : AUTOSCALE_Y_JS les garde visibles au zoom (D-013).
for v, name, state in ((rule["full_charge_v"], "full charge", "FULL_CHARGE"),
(rule["plateau_floor_v"], "plateau floor", "BELOW_PLATEAU")):
fig.add_scatter(x=x, y=[v] * len(x), name=f"{name} {v} V", mode="lines", hoverinfo="skip",
line=dict(color=STATE_STYLE[state][1], dash="dash", width=1), row=row, col=1, **group)
for state, (label, color, symbol) in STATE_STYLE.items():
sel = [p for p in pts if p["facts"]["state"] == state and p["facts"].get("max_v") is not None]
if not sel:
continue
hover = [f"{p['t']:%Y-%m-%d}: {label}<br>" + "<br>".join(f"{k}={val}" for k, val in p["facts"].items())
for p in sel]
fig.add_scatter(x=[noon(p["t"].date()) for p in sel], y=[p["facts"]["max_v"] for p in sel], name=label,
mode="markers",
marker=dict(color=color, symbol=symbol, size=8, line=dict(color="white", width=1)),
text=hover, hovertemplate="%{text}<extra></extra>", row=row, col=1, **group)
fig.update_yaxes(title_text="BusVoltage [V]", row=row, col=1)
# Tendances de DAILY_BATTERY_TREND : paire divergente bleu/rouge et gris neutre du skill dataviz
# (vert/rouge refusé par son validateur : deutéranopie, ΔE 5,9). Chaque tendance a aussi sa légende et son survol.
TREND_STYLE = {"BETTER": ("better", "#2a78d6"), "STABLE": ("stable", "#8c8c8c"), "WORSE": ("worse", "#d03b3b")}
def daily_trend_panel(fig, result, row):
"""Panneau DAILY_BATTERY_TREND : écart de fin de nuit J − J−1 en barres, couleur = tendance (fiche analyse_04)."""
group = dict(legendgroup=result["analysis"], legendgrouptitle_text=f"Analysis {result['analysis']}")
pts, db = result["points"], result["rule"]["night_end_deadband_v"]
x = [noon(p["t"].date()) for p in pts]
# Zone morte tracée comme une courbe (un point par jour, None entre +db et −db) : visible au zoom (D-013).
fig.add_scatter(x=x + [None] + x, y=[db] * len(x) + [None] + [-db] * len(x), name=f"dead band ±{db} V",
mode="lines", hoverinfo="skip", line=dict(color="#52514e", dash="dash", width=1),
row=row, col=1, **group)
for trend, (label, color) in TREND_STYLE.items():
sel = [p for p in pts if p["facts"]["trend"] == trend and p["value"] is not None]
if not sel:
continue
hover = [f"{p['t']:%Y-%m-%d}: {label} (decided by {p['facts']['decided_by']})<br>"
f"{p['facts']['state_prev']} → {p['facts']['state']}<br>"
+ "<br>".join(f"{k}={v}" for k, v in p["facts"].items() if k.startswith("delta")) for p in sel]
fig.add_bar(x=[noon(p["t"].date()) for p in sel], y=[p["value"] for p in sel], name=label,
marker_color=color, width=0.8 * 86_400_000, hovertext=hover, hovertemplate="%{hovertext}<extra></extra>",
row=row, col=1, **group)
fig.update_yaxes(title_text="Δ night end [V]", row=row, col=1)
def daily_figure(station, series):
"""Graphique pour le site de monitoring (D-021) : analyse 03 en haut, analyse 04 dessous, même axe du temps."""
fig = make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.08,
subplot_titles=[f"Analysis {bs.RULE['id']}", f"Analysis {bt.RULE['id']}"])
daily_state_panel(fig, bs.run(series), 1)
daily_trend_panel(fig, bt.run(series), 2)
fig.update_xaxes(showticklabels=True)
fig.update_yaxes(fixedrange=True) # zoom horizontal seulement : l'ordonnée est recalée par AUTOSCALE_Y_JS
fig.update_layout(title=f"{station}: battery state and trend", height=700, barmode="overlay",
legend=dict(groupclick="toggleitem"))
return fig
def gaps(v):
"""Plages de créneaux sans BusVoltage : [(premier créneau vide, première valeur suivante)]."""
out, start = [], None
for t in sorted(v):
if v[t] is None and start is None:
start = t
elif v[t] is not None and start is not None:
out.append((start, t))
start = None
if start is not None:
out.append((start, max(v)))
return out
def figure(station, series, results):
"""Haut : une analyse par graphique, courbes superposées. Dessous : une série temporelle par graphique."""
metrics = ["BusVoltage"] + sorted(m for m in series if m != "BusVoltage")
n = len(results)
fig = make_subplots(rows=n + len(metrics), cols=1, shared_xaxes=True, vertical_spacing=0.03,
row_heights=[1.5] * n + [1] * len(metrics),
subplot_titles=[f"Analyse {r['analysis']}" for r in results] + metrics)
for row, result in enumerate(results, start=1):
if result["analysis"] == bs.RULE["id"]:
daily_state_panel(fig, result, row)
continue
if result["analysis"] == bt.RULE["id"]:
daily_trend_panel(fig, result, row)
continue
analyse = dict(legendgroup=result["analysis"], legendgrouptitle_text=f"Analyse {result['analysis']}")
thr = result["rule"]["threshold_v"]
pts = result["points"]
x = [p["t"] for p in pts]
ref = [p["facts"].get("ref_v") for p in pts]
hover = [f"écart={p['value'] if p['value'] is None else round(p['value'], 3)}<br>anomalie={p['anomaly']}<br>"
+ "<br>".join(f"{k}={val}" for k, val in p["facts"].items()) for p in pts]
fig.add_scatter(x=x, y=[p["facts"].get("ma_v") for p in pts], name="moyenne 7 j", text=hover,
hovertemplate="%{y:.3f} V<br>%{text}", line=dict(color="royalblue"), row=row, col=1,
**analyse)
fig.add_scatter(x=x, y=[None if r is None else r + thr for r in ref], name=f"seuil (max 30 j {thr:+} V)",
line=dict(color="red", dash="dash"), row=row, col=1, **analyse)
for p in ec.periods(pts):
fig.add_vrect(x0=p["start"], x1=p["end"] or x[-1], fillcolor="red", opacity=0.15,
line_width=0, row=row, col=1)
fig.update_yaxes(title_text="BusVoltage moyennée [V]", row=row, col=1)
mesures = dict(legendgroup="mesures", legendgrouptitle_text="Mesures")
for row, m in enumerate(metrics, start=n + 1):
s = series[m]
fig.add_scatter(x=sorted(s), y=[s[t] for t in sorted(s)], name=m, line=dict(width=1),
row=row, col=1, **mesures)
for a, b in gaps(series["BusVoltage"]):
fig.add_vrect(x0=a, x1=b, fillcolor="gray", opacity=0.3, line_width=0, row=n + 1, col=1)
for name, lo, hi, color, pos in ZONES:
fig.add_hrect(y0=lo, y1=hi, fillcolor=color, opacity=0.15, line_width=0, layer="below",
annotation_text=f"{name} {lo}-{hi} V", annotation_position=pos, row=n + 1, col=1)
fig.update_xaxes(showticklabels=True) # dates sous chaque graphique, pas seulement le dernier
fig.update_yaxes(fixedrange=True) # zoom horizontal seulement : l'ordonnée est recalée par AUTOSCALE_Y_JS
fig.update_layout(title=f"{station} : analyses " + ", ".join(r["analysis"] for r in results),
height=300 * (n + len(metrics)),
barmode="overlay", legend=dict(groupclick="toggleitem"))
return fig
def main(argv=None):
ap = argparse.ArgumentParser(description="Graphique HTML des analyses énergétiques")
ap.add_argument("paths", nargs="+", help="fichiers CSV et/ou dossiers")
ap.add_argument("--out", default="out", help="dossier de sortie (défaut : out/)")
a = ap.parse_args(argv)
os.makedirs(a.out, exist_ok=True)
paths = []
for station, series in sorted(ec.load_stations(a.paths).items()):
if "BusVoltage" not in series:
print(f"{station} : ignorée, pas de BusVoltage")
continue
path = os.path.join(a.out, f"{station}_energy.html")
figure(station, series, [ec.run(series), ec.run_midnight(series), bs.run(series), bt.run(series)]).write_html(
path, include_plotlyjs="cdn", post_script=AUTOSCALE_Y_JS, config={"scrollZoom": True})
print(path)
paths.append(path)
# Analyses 03 et 04 : page de démo, et figure Plotly en JSON pour le site (Plotly.newPlot).
path = os.path.join(a.out, f"{station}_daily.html")
fig = daily_figure(station, series)
fig.write_html(path, include_plotlyjs="cdn", post_script=AUTOSCALE_Y_JS, config={"scrollZoom": True})
fig.write_json(path.removesuffix(".html") + ".json")
print(path)
paths.append(path)
return paths
if __name__ == "__main__":
main()