Pumping of the differential images: extraction/pump_diff.sh into data_diff/<SERIAL>/, in the tree of data/ (never data/ itself)
The OPTICAL_SHIFT-DIFF images full size, with what measuring them again needs under images_diff/ (index of every measurement per day with its exact time and why an image is missing, sensors.json with every pattern archived included, unit.json), and the metrics, IRON images, weather and targets.csv as pump.py and weather.py write them into data/. Its own lock and log; a day is indexed last, and not when its metrics failed; --min-free-gb. pump_metrics now returns whether every day succeeded; weather.py takes --data. Tried on JUNGFRAU 2026-09-01 and 2026-10-12..16: DIFF images byte-identical to pump.py --diff, metrics, IRON and weather byte-identical to data/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1b918facd9
commit
81908ca370
9 files changed
+660
-8
No files matched your search
+5
-1
@@ -124,10 +124,12 @@ def pump_metrics(conn, system_id, station, start, end, out, replace):
|
||||
"""Like the download of the System page (Dashboard/System.py, get_metrics(..., bucketing="1 hour")): per hour,
|
||||
mean * transform_factor + transform_offset, or the list of values of a text metric; the labelled series of one
|
||||
metric merged; an empty row for an hour without measurement. get_metrics counts in hours before now: its query is
|
||||
redone here on a fixed day, one query per day for all the metrics."""
|
||||
redone here on a fixed day, one query per day for all the metrics. -> True when no day failed (pump_diff.py: a
|
||||
day whose metrics failed is not marked done)."""
|
||||
names = sorted(conn.execute(text("SELECT DISTINCT dt.name FROM datatype dt JOIN metric_series se "
|
||||
"ON se.datatype_id = dt.id WHERE se.system_id = :s"), {"s": system_id}).scalars())
|
||||
conn.rollback()
|
||||
ok = True
|
||||
for k in range((end - start).days):
|
||||
a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC)
|
||||
paths = {n: out / station / "metrics"
|
||||
@@ -148,6 +150,7 @@ def pump_metrics(conn, system_id, station, start, end, out, replace):
|
||||
GROUP BY 1, dt.name""", s=system_id, a=a, b=a + timedelta(days=1))
|
||||
except Exception as e: # ponytail: like the displacements, a failed day is skipped, a new run picks it up
|
||||
print(f"{station} {a:%Y-%m-%d} metrics: failed ({root_cause(e)}), run again")
|
||||
ok = False
|
||||
continue
|
||||
finally:
|
||||
conn.rollback()
|
||||
@@ -163,6 +166,7 @@ def pump_metrics(conn, system_id, station, start, end, out, replace):
|
||||
pd.DataFrame({"time_utc": hours, "metric": col.reindex(hours).to_numpy()}).to_csv(path)
|
||||
print(f"{station} {a:%Y-%m-%d} metrics: {len(names)} files, {df['name'].nunique()} with data, "
|
||||
f"{time.time() - t0:.0f} s")
|
||||
return ok
|
||||
|
||||
|
||||
# --- the "advanced" export: build_advanced_frame (app/datapoints.py) ---
|
||||
|
||||
Reference in new issue
Block a user