From 81908ca370df66a8ed16588d1a72a94259309884 Mon Sep 17 00:00:00 2001 From: maxime Date: Thu, 19 Nov 2026 09:59:50 +0100 Subject: [PATCH] Pumping of the differential images: extraction/pump_diff.sh into data_diff//, in the tree of data/ (never data/ itself) The OPTICAL_SHIFT-DIFF images full size, with what measuring them again needs under images_diff/ (index of every measurement per day with its exact time and why an image is missing, sensors.json with every pattern archived included, unit.json), and the metrics, IRON images, weather and targets.csv as pump.py and weather.py write them into data/. Its own lock and log; a day is indexed last, and not when its metrics failed; --min-free-gb. pump_metrics now returns whether every day succeeded; weather.py takes --data. Tried on JUNGFRAU 2026-09-01 and 2026-10-12..16: DIFF images byte-identical to pump.py --diff, metrics, IRON and weather byte-identical to data/. Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-19-pump-diff-data-layout.md | 48 +++ .../superpowers/plans/2026-11-19-pump-diff.md | 86 +++++ .../specs/2026-11-19-configurations-design.md | 70 ++++ .../specs/2026-11-19-pump-diff-design.md | 67 ++++ extraction/README.md | 22 ++ extraction/pump.py | 6 +- extraction/pump_diff.py | 326 ++++++++++++++++++ extraction/pump_diff.sh | 24 ++ extraction/weather.py | 19 +- 9 files changed, 660 insertions(+), 8 deletions(-) create mode 100644 docs/superpowers/plans/2026-11-19-pump-diff-data-layout.md create mode 100644 docs/superpowers/plans/2026-11-19-pump-diff.md create mode 100644 docs/superpowers/specs/2026-11-19-configurations-design.md create mode 100644 docs/superpowers/specs/2026-11-19-pump-diff-design.md create mode 100644 extraction/pump_diff.py create mode 100755 extraction/pump_diff.sh diff --git a/docs/superpowers/plans/2026-11-19-pump-diff-data-layout.md b/docs/superpowers/plans/2026-11-19-pump-diff-data-layout.md new file mode 100644 index 0000000..5ffca7f --- /dev/null +++ b/docs/superpowers/plans/2026-11-19-pump-diff-data-layout.md @@ -0,0 +1,48 @@ +# pump_diff: the layout of data/ Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `data_diff//` read by the other repositories as `data//`: images, metrics, weather, +targets.csv as in `data/`; the DIFF images' extras under `images_diff/`. + +**Architecture:** `pump_diff.py` calls `pump.py`'s `pump_metrics`, `pump_images`, `sensors`, `write_targets` per +day / station, then `weather.main` on its `--out`; `weather.py` gets a root folder parameter. + +**Tech Stack:** as `pump_diff.sh` (uv, sqlalchemy, psycopg2-binary, pandas, boto3). + +**Spec:** `docs/superpowers/specs/2026-11-19-configurations-design.md` (section "Work, in order", 1) + +## Global Constraints + +- `data/` never written; `pump.py`'s behaviour unchanged for `pump.sh` (only a return value added). +- Extras paths: `images_diff/index/_diff_.csv`, `images_diff/sensors.json`, `images_diff/unit.json`. +- IRON `--every` default 60; metrics replaced for every day pumped. + +## Review Focus + +- Metrics failed for a day: no index, the day is read again next run. +- `weather.py` without `--data`: writes into `data/` as before (`pump.sh` unchanged). +- A station without position: no weather, the pumping goes on. +- Weather raising something unexpected (network): printed, the pumping's result kept. +- Old layout on disk (`index/` at the station root): migrated by hand once, then regenerated. + +--- + +### Task 1: pump_diff and weather + +**Files:** Modify `extraction/pump_diff.py`, `extraction/pump.py` (`pump_metrics` returns `bool`), +`extraction/weather.py` (`main(serials, data=DATA)`, `station(..., data)`, `--data DIR`), `extraction/README.md`, +`README.md`. + +- [ ] **Step 1:** `check()`: the index and `indexed_days` under `images_diff/index/`; fails. +- [ ] **Step 2:** move the extras; `--every`; per day `pump_metrics` (skip the day when it returns False), + `pump_images(..., "images", every, out)`, then the DIFF day; per station `write_targets(folder, sensors(conn, unit), + seen)` with `seen` = {sensor_name: id} of the non-archived DIFF sensors with a pattern; at the end `weather.main( + stations, a.out)` in a try. +- [ ] **Step 3:** `--check` ok; `python3 extraction/pump.py --check` ok (uv); `python3 extraction/weather.py --data + /nonexistent` prints nothing and exits 0. +- [ ] **Step 4:** migrate `data_diff/` (old `index/` removed, `sensors.json` / `unit.json` moved; LENZSPITZE, empty, + removed); `pump_diff.sh --from 2026-10-12 --to 2026-10-17 JUNGFRAU` and `--from 2026-09-01 --to 2026-09-02`; + expected: no DIFF image downloaded again, metrics / images / weather / targets.csv present, `data/` unchanged. +- [ ] **Step 5:** `data.py` on `data_diff/JUNGFRAU`: `find_files` without `others` from the extras; metrics, + temperature and weather loaded; same counts as `data/JUNGFRAU` on those days. diff --git a/docs/superpowers/plans/2026-11-19-pump-diff.md b/docs/superpowers/plans/2026-11-19-pump-diff.md new file mode 100644 index 0000000..49a097b --- /dev/null +++ b/docs/superpowers/plans/2026-11-19-pump-diff.md @@ -0,0 +1,86 @@ +# Pumping the differential images Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `extraction/pump_diff.py` + `pump_diff.sh` pump a station's `OPTICAL_SHIFT-DIFF` images, their index and +the pattern / unit properties into `data_diff//`, read-only, without touching `data/` or `pump.py`. + +**Architecture:** One script beside `pump.py`, importing `acquisition_unit`, `root_cause`, `DB_NAMES`, `TYPES`, `UTC` +from it; pure range logic and a per-day function testable through stubs; a wrapper copied from `pump.sh` minus +`absolute.py` / `weather.py`. + +**Tech Stack:** Python 3.12 in `uv` (sqlalchemy, psycopg2-binary, pandas, boto3, as `pump.sh`), standard `csv` / +`json`. + +**Spec:** `docs/superpowers/specs/2026-11-19-pump-diff-design.md` + +## Global Constraints + +- Read-only: `default_transaction_read_only=on`, `statement_timeout=--delay` (default 60 s), as `pump.py`. +- Never read or print the `.env`; variable names only. +- Image names `TZ_.`, as `pump.py`. +- `--min-free-gb` default 50, checked before each day. +- Lock `out/pump_diff.lock`, log `out/pump_diff.log`. + +## Review Focus + +- A crash in the middle of a day: no index file for it (temporary file + `os.replace`), so the next run reads it again. +- Two measurements in the same minute (bursts): distinct files thanks to the id; both in the index. +- A station without acquisition unit or unknown serial: a message, the next station goes on. +- No `--from` and nothing indexed: the station is skipped with a message, nothing downloaded. +- Disk below the limit: the run stops before the day, with the free space printed. + +--- + +### Task 1: range logic and one day, with `--check` + +**Files:** +- Create: `extraction/pump_diff.py` + +**Interfaces:** +- Produces: `indexed_days(folder: Path) -> set[date]`; `days_to_pump(indexed: set[date], start: date | None, end: date) + -> list[date] | None` (None: no `--from` and nothing indexed); `pump_day(rows, client, bucket: str, ext: str, + folder: Path, station: str, day: date) -> Counter` (rows: the day's measurements, queried by `main`; counts per + status, plus `new` and `bytes`); `check()`. + +- [ ] **Step 1:** write `check()` with these assertions (days 1..19 of 2026-11): + - `days_to_pump(set(), d(1), d(4)) == [d(1), d(2), d(3)]`; + - `days_to_pump({d(1), d(2), d(5)}, d(1), d(7)) == [d(3), d(4), d(5), d(6)]` (before the last: missing only; the + last and after: again); + - `days_to_pump({d(2)}, None, d(5)) == [d(2), d(3), d(4)]`; `days_to_pump(set(), None, d(5)) is None`; + - `pump_day` on stub `conn` / `client` with 4 measurements (one image to fetch, one already on disk, one + `measurement_uuid` None, one where `get_object` raises): returns `{"ok": 2, "no_uuid": 1, "s3_missing": 1}`, writes + one new file, leaves the existing one untouched, and the index has 4 rows with the right `status`, `file` empty for + the two missing, `properties` as JSON. +- [ ] **Step 2:** `python3 extraction/pump_diff.py --check` fails (functions missing). +- [ ] **Step 3:** implement the three functions. `pump_day` queries `id, time_utc, measurement_group_id, + measurement_uuid, properties` of `measurement` for the unit, type and UTC day, ordered by time; index columns + `measurement_id, time_utc, measurement_group_id, file, status, properties`; written to `index/_diff_.csv` + through `.tmp` and `os.replace` after the images. +- [ ] **Step 4:** `--check` prints `ok`. + +### Task 2: database wiring, wrapper, docs, real trial + +**Files:** +- Modify: `extraction/pump_diff.py` (add `sensors_json`, `unit_json`, `main`) +- Create: `extraction/pump_diff.sh` +- Modify: `extraction/README.md`, `README.md`, `.gitignore` (`data_diff/`) + +**Interfaces:** +- Consumes: Task 1's functions. +- Produces: `sensors_json(conn, unit: int) -> list[dict]` (`id, name, type, time_utc, properties: [{name, value, + time_utc, status}]`, every sensor of the unit, archived included); `unit_json(conn, unit: int) -> dict` (`id, + serial_number, product_number, properties: [...]`). + +- [ ] **Step 1:** `main()`: arguments `stations*`, `--from`, `--to`, `--delay`, `--min-free-gb`, `--out` (default + `/data_diff`); without stations, the folders of `--out`; engine and S3 client as `pump.py`'s `main`; per + station: system, unit, measurement type `OPTICAL_SHIFT-DIFF`, `sensors.json` / `unit.json` written, then each day of + `days_to_pump` after the disk check, one line printed per day (counts, MB). +- [ ] **Step 2:** `pump_diff.sh` from `pump.sh`: same `.env` load and `uv` packages, its own lock and log, no + `absolute.py` / `weather.py`, no default arguments. +- [ ] **Step 3:** `--check` still `ok`; `pump_diff.sh -h` shows the help. +- [ ] **Step 4:** real trial: `extraction/pump_diff.sh --from 2026-09-01 --to 2026-09-02 JUNGFRAU`; expected: 94 + images, byte-identical to `data/JUNGFRAU/images_diff/20260901*` (`cmp`), index of the day, `sensors.json`, + `unit.json`; `data/` unchanged (`find data -newer` the start mark finds nothing). +- [ ] **Step 5:** docs: a `pump_diff` section in `extraction/README.md`, one line in `README.md`, `data_diff/` in + `.gitignore`. diff --git a/docs/superpowers/specs/2026-11-19-configurations-design.md b/docs/superpowers/specs/2026-11-19-configurations-design.md new file mode 100644 index 0000000..9216390 --- /dev/null +++ b/docs/superpowers/specs/2026-11-19-configurations-design.md @@ -0,0 +1,70 @@ +# Configurations: recomputing the stations' data from the differential images + +2026-11-19. Approved in conversation. Extends `2026-11-19-pump-diff-design.md` (its layout of `data_diff/` changes: +section "data_diff"). + +## Intent + +Recompute a station's data from its differential images under configurations of our own (patterns redrawn, +thresholds, references, camera correction), compare them, and let every other repository (aimsight-app, tools, +detectors, models, optimizer) read the result exactly as it reads `data/`. `data/` stays as it is: the cloud's +version, the reference to compare with. + +## Layout + +``` +aimsight-data/ + data/ the cloud, unchanged (pump.sh) + data_diff// raw material shared by every configuration (pump_diff.sh), out of git + images_diff/ the DIFF images; and, invisible to the tools (data.find_files skips images_diff/): + index/_diff_.csv, sensors.json, unit.json + images/ metrics/ weather/ targets.csv as in data/ (targets.csv: the cloud's patterns) + configs/.yaml one configuration per file, in git + runs// the result of a configuration, out of git, the same tree as data/ + / + displacement/ camera/ targets.csv recomputed, the columns of data/ + images/ metrics/ weather/ symbolic links to data_diff// (nothing copied) + run.yaml the configuration used, optical's commit, the date (outside the station folders: + invisible to the tools) + diagnostics/ the emulator's raw rows (edge margin, second spot) +``` + +A component reads a configuration by pointing its `data` link at `~/aimsight-data/runs/` instead of +`~/aimsight-data/data`. Checked on 2026-11-19: `data.py` (aimsight-tools) reads through symbolic links (images, +metrics, temperature, weather). + +## Configuration + +A configuration states only what differs from the cloud's patterns (`data_diff//images_diff/sensors.json`) +and from the unit's properties (`unit.json`); the configuration `cloud`, with nothing in it, must reproduce `data/` +(the reference: on JUNGFRAU 2026-10-12..16 the emulator matched the cloud within 0.05 px, errors identical). + +```yaml +stations: + JUNGFRAU: + patterns: # by sensor_name, over sensors.json + PM_G: {error_threshold: 0.7} + BL_C: {corners: [[1700, 929], [1724, 953]]} + REF_CT: {pattern_type: target} + camera: {focal_distance: 6.3} # over unit.json and the common camera block +camera: # common to the stations of the file + algorithm: ransac # ransac | danish_hybrid | trusted_danish + mode: "off" # off | rotation | rotation_scale | relative +``` + +Running a configuration again replaces `runs//`: to compare two settings, two files. No cache between the +re-measurement and the correction (431 images in 11 s; about 15 min for a year of one station): added if it gets slow. + +Rejected: a full copy per configuration (the DIFF images duplicated, 0.6 GB per station-day); a configuration +suffix in file names (the tools recognise files by their columns and would mix configurations). + +## Work, in order + +1. **`pump_diff`** (this repository's `extraction/`): the extras of the DIFF images move under `images_diff/`; per day, + before the DIFF images and their index, the metrics (`pump.py`'s `pump_metrics`) and the IRON images + (`pump_images`, `--every`, default 60 min); per station, `targets.csv` (`write_targets`, on the DIFF patterns); + at the end, the weather (`weather.py`, given the root folder). A day whose metrics failed gets no index: the next + run reads it again. `pump.py`'s only change: `pump_metrics` returns whether every day succeeded. +2. **`run_config`**: re-measurement (optical's emulator), camera correction (optical's `remove_camera_movement` and + the all-point modes), written in the format of `data/`. Its own design once optical's correction outputs are mapped + to the columns of `data/` (`relative_x/y, corr_x/y, reproj_err, inlier, sigma0`, the camera pose). diff --git a/docs/superpowers/specs/2026-11-19-pump-diff-design.md b/docs/superpowers/specs/2026-11-19-pump-diff-design.md new file mode 100644 index 0000000..449a8fc --- /dev/null +++ b/docs/superpowers/specs/2026-11-19-pump-diff-design.md @@ -0,0 +1,67 @@ +# Pumping the differential images into `data_diff/` + +2026-11-19. Short design, approved in conversation ("A", "go with your recommendations"). + +## Intent + +Recompute every station's data from the differential images with configurations of our own (patterns redrawn, +thresholds, camera-correction modes), recover the frames the cloud rejected or left uncorrected, and in the end +replace `data/`. This design covers only the first of four steps: **pump the raw material**. The next ones +(re-measure the images with the device chain of optical, correct the camera motion, write the result in the format of +`data/`) get their own designs. + +## Goals, non-goals + +- Every `OPTICAL_SHIFT-DIFF` image of a station over a range, full size, as stored on S3 (patterns may be redrawn later, + so no crop). +- Everything needed to redo the measurement from them: all the patterns, archived ones included, with every property + row as stored (value, time, status); the acquisition unit's properties (focal distance, pixel size, algorithm...); + each measurement's exact time, group and properties. +- Gaps explained, not only seen: every measurement of the database is listed, with its file or why it has none. +- Non-goal: touching `data/` or `pump.py` (the nightly pump stays as it is). Non-goal: scheduling (later, once the + history is in). +- Anti-goal: filling the disk (about 0.6 GB per station-day at 15 min; 802 GB free on 2026-11-19). + +## Approaches + +- **A, chosen: a separate script `extraction/pump_diff.py` + `pump_diff.sh`.** Imports what it needs from `pump.py` + (acquisition unit, error formatting); own lock and log, so a backfill of days does not block the nightly pump; does + not run `absolute.py` / `weather.py`. +- B: a "diff only" mode in `pump.py`: least code, but touches the nightly pump and shares its lock. +- C: optical's `scripts/pcc_emulator/fetch_cloud_images.py`: needs monitoring-dashboard's code and environment, + against this repository's rule of no dependency on the app. + +## Design + +Output, `data_diff//` (out of git): + +- `images_diff/TZ_.png`: the S3 object as is, named as in `data/`. +- `index/_diff_.csv`, one per UTC day: `measurement_id, time_utc` (exact, ISO), + `measurement_group_id, file` (empty without image), `status` (`ok`, `no_uuid`: never reached S3, `s3_missing`: S3 + returned nothing), `properties` (the measurement's JSON). Written after the day's images, through a temporary file: + an index on disk means its day was read through. +- `sensors.json`: every sensor of the acquisition unit (any type, archived included): `id, name, type, time_utc` and + all its property rows `{name, value, time_utc, status}`. Rewritten at each run (the database keeps archived + sensors, so nothing is lost by rewriting). +- `unit.json`: the acquisition unit (`id, serial_number, product_number`) and its property rows, same shape. + +Runs: + +- `pump_diff.sh --from D [--to E] SERIAL...`: the days of `[D, E)`. A day whose index exists and is before the last + indexed day is skipped; the last indexed day and the later ones are read again (a day pumped while it was running + is incomplete). An image already on disk is never downloaded again. +- `pump_diff.sh [SERIAL...]` without `--from`: from the last indexed day of each station up to now. Without station: + the stations already in `data_diff/`. A station with nothing indexed is skipped with a message (its history starts + only on an explicit `--from`, so no 2.6 TB by accident). +- `--min-free-gb` (default 50): checked before each day; below it, the run stops cleanly. +- `--delay` as in `pump.py`; read-only connection; one query per day for the measurements; images one at a time. +- Lock `out/pump_diff.lock`, log appended to `out/pump_diff.log`. Same `.env` and `uv` environment as `pump.sh`. + +Check: `python3 extraction/pump_diff.py --check`, without database or S3: the range logic and one day pumped through +stubs (an image downloaded, one already there, one without uuid, one refused by S3, the index written). + +## Left to the builder + +- The SQL of the sensors and unit properties (`property` joined to `datatype`, as `pump.py`'s `sensors`). +- Whether a pattern's property has several rows over time is to be seen on the first real run; the format keeps them + all either way. diff --git a/extraction/README.md b/extraction/README.md index 765fad0..bea53e7 100644 --- a/extraction/README.md +++ b/extraction/README.md @@ -55,3 +55,25 @@ Claude may run `pump.sh` itself (user's agreement, 2026-11-12): read-only, small - An hour without an image can have its measurement: the image never reached S3 (empty `measurement_uuid`, unfinished upload). The script skips it and takes the next image of the slot if there is one. - A past day, pumped before the station had finished sending its data (outage, late synchronisation), stays incomplete: delete its files in `data//` and run again. - Relative `x` / `y` (LENZSPITZE, SUSTENHORN, NADELHORN `c*_mon`): after each pumping, `pump.sh` runs `absolute.py`, which converts every sensor whose `x` / `y` medians are under 100 px, from `data//*_patterns.json` and the offset of the station (only LENZSPITZE has both today); a relative sensor it cannot place is dropped (details in `~/aimsight-labeling/knowledge/displacement-columns.md`). + +## The differential images, into `data_diff/` (`pump_diff.sh`) + +`pump_diff.py`, through `pump_diff.sh`, fetches the `OPTICAL_SHIFT-DIFF` images of a station, full size, with what is needed to measure them again, into `data_diff//` (out of git), in the tree of `data/`, so that the other repositories read it the same way; it never touches `data/` and does not run `absolute.py` (the recomputed positions are absolute). Its own lock (`out/pump_diff.lock`) and log (`out/pump_diff.log`): a long history does not block the nightly `pump.sh`. Designs: `docs/superpowers/specs/2026-11-19-pump-diff-design.md`, `2026-11-19-configurations-design.md` (the configurations, `runs//`, read like `data/`). + +- `images_diff/TZ_.png`: the S3 object as is; +- `images_diff/index/_diff_.csv`: every DIFF measurement of the day in the database, `measurement_id, time_utc` (exact), `measurement_group_id, file, status` (`ok`; `no_uuid`: never reached S3; `s3_missing`: S3 returned nothing), `properties` (JSON); +- `images_diff/sensors.json`: every sensor of the unit, archived included, with all its property rows (pattern, corners, x, y, error_threshold, distance_convertion_factor, start_date…); +- `images_diff/unit.json`: the acquisition unit and its properties (`focal_distance`, `pixel_size`, `image_shape` when filled in; none: the platform takes 6.0 mm and 0.0024 mm); +- (under `images_diff/`, the reading tools do not see these files: `data.find_files` skips the folder) +- `metrics/`, `images/` (IRON, one per `--every` minutes, default 60), `weather/`: with the functions of `pump.py` and `weather.py` (`weather.py --data data_diff`), the same files as in `data/`; +- `targets.csv`: as in `data/`, for the patterns the DIFF images are measured with (not archived); `displacement/` and `camera/` come only from the recomputation of a configuration (`runs//`), never from the cloud. + +```bash +extraction/pump_diff.sh --from 2026-09-01 --to 2026-09-08 JUNGFRAU # a range; the history of a station starts here +extraction/pump_diff.sh JUNGFRAU # from its last indexed day up to now +extraction/pump_diff.sh # the same for every station of data_diff/ +``` + +Resuming: a day indexed before the last indexed day is not read again, the last one and later ones are; an image on disk is never downloaded again; every DIFF file is written through a temporary one; the index of a day is written last, after its metrics, IRON and DIFF images, and not at all when its metrics failed (the next run reads it again). `--min-free-gb` (default 50) stops the run before a day. Check without database: `python3 extraction/pump_diff.py --check`. Tried on 2026-11-19: JUNGFRAU 2026-09-01, 94 images (615 MB, 1 min) byte-identical to those of `pump.py --diff`; JUNGFRAU 2026-09-01 and 2026-10-12..16: metrics (168 files), IRON images (133) and weather (6) byte-identical to `data/`; `targets.csv` the same without the two non-DIFF sensors `target_01`, `target_02`. + +Not every station sends its differential images (2026-11-19, last day of each unit): one per measurement on BREITHORN, CLARIDEN, FLORIDA, JUNGFRAU, MICHIGAN, NADELHORN, NORDEND, SNCF01, and ALASKA / WYOMING since 2026-10-30 (new units); one per hour on LENZSPITZE, and none after 2026-10-13; none on SUSTENHORN; WILDSTRUBEL none since 2026-11-16. The script prints, per station, the range the database holds. Only the unit `pump.py` picks is read: the older units of ALASKA (until 2025-08-21) and WYOMING (until 2025-12-08) are not. diff --git a/extraction/pump.py b/extraction/pump.py index 51d7991..85211ae 100644 --- a/extraction/pump.py +++ b/extraction/pump.py @@ -124,10 +124,12 @@ def pump_metrics(conn, system_id, station, start, end, out, replace): """Like the download of the System page (Dashboard/System.py, get_metrics(..., bucketing="1 hour")): per hour, mean * transform_factor + transform_offset, or the list of values of a text metric; the labelled series of one metric merged; an empty row for an hour without measurement. get_metrics counts in hours before now: its query is - redone here on a fixed day, one query per day for all the metrics.""" + redone here on a fixed day, one query per day for all the metrics. -> True when no day failed (pump_diff.py: a + day whose metrics failed is not marked done).""" names = sorted(conn.execute(text("SELECT DISTINCT dt.name FROM datatype dt JOIN metric_series se " "ON se.datatype_id = dt.id WHERE se.system_id = :s"), {"s": system_id}).scalars()) conn.rollback() + ok = True for k in range((end - start).days): a = datetime.combine(start + timedelta(days=k), datetime.min.time(), UTC) paths = {n: out / station / "metrics" @@ -148,6 +150,7 @@ def pump_metrics(conn, system_id, station, start, end, out, replace): GROUP BY 1, dt.name""", s=system_id, a=a, b=a + timedelta(days=1)) except Exception as e: # ponytail: like the displacements, a failed day is skipped, a new run picks it up print(f"{station} {a:%Y-%m-%d} metrics: failed ({root_cause(e)}), run again") + ok = False continue finally: conn.rollback() @@ -163,6 +166,7 @@ def pump_metrics(conn, system_id, station, start, end, out, replace): pd.DataFrame({"time_utc": hours, "metric": col.reindex(hours).to_numpy()}).to_csv(path) print(f"{station} {a:%Y-%m-%d} metrics: {len(names)} files, {df['name'].nunique()} with data, " f"{time.time() - t0:.0f} s") + return ok # --- the "advanced" export: build_advanced_frame (app/datapoints.py) --- diff --git a/extraction/pump_diff.py b/extraction/pump_diff.py new file mode 100644 index 0000000..fe4a881 --- /dev/null +++ b/extraction/pump_diff.py @@ -0,0 +1,326 @@ +"""Pumps the differential images (OPTICAL_SHIFT-DIFF) of stations from the AIMsight platform, full size, with what is +needed to measure them again: every pattern of the unit (archived ones included) and the unit's properties. READ-ONLY. +Never touches data/ nor pump.py's pumping; design: docs/superpowers/specs/2026-11-19-pump-diff-design.md. + +Run through extraction/pump_diff.sh, under WSL, from anywhere (see README.md): + + extraction/pump_diff.sh --from 2026-09-01 --to 2026-09-08 JUNGFRAU a range (its history starts here) + extraction/pump_diff.sh JUNGFRAU from its last indexed day up to now + extraction/pump_diff.sh the same for every station of data_diff/ + +Output (default: the repository's data_diff/, next to this folder), the tree of data/ so that the other repositories +read it the same way (docs/superpowers/specs/2026-11-19-configurations-design.md): + /images_diff/TZ_.png the S3 object as is, named as in data/ + /images_diff/index/_diff_.csv every measurement of the day in the database: + measurement_id, time_utc (exact), measurement_group_id, file (empty without image), status (ok; no_uuid: never + reached S3; s3_missing: S3 returned nothing), properties (the measurement's JSON) + /images_diff/sensors.json every sensor of the unit: id, name, type, time_utc, properties [{name, value, + time_utc, status}] (every property row as stored), rewritten at each run + /images_diff/unit.json the acquisition unit: id, serial_number, product_number, properties + (under images_diff/, the tools of aimsight-tools do not see them: data.find_files skips that folder) + /metrics/, images/ (IRON, one per --every minutes), targets.csv (the DIFF patterns), weather/: as pump.py + and weather.py write them into data/, with their own functions + +Resumable: a day whose index exists and is before the last indexed day is not read again; the last indexed day and +the later ones are (pumped while running, a day is incomplete); an image on disk is never downloaded again. An index +is written after its day's metrics, IRON and DIFF images, every DIFF file through a temporary one: an indexed day is +complete; a day whose metrics failed gets no index. Without --from, a station with nothing indexed is skipped (its +history starts only on an explicit --from). Stops before a day when the disk has less than --min-free-gb left. + +Same reads as pump.py (measurement, acquisition_unit; the copied queries of the app, branch `explo`), plus the +`property` rows of the sensors and of the unit (AcquisitionUnit.list_sensors, Sensor.properties of +app/aimsight_database.py). Variables read (names only, none is printed): AIMSIGHT_DATABASE_USER / _PSWD / _TURL / +_NAME / _PORT, AIMSIGHT_S3_URL / _ACCESS_KEY / _SECRET_KEY / _BUCKET. +""" +import argparse +import csv +import io +import json +import os +import shutil +import sys +import time +from collections import Counter +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +UTC = timezone.utc +MTYPE = "OPTICAL_SHIFT-DIFF" +INDEX = ["measurement_id", "time_utc", "measurement_group_id", "file", "status", "properties"] + + +def write_atomic(path, data): + """A file is either complete or absent: a run stopped in the middle leaves nothing that looks done.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + tmp.write_bytes(data) + os.replace(tmp, path) + + +def indexed_days(folder): + return {datetime.strptime(p.stem.rsplit("_", 1)[1], "%Y%m%d").date() + for p in (folder / "images_diff" / "index").glob(f"{folder.name}_diff_*.csv")} + + +def days_to_pump(indexed, start, end): + """The days of [start, end) to read: the missing ones, and from the last indexed day on (perhaps incomplete). + Without start: from the last indexed day; None when nothing is indexed.""" + last = max(indexed, default=None) + if start is None: + if last is None: + return None + start = last + days = (start + timedelta(days=k) for k in range((end - start).days)) + return [d for d in days if last is None or d not in indexed or d >= last] + + +def pump_day(rows, client, bucket, ext, folder, station, day): + """The images of one day's measurements (rows: id, time_utc, measurement_group_id, measurement_uuid, properties), + then its index. Returns the counts per status, plus the new images and their bytes.""" + c, index = Counter(), [] + for m in rows: + t = m.time_utc.astimezone(UTC) + name, status = f"{t:%Y%m%dT%H%M}Z_{m.id}.{ext}", "ok" + path = folder / "images_diff" / name + if m.measurement_uuid is None: # the image never reached S3 (unfinished multipart upload) + name, status = "", "no_uuid" + elif not path.exists(): + try: + data = client.get_object(Bucket=bucket, Key=m.measurement_uuid)["Body"].read() + except Exception: # like the app's get_measurement_payload: S3 not answering, the image is missing + data = None + if data: + write_atomic(path, data) + c["new"], c["bytes"] = c["new"] + 1, c["bytes"] + len(data) + else: + name, status = "", "s3_missing" + c[status] += 1 + index.append([m.id, t.isoformat(), "" if m.measurement_group_id is None else m.measurement_group_id, name, + status, "" if m.properties is None else json.dumps(m.properties, sort_keys=True)]) + out = io.StringIO() + w = csv.writer(out, lineterminator="\n") + w.writerow(INDEX) + w.writerows(index) + write_atomic(folder / "images_diff" / "index" / f"{station}_diff_{day:%Y%m%d}.csv", out.getvalue().encode()) + return c + + +def prop(name, value, t, status): + return {"name": name, "value": value, "time_utc": t.isoformat(), "status": status} + + +def sensors_json(conn, unit): + """Every sensor of the unit, any type, archived included (`archive` is one of their properties), with every + property row: a pattern redrawn on the platform is a new sensor, the old one stays.""" + from sqlalchemy import text + rows = conn.execute(text(""" + SELECT s.id, s.name, s.type, s.time_utc, dt.name AS prop, p.value, p.time_utc AS ptime, p.status + FROM sensor s LEFT JOIN property p ON p.sensor_id = s.id LEFT JOIN datatype dt ON dt.id = p.datatype_id + WHERE s.acquisition_unit_id = :au ORDER BY s.id, dt.name, p.time_utc"""), {"au": unit}).all() + sensors = {} + for r in rows: + s = sensors.setdefault(r.id, {"id": r.id, "name": r.name, "type": r.type, "time_utc": r.time_utc.isoformat(), + "properties": []}) + if r.prop is not None: + s["properties"].append(prop(r.prop, r.value, r.ptime, r.status)) + return list(sensors.values()) + + +def unit_json(conn, unit): + """The acquisition unit and its own property rows (focal_distance, pixel_size, algorithm...), not its sensors'.""" + from sqlalchemy import text + u = conn.execute(text("SELECT id, serial_number, product_number FROM acquisition_unit WHERE id = :au"), + {"au": unit}).one() + rows = conn.execute(text(""" + SELECT dt.name, p.value, p.time_utc, p.status FROM property p JOIN datatype dt ON dt.id = p.datatype_id + WHERE p.acquisition_unit_id = :au AND p.sensor_id IS NULL ORDER BY dt.name, p.time_utc"""), {"au": unit}).all() + return {"id": u.id, "serial_number": u.serial_number, "product_number": u.product_number, + "properties": [prop(*r) for r in rows]} + + +def diff_patterns(sensors): + """{sensor_name: sensor_id} of the patterns the DIFF images are measured with (sensors_json rows): type + OPTICAL_SHIFT-DIFF, a pattern, not archived; for targets.csv.""" + out = {} + for s in sensors: + p = {r["name"]: r["value"] for r in s["properties"]} + if s["type"] == MTYPE and p.get("pattern") and not p.get("archive"): + out[p.get("sensor_name") or s["name"]] = s["id"] + return out + + +def to_json(obj): + return json.dumps(obj, indent=1, ensure_ascii=False, default=str).encode() + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("stations", nargs="*", help="serial number of the system, as in the URL of the System page; " + "none: the stations already in --out") + p.add_argument("--from", dest="start", help="first UTC day, YYYY-MM-DD; without: the last indexed day of each " + "station (a station with nothing indexed is skipped)") + p.add_argument("--to", dest="end", help="last UTC day, excluded, YYYY-MM-DD; without: up to now") + p.add_argument("--delay", type=int, default=60, help="maximum duration of a query, in s (default 60: database " + "shared with production)") + p.add_argument("--every", type=int, default=60, help="IRON images: one per slot of N minutes, 0 = all (default 60, " + "as pump.sh)") + p.add_argument("--min-free-gb", type=float, default=50, help="stop before a day when less is free (default 50)") + p.add_argument("--out", type=Path, default=Path(__file__).resolve().parent.parent / "data_diff") + a = p.parse_args() + + import boto3 + from sqlalchemy import create_engine, text + from sqlalchemy.engine import URL + + from pump import DB_NAMES, acquisition_unit, pump_images, pump_metrics, root_cause, sensors, write_targets + + tomorrow = datetime.now(UTC).date() + timedelta(days=1) + start = datetime.strptime(a.start, "%Y-%m-%d").date() if a.start else None + end = min(datetime.strptime(a.end, "%Y-%m-%d").date(), tomorrow) if a.end else tomorrow + stations = a.stations or sorted(f.name for f in a.out.glob("*") if f.is_dir()) + if not stations: + sys.exit(f"No station: name one, with --from (nothing in {a.out} yet)") + missing = [n for n in DB_NAMES if not os.environ.get(n)] + if missing: + sys.exit(f"Missing variables (names only): {', '.join(missing)}") + + # the engine and the S3 client of pump.py's main + url = URL.create("postgresql+psycopg2", username=os.environ["AIMSIGHT_DATABASE_USER"], + password=os.environ["AIMSIGHT_DATABASE_PSWD"], host=os.environ["AIMSIGHT_DATABASE_TURL"], + port=int(os.environ.get("AIMSIGHT_DATABASE_PORT", "5432")), + database=os.environ["AIMSIGHT_DATABASE_NAME"]) + engine = create_engine(url, connect_args={ + "options": f"-c timezone=utc -c default_transaction_read_only=on -c statement_timeout={a.delay}s"}) + client = boto3.client("s3", endpoint_url=os.environ.get("AIMSIGHT_S3_URL", "http://s3:3900"), + aws_access_key_id=os.environ.get("AIMSIGHT_S3_ACCESS_KEY", ""), + aws_secret_access_key=os.environ.get("AIMSIGHT_S3_SECRET_KEY", ""), + region_name="garage", use_ssl=False) + bucket = os.environ.get("AIMSIGHT_S3_BUCKET", "measurements-assets") + try: + with engine.connect() as conn: + mt = conn.execute(text("SELECT id, extension FROM measurement_type WHERE name = :n"), {"n": MTYPE}).first() + conn.rollback() + if mt is None: + sys.exit(f"Unknown measurement type {MTYPE}") + for station in stations: + system_id = conn.execute(text("SELECT id FROM system WHERE serial_number = :s"), + {"s": station}).scalar() + unit = system_id and acquisition_unit(conn, system_id) + conn.rollback() + if not unit: + print(f"{station}: " + ("unknown system" if system_id is None else "no recognised acquisition unit")) + continue + folder = a.out / station + days = days_to_pump(indexed_days(folder), start, end) + if days is None: + print(f"{station}: nothing indexed yet, its history starts with --from") + continue + try: # what the database holds, so that an empty day is explained (LENZSPITZE: none after 2026-10-13) + first, last = (conn.execute(text( + "SELECT time_utc FROM measurement WHERE acquisition_unit_id = :au AND measurement_type_id = :mt " + f"ORDER BY time_utc {o} LIMIT 1"), {"au": unit, "mt": mt.id}).scalar() for o in ("ASC", "DESC")) + print(f"{station}: differential images in the database from {first:%Y-%m-%d} to {last:%Y-%m-%d}" + if first else f"{station}: no differential image in the database") + sens = sensors_json(conn, unit) + write_atomic(folder / "images_diff" / "sensors.json", to_json(sens)) + write_atomic(folder / "images_diff" / "unit.json", to_json(unit_json(conn, unit))) + t = write_targets(folder, sensors(conn, unit), diff_patterns(sens)) + print(f"{station}: targets.csv, {len(t)} sensors") + except Exception as e: # the next run writes them + print(f"{station} images range, sensors.json, unit.json, targets.csv: failed ({root_cause(e)}), " + "run again") + finally: + conn.rollback() + print(f"{station}: {len(days)} days, from {days[0] if days else '-'} to {days[-1] if days else '-'}") + for day in days: + free = shutil.disk_usage(a.out).free / 1e9 + if free < a.min_free_gb: + sys.exit(f"Stopped before {station} {day}: {free:.0f} GB free, under --min-free-gb " + f"{a.min_free_gb:g}") + nxt = day + timedelta(days=1) + try: # as pump.py into data/; a day whose metrics or IRON images failed stays open (no index) + if not pump_metrics(conn, system_id, station, day, nxt, a.out, True): + print(f"{station} {day}: left open (metrics failed), run again") + continue + pump_images(conn, (client, bucket), unit, station, day, nxt, "images", a.every, a.out) + except Exception as e: + print(f"{station} {day}: left open ({root_cause(e)}), run again") + continue + finally: + conn.rollback() + a0 = datetime.combine(day, datetime.min.time(), UTC) + try: + rows = conn.execute(text( + "SELECT id, time_utc, measurement_group_id, measurement_uuid, properties FROM measurement " + "WHERE acquisition_unit_id = :au AND measurement_type_id = :mt AND time_utc >= :a " + "AND time_utc < :b ORDER BY time_utc"), + {"au": unit, "mt": mt.id, "a": a0, "b": a0 + timedelta(days=1)}).all() + except Exception as e: # no index for the day: the next run reads it again + print(f"{station} {day}: failed ({root_cause(e)}), run again") + continue + finally: + conn.rollback() + c = pump_day(rows, client, bucket, mt.extension, folder, station, day) + print(f"{station} {day}: {c['new']} new images, {c['bytes'] / 1e6:.0f} MB ({len(rows)} in " + f"database: {c['ok']} on disk, {c['no_uuid']} never reached S3, {c['s3_missing']} not " + "returned by S3)") + time.sleep(0.5) + try: # the MeteoSwiss weather of the days now in --out, as pump.sh does for data/ + import weather + weather.main(stations, a.out) + except Exception as e: # the next run fetches it + print(f"weather: failed ({type(e).__name__}: {e}), run again") + finally: + engine.dispose() + + +def check(): + """Without database or S3: `python3 extraction/pump_diff.py --check`.""" + import tempfile + from types import SimpleNamespace + + d = lambda n: date(2026, 11, n) # noqa: E731 + assert days_to_pump(set(), d(1), d(4)) == [d(1), d(2), d(3)] + assert days_to_pump({d(1), d(2), d(5)}, d(1), d(7)) == [d(3), d(4), d(5), d(6)] # the last indexed day: again + assert days_to_pump({d(2)}, None, d(5)) == [d(2), d(3), d(4)] + assert days_to_pump(set(), None, d(5)) is None # no --from, nothing indexed: skipped + + class S3: + def get_object(self, Bucket, Key): + if Key == "refused": + raise OSError("NoSuchKey") + return {"Body": io.BytesIO(b"png")} + + t = lambda h, m: datetime(2026, 11, 3, h, m, 7, 250000, tzinfo=UTC) # noqa: E731 + rows = [SimpleNamespace(id=1, time_utc=t(10, 0), measurement_group_id=5, measurement_uuid="a", properties={"e": 1}), + SimpleNamespace(id=2, time_utc=t(10, 0), measurement_group_id=5, measurement_uuid="b", properties=None), + SimpleNamespace(id=3, time_utc=t(10, 15), measurement_group_id=None, measurement_uuid=None, properties={}), + SimpleNamespace(id=4, time_utc=t(10, 30), measurement_group_id=None, measurement_uuid="refused", + properties={})] + with tempfile.TemporaryDirectory() as tmp: + folder = Path(tmp) / "S" + (folder / "images_diff").mkdir(parents=True) + (folder / "images_diff" / "20261103T1000Z_2.png").write_bytes(b"old") # a burst: same minute, other id + c = pump_day(rows, S3(), "bucket", "png", folder, "S", d(3)) + assert dict(c) == {"ok": 2, "new": 1, "bytes": 3, "no_uuid": 1, "s3_missing": 1}, c + assert (folder / "images_diff" / "20261103T1000Z_1.png").read_bytes() == b"png" + assert (folder / "images_diff" / "20261103T1000Z_2.png").read_bytes() == b"old" # never downloaded again + assert not list(folder.rglob("*.tmp")) + with open(folder / "images_diff" / "index" / "S_diff_20261103.csv", newline="") as fp: + index = list(csv.DictReader(fp)) + assert [r["status"] for r in index] == ["ok", "ok", "no_uuid", "s3_missing"], index + assert [r["file"] for r in index] == ["20261103T1000Z_1.png", "20261103T1000Z_2.png", "", ""], index + assert index[0]["time_utc"] == "2026-11-03T10:00:07.250000+00:00" and index[0]["properties"] == '{"e": 1}' + assert index[1]["properties"] == "" and index[2]["measurement_group_id"] == "", index + assert indexed_days(folder) == {d(3)} + row = lambda name, typ, **p: {"name": name, "type": typ, "id": len(p), # noqa: E731 + "properties": [{"name": k, "value": v} for k, v in p.items()]} + assert diff_patterns([row("u1", MTYPE, pattern="b64", sensor_name="T1"), row("u2", MTYPE, pattern="b64"), + row("u3", MTYPE, pattern="b64", sensor_name="OLD", archive="2026-01-01"), + row("u4", "camera_movement", sensor_name="CAM"), row("u5", MTYPE, sensor_name="NOPAT")]) \ + == {"T1": 2, "u2": 1}, "DIFF patterns: by sensor_name, else the sensor's name; archived and others left out" + print("ok") + + +if __name__ == "__main__": + check() if sys.argv[1:] == ["--check"] else main() diff --git a/extraction/pump_diff.sh b/extraction/pump_diff.sh new file mode 100755 index 0000000..68bbd84 --- /dev/null +++ b/extraction/pump_diff.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Pumping of the differential images under WSL, into data_diff/ (never data/). Reads the credentials from the .env at +# the root of the repository (never versioned) and runs pump_diff.py in the throwaway uv environment of pump.sh; no +# absolute.py nor weather.py (they work on data/). +# extraction/pump_diff.sh --from D [--to E] SERIAL... a range; the history of a station starts here +# extraction/pump_diff.sh [SERIAL...] from the last indexed day up to now +# Output appended to out/pump_diff.log; one at a time (lock out/pump_diff.lock), independent of pump.sh's. +set -u +export PATH="$HOME/.local/bin:$PATH" # uv, missing from the PATH when the Task Scheduler starts wsl.exe +here=$(cd "$(dirname "$0")" && pwd) +root=$(dirname "$here") +mkdir -p "$root/out" + +exec 9>"$root/out/pump_diff.lock" +flock -n 9 || { echo "A pumping of the differential images is already running." >&2; exit 1; } + +{ + echo "=== $(date -u +%Y-%m-%dT%H:%MZ) pump_diff.py $*" + (set -a +u && . "$root/.env" && set +a -u && # +u: an unquoted $ in .env does not stop the load + uv run --no-project --quiet --python 3.12 \ + --with sqlalchemy --with psycopg2-binary --with pandas --with boto3 \ + python "$here/pump_diff.py" "$@") + echo "=== $(date -u +%Y-%m-%dT%H:%MZ) end" +} 2>&1 | tee -a "$root/out/pump_diff.log" diff --git a/extraction/weather.py b/extraction/weather.py index 82bb960..b126095 100644 --- a/extraction/weather.py +++ b/extraction/weather.py @@ -7,7 +7,8 @@ hourly values of the MeteoSwiss station nearest to the position (weather/meteosw Fetched again: a missing day, a day written for another position, the last 2 days (incomplete, not yet quality-checked). A station without a position (an indoor site) or whose place is not found: no weather/ folder, a line in the log. -Usage: python3 extraction/weather.py [SERIAL...] (default: every tracked station) +Usage: python3 extraction/weather.py [--data DIR] [SERIAL...] (default: every tracked station, into data/; --data: +another root with the tree of data/, e.g. data_diff/, which pump_diff.py passes) """ import csv import functools @@ -40,8 +41,8 @@ def position_of(path): return next(csv.DictReader(f), {}).get("position") -def station(serial, position, now): - folder = DATA / serial +def station(serial, position, now, data=DATA): + folder = data / serial wanted = sorted(d for d in days_with_data(folder) if d <= now) out = folder / "weather" path = lambda d: out / f"{serial}_weather_{d:%Y%m%d}.csv" # noqa: E731 @@ -63,19 +64,23 @@ def station(serial, position, now): f"{where['distance_km']:.1f} km") -def main(serials): +def main(serials, data=DATA): tracked, now = pumping.tracked(), datetime.now(timezone.utc) for serial in serials or tracked: - if not (DATA / serial).is_dir(): + if not (Path(data) / serial).is_dir(): continue if not (position := tracked.get(serial)): print(f"{serial}: no position in stations.txt, no weather") continue try: - station(serial, position, now) + station(serial, position, now, Path(data)) except (LookupError, OSError) as e: # the pumping goes on without the weather of this station print(f"{serial}: no weather: {e}") if __name__ == "__main__": - main(sys.argv[1:]) + args = sys.argv[1:] + if args[:1] == ["--data"]: + main(args[2:], Path(args[1])) + else: + main(args)