Files
maximeandClaude Opus 5.5 81908ca370 Pumping of the differential images: extraction/pump_diff.sh into data_diff/<SERIAL>/, in the tree of data/ (never data/ itself)
The OPTICAL_SHIFT-DIFF images full size, with what measuring them again needs under images_diff/ (index of every
measurement per day with its exact time and why an image is missing, sensors.json with every pattern archived
included, unit.json), and the metrics, IRON images, weather and targets.csv as pump.py and weather.py write them into
data/. Its own lock and log; a day is indexed last, and not when its metrics failed; --min-free-gb. pump_metrics now
returns whether every day succeeded; weather.py takes --data. Tried on JUNGFRAU 2026-09-01 and 2026-10-12..16: DIFF
images byte-identical to pump.py --diff, metrics, IRON and weather byte-identical to data/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 09:59:50 +01:00

87 lines
4.1 KiB
Python

"""The weather of each tracked station with a position, into data/<STATION>/weather/. Run by pump.sh after each pumping.
One CSV per day that has data in the station's folder: data/<STATION>/weather/<STATION>_weather_<YYYYMMDD>.csv, the
hourly values of the MeteoSwiss station nearest to the position (weather/meteoswiss.py), UTC, columns: time_utc, position
(the line of stations.txt), place (found), station, station_name, distance_km, height_m, then the measurements
(meteoswiss.PARAMETERS). A day without MeteoSwiss values gets the header only.
Fetched again: a missing day, a day written for another position, the last 2 days (incomplete, not yet quality-checked).
A station without a position (an indoor site) or whose place is not found: no weather/ folder, a line in the log.
Usage: python3 extraction/weather.py [--data DIR] [SERIAL...] (default: every tracked station, into data/; --data:
another root with the tree of data/, e.g. data_diff/, which pump_diff.py passes)
"""
import csv
import functools
import re
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / "extraction"), str(ROOT / "weather")]
import meteoswiss # noqa: E402
import pumping # noqa: E402
DATA = ROOT / "data"
DAY = re.compile(r"(?<!\d)(20\d\d)_?(\d\d)_?(\d\d)(?!\d)") # ALASKA_advanced_20261017.csv, _2026_10_17_00_00 to …, 20261029T1617Z_…
RECENT = timedelta(days=2)
# The files of MeteoSwiss (one per station, decade, recent, now) are read once per run, for every station that shares them.
meteoswiss.fetch = functools.lru_cache(maxsize=None)(meteoswiss.fetch)
def days_with_data(folder):
"""The days (UTC dates) of the files of the station, its weather aside."""
return {datetime(*map(int, m.groups()), tzinfo=timezone.utc) for p in folder.rglob("*")
if p.is_file() and "weather" not in p.relative_to(folder).parts and (m := DAY.search(p.name))}
def position_of(path):
"""The position a weather file was written for; None without a row."""
with open(path, newline="") as f:
return next(csv.DictReader(f), {}).get("position")
def station(serial, position, now, data=DATA):
folder = data / serial
wanted = sorted(d for d in days_with_data(folder) if d <= now)
out = folder / "weather"
path = lambda d: out / f"{serial}_weather_{d:%Y%m%d}.csv" # noqa: E731
todo = [d for d in wanted if not path(d).is_file() or now - d < RECENT or position_of(path(d)) not in (position, None)]
if not todo:
return
where, hours = meteoswiss.hourly(position, todo[0], todo[-1] + timedelta(days=1))
meta = [position, where["place"], where["station"], where["name"], f"{where['distance_km']:.1f}", f"{where['height_m']:.0f}"]
out.mkdir(exist_ok=True)
for d in todo:
with open(path(d), "w", newline="") as f:
w = csv.writer(f)
w.writerow(["time_utc", "position", "place", "station", "station_name", "distance_km", "height_m",
*meteoswiss.PARAMETERS.values()])
for t, values in hours.items():
if d <= t < d + timedelta(days=1):
w.writerow([t.strftime("%Y-%m-%dT%H:%MZ"), *meta, *("" if v is None else f"{v:g}" for v in values.values())])
print(f"{serial}: weather of {len(todo)} days, {where['place']} -> MeteoSwiss {where['station']} {where['name']}, "
f"{where['distance_km']:.1f} km")
def main(serials, data=DATA):
tracked, now = pumping.tracked(), datetime.now(timezone.utc)
for serial in serials or tracked:
if not (Path(data) / serial).is_dir():
continue
if not (position := tracked.get(serial)):
print(f"{serial}: no position in stations.txt, no weather")
continue
try:
station(serial, position, now, Path(data))
except (LookupError, OSError) as e: # the pumping goes on without the weather of this station
print(f"{serial}: no weather: {e}")
if __name__ == "__main__":
args = sys.argv[1:]
if args[:1] == ["--data"]:
main(args[2:], Path(args[1]))
else:
main(args)