The OPTICAL_SHIFT-DIFF images full size, with what measuring them again needs under images_diff/ (index of every measurement per day with its exact time and why an image is missing, sensors.json with every pattern archived included, unit.json), and the metrics, IRON images, weather and targets.csv as pump.py and weather.py write them into data/. Its own lock and log; a day is indexed last, and not when its metrics failed; --min-free-gb. pump_metrics now returns whether every day succeeded; weather.py takes --data. Tried on JUNGFRAU 2026-09-01 and 2026-10-12..16: DIFF images byte-identical to pump.py --diff, metrics, IRON and weather byte-identical to data/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
87 lines
4.1 KiB
Python
87 lines
4.1 KiB
Python
"""The weather of each tracked station with a position, into data/<STATION>/weather/. Run by pump.sh after each pumping.
|
|
|
|
One CSV per day that has data in the station's folder: data/<STATION>/weather/<STATION>_weather_<YYYYMMDD>.csv, the
|
|
hourly values of the MeteoSwiss station nearest to the position (weather/meteoswiss.py), UTC, columns: time_utc, position
|
|
(the line of stations.txt), place (found), station, station_name, distance_km, height_m, then the measurements
|
|
(meteoswiss.PARAMETERS). A day without MeteoSwiss values gets the header only.
|
|
Fetched again: a missing day, a day written for another position, the last 2 days (incomplete, not yet quality-checked).
|
|
A station without a position (an indoor site) or whose place is not found: no weather/ folder, a line in the log.
|
|
|
|
Usage: python3 extraction/weather.py [--data DIR] [SERIAL...] (default: every tracked station, into data/; --data:
|
|
another root with the tree of data/, e.g. data_diff/, which pump_diff.py passes)
|
|
"""
|
|
import csv
|
|
import functools
|
|
import re
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path[:0] = [str(ROOT / "extraction"), str(ROOT / "weather")]
|
|
import meteoswiss # noqa: E402
|
|
import pumping # noqa: E402
|
|
|
|
DATA = ROOT / "data"
|
|
DAY = re.compile(r"(?<!\d)(20\d\d)_?(\d\d)_?(\d\d)(?!\d)") # ALASKA_advanced_20261017.csv, _2026_10_17_00_00 to …, 20261029T1617Z_…
|
|
RECENT = timedelta(days=2)
|
|
# The files of MeteoSwiss (one per station, decade, recent, now) are read once per run, for every station that shares them.
|
|
meteoswiss.fetch = functools.lru_cache(maxsize=None)(meteoswiss.fetch)
|
|
|
|
|
|
def days_with_data(folder):
|
|
"""The days (UTC dates) of the files of the station, its weather aside."""
|
|
return {datetime(*map(int, m.groups()), tzinfo=timezone.utc) for p in folder.rglob("*")
|
|
if p.is_file() and "weather" not in p.relative_to(folder).parts and (m := DAY.search(p.name))}
|
|
|
|
|
|
def position_of(path):
|
|
"""The position a weather file was written for; None without a row."""
|
|
with open(path, newline="") as f:
|
|
return next(csv.DictReader(f), {}).get("position")
|
|
|
|
|
|
def station(serial, position, now, data=DATA):
|
|
folder = data / serial
|
|
wanted = sorted(d for d in days_with_data(folder) if d <= now)
|
|
out = folder / "weather"
|
|
path = lambda d: out / f"{serial}_weather_{d:%Y%m%d}.csv" # noqa: E731
|
|
todo = [d for d in wanted if not path(d).is_file() or now - d < RECENT or position_of(path(d)) not in (position, None)]
|
|
if not todo:
|
|
return
|
|
where, hours = meteoswiss.hourly(position, todo[0], todo[-1] + timedelta(days=1))
|
|
meta = [position, where["place"], where["station"], where["name"], f"{where['distance_km']:.1f}", f"{where['height_m']:.0f}"]
|
|
out.mkdir(exist_ok=True)
|
|
for d in todo:
|
|
with open(path(d), "w", newline="") as f:
|
|
w = csv.writer(f)
|
|
w.writerow(["time_utc", "position", "place", "station", "station_name", "distance_km", "height_m",
|
|
*meteoswiss.PARAMETERS.values()])
|
|
for t, values in hours.items():
|
|
if d <= t < d + timedelta(days=1):
|
|
w.writerow([t.strftime("%Y-%m-%dT%H:%MZ"), *meta, *("" if v is None else f"{v:g}" for v in values.values())])
|
|
print(f"{serial}: weather of {len(todo)} days, {where['place']} -> MeteoSwiss {where['station']} {where['name']}, "
|
|
f"{where['distance_km']:.1f} km")
|
|
|
|
|
|
def main(serials, data=DATA):
|
|
tracked, now = pumping.tracked(), datetime.now(timezone.utc)
|
|
for serial in serials or tracked:
|
|
if not (Path(data) / serial).is_dir():
|
|
continue
|
|
if not (position := tracked.get(serial)):
|
|
print(f"{serial}: no position in stations.txt, no weather")
|
|
continue
|
|
try:
|
|
station(serial, position, now, Path(data))
|
|
except (LookupError, OSError) as e: # the pumping goes on without the weather of this station
|
|
print(f"{serial}: no weather: {e}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = sys.argv[1:]
|
|
if args[:1] == ["--data"]:
|
|
main(args[2:], Path(args[1]))
|
|
else:
|
|
main(args)
|