Files
aimsight-data/extraction/weather.py
T

81 lines
3.8 KiB
Python
Raw Normal View History

"""The weather of each tracked station with a position, into data/<STATION>/weather/. Run by pump.sh after each pumping.
One CSV per day that has data in the station's folder: data/<STATION>/weather/<STATION>_weather_<YYYYMMDD>.csv, the
hourly values of the MeteoSwiss station nearest to the position (weather/meteoswiss.py), UTC, columns: time_utc, position
(the line of stations.txt), place (found), station, station_name, distance_km, height_m, then the measurements
(meteoswiss.PARAMETERS). A day without MeteoSwiss values gets the header only.
Fetched again: a missing day, a day written for another position, the last 2 days (incomplete, not yet quality-checked).
A station without a position (an indoor site) or whose place is not found: no weather/ folder, a line in the log.
Usage: python3 extraction/weather.py [SERIAL...] (default: every tracked station)
"""
import csv
import functools
import re
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / "extraction"), str(ROOT / "weather")]
import meteoswiss # noqa: E402
import pumping # noqa: E402
DATA = ROOT / "data"
DAY = re.compile(r"(?<!\d)(20\d\d)_?(\d\d)_?(\d\d)(?!\d)") # ALASKA_advanced_20261017.csv, _2026_10_17_00_00 to …, 20261029T1617Z_…
RECENT = timedelta(days=2)
# The files of MeteoSwiss (one per station, decade, recent, now) are read once per run, for every station that shares them.
meteoswiss.fetch = functools.lru_cache(maxsize=None)(meteoswiss.fetch)
def days_with_data(folder):
"""The days (UTC dates) of the files of the station, its weather aside."""
return {datetime(*map(int, m.groups()), tzinfo=timezone.utc) for p in folder.rglob("*")
if p.is_file() and "weather" not in p.relative_to(folder).parts and (m := DAY.search(p.name))}
def position_of(path):
"""The position a weather file was written for; None without a row."""
with open(path, newline="") as f:
return next(csv.DictReader(f), {}).get("position")
def station(serial, position, now):
folder = DATA / serial
wanted = sorted(d for d in days_with_data(folder) if d <= now)
out = folder / "weather"
path = lambda d: out / f"{serial}_weather_{d:%Y%m%d}.csv" # noqa: E731
todo = [d for d in wanted if not path(d).is_file() or now - d < RECENT or position_of(path(d)) not in (position, None)]
if not todo:
return
where, hours = meteoswiss.hourly(position, todo[0], todo[-1] + timedelta(days=1))
meta = [position, where["place"], where["station"], where["name"], f"{where['distance_km']:.1f}", f"{where['height_m']:.0f}"]
out.mkdir(exist_ok=True)
for d in todo:
with open(path(d), "w", newline="") as f:
w = csv.writer(f)
w.writerow(["time_utc", "position", "place", "station", "station_name", "distance_km", "height_m",
*meteoswiss.PARAMETERS.values()])
for t, values in hours.items():
if d <= t < d + timedelta(days=1):
w.writerow([t.strftime("%Y-%m-%dT%H:%MZ"), *meta, *("" if v is None else f"{v:g}" for v in values.values())])
print(f"{serial}: weather of {len(todo)} days, {where['place']} -> MeteoSwiss {where['station']} {where['name']}, "
f"{where['distance_km']:.1f} km")
def main(serials):
tracked, now = pumping.tracked(), datetime.now(timezone.utc)
for serial in serials or tracked:
if not (DATA / serial).is_dir():
continue
if not (position := tracked.get(serial)):
print(f"{serial}: no position in stations.txt, no weather")
continue
try:
station(serial, position, now)
except (LookupError, OSError) as e: # the pumping goes on without the weather of this station
print(f"{serial}: no weather: {e}")
if __name__ == "__main__":
main(sys.argv[1:])