81 lines
3.8 KiB
Python
81 lines
3.8 KiB
Python
"""The weather of each tracked station with a position, into data/<STATION>/weather/. Run by pump.sh after each pumping.
|
|||
|
|
|
||
|
|
One CSV per day that has data in the station's folder: data/<STATION>/weather/<STATION>_weather_<YYYYMMDD>.csv, the
|
||
|
|
hourly values of the MeteoSwiss station nearest to the position (weather/meteoswiss.py), UTC, columns: time_utc, position
|
||
|
|
(the line of stations.txt), place (found), station, station_name, distance_km, height_m, then the measurements
|
||
|
|
(meteoswiss.PARAMETERS). A day without MeteoSwiss values gets the header only.
|
||
|
|
Fetched again: a missing day, a day written for another position, the last 2 days (incomplete, not yet quality-checked).
|
||
|
|
A station without a position (an indoor site) or whose place is not found: no weather/ folder, a line in the log.
|
||
|
|
|
||
|
|
Usage: python3 extraction/weather.py [SERIAL...] (default: every tracked station)
|
||
|
|
"""
|
||
|
|
import csv
|
||
|
|
import functools
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
from datetime import datetime, timedelta, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parents[1]
|
||
|
|
sys.path[:0] = [str(ROOT / "extraction"), str(ROOT / "weather")]
|
||
|
|
import meteoswiss # noqa: E402
|
||
|
|
import pumping # noqa: E402
|
||
|
|
|
||
|
|
DATA = ROOT / "data"
|
||
|
|
DAY = re.compile(r"(?<!\d)(20\d\d)_?(\d\d)_?(\d\d)(?!\d)") # ALASKA_advanced_20261017.csv, _2026_10_17_00_00 to …, 20261029T1617Z_…
|
||
|
|
RECENT = timedelta(days=2)
|
||
|
|
# The files of MeteoSwiss (one per station, decade, recent, now) are read once per run, for every station that shares them.
|
||
|
|
meteoswiss.fetch = functools.lru_cache(maxsize=None)(meteoswiss.fetch)
|
||
|
|
|
||
|
|
|
||
|
|
def days_with_data(folder):
|
||
|
|
"""The days (UTC dates) of the files of the station, its weather aside."""
|
||
|
|
return {datetime(*map(int, m.groups()), tzinfo=timezone.utc) for p in folder.rglob("*")
|
||
|
|
if p.is_file() and "weather" not in p.relative_to(folder).parts and (m := DAY.search(p.name))}
|
||
|
|
|
||
|
|
|
||
|
|
def position_of(path):
|
||
|
|
"""The position a weather file was written for; None without a row."""
|
||
|
|
with open(path, newline="") as f:
|
||
|
|
return next(csv.DictReader(f), {}).get("position")
|
||
|
|
|
||
|
|
|
||
|
|
def station(serial, position, now):
|
||
|
|
folder = DATA / serial
|
||
|
|
wanted = sorted(d for d in days_with_data(folder) if d <= now)
|
||
|
|
out = folder / "weather"
|
||
|
|
path = lambda d: out / f"{serial}_weather_{d:%Y%m%d}.csv" # noqa: E731
|
||
|
|
todo = [d for d in wanted if not path(d).is_file() or now - d < RECENT or position_of(path(d)) not in (position, None)]
|
||
|
|
if not todo:
|
||
|
|
return
|
||
|
|
where, hours = meteoswiss.hourly(position, todo[0], todo[-1] + timedelta(days=1))
|
||
|
|
meta = [position, where["place"], where["station"], where["name"], f"{where['distance_km']:.1f}", f"{where['height_m']:.0f}"]
|
||
|
|
out.mkdir(exist_ok=True)
|
||
|
|
for d in todo:
|
||
|
|
with open(path(d), "w", newline="") as f:
|
||
|
|
w = csv.writer(f)
|
||
|
|
w.writerow(["time_utc", "position", "place", "station", "station_name", "distance_km", "height_m",
|
||
|
|
*meteoswiss.PARAMETERS.values()])
|
||
|
|
for t, values in hours.items():
|
||
|
|
if d <= t < d + timedelta(days=1):
|
||
|
|
w.writerow([t.strftime("%Y-%m-%dT%H:%MZ"), *meta, *("" if v is None else f"{v:g}" for v in values.values())])
|
||
|
|
print(f"{serial}: weather of {len(todo)} days, {where['place']} -> MeteoSwiss {where['station']} {where['name']}, "
|
||
|
|
f"{where['distance_km']:.1f} km")
|
||
|
|
|
||
|
|
|
||
|
|
def main(serials):
|
||
|
|
tracked, now = pumping.tracked(), datetime.now(timezone.utc)
|
||
|
|
for serial in serials or tracked:
|
||
|
|
if not (DATA / serial).is_dir():
|
||
|
|
continue
|
||
|
|
if not (position := tracked.get(serial)):
|
||
|
|
print(f"{serial}: no position in stations.txt, no weather")
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
station(serial, position, now)
|
||
|
|
except (LookupError, OSError) as e: # the pumping goes on without the weather of this station
|
||
|
|
print(f"{serial}: no weather: {e}")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main(sys.argv[1:])
|