Pumping: the weather of the stations with a position written into data/<SERIAL>/weather/ (extraction/weather.py, after pump.py); the mm (corr_x_mm, corr_y_mm by the sensor_id of each row) and targets.csv with the references (pattern_type, distance_mm, mm_per_px); --replace pumps a range again
pump.py --check passes; the mm and targets.csv parts are not yet run against the database (no VPN). stations.txt: NIEDERHORN and NORDEND without a position (outside Switzerland). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9adc38f9b9
commit
ea0228a73a
8 files changed
+205
-22
No files matched your search
@@ -0,0 +1,81 @@
|
||||
"""The weather of each tracked station with a position, into data/<STATION>/weather/. Run by pump.sh after each pumping.
|
||||
|
||||
One CSV per day that has data in the station's folder: data/<STATION>/weather/<STATION>_weather_<YYYYMMDD>.csv, the
|
||||
hourly values of the MeteoSwiss station nearest to the position (weather/meteoswiss.py), UTC, columns: time_utc, position
|
||||
(the line of stations.txt), place (found), station, station_name, distance_km, height_m, then the measurements
|
||||
(meteoswiss.PARAMETERS). A day without MeteoSwiss values gets the header only.
|
||||
Fetched again: a missing day, a day written for another position, the last 2 days (incomplete, not yet quality-checked).
|
||||
A station without a position (an indoor site) or whose place is not found: no weather/ folder, a line in the log.
|
||||
|
||||
Usage: python3 extraction/weather.py [SERIAL...] (default: every tracked station)
|
||||
"""
|
||||
import csv
|
||||
import functools
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path[:0] = [str(ROOT / "extraction"), str(ROOT / "weather")]
|
||||
import meteoswiss # noqa: E402
|
||||
import pumping # noqa: E402
|
||||
|
||||
DATA = ROOT / "data"
|
||||
DAY = re.compile(r"(?<!\d)(20\d\d)_?(\d\d)_?(\d\d)(?!\d)") # ALASKA_advanced_20261017.csv, _2026_10_17_00_00 to …, 20261029T1617Z_…
|
||||
RECENT = timedelta(days=2)
|
||||
# The files of MeteoSwiss (one per station, decade, recent, now) are read once per run, for every station that shares them.
|
||||
meteoswiss.fetch = functools.lru_cache(maxsize=None)(meteoswiss.fetch)
|
||||
|
||||
|
||||
def days_with_data(folder):
|
||||
"""The days (UTC dates) of the files of the station, its weather aside."""
|
||||
return {datetime(*map(int, m.groups()), tzinfo=timezone.utc) for p in folder.rglob("*")
|
||||
if p.is_file() and "weather" not in p.relative_to(folder).parts and (m := DAY.search(p.name))}
|
||||
|
||||
|
||||
def position_of(path):
|
||||
"""The position a weather file was written for; None without a row."""
|
||||
with open(path, newline="") as f:
|
||||
return next(csv.DictReader(f), {}).get("position")
|
||||
|
||||
|
||||
def station(serial, position, now):
|
||||
folder = DATA / serial
|
||||
wanted = sorted(d for d in days_with_data(folder) if d <= now)
|
||||
out = folder / "weather"
|
||||
path = lambda d: out / f"{serial}_weather_{d:%Y%m%d}.csv" # noqa: E731
|
||||
todo = [d for d in wanted if not path(d).is_file() or now - d < RECENT or position_of(path(d)) not in (position, None)]
|
||||
if not todo:
|
||||
return
|
||||
where, hours = meteoswiss.hourly(position, todo[0], todo[-1] + timedelta(days=1))
|
||||
meta = [position, where["place"], where["station"], where["name"], f"{where['distance_km']:.1f}", f"{where['height_m']:.0f}"]
|
||||
out.mkdir(exist_ok=True)
|
||||
for d in todo:
|
||||
with open(path(d), "w", newline="") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["time_utc", "position", "place", "station", "station_name", "distance_km", "height_m",
|
||||
*meteoswiss.PARAMETERS.values()])
|
||||
for t, values in hours.items():
|
||||
if d <= t < d + timedelta(days=1):
|
||||
w.writerow([t.strftime("%Y-%m-%dT%H:%MZ"), *meta, *("" if v is None else f"{v:g}" for v in values.values())])
|
||||
print(f"{serial}: weather of {len(todo)} days, {where['place']} -> MeteoSwiss {where['station']} {where['name']}, "
|
||||
f"{where['distance_km']:.1f} km")
|
||||
|
||||
|
||||
def main(serials):
|
||||
tracked, now = pumping.tracked(), datetime.now(timezone.utc)
|
||||
for serial in serials or tracked:
|
||||
if not (DATA / serial).is_dir():
|
||||
continue
|
||||
if not (position := tracked.get(serial)):
|
||||
print(f"{serial}: no position in stations.txt, no weather")
|
||||
continue
|
||||
try:
|
||||
station(serial, position, now)
|
||||
except (LookupError, OSError) as e: # the pumping goes on without the weather of this station
|
||||
print(f"{serial}: no weather: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
Reference in new issue
Block a user