The methods that bring data in, for the callers: extraction/pumping.py (tracked stations, pump.sh, its state), weather/meteoswiss.py (from aimsight-app, with hourly())
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
088484c376
commit
b1790fc422
6 files changed
+302
-5
No files matched your search
@@ -0,0 +1,17 @@
|
||||
# weather
|
||||
|
||||
The weather really measured at a place over a past range: the hourly values of the MeteoSwiss automatic weather station (SwissMetNet) nearest to the place. Measurements, not a model output.
|
||||
|
||||
```bash
|
||||
python3 weather/meteoswiss.py <place> <start> <end> [--out FILE]
|
||||
python3 weather/meteoswiss.py "Genève" 2026-11-15 2026-11-16 --out gve.csv
|
||||
python3 weather/meteoswiss.py 46.52,6.63 2025-06-01T12:00 2025-06-01T18:00
|
||||
```
|
||||
|
||||
A caller (the Data page of aimsight-app) calls `hourly(place, start, end)` -> `(where, {time: {column: value or None}})`, `where` the place found and the station chosen; `LookupError` for a place not found, `OSError` when MeteoSwiss or swisstopo do not answer.
|
||||
|
||||
- `place`: a Swiss place name, resolved by the swisstopo search (`api3.geo.admin.ch`), or coordinates `lat,lon` (WGS84). The resolved place and the chosen station (name, distance, altitude) are written on stderr: check them, the search always returns its best match.
|
||||
- `start`, `end`: `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM`, in UTC, `end` excluded. The range is in the past.
|
||||
- The output: a CSV, one row per hour, `time_utc` (start of the hour) and `station`, then `precipitation_mm`, `temperature_c`, `humidity_pct`, `wind_speed_kmh`, `wind_gust_kmh`, `wind_direction_deg`, `pressure_hpa` (at the station, QFE), `sunshine_min`, `global_radiation_wm2`, `snow_depth_cm`. An empty cell: not measured at this station or at this hour.
|
||||
|
||||
The source is the Open Data of MeteoSwiss, free, without key ([terms of use](https://opendatadocs.meteoswiss.ch/fr/general/terms-of-use), [documentation of the dataset](https://opendatadocs.meteoswiss.ch/a-data-groundbased/a1-automatic-weather-stations)); the files of a station are listed by the STAC API (`data.geo.admin.ch/api/stac/v1/collections/ch.meteoschweiz.ogd-smn/items/<station>`). The values are quality-checked by MeteoSwiss, except those of the current day (`now` file). Hourly series from 1980 at most, depending on the station. Standard library only.
|
||||
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The weather measured by MeteoSwiss at the station nearest to a place, over a past range.
|
||||
|
||||
python3 weather/meteoswiss.py <place> <start> <end> [--out FILE]
|
||||
hourly(place, start, end) for a caller (aimsight-app): where, and the values per hour
|
||||
|
||||
<place> is a Swiss place name ("Genève", "Lausanne") or coordinates "lat,lon" (WGS84);
|
||||
<start> and <end> are dates YYYY-MM-DD or YYYY-MM-DDTHH:MM, in UTC, <end> excluded.
|
||||
The output is an hourly CSV (stdout by default), one row per hour.
|
||||
|
||||
The source is the Open Data of MeteoSwiss: the automatic weather stations SwissMetNet,
|
||||
the real measurements (not a model), quality-checked except for the current day
|
||||
(https://opendatadocs.meteoswiss.ch/a-data-groundbased/a1-automatic-weather-stations).
|
||||
The place name is resolved by the search service of swisstopo (api3.geo.admin.ch).
|
||||
No key, no dependency outside the standard library.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
BASE = "https://data.geo.admin.ch/ch.meteoschweiz.ogd-smn"
|
||||
STAC_ITEM = "https://data.geo.admin.ch/api/stac/v1/collections/ch.meteoschweiz.ogd-smn/items/{}"
|
||||
GEOCODER = "https://api3.geo.admin.ch/rest/services/api/SearchServer"
|
||||
|
||||
# MeteoSwiss parameter -> column of the output (hourly values).
|
||||
PARAMETERS = {
|
||||
"rre150h0": "precipitation_mm", # hourly total
|
||||
"tre200h0": "temperature_c", # air at 2 m, hourly mean
|
||||
"ure200h0": "humidity_pct", # relative, at 2 m, hourly mean
|
||||
"fu3010h0": "wind_speed_kmh", # hourly mean
|
||||
"fu3010h1": "wind_gust_kmh", # hourly max of the gusts
|
||||
"dkl010h0": "wind_direction_deg", # hourly mean
|
||||
"prestah0": "pressure_hpa", # at the station (QFE), hourly mean
|
||||
"sre000h0": "sunshine_min", # hourly total
|
||||
"gre000h0": "global_radiation_wm2", # hourly mean
|
||||
"htoauths": "snow_depth_cm", # instantaneous, not at every station
|
||||
}
|
||||
|
||||
|
||||
def fetch(url):
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "aimsight-weather"})
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
return response.read()
|
||||
|
||||
|
||||
def parse_time(text):
|
||||
for form in ("%Y-%m-%dT%H:%M", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(text, form).replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
pass
|
||||
raise argparse.ArgumentTypeError(f"not a date YYYY-MM-DD[THH:MM]: {text}")
|
||||
|
||||
|
||||
def locate(place):
|
||||
"""(lat, lon, label) of a place name or of "lat,lon"."""
|
||||
parts = place.split(",")
|
||||
if len(parts) == 2:
|
||||
try:
|
||||
lat, lon = float(parts[0]), float(parts[1])
|
||||
return lat, lon, f"{lat},{lon}"
|
||||
except ValueError:
|
||||
pass
|
||||
query = urllib.parse.urlencode({"searchText": place, "type": "locations", "limit": 1, "sr": 4326})
|
||||
results = json.loads(fetch(f"{GEOCODER}?{query}"))["results"]
|
||||
if not results:
|
||||
raise LookupError(f"place not found: {place}")
|
||||
attrs = results[0]["attrs"]
|
||||
label = re.sub(r"<[^>]+>", "", attrs["label"]).split("\n")[0].strip()
|
||||
return attrs["lat"], attrs["lon"], label
|
||||
|
||||
|
||||
def distance_km(lat1, lon1, lat2, lon2):
|
||||
p1, p2 = math.radians(lat1), math.radians(lat2)
|
||||
a = (math.sin((p2 - p1) / 2) ** 2
|
||||
+ math.cos(p1) * math.cos(p2) * math.sin(math.radians(lon2 - lon1) / 2) ** 2)
|
||||
return 2 * 6371 * math.asin(math.sqrt(a))
|
||||
|
||||
|
||||
def nearest_station(lat, lon):
|
||||
text = fetch(f"{BASE}/ogd-smn_meta_stations.csv").decode("latin-1")
|
||||
stations = list(csv.DictReader(io.StringIO(text), delimiter=";"))
|
||||
station = min(stations, key=lambda s: distance_km(
|
||||
lat, lon, float(s["station_coordinates_wgs84_lat"]), float(s["station_coordinates_wgs84_lon"])))
|
||||
station["distance_km"] = distance_km(
|
||||
lat, lon, float(station["station_coordinates_wgs84_lat"]), float(station["station_coordinates_wgs84_lon"]))
|
||||
return station
|
||||
|
||||
|
||||
def files_for(station, start, end):
|
||||
"""The hourly files of the station that cover [start, end): historical by decade, recent, now."""
|
||||
item = json.loads(fetch(STAC_ITEM.format(station.lower())))
|
||||
names = [n for n in item["assets"] if f"_{station.lower()}_h_" in n]
|
||||
chosen = []
|
||||
for name in names:
|
||||
stem = name.rsplit(".", 1)[0]
|
||||
if "_historical_" in stem:
|
||||
first, last = (int(y) for y in stem.rsplit("_", 1)[1].split("-"))
|
||||
if start.year <= last and end.year >= first:
|
||||
chosen.append(name)
|
||||
elif stem.endswith("_recent"): # from January 1 of the current year to yesterday
|
||||
if end.year >= datetime.now(timezone.utc).year - 1:
|
||||
chosen.append(name)
|
||||
elif stem.endswith("_now"): # since midnight, not yet quality-checked
|
||||
if end > datetime.now(timezone.utc) - timedelta(days=1):
|
||||
chosen.append(name)
|
||||
return [item["assets"][n]["href"] for n in sorted(chosen)]
|
||||
|
||||
|
||||
def read_rows(urls, start, end):
|
||||
"""{time: row} over [start, end); a later file (recent, now) completes an earlier one."""
|
||||
rows = {}
|
||||
for url in urls:
|
||||
text = fetch(url).decode("latin-1")
|
||||
for row in csv.DictReader(io.StringIO(text), delimiter=";"):
|
||||
time = datetime.strptime(row["reference_timestamp"], "%d.%m.%Y %H:%M").replace(tzinfo=timezone.utc)
|
||||
if start <= time < end:
|
||||
rows.setdefault(time, row)
|
||||
return rows
|
||||
|
||||
|
||||
def number(text):
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def hourly(place, start, end):
|
||||
"""-> (where, {time: {column: float or None}}) over [start, end) (UTC datetimes), the times sorted; where: the place
|
||||
found (place, lat, lon) and the station chosen (station, name, distance_km, height_m). Raises LookupError (place not
|
||||
found) or OSError (MeteoSwiss or swisstopo unreachable)."""
|
||||
lat, lon, label = locate(place)
|
||||
station = nearest_station(lat, lon)
|
||||
rows = read_rows(files_for(station["station_abbr"], start, end), start, end)
|
||||
where = {"place": label, "lat": lat, "lon": lon, "station": station["station_abbr"], "name": station["station_name"],
|
||||
"distance_km": station["distance_km"], "height_m": number(station["station_height_masl"])}
|
||||
return where, {t: {c: number(rows[t].get(p, "")) for p, c in PARAMETERS.items()} for t in sorted(rows)}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("place", help='a Swiss place name, or "lat,lon" (WGS84)')
|
||||
parser.add_argument("start", type=parse_time, help="YYYY-MM-DD[THH:MM], UTC, included")
|
||||
parser.add_argument("end", type=parse_time, help="YYYY-MM-DD[THH:MM], UTC, excluded")
|
||||
parser.add_argument("--out", help="the CSV file (stdout by default)")
|
||||
args = parser.parse_args()
|
||||
if args.end <= args.start:
|
||||
sys.exit("the end must be after the start")
|
||||
if args.start >= datetime.now(timezone.utc):
|
||||
sys.exit("the range must be in the past")
|
||||
|
||||
try:
|
||||
where, hours = hourly(args.place, args.start, args.end)
|
||||
except LookupError as e:
|
||||
sys.exit(str(e))
|
||||
print(f"{where['place']} ({where['lat']:.4f}, {where['lon']:.4f}) -> station {where['station']} {where['name']}, "
|
||||
f"{where['distance_km']:.1f} km, {where['height_m']:.0f} m", file=sys.stderr)
|
||||
|
||||
out = open(args.out, "w", newline="", encoding="utf-8") if args.out else sys.stdout
|
||||
writer = csv.writer(out)
|
||||
writer.writerow(["time_utc", "station", *PARAMETERS.values()])
|
||||
for time, values in hours.items():
|
||||
writer.writerow([time.strftime("%Y-%m-%dT%H:%MZ"), where["station"], *("" if v is None else f"{v:g}" for v in values.values())])
|
||||
if args.out:
|
||||
out.close()
|
||||
print(f"{len(hours)} hours", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user