Conversations : le pompage en lecture seule (incrémental, pièces jointes, garde-fou GET)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
116488ddd4
commit
2a03b57b49
3 files changed
+344
No files matched your search
@@ -0,0 +1,154 @@
|
||||
"""Pompe les conversations de LibreDesk, en lecture seule, dans un dossier par conversation (labeling/auto/conversations/
|
||||
d'aimsight-labeling, par l'onglet Conversations d'aimsight-app) : conversation.json, messages.json, attachments/, et
|
||||
transcript.md, le fil lisible.
|
||||
|
||||
Que des GET : la liste des conversations, leurs messages, et les URL (signées) des pièces jointes que portent les messages ;
|
||||
jamais une adresse trouvée dans le texte d'un message. La clé ne va qu'à LibreDesk ; une pièce jointe ailleurs (un bucket S3)
|
||||
est téléchargée sans elle. Il faut une clé de lecture (un agent dont le rôle n'a que conversations:read_all et messages:read).
|
||||
|
||||
Sans dates : ce qui a changé depuis le dernier pompage complet (pumped.json, écrit seulement à la fin ; la première fois,
|
||||
tout l'historique). Avec --depuis/--jusqu-a : les conversations mises à jour dans la plage (--jusqu-a exclu), repère inchangé.
|
||||
|
||||
LIBREDESK_URL=… LIBREDESK_LECTURE_KEY=… LIBREDESK_LECTURE_SECRET=… python3 -m conversations.pomper --sortie <dossier>
|
||||
[--depuis AAAA-MM-JJ --jusqu-a AAAA-MM-JJ]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from conversations.transcript import transcript
|
||||
|
||||
PAGE = 100
|
||||
MARGE = timedelta(hours=1) # le repère moins une heure : une conversation mise à jour pendant le pompage n'est pas perdue
|
||||
REFERENCE = re.compile(r"[A-Za-z0-9_-]+")
|
||||
|
||||
|
||||
class Lecteur:
|
||||
"""Le client de LibreDesk, restreint à la lecture des conversations."""
|
||||
|
||||
def __init__(self, url: str, cle: str, secret: str):
|
||||
self.url = url.rstrip("/")
|
||||
self._auth = f"token {cle}:{secret}"
|
||||
|
||||
def _ouvrir(self, url: str) -> bytes:
|
||||
url = urllib.parse.urljoin(self.url + "/", url)
|
||||
r = urllib.request.Request(url)
|
||||
if urllib.parse.urlsplit(url)[:2] == urllib.parse.urlsplit(self.url)[:2]: # la clé ne va qu'à LibreDesk
|
||||
r.add_unredirected_header("Authorization", self._auth)
|
||||
with urllib.request.urlopen(r, timeout=60) as rep:
|
||||
return rep.read()
|
||||
|
||||
def _pages(self, chemin: str, **params):
|
||||
page = 1
|
||||
while True:
|
||||
q = urllib.parse.urlencode({"page": page, "page_size": PAGE, **params})
|
||||
data = json.loads(self._ouvrir(f"{chemin}?{q}"))["data"]
|
||||
yield from data.get("results") or []
|
||||
if page >= (data.get("total_pages") or 1):
|
||||
return
|
||||
page += 1
|
||||
|
||||
def conversations(self):
|
||||
"""Toutes les conversations, la plus récemment mise à jour d'abord."""
|
||||
return self._pages("/api/v1/conversations/all", order_by="updated_at", order="desc")
|
||||
|
||||
def messages(self, uuid: str) -> list[dict]:
|
||||
"""Les messages d'une conversation (notes privées comprises), le plus ancien d'abord."""
|
||||
uuid = urllib.parse.quote(uuid, safe="")
|
||||
return sorted(self._pages(f"/api/v1/conversations/{uuid}/messages"), key=lambda m: _date(m["created_at"]))
|
||||
|
||||
def telecharger(self, url: str, chemin: Path) -> None:
|
||||
chemin.parent.mkdir(parents=True, exist_ok=True)
|
||||
partiel = chemin.with_name(chemin.name + ".part")
|
||||
partiel.write_bytes(self._ouvrir(url))
|
||||
partiel.replace(chemin)
|
||||
|
||||
|
||||
def _date(t: str) -> datetime:
|
||||
d = datetime.fromisoformat(t)
|
||||
return (d if d.tzinfo else d.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _nom_local(nom: str, uuid: str) -> str:
|
||||
"""Un nom de fichier sûr, dans attachments/ seulement ; vide : l'uuid de la pièce jointe."""
|
||||
sur = lambda s: re.sub(r"[^\w.\- ]", "_", Path(s or "").name).strip(" .") # noqa: E731
|
||||
return sur(nom) or sur(uuid) or "piece"
|
||||
|
||||
|
||||
def _ecrire(lecteur: Lecteur, sortie: Path, c: dict) -> tuple[str, int, int]:
|
||||
ref = str(c.get("reference_number") or "")
|
||||
dossier = sortie / (ref if REFERENCE.fullmatch(ref) else _nom_local(c["uuid"], "conversation"))
|
||||
messages, pris, n_pieces = lecteur.messages(c["uuid"]), set(), 0
|
||||
for m in messages:
|
||||
for a in m.get("attachments") or []:
|
||||
nom = _nom_local(a.get("name"), a.get("uuid"))
|
||||
if nom in pris:
|
||||
nom = _nom_local(f"{a.get('uuid')}_{nom}", "")
|
||||
pris.add(nom)
|
||||
f, url = dossier / "attachments" / nom, a.get("url") or ""
|
||||
if url and not (f.exists() and f.stat().st_size == a.get("size")):
|
||||
lecteur.telecharger(url, f)
|
||||
if url: # l'URL signée expire : le chemin local, dans la pièce et dans le contenu (images en ligne)
|
||||
m["content"] = (m.get("content") or "").replace(url.replace("&", "&"), url).replace(url, f"attachments/{nom}")
|
||||
a["url"] = f"attachments/{nom}"
|
||||
n_pieces += 1
|
||||
c = {k: v for k, v in c.items() if k != "previous_conversations"}
|
||||
dossier.mkdir(parents=True, exist_ok=True)
|
||||
for nom, contenu in (("conversation.json", c), ("messages.json", messages)):
|
||||
(dossier / nom).write_text(json.dumps(contenu, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||||
(dossier / "transcript.md").write_text(transcript(c, messages), encoding="utf-8")
|
||||
return dossier.name, sum(m.get("type") != "activity" for m in messages), n_pieces
|
||||
|
||||
|
||||
def pomper(lecteur: Lecteur, sortie: Path, depuis: str | None = None, jusqu_a: str | None = None, journal=print) -> int:
|
||||
debut, repere, haut = datetime.now(timezone.utc), sortie / "pumped.json", None
|
||||
if depuis:
|
||||
bas, haut = _date(depuis), _date(jusqu_a)
|
||||
else:
|
||||
bas = _date(json.loads(repere.read_text())["until"]) - MARGE if repere.exists() else None
|
||||
n = 0
|
||||
for c in lecteur.conversations():
|
||||
t = _date(c["updated_at"])
|
||||
if haut and t >= haut:
|
||||
continue
|
||||
if bas and t < bas:
|
||||
break
|
||||
nom, n_messages, n_pieces = _ecrire(lecteur, sortie, c)
|
||||
journal(f"#{nom} {c.get('subject') or ''} : {n_messages} messages, {n_pieces} pièces jointes")
|
||||
n += 1
|
||||
if not depuis: # seulement un pompage complet : un pompage interrompu est repris au suivant
|
||||
sortie.mkdir(parents=True, exist_ok=True)
|
||||
repere.write_text(json.dumps({"until": debut.isoformat()}) + "\n")
|
||||
journal(f"{n} conversation(s) pompée(s) dans {sortie}")
|
||||
return n
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
p.add_argument("--sortie", required=True, type=Path)
|
||||
p.add_argument("--depuis")
|
||||
p.add_argument("--jusqu-a")
|
||||
a = p.parse_args()
|
||||
if bool(a.depuis) != bool(a.jusqu_a):
|
||||
sys.exit("--depuis et --jusqu-a vont ensemble")
|
||||
cle, secret = os.environ.get("LIBREDESK_LECTURE_KEY"), os.environ.get("LIBREDESK_LECTURE_SECRET")
|
||||
if not (cle and secret):
|
||||
sys.exit("Pas de clé de lecture : LIBREDESK_LECTURE_KEY et LIBREDESK_LECTURE_SECRET dans .env (README)")
|
||||
lecteur = Lecteur(os.environ.get("LIBREDESK_URL", "https://helpdesk.sensima.ch"), cle, secret)
|
||||
try:
|
||||
pomper(lecteur, a.sortie, a.depuis, a.jusqu_a, journal=lambda l: print(l, flush=True))
|
||||
except urllib.error.HTTPError as e:
|
||||
sys.exit(f"LibreDesk : {e.code} sur {e.url}".split("?")[0]) # sans la signature d'une URL de pièce jointe
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
sys.exit(f"LibreDesk injoignable : {getattr(e, 'reason', e)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Le paquet conversations ne fait que lire LibreDesk : des GET, sur les deux routes des conversations (design aimsight-app
|
||||
2026-11-18-conversations-tab), plus les URL des pièces jointes que les messages portent."""
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
PAQUET = Path(__file__).resolve().parents[1] / "conversations"
|
||||
AUTORISEES = {"/api/v1/conversations/all", "/api/v1/conversations/{uuid}/messages"}
|
||||
|
||||
|
||||
class TestGardeFouConversations(unittest.TestCase):
|
||||
def test_seulement_des_lectures(self):
|
||||
for f in PAQUET.glob("*.py"):
|
||||
texte = f.read_text()
|
||||
self.assertEqual(set(re.findall(r"/api/v1/[a-z0-9_/{}-]+", texte)) - AUTORISEES, set(), f.name)
|
||||
for interdit in ('"POST"', '"PUT"', '"DELETE"', '"PATCH"', "method=", "data="):
|
||||
self.assertNotIn(interdit, texte, f"{interdit} dans {f.name}")
|
||||
|
||||
def test_le_paquet_a_bien_un_client(self): # sans pomper.py, le test ci-dessus passerait à vide
|
||||
self.assertTrue((PAQUET / "pomper.py").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,166 @@
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
from conversations.pomper import Lecteur, pomper
|
||||
from faux_libredesk import FauxLibreDesk
|
||||
|
||||
LISTE = ("GET", "/api/v1/conversations/all")
|
||||
|
||||
|
||||
def conv(ref, uuid, updated_at):
|
||||
return {"uuid": uuid, "reference_number": ref, "subject": f"Sujet {ref}", "status": "Open", "inbox_name": "Support",
|
||||
"created_at": "2026-11-01T08:00:00Z", "updated_at": updated_at, "previous_conversations": [{"uuid": "x"}],
|
||||
"contact": {"first_name": "Jean", "last_name": "Dupont", "email": "j@x.ch"}}
|
||||
|
||||
|
||||
def piece(nom, url, uuid="a1", taille=3, type_="image/png"):
|
||||
return {"name": nom, "url": url, "uuid": uuid, "size": taille, "content_type": type_}
|
||||
|
||||
|
||||
def message(heure, pieces=(), contenu="<p>Bonjour</p>"):
|
||||
return {"uuid": "m" + heure, "created_at": f"2026-11-18T{heure}:00Z", "type": "incoming", "sender_type": "contact",
|
||||
"private": False, "content": contenu, "text_content": "", "attachments": list(pieces), "author": None}
|
||||
|
||||
|
||||
def pages(elements, taille=2):
|
||||
"""Une route paginée comme LibreDesk : ?page=N, data = {results, total, per_page, total_pages, page}."""
|
||||
def reponse(chemin):
|
||||
page = int(urllib.parse.parse_qs(urllib.parse.urlsplit(chemin).query).get("page", ["1"])[0])
|
||||
total_pages = max(1, -(-len(elements) // taille))
|
||||
return {"results": elements[(page - 1) * taille:page * taille], "total": len(elements), "per_page": taille,
|
||||
"total_pages": total_pages, "page": page}
|
||||
return reponse
|
||||
|
||||
|
||||
class TestPomper(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.faux, self.s3 = FauxLibreDesk(), FauxLibreDesk()
|
||||
self.addCleanup(self.faux.arreter)
|
||||
self.addCleanup(self.s3.arreter)
|
||||
self.sortie = Path(tempfile.mkdtemp())
|
||||
self.addCleanup(shutil.rmtree, self.sortie)
|
||||
self.lignes = []
|
||||
|
||||
def lecteur(self):
|
||||
return Lecteur(self.faux.url, "k", "s")
|
||||
|
||||
def pomper(self, **kw):
|
||||
return pomper(self.lecteur(), self.sortie, journal=self.lignes.append, **kw)
|
||||
|
||||
def servir(self, convs, messages=None):
|
||||
self.faux.reponses[LISTE] = pages(convs)
|
||||
for c in convs:
|
||||
self.faux.reponses[("GET", f"/api/v1/conversations/{c['uuid']}/messages")] = pages(
|
||||
(messages or {}).get(c["uuid"], [message("08:00")]))
|
||||
|
||||
def demandes(self, prefixe):
|
||||
return [c for m, c, _ in self.faux.requetes if urllib.parse.urlsplit(c).path.startswith(prefixe)]
|
||||
|
||||
def repere(self):
|
||||
return json.loads((self.sortie / "pumped.json").read_text())["until"]
|
||||
|
||||
def test_tout_au_premier_pompage(self):
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z"), conv("2", "u2", "2026-11-17T12:00:00Z"),
|
||||
conv("1", "u1", "2026-11-16T12:00:00Z")])
|
||||
self.assertEqual(self.pomper(), 3)
|
||||
self.assertEqual(sorted(p.name for p in self.sortie.iterdir() if p.is_dir()), ["1", "2", "3"])
|
||||
c = json.loads((self.sortie / "3" / "conversation.json").read_text())
|
||||
self.assertEqual(c["subject"], "Sujet 3")
|
||||
self.assertNotIn("previous_conversations", c)
|
||||
self.assertIn("Bonjour", (self.sortie / "3" / "transcript.md").read_text())
|
||||
self.assertTrue(self.repere())
|
||||
self.assertIn("order_by=updated_at", self.demandes("/api/v1/conversations/all")[0])
|
||||
|
||||
def test_incremental_s_arrete_sous_le_repere(self):
|
||||
(self.sortie / "pumped.json").write_text(json.dumps({"until": "2026-11-18T10:00:00+00:00"}))
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z"), conv("2", "u2", "2026-11-18T09:30:00Z"),
|
||||
conv("1", "u1", "2026-11-18T08:00:00Z")])
|
||||
self.assertEqual(self.pomper(), 2)
|
||||
self.assertFalse((self.sortie / "1").exists())
|
||||
self.assertEqual(self.demandes("/api/v1/conversations/u1"), [])
|
||||
self.assertNotEqual(self.repere(), "2026-11-18T10:00:00+00:00")
|
||||
|
||||
def test_interrompu_le_repere_ne_bouge_pas(self):
|
||||
(self.sortie / "pumped.json").write_text(json.dumps({"until": "2026-11-01T00:00:00+00:00"}))
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z"), conv("2", "u2", "2026-11-17T12:00:00Z")])
|
||||
self.faux.reponses[("GET", "/api/v1/conversations/u2/messages")] = 500
|
||||
with self.assertRaises(Exception) as erreur:
|
||||
self.pomper()
|
||||
erreur.exception.close()
|
||||
self.assertTrue((self.sortie / "3" / "transcript.md").exists())
|
||||
self.assertEqual(self.repere(), "2026-11-01T00:00:00+00:00")
|
||||
|
||||
def test_plage_ne_touche_pas_le_repere(self):
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z"), conv("2", "u2", "2026-11-17T12:00:00Z"),
|
||||
conv("1", "u1", "2026-11-16T12:00:00Z")])
|
||||
self.assertEqual(self.pomper(depuis="2026-11-17", jusqu_a="2026-11-18"), 1)
|
||||
self.assertTrue((self.sortie / "2").exists())
|
||||
self.assertFalse((self.sortie / "3").exists() or (self.sortie / "1").exists())
|
||||
self.assertFalse((self.sortie / "pumped.json").exists())
|
||||
|
||||
def test_pieces_jointes_locales(self):
|
||||
url = f"{self.faux.url}/uploads/f1?signature=abc&expires=1"
|
||||
self.faux.reponses[("GET", "/uploads/f1")] = b"PNG"
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z")],
|
||||
{"u3": [message("08:00", [piece("photo.png", url)], contenu=f'<img src="{url.replace("&", "&")}">')]})
|
||||
self.pomper()
|
||||
m = json.loads((self.sortie / "3" / "messages.json").read_text())[0]
|
||||
self.assertEqual(m["attachments"][0]["url"], "attachments/photo.png")
|
||||
self.assertEqual(m["content"], '<img src="attachments/photo.png">')
|
||||
self.assertEqual((self.sortie / "3" / "attachments" / "photo.png").read_bytes(), b"PNG")
|
||||
|
||||
def test_url_relative(self):
|
||||
self.faux.reponses[("GET", "/uploads/f1")] = b"PNG"
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z")], {"u3": [message("08:00", [piece("p.png", "/uploads/f1?s=1")])]})
|
||||
self.pomper()
|
||||
self.assertEqual((self.sortie / "3" / "attachments" / "p.png").read_bytes(), b"PNG")
|
||||
|
||||
def test_nom_de_piece_jointe_assaini(self):
|
||||
self.faux.reponses[("GET", "/uploads/f1")] = b"abc"
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z")],
|
||||
{"u3": [message("08:00", [piece("../../x.sh", "/uploads/f1"), piece("..", "/uploads/f1", uuid="a2")])]})
|
||||
self.pomper()
|
||||
self.assertEqual(sorted(p.name for p in (self.sortie / "3" / "attachments").iterdir()), ["a2", "x.sh"])
|
||||
|
||||
def test_noms_en_double(self):
|
||||
self.faux.reponses[("GET", "/uploads/f1")] = b"abc"
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z")],
|
||||
{"u3": [message("08:00", [piece("a.png", "/uploads/f1")]),
|
||||
message("09:00", [piece("a.png", "/uploads/f1", uuid="a2")])]})
|
||||
self.pomper()
|
||||
self.assertEqual(sorted(p.name for p in (self.sortie / "3" / "attachments").iterdir()), ["a.png", "a2_a.png"])
|
||||
|
||||
def test_deja_telechargee_pas_de_requete(self):
|
||||
self.faux.reponses[("GET", "/uploads/f1")] = b"abc"
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z")], {"u3": [message("08:00", [piece("a.png", "/uploads/f1")])]})
|
||||
self.pomper(depuis="2026-11-18", jusqu_a="2026-11-19")
|
||||
self.pomper(depuis="2026-11-18", jusqu_a="2026-11-19")
|
||||
self.assertEqual(len(self.demandes("/uploads/")), 1)
|
||||
|
||||
def test_reference_bizarre_dossier_par_uuid(self):
|
||||
self.servir([conv("../42", "u42", "2026-11-18T12:00:00Z")])
|
||||
self.pomper()
|
||||
self.assertTrue((self.sortie / "u42" / "conversation.json").exists())
|
||||
|
||||
def test_cle_seulement_vers_libredesk(self):
|
||||
self.s3.reponses[("GET", "/bucket/f1")] = b"abc"
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z")],
|
||||
{"u3": [message("08:00", [piece("a.png", f"{self.s3.url}/bucket/f1?X-Amz-Signature=z")])]})
|
||||
self.pomper()
|
||||
self.assertEqual((self.sortie / "3" / "attachments" / "a.png").read_bytes(), b"abc")
|
||||
self.assertEqual(len(self.s3.entetes), 1)
|
||||
self.assertNotIn("Authorization", self.s3.entetes[0])
|
||||
self.assertEqual(self.faux.entetes[0]["Authorization"], "token k:s")
|
||||
|
||||
def test_aucune_ecriture(self):
|
||||
self.servir([conv("3", "u3", "2026-11-18T12:00:00Z")])
|
||||
self.pomper()
|
||||
self.assertEqual(self.faux.ecritures(), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user