Coverage for lode / api.py: 71%
241 statements
« prev ^ index » next coverage.py v7.13.0, created at 2026-10-05 16:02 +0000
1# lode/api.py
2"""LODE 2.0 — HTTP API.
4Endpoints:
5 GET / web form
6 GET /health liveness probe
7 GET /extract render an artefact (HTML, or RDF via content negotiation)
8 POST /extract same, from an uploaded file
9 GET /build static documentation site, as a ZIP archive
10 POST /build same, from an uploaded file
11"""
13# ==========================================================================
14# 1. IMPORTS
15# ==========================================================================
16from enum import Enum
17from functools import lru_cache
18from pathlib import Path
19from typing import Optional
20from urllib.parse import urlencode
21from uuid import uuid4
22import hashlib
23import logging
24import os
25import re
26import shutil
27import tempfile
28import threading
29import time
30import traceback
31from pathlib import Path
33from fastapi import FastAPI, File, Form, HTTPException, Query, Request, UploadFile
34from fastapi.responses import FileResponse, HTMLResponse, Response
35from fastapi.staticfiles import StaticFiles
36from fastapi.templating import Jinja2Templates
37from starlette.background import BackgroundTask
38from starlette.concurrency import run_in_threadpool
39from starlette.middleware.gzip import GZipMiddleware
40from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
42# Internal modules
43from lode.helpers.formats import (
44 ENABLED_FORMATS, ReadAsFormat,
45 check_format_enabled as _check_format_enabled,
46)
47from lode.helpers.html_utils import minify as _minify
48import lode.helpers.spool as spool
49from lode.reader import Reader
50from lode.reader import security
51from lode.exceptions import LODEError, ArtefactValidationError
52from lode.viewer.base_viewer import formats_for
53from lode.reader.provenance_graph import ProvenanceGraph
54from lode.helpers.rdf import iri_local_name
57# ==========================================================================
58# 2. APP SETUP
59# ==========================================================================
60logging.basicConfig(
61 level=logging.DEBUG,
62 format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
63)
64logger = logging.getLogger(__name__)
66# When enabled, error pages include the full traceback (development only).
67DEBUG = os.getenv("LODE_DEBUG", "").strip().lower() in ("1", "true", "yes", "on")
69TAGS_METADATA = [
70 {"name": "extract", "description": "Render a semantic artefact as HTML or serialized RDF."},
71 {"name": "build", "description": "Generate a full static documentation site, delivered as a ZIP."},
72 {"name": "ops", "description": "Operational endpoints: web form and health probe."},
73]
75app = FastAPI(
76 title="LODE 2.0 API",
77 version="0.3.12",
78 description="Live OWL Documentation Environment — extract and document "
79 "RDF/OWL/SKOS semantic artefacts as browsable HTML.",
80 license_info={"name": "ISC"},
81 contact={"name": "OpenCitations", "url": "https://github.com/opencitations/lode"},
82 openapi_tags=TAGS_METADATA,
83)
85# Compress HTML responses on the wire.
86app.add_middleware(GZipMiddleware, minimum_size=1000)
87# Fix "blocked loading mixed active content" on style.css behind a proxy.
88app.add_middleware(ProxyHeadersMiddleware, trusted_hosts="*")
90_PKG_DIR = Path(__file__).resolve().parent
91templates = Jinja2Templates(directory=str(_PKG_DIR / "templates"))
92app.mount("/static", StaticFiles(directory=str(_PKG_DIR / "static")), name="static")
94# Beautify produced HTML.
95templates.env.trim_blocks = True
96templates.env.lstrip_blocks = True
98# ==========================================================================
99# 5. BUILD WORKSPACE — quota and concurrency for /build
100# ==========================================================================
101# Builds write hundreds of files each, so they get their own directory with an
102# explicit budget instead of filling up /tmp.
103BUILD_DIR = os.path.realpath(
104 os.getenv("LODE_BUILD_DIR", os.path.join(tempfile.gettempdir(), "lode-builds"))
105)
106os.makedirs(BUILD_DIR, exist_ok=True)
107_BUILD_TTL = 30 * 60 # orphans (aborted downloads) age out
108_BUILD_MAX_BYTES = 2 * 1024 ** 3 # 2 GB for all pending builds
109_BUILD_SLOTS = threading.BoundedSemaphore(2) # concurrent builds, per worker process
110_BUILD_WAIT = 5 # seconds a request waits for a slot
113def _prune_builds() -> int:
114 """Drop orphaned build directories and return the bytes still in use.
116 A build is orphaned when its BackgroundTask never fired, i.e. the client
117 opened the connection and never finished reading the archive.
118 """
119 cutoff = time.time() - _BUILD_TTL
120 total = 0
121 for name in os.listdir(BUILD_DIR):
122 p = os.path.join(BUILD_DIR, name)
123 try:
124 st = os.stat(p)
125 except OSError:
126 continue
127 if st.st_mtime < cutoff:
128 shutil.rmtree(p, ignore_errors=True)
129 continue
130 for root, _dirs, files in os.walk(p):
131 for f in files:
132 try:
133 total += os.path.getsize(os.path.join(root, f))
134 except OSError:
135 pass
136 return total
139# ==========================================================================
140# 6. HTML HELPERS
141# ==========================================================================
143def _nav_qs(read_as: str, url, upload_id, lang, imported, closure, partial_import) -> str:
144 p = {"read_as": read_as, "lang": lang or ""}
145 p["upload_id" if upload_id else "url"] = upload_id or (url or "")
146 if imported:
147 p["imported"] = "true"
148 if closure:
149 p["closure"] = "true"
150 if partial_import:
151 p["partial_import"] = "true"
153 return urlencode(p)
155def _render_view(request, reader, *, resource, lang, source_url, upload_id, read_as,
156 imported=None, closure=None, partial_import=None):
157 viewer = reader.get_viewer()
158 data = viewer.get_view_data(resource_uri=resource, language=lang)
159 data["warnings"] = reader.get_warnings()
160 resp = templates.TemplateResponse(request, "viewer.html", {
161 "source_url": source_url,
162 "upload_id": upload_id,
163 "nav_qs": _nav_qs(read_as, source_url, upload_id, lang, imported, closure, partial_import),
164 **data,
165 })
166 resp.body = _minify(resp.body.decode("utf-8")).encode("utf-8")
167 resp.headers["content-length"] = str(len(resp.body))
168 return resp
171# ==========================================================================
172# 7. READER RESOLUTION
173# ==========================================================================
174def _url_token(url, read_as, imported, closure, partial_import) -> str:
175 key = f"{url}|{read_as}|{imported}|{closure}|{partial_import}".encode()
176 return "url_" + hashlib.sha256(key).hexdigest()[:32]
179def _load_url(url, read_as, imported, closure, partial_import, warnings, use_cache=True):
180 # Enforce http(s)://host up front: a non-URL value (local path, file://, ...)
181 # must never reach the loader and be opened as a local file.
182 security.check_url_safe(url)
183 spool.prune()
184 token = _url_token(url, read_as, imported, closure, partial_import)
185 path = spool.get_path(token)
186 if use_cache and os.path.exists(path):
187 # cache hit: ricostruisci dal Turtle salvato
188 reader = Reader()
189 reader.load_instances(path, read_as, imported=imported, closure=closure, partial_import=partial_import,
190 warnings=warnings)
191 return reader
192 if not use_cache:
193 # cache=false: drop the stale copy so the fresh fetch replaces it
194 try:
195 os.unlink(path)
196 except OSError:
197 pass
198 # cache miss (or forced refresh): scarica e processa dalla URL
199 reader = Reader()
200 reader.load_instances(url, read_as, imported=imported, closure=closure, partial_import=partial_import,
201 warnings=warnings)
202 # persisti SOLO il documento principale: i moduli vengono riapplicati a ogni hit,
203 try:
204 g = reader._graph
205 main = g.graph(reader._ontology_iri) if isinstance(g, ProvenanceGraph) else g
206 with open(path, "wb") as f:
207 f.write(main.serialize(format="turtle").encode("utf-8"))
208 except OSError:
209 pass
210 return reader
212def _resolve_reader(read_as: str, url, upload_id, imported, closure, partial_import, warnings,
213 use_cache=True):
214 if upload_id:
215 # Uploads are not re-fetched, so the cache flag does not apply to them.
216 path = spool.get_path(upload_id)
217 if not os.path.exists(path):
218 raise ArtefactValidationError("Upload expired, please re-upload",
219 context={"upload_id": upload_id})
220 reader = Reader()
221 reader.load_instances(path, read_as, imported=imported, closure=closure, partial_import=partial_import,
222 warnings=warnings)
223 return reader
224 if url:
225 return _load_url(url, read_as, imported, closure, partial_import, warnings, use_cache=use_cache)
226 raise ArtefactValidationError("Missing 'url' or 'upload_id'")
229async def _read_validated_upload(file: UploadFile) -> bytes:
230 """Run every upload check in one place, then return the raw bytes."""
231 security.check_extension(file.filename)
232 content = await security.read_upload_capped(file)
233 security.check_is_text(content)
234 security.check_safe_xml(content.decode("utf-8-sig"))
235 return content
238# ==========================================================================
239# 8. ZIP BUILD HELPERS
240# ==========================================================================
241def _slug(name: Optional[str]) -> str:
242 """Safe directory/file stem derived from a URL or filename.
244 Must never yield '.' or '..': those would let make_archive(base_dir=...)
245 escape the build directory and archive its parent.
246 """
247 if not name:
248 return "ontology"
249 raw = name.rstrip("/").split("#")[0].split("/")[-1]
250 raw = re.sub(r"\.(ttl|rdf|owl|n3|nt|jsonld|xml)$", "", raw, flags=re.I)
251 raw = re.sub(r"[^A-Za-z0-9._-]", "-", raw).strip("-.")
252 if not raw or raw in {".", ".."}:
253 return "ontology"
254 return raw[:64]
257def _zip_response(reader, slug: str, lang: Optional[str] = None) -> FileResponse:
258 """Render the whole static site and return it as a ZIP download.
260 Blocking and disk-heavy: callers must keep it off the event loop.
261 """
262 from lode.builder import build_html # lazy: evita l'import circolare
264 if not _BUILD_SLOTS.acquire(timeout=_BUILD_WAIT):
265 raise HTTPException(503, "Server busy: too many concurrent builds",
266 headers={"Retry-After": "30"})
267 workdir = None
268 try:
269 if _prune_builds() > _BUILD_MAX_BYTES:
270 raise HTTPException(503, "Build storage full, retry shortly",
271 headers={"Retry-After": "60"})
273 workdir = tempfile.mkdtemp(prefix="build_", dir=BUILD_DIR)
274 site_dir = Path(workdir) / slug
275 site_dir.mkdir(parents=True, exist_ok=True)
277 build_html(reader.get_viewer(), site_dir, lang=lang or "en", reader=reader)
279 zip_path = shutil.make_archive(
280 os.path.join(workdir, f"{slug}-lode"), "zip",
281 root_dir=workdir, base_dir=slug,
282 )
283 # The loose pages are redundant once archived: halve the disk peak.
284 shutil.rmtree(site_dir, ignore_errors=True)
285 except BaseException:
286 if workdir:
287 shutil.rmtree(workdir, ignore_errors=True)
288 raise
289 finally:
290 # Free the slot as soon as the build ends, not when the client is done
291 # downloading: a slow reader must not hold a build slot hostage.
292 _BUILD_SLOTS.release()
294 return FileResponse(
295 zip_path,
296 media_type="application/zip",
297 filename=f"{slug}-lode.zip",
298 background=BackgroundTask(shutil.rmtree, workdir, True),
299 )
302# ==========================================================================
303# 9. ERROR RENDERING
304# ==========================================================================
305def _error_payload(exc: Exception) -> dict:
306 is_lode = isinstance(exc, LODEError)
307 return {
308 "type": type(exc).__name__,
309 "message": str(exc),
310 "context": exc.context if is_lode else {},
311 "request_id": exc.request_id if is_lode else None,
312 # Never leak the traceback to the user in production: it is always logged
313 # server-side; it is only surfaced in the page when LODE_DEBUG is set.
314 "traceback": traceback.format_exc() if (not is_lode and DEBUG) else None,
315 }
318@app.exception_handler(LODEError)
319async def lode_error_handler(request: Request, exc: LODEError):
320 logger.error(f"{type(exc).__name__}: {exc}")
321 return templates.TemplateResponse(
322 "error.html", {"request": request, "error": _error_payload(exc)}, status_code=400
323 )
326@app.exception_handler(Exception)
327async def unexpected_error_handler(request: Request, exc: Exception):
328 logger.exception("Unexpected error")
329 return templates.TemplateResponse(
330 "error.html", {"request": request, "error": _error_payload(exc)}, status_code=500
331 )
334# ==========================================================================
335# 10. MIDDLEWARE
336# ==========================================================================
337@app.middleware("http")
338async def limit_upload_size(request: Request, call_next):
339 """Reject oversized POST bodies early, before reading them, when the client
340 declares a Content-Length. (Chunked bodies omit it: those are still capped
341 in read_upload_capped; the hard limit belongs at the reverse proxy.)"""
342 if request.method == "POST":
343 declared = request.headers.get("content-length")
344 if declared and declared.isdigit() and int(declared) > security.MAX_BYTES:
345 exc = ArtefactValidationError(
346 "File too large",
347 context={"bytes": int(declared), "max": security.MAX_BYTES},
348 )
349 return templates.TemplateResponse(
350 "error.html", {"request": request, "error": _error_payload(exc)},
351 status_code=400,
352 )
353 return await call_next(request)
356# ==========================================================================
357# 11. ROUTES — pages
358# ==========================================================================
359@app.get("/", response_class=HTMLResponse, tags=["ops"], summary="Web input form")
360async def input_web_interface(request: Request):
361 """Interfaccia web per l'API"""
362 return templates.TemplateResponse("index.html", {
363 "request": request,
364 "formats": [format.value for format in ReadAsFormat],
365 "enabled_formats": sorted(ENABLED_FORMATS),
366 })
369@app.get("/health", tags=["ops"], summary="Liveness probe")
370async def health_check():
371 """Return 200 with a fixed body. Used by Docker/K8s readiness checks."""
372 return {"status": "ok"}
375# ==========================================================================
376# 12. ROUTES — /extract
377# ==========================================================================
378@app.get("/extract", tags=["extract"], summary="Render an artefact as HTML or RDF")
379def extract_get(
380 request: Request,
381 read_as: ReadAsFormat = Query(description="Read the source as this artefact type"),
382 url: Optional[str] = Query(None, description="Public http(s) URL of the artefact"),
383 upload_id: Optional[str] = Query(None, description="Token of a previously uploaded file"),
384 resource: Optional[str] = Query(None, description="IRI of a single resource; omit for the whole ontology"),
385 lang: Optional[str] = Query(None, description="Language tag for labels, e.g. 'en', 'it'"),
386 imported: Optional[bool] = Query(None, description="Also load owl:imports (depth 1)"),
387 closure: Optional[bool] = Query(None, description="Load the full owl:imports closure"),
388 partial_import: Optional[bool] = Query(None, description="Additiornally load only explicitly mentioned entities from other ontologies"),
389 format: Optional[str] = Query(None, description="Force a serialization (ttl, rdf, jsonld…) instead of HTML"),
390 warnings: bool = Query(False, description="Collect and show reader warnings"),
391 cache: bool = Query(True, description="Use the disk cache for fetched URLs"),
392):
393 """Render an artefact. Returns HTML by default, or RDF when a serialization
394 is requested via the `format` query or the `Accept` header."""
395 _check_format_enabled(read_as)
397 reader = _resolve_reader(read_as.value, url, upload_id, imported, closure, partial_import,
398 warnings, use_cache=cache)
400 # Content negotiation
401 accept = request.headers.get("accept", "text/html")
402 serial = None
404 # Content negotiation: i formati disponibili dipendono dal grafo (quadruple solo con moduli)
405 accept = request.headers.get("accept", "text/html")
406 available = formats_for(reader._graph)
407 serial = None
409 if format:
410 fmt_l = format.lower()
411 serial = next((f for f in available if f["ext"] == fmt_l), None)
412 if serial is None:
413 raise HTTPException(406, f"Format '{format}' not available for this request")
414 else:
415 for part in accept.split(","):
416 mime = part.split(";", 1)[0].strip()
417 serial = next((f for f in available if f["mime"] == mime), None)
418 if serial:
419 break
421 if serial:
422 if resource:
423 serialized = reader.get_viewer().export_resource(resource, serial["fmt"])
424 filename = iri_local_name(resource) or "resource"
425 else:
426 serialized = reader._graph.serialize(format=serial["fmt"])
427 filename = (iri_local_name(url) if url else "") or "graph"
428 return Response(content=serialized, media_type=serial["mime"],
429 headers={"Content-Disposition": f'inline; filename="{filename}.{serial["ext"]}"'})
431 logger.info("=== REQUEST SUCCESS ===")
432 return _render_view(request, reader, resource=resource, lang=lang,
433 source_url=url, upload_id=upload_id, read_as=read_as.value,
434 imported=imported, closure=closure, partial_import=partial_import)
437@app.post("/extract", response_class=HTMLResponse, tags=["extract"],
438 summary="Render an uploaded artefact as HTML")
439async def extract_post(
440 request: Request,
441 read_as: ReadAsFormat = Form(..., description="Read the uploaded file as this artefact type"),
442 file: UploadFile = File(..., description="The artefact file (ttl, rdf, owl…)"),
443 resource: Optional[str] = Form(None, description="IRI of a single resource; omit for the whole ontology"),
444 lang: Optional[str] = Form(None, description="Language tag for labels, e.g. 'en', 'it'"),
445 imported: Optional[bool] = Form(None, description="Also load owl:imports (depth 1)"),
446 partial_import: Optional[bool] = Form(None, description="Additiornally load only explicitly mentioned entities from other ontologies"),
447 closure: Optional[bool] = Form(None, description="Load the full owl:imports closure"),
448 warnings: bool = Form(False, description="Collect and show reader warnings"),
449):
450 """Render an artefact from an uploaded file instead of a URL."""
451 logger.info("=== FILE UPLOAD START ===")
452 logger.info(f"Filename: {file.filename}")
453 logger.info(f"Format: {read_as.value}")
455 _check_format_enabled(read_as)
456 content = await _read_validated_upload(file)
457 token = spool.save(content)
459 reader = Reader()
460 await run_in_threadpool(
461 reader.load_instances, spool.get_path(token), read_as.value,
462 imported=imported, closure=closure, partial_import=partial_import, warnings=warnings,
463 )
464 return _render_view(request, reader, resource=resource, lang=lang,
465 source_url=None, upload_id=token, read_as=read_as.value,
466 imported=imported, closure=closure, partial_import=partial_import)
469# ==========================================================================
470# 13. ROUTES — /build
471# ==========================================================================
472@app.get("/build", tags=["build"], summary="Build a static site (ZIP) from a URL")
473def build_get(
474 read_as: ReadAsFormat = Query(description="Read the source as this artefact type"),
475 url: Optional[str] = Query(None, description="Public http(s) URL of the artefact"),
476 upload_id: Optional[str] = Query(None, description="Token of a previously uploaded file"),
477 lang: Optional[str] = Query(None, description="Language tag for labels, e.g. 'en', 'it'"),
478 imported: Optional[bool] = Query(None, description="Also load owl:imports (depth 1)"),
479 closure: Optional[bool] = Query(None, description="Load the full owl:imports closure"),
480 partial_import: Optional[bool] = Query(None, description="Additiornally load only explicitly mentioned entities from other ontologies"),
481 warnings: bool = Query(False, description="Collect and show reader warnings"),
482 cache: bool = Query(True, description="Use the disk cache for fetched URLs"),
483):
484 """Static documentation site as a ZIP. Sync on purpose: FastAPI runs
485 non-async endpoints in the threadpool, keeping the event loop free."""
486 _check_format_enabled(read_as)
487 reader = _resolve_reader(read_as.value, url, upload_id, imported, closure, partial_import,
488 warnings, use_cache=cache)
489 return _zip_response(reader, _slug(url), lang)
492@app.post("/build", tags=["build"], summary="Build a static site (ZIP) from an upload")
493async def build_post(
494 read_as: ReadAsFormat = Form(..., description="Read the uploaded file as this artefact type"),
495 file: UploadFile = File(..., description="The artefact file (ttl, rdf, owl…)"),
496 lang: Optional[str] = Form(None, description="Language tag for labels, e.g. 'en', 'it'"),
497 imported: Optional[bool] = Form(None, description="Also load owl:imports (depth 1)"),
498 closure: Optional[bool] = Form(None, description="Load the full owl:imports closure"),
499 partial_import: Optional[bool] = Form(None, description="Additiornally load only explicitly mentioned entities from other ontologies"),
500 warnings: bool = Form(False, description="Collect and show reader warnings"),
501):
502 """Build the static site from an uploaded file instead of a URL."""
503 _check_format_enabled(read_as)
504 content = await _read_validated_upload(file)
505 token = spool.save(content)
507 reader = Reader()
508 await run_in_threadpool(
509 reader.load_instances, spool.get_path(token), read_as.value,
510 imported=imported, closure=closure, partial_import=partial_import, warnings=warnings,
511 )
512 return await run_in_threadpool(_zip_response, reader, _slug(file.filename), lang)
515# ==========================================================================
516# 14. ENTRY POINT
517# ==========================================================================
518if __name__ == "__main__":
519 import uvicorn
520 uvicorn.run(app, host="0.0.0.0", port=8000)