Coverage for heritrace/utils/sparql_utils.py: 95%

1# SPDX-FileCopyrightText: 2024-2026 Arcangelo Massari <arcangelo.massari@unibo.it> 

2# 

3# SPDX-License-Identifier: ISC 

4 

5import atexit 

6import logging 

7import os 

8import re 

9import time 

10from collections import defaultdict, deque 

11from collections.abc import Generator 

12from concurrent.futures import ProcessPoolExecutor 

13from dataclasses import dataclass 

14 

15from flask import current_app 

16from rdflib import RDF, Dataset, Graph, Literal, URIRef 

17from rdflib.plugins.sparql.algebra import translateUpdate 

18from rdflib.plugins.sparql.parser import parseUpdate 

19from rdflib.term import Node 

20from rdflib.util import from_n3 

21from SPARQLWrapper import JSON 

22from SPARQLWrapper.SPARQLExceptions import SPARQLWrapperException 

23 

24from heritrace.editor import Editor 

25from heritrace.extensions import ( 

26 get_classes_with_multiple_shapes, 

27 get_custom_filter, 

28 get_dataset_is_quadstore, 

29 get_display_rules, 

30 get_provenance_sparql, 

31 get_shacl_graph, 

32 get_sparql, 

33) 

34from heritrace.sparql import get_sparql_bindings 

35from heritrace.utils.display_rules_utils import ( 

36 find_matching_rule, 

37 get_highest_priority_class, 

38 get_sortable_properties, 

39 is_entity_type_visible, 

40) 

41from heritrace.utils.shacl_utils import ( 

42 determine_shape_for_classes, 

43 determine_shape_for_entity_triples, 

44) 

45from heritrace.utils.virtuoso_utils import VIRTUOSO_EXCLUDED_GRAPHS, is_virtuoso 

46 

47_cache: dict[str, tuple[list[dict[str, str | int]], float] | None] = { 

48 "available_classes": None, 

49 "deleted_classes": None, 

50} 

51AVAILABLE_CLASSES_TTL_SECONDS = 60 

52 

53 

54def _parse_n3(value: str) -> Node: 

55 result = from_n3(value) 

56 if not isinstance(result, Node): 

57 msg = f"Cannot parse N3 value: {value}" 

58 raise TypeError(msg) 

59 return result 

60 

61 

62def n3_set_to_graph( 

63 n3_set: set[tuple[str, ...]], 

64 *, 

65 is_quadstore: bool, 

66) -> Graph | Dataset: 

67 if is_quadstore: 

68 g = Dataset(default_union=True) 

69 for tup in n3_set: 

70 quad = ( 

71 _parse_n3(tup[0]), 

72 _parse_n3(tup[1]), 

73 _parse_n3(tup[2]), 

74 _parse_n3(tup[3]), 

75 ) 

76 g.add(quad) # type: ignore[arg-type] 

77 else: 

78 g = Graph() 

79 for tup in n3_set: 

80 g.add((_parse_n3(tup[0]), _parse_n3(tup[1]), _parse_n3(tup[2]))) 

81 return g 

82 

83 

84def convert_to_rdflib_graphs(snapshots: dict, *, is_quadstore: bool) -> dict: 

85 converted = {} 

86 for entity_uri, timestamps in snapshots.items(): 

87 converted[entity_uri] = {} 

88 for ts, n3_set in timestamps.items(): 

89 converted[entity_uri][ts] = n3_set_to_graph( 

90 n3_set, is_quadstore=is_quadstore 

91 ) 

92 return converted 

93 

94 

95def get_triples_from_graph( 

96 graph_or_dataset: Graph | Dataset, 

97 pattern: tuple[URIRef | None, URIRef | None, Node | None], 

98) -> Generator[tuple[Node, Node, Node]]: 

99 """ 

100 Get triples from a Graph or Dataset, handling both cases correctly. 

101 

102 For Dataset (quadstore), converts quads to triples by extracting (s, p, o). 

103 For Graph (triplestore), uses triples() directly. 

104 

105 Args: 

106 graph_or_dataset: Graph or Dataset instance 

107 pattern: Triple pattern tuple (s, p, o) where each can be None 

108 

109 Returns: 

110 Generator of triples (s, p, o) 

111 """ 

112 if isinstance(graph_or_dataset, Dataset): 

113 # For Dataset, use quads() and extract only (s, p, o) 

114 for s, p, o, _g in graph_or_dataset.quads(pattern): 

115 yield (s, p, o) 

116 else: 

117 # For Graph, use triples() directly 

118 yield from graph_or_dataset.triples(pattern) 

119 

120 

121COUNT_LIMIT = int(os.getenv("COUNT_LIMIT", "10000")) 

122 

123 

124@dataclass(slots=True) 

125class _WorkerPool: 

126 executor: ProcessPoolExecutor | None = None 

127 

128 

129_worker_pool = _WorkerPool() 

130 

131 

132def configure_worker_pool(max_workers: int, gunicorn_workers: int) -> None: 

133 if max_workers < 1: 

134 msg = "MAX_WORKERS must be at least 1" 

135 raise ValueError(msg) 

136 if gunicorn_workers < 1: 

137 msg = "GUNICORN_WORKERS must be at least 1" 

138 raise ValueError(msg) 

139 

140 if _worker_pool.executor is not None: 

141 _worker_pool.executor.shutdown() 

142 

143 workers_per_server = max_workers // gunicorn_workers 

144 _worker_pool.executor = ( 

145 ProcessPoolExecutor(max_workers=workers_per_server) 

146 if workers_per_server > 0 

147 else None 

148 ) 

149 

150 

151def shutdown_worker_pool() -> None: 

152 if _worker_pool.executor is not None: 

153 _worker_pool.executor.shutdown() 

154 _worker_pool.executor = None 

155 

156 

157atexit.register(shutdown_worker_pool) 

158 

159 

160def _wrap_virtuoso_graph_pattern(pattern: str) -> str: 

161 """Wrap a SPARQL pattern with Virtuoso GRAPH clause if needed.""" 

162 if is_virtuoso(): 

163 return f""" 

164 GRAPH ?g {{ 

165 {pattern} 

166 }} 

167 FILTER(?g NOT IN (<{">, <".join(VIRTUOSO_EXCLUDED_GRAPHS)}>)) 

168 """ 

169 return pattern 

170 

171 

172def _build_count_query_with_limit(class_uri: str, limit: int) -> str: 

173 """Build a COUNT query with LIMIT for a specific class.""" 

174 

175 return f""" 

176 SELECT (COUNT(?subject) as ?count) 

177 WHERE {{ 

178 {{ 

179 SELECT DISTINCT ?subject 

180 WHERE {{ 

181 ?subject a <{class_uri}> . 

182 }} 

183 LIMIT {limit} 

184 }} 

185 }} 

186 """ 

187 

188 

189def _count_class_instances(class_uri: str, limit: int = COUNT_LIMIT) -> tuple: 

190 """ 

191 Count instances of a class up to a limit. 

192 

193 Returns: 

194 tuple: (display_count, numeric_count) where display_count may be "LIMIT+" 

195 """ 

196 sparql = get_sparql() 

197 query = _build_count_query_with_limit(class_uri, limit + 1) 

198 

199 sparql.setQuery(query) 

200 sparql.setReturnFormat(JSON) 

201 bindings = get_sparql_bindings(sparql.query().convert()) 

202 

203 count = int(bindings[0]["count"]["value"]) 

204 

205 if count > limit: 

206 return f"{limit}+", limit 

207 return str(count), count 

208 

209 

210def _get_entities_with_enhanced_shape_detection( 

211 class_uri: str, classes_with_multiple_shapes: set[str], limit: int = COUNT_LIMIT 

212) -> defaultdict[str, list[dict[str, str]]]: 

213 """ 

214 Get entities for a class using enhanced shape detection 

215 for classes with multiple shapes. 

216 Uses LIMIT to avoid loading all entities. 

217 """ 

218 # Early exit if no classes have multiple shapes 

219 if ( 

220 not classes_with_multiple_shapes 

221 or class_uri not in classes_with_multiple_shapes 

222 ): 

223 return defaultdict(list) 

224 

225 sparql = get_sparql() 

226 

227 subjects_query = f""" 

228 SELECT DISTINCT ?subject 

229 WHERE {{ 

230 ?subject a <{class_uri}> . 

231 }} 

232 LIMIT {limit} 

233 """ 

234 

235 sparql.setQuery(subjects_query) 

236 sparql.setReturnFormat(JSON) 

237 subjects_bindings = get_sparql_bindings(sparql.query().convert()) 

238 

239 subjects = [r["subject"]["value"] for r in subjects_bindings] 

240 

241 if not subjects: 

242 return defaultdict(list) 

243 

244 # Fetch triples only for these specific subjects 

245 subjects_filter = " ".join([f"(<{s}>)" for s in subjects]) 

246 pattern_with_filter = ( 

247 f"?subject a <{class_uri}> . ?subject ?p ?o" 

248 f" . VALUES (?subject) {{ {subjects_filter} }}" 

249 ) 

250 

251 triples_query = f""" 

252 SELECT ?subject ?p ?o 

253 WHERE {{ 

254 {pattern_with_filter} 

255 }} 

256 """ 

257 

258 sparql.setQuery(triples_query) 

259 sparql.setReturnFormat(JSON) 

260 triples_bindings = get_sparql_bindings(sparql.query().convert()) 

261 

262 entities_triples = defaultdict(list) 

263 for binding in triples_bindings: 

264 subject = binding["subject"]["value"] 

265 predicate = binding["p"]["value"] 

266 obj = binding["o"]["value"] 

267 entities_triples[subject].append((subject, predicate, obj)) 

268 

269 shape_to_entities = defaultdict(list) 

270 for subject_uri, triples in entities_triples.items(): 

271 shape_uri = determine_shape_for_entity_triples(triples) 

272 if shape_uri: 

273 entity_key = (class_uri, shape_uri) 

274 if is_entity_type_visible(entity_key): 

275 shape_to_entities[shape_uri].append( 

276 {"uri": subject_uri, "class": class_uri, "shape": shape_uri} 

277 ) 

278 

279 return shape_to_entities 

280 

281 

282def get_classes_from_shacl_or_display_rules() -> list[str]: 

283 """Extract classes from SHACL shapes or display_rules configuration.""" 

284 sh_target_class = URIRef("http://www.w3.org/ns/shacl#targetClass") 

285 classes = set() 

286 

287 shacl_graph = get_shacl_graph() 

288 if shacl_graph: 

289 for shape in shacl_graph.subjects(sh_target_class, None, unique=True): 

290 for target_class in shacl_graph.objects( 

291 shape, sh_target_class, unique=True 

292 ): 

293 classes.add(str(target_class)) 

294 

295 if not classes: 

296 display_rules = get_display_rules() 

297 if display_rules: 

298 for rule in display_rules: 

299 if "target" in rule and "class" in rule["target"]: 

300 classes.add(rule["target"]["class"]) 

301 

302 return list(classes) 

303 

304 

305def _get_classes_from_config() -> list[str]: 

306 classes_from_config = get_classes_from_shacl_or_display_rules() 

307 if classes_from_config: 

308 return classes_from_config 

309 

310 return _get_classes_from_sparql() 

311 

312 

313def _get_classes_from_sparql() -> list[str]: 

314 sparql = get_sparql() 

315 pattern = "?subject a ?class ." 

316 wrapped_pattern = _wrap_virtuoso_graph_pattern(pattern) 

317 

318 query = f""" 

319 SELECT DISTINCT ?class 

320 WHERE {{ 

321 {wrapped_pattern} 

322 }} 

323 """ 

324 

325 sparql.setQuery(query) 

326 sparql.setReturnFormat(JSON) 

327 class_bindings = get_sparql_bindings(sparql.query().convert()) 

328 return [r["class"]["value"] for r in class_bindings] 

329 

330 

331def get_available_classes() -> list[dict[str, str | int]]: 

332 cached = _cache["available_classes"] 

333 if cached is not None: 

334 available_classes, computed_at = cached 

335 if time.monotonic() - computed_at < AVAILABLE_CLASSES_TTL_SECONDS: 

336 return available_classes 

337 

338 custom_filter = get_custom_filter() 

339 class_uris = _get_classes_from_config() 

340 

341 classes_with_counts = [] 

342 for class_uri in class_uris: 

343 display_count, numeric_count = _count_class_instances(class_uri) 

344 classes_with_counts.append( 

345 { 

346 "uri": class_uri, 

347 "display_count": display_count, 

348 "numeric_count": numeric_count, 

349 } 

350 ) 

351 

352 classes_with_counts.sort(key=lambda x: x["numeric_count"], reverse=True) 

353 

354 available_classes = [] 

355 classes_with_multiple_shapes = get_classes_with_multiple_shapes() 

356 

357 for class_data in classes_with_counts: 

358 class_uri = class_data["uri"] 

359 

360 if classes_with_multiple_shapes and class_uri in classes_with_multiple_shapes: 

361 shape_to_entities = _get_entities_with_enhanced_shape_detection( 

362 class_uri, classes_with_multiple_shapes, limit=COUNT_LIMIT 

363 ) 

364 

365 for shape_uri, entities in shape_to_entities.items(): 

366 if entities: 

367 entity_key = (class_uri, shape_uri) 

368 available_classes.append( 

369 { 

370 "uri": class_uri, 

371 "label": custom_filter.human_readable_class(entity_key), 

372 "count": f"{len(entities)}+" 

373 if len(entities) >= COUNT_LIMIT 

374 else str(len(entities)), 

375 "count_numeric": len(entities), 

376 "shape": shape_uri, 

377 } 

378 ) 

379 else: 

380 shape_uri = determine_shape_for_classes([class_uri]) 

381 entity_key = (class_uri, shape_uri) 

382 

383 if is_entity_type_visible(entity_key): 

384 available_classes.append( 

385 { 

386 "uri": class_uri, 

387 "label": custom_filter.human_readable_class(entity_key), 

388 "count": class_data["display_count"], 

389 "count_numeric": class_data["numeric_count"], 

390 "shape": shape_uri, 

391 } 

392 ) 

393 

394 available_classes.sort(key=lambda x: x["label"].lower()) 

395 _cache["available_classes"] = (available_classes, time.monotonic()) 

396 return available_classes 

397 

398 

399def build_sort_clause( 

400 sort_property: str, entity_type: str, shape_uri: str | None = None 

401) -> str: 

402 """ 

403 Build a SPARQL sort clause based on the sortableBy configuration. 

404 

405 Args: 

406 sort_property: The property to sort by 

407 entity_type: The entity type URI 

408 shape_uri: Optional shape URI for more specific sorting rules 

409 

410 Returns: 

411 SPARQL sort clause or empty string 

412 """ 

413 if not sort_property or not entity_type: 

414 return "" 

415 

416 rule = find_matching_rule(entity_type, shape_uri) 

417 

418 if not rule or "sortableBy" not in rule: 

419 return "" 

420 

421 sort_config = next( 

422 (s for s in rule["sortableBy"] if s.get("property") == sort_property), None 

423 ) 

424 

425 if not sort_config: 

426 return "" 

427 

428 return f"OPTIONAL {{ ?subject <{sort_property}> ?sortValue }}" 

429 

430 

431@dataclass(frozen=True, slots=True) 

432class CatalogQuery: 

433 selected_class: str | None 

434 page: int 

435 per_page: int 

436 sort_property: str | None = None 

437 sort_direction: str = "ASC" 

438 selected_shape: str | None = None 

439 

440 

441def _fetch_entity_labels( 

442 subject_uris: list[str], entity_key: tuple[str | None, str | None] 

443) -> list[str]: 

444 if not subject_uris: 

445 return [] 

446 

447 if _worker_pool.executor is None: 

448 return [_fetch_entity_label(uri, entity_key) for uri in subject_uris] 

449 return list( 

450 _worker_pool.executor.map( 

451 _fetch_entity_label, subject_uris, [entity_key] * len(subject_uris) 

452 ) 

453 ) 

454 

455 

456def _fetch_entity_label(uri: str, entity_key: tuple[str | None, str | None]) -> str: 

457 return get_custom_filter().human_readable_entity(uri, entity_key, None) 

458 

459 

460def _get_entities_with_shape_filtering( 

461 query: CatalogQuery, 

462) -> tuple[list[dict[str, str]], int]: 

463 sparql = get_sparql() 

464 selected_class = query.selected_class 

465 selected_shape = query.selected_shape 

466 offset = (query.page - 1) * query.per_page 

467 fetch_limit = query.per_page * 5 

468 

469 subjects_query = f""" 

470 SELECT DISTINCT ?subject 

471 WHERE {{ 

472 ?subject a <{selected_class}> . 

473 }} 

474 LIMIT {fetch_limit} 

475 OFFSET {offset} 

476 """ 

477 

478 sparql.setQuery(subjects_query) 

479 sparql.setReturnFormat(JSON) 

480 subjects_bindings = get_sparql_bindings(sparql.query().convert()) 

481 

482 subjects = [r["subject"]["value"] for r in subjects_bindings] 

483 

484 if not subjects: 

485 return [], 0 

486 

487 subjects_filter = " ".join([f"(<{s}>)" for s in subjects]) 

488 

489 triples_query = f""" 

490 SELECT ?subject ?p ?o 

491 WHERE {{ 

492 ?subject a <{selected_class}> . ?subject ?p ?o . VALUES (?subject) {{ 

493 {subjects_filter} }} 

494 }} 

495 """ 

496 

497 sparql.setQuery(triples_query) 

498 sparql.setReturnFormat(JSON) 

499 triples_bindings = get_sparql_bindings(sparql.query().convert()) 

500 

501 entities_triples = defaultdict(list) 

502 for binding in triples_bindings: 

503 subject = binding["subject"]["value"] 

504 predicate = binding["p"]["value"] 

505 obj = binding["o"]["value"] 

506 entities_triples[subject].append((subject, predicate, obj)) 

507 

508 matching_uris = [ 

509 subject_uri 

510 for subject_uri, triples in entities_triples.items() 

511 if determine_shape_for_entity_triples(list(triples)) == selected_shape 

512 ] 

513 labels = _fetch_entity_labels(matching_uris, (selected_class, selected_shape)) 

514 filtered_entities = [ 

515 {"uri": uri, "label": label} 

516 for uri, label in zip(matching_uris, labels, strict=True) 

517 ] 

518 

519 if query.sort_property and query.sort_direction: 

520 reverse_sort = query.sort_direction.upper() == "DESC" 

521 filtered_entities.sort(key=lambda x: x["label"].lower(), reverse=reverse_sort) 

522 

523 total_count = len(filtered_entities) 

524 return filtered_entities[: query.per_page], total_count 

525 

526 

527def get_entities_for_class( 

528 query: CatalogQuery, 

529 available_classes: list[dict[str, str | int]], 

530) -> tuple[list[dict[str, str]], int]: 

531 if query.selected_class is None: 

532 msg = "selected_class must not be None" 

533 raise ValueError(msg) 

534 sparql = get_sparql() 

535 classes_with_multiple_shapes = get_classes_with_multiple_shapes() 

536 

537 selected_class: str = query.selected_class 

538 selected_shape = query.selected_shape 

539 page = query.page 

540 per_page = query.per_page 

541 sort_property = query.sort_property 

542 sort_direction = query.sort_direction 

543 

544 use_shape_filtering = ( 

545 selected_shape and selected_class in classes_with_multiple_shapes 

546 ) 

547 

548 if use_shape_filtering: 

549 return _get_entities_with_shape_filtering(query) 

550 

551 offset = (page - 1) * per_page 

552 sort_clause = "" 

553 order_clause = "" 

554 

555 if sort_property: 

556 sort_clause = build_sort_clause(sort_property, selected_class, selected_shape) 

557 if sort_clause: 

558 order_clause = f"ORDER BY {sort_direction}(?sortValue)" 

559 

560 entities_query = f""" 

561 SELECT ?subject {"?sortValue" if sort_property else ""} 

562 WHERE {{ 

563 ?subject a <{selected_class}> . {sort_clause} 

564 }} 

565 {order_clause} 

566 LIMIT {per_page} 

567 OFFSET {offset} 

568 """ 

569 

570 class_info = next( 

571 ( 

572 c 

573 for c in available_classes 

574 if c["uri"] == selected_class and c.get("shape") == selected_shape 

575 ), 

576 None, 

577 ) 

578 total_count = int(class_info["count_numeric"]) if class_info else 0 

579 

580 sparql.setQuery(entities_query) 

581 sparql.setReturnFormat(JSON) 

582 entities_bindings = get_sparql_bindings(sparql.query().convert()) 

583 

584 shape = selected_shape or determine_shape_for_classes([selected_class]) 

585 subject_uris = [result["subject"]["value"] for result in entities_bindings] 

586 labels = _fetch_entity_labels(subject_uris, (selected_class, shape)) 

587 entities = [ 

588 {"uri": uri, "label": label} 

589 for uri, label in zip(subject_uris, labels, strict=True) 

590 ] 

591 

592 return entities, total_count 

593 

594 

595def get_catalog_data( 

596 query: CatalogQuery, 

597 available_classes: list[dict[str, str | int]], 

598) -> dict: 

599 entities = [] 

600 total_count = 0 

601 sortable_properties = [] 

602 sort_property = query.sort_property 

603 

604 if query.selected_class: 

605 sortable_properties = get_sortable_properties( 

606 (query.selected_class, query.selected_shape) 

607 ) 

608 

609 if not sort_property and sortable_properties: 

610 sort_property = sortable_properties[0]["property"] 

611 

612 inner_query = CatalogQuery( 

613 selected_class=query.selected_class, 

614 page=query.page, 

615 per_page=query.per_page, 

616 sort_property=sort_property, 

617 sort_direction=query.sort_direction, 

618 selected_shape=query.selected_shape, 

619 ) 

620 entities, total_count = get_entities_for_class(inner_query, available_classes) 

621 

622 return { 

623 "entities": entities, 

624 "total_pages": ( 

625 (total_count + query.per_page - 1) // query.per_page 

626 if total_count > 0 

627 else 0 

628 ), 

629 "current_page": query.page, 

630 "per_page": query.per_page, 

631 "total_count": total_count, 

632 "sort_property": sort_property, 

633 "sort_direction": query.sort_direction, 

634 "sortable_properties": sortable_properties, 

635 "selected_class": query.selected_class, 

636 "selected_shape": query.selected_shape, 

637 } 

638 

639 

640def warm_catalogue( 

641 available_classes: list[dict[str, str | int]], per_page: int 

642) -> None: 

643 total_started_at = time.monotonic() 

644 

645 for class_info in available_classes: 

646 if int(class_info["count_numeric"]) == 0: 

647 continue 

648 

649 class_uri = str(class_info["uri"]) 

650 shape_value = class_info["shape"] 

651 shape_uri = str(shape_value) if shape_value is not None else None 

652 category_started_at = time.monotonic() 

653 

654 get_catalog_data( 

655 CatalogQuery( 

656 selected_class=class_uri, 

657 selected_shape=shape_uri, 

658 page=1, 

659 per_page=per_page, 

660 ), 

661 available_classes, 

662 ) 

663 

664 current_app.logger.info( 

665 "[STARTUP] Warmed catalogue category class=%s shape=%s in %.3f seconds", 

666 class_uri, 

667 shape_uri, 

668 time.monotonic() - category_started_at, 

669 ) 

670 

671 current_app.logger.info( 

672 "[STARTUP] Catalogue warm-up completed in %.3f seconds", 

673 time.monotonic() - total_started_at, 

674 ) 

675 

676 

677def _binding_to_node(binding: dict[str, str]) -> Literal | URIRef: 

678 if binding["type"] not in {"literal", "typed-literal"}: 

679 return URIRef(binding["value"]) 

680 if "datatype" in binding: 

681 return Literal(binding["value"], datatype=URIRef(binding["datatype"])) 

682 # Omit explicit datatype to match Reader's import behavior 

683 return Literal(binding["value"]) 

684 

685 

686def fetch_data_graph_for_subject(subject: URIRef) -> Graph | Dataset: 

687 g = Dataset() if get_dataset_is_quadstore() else Graph() 

688 sparql = get_sparql() 

689 

690 if is_virtuoso(): 

691 # For virtuoso we need to explicitly query the graph 

692 query = f""" 

693 SELECT ?predicate ?object ?g WHERE {{ 

694 GRAPH ?g {{ 

695 <{subject}> ?predicate ?object. 

696 }} 

697 FILTER(?g NOT IN (<{">, <".join(VIRTUOSO_EXCLUDED_GRAPHS)}>)) 

698 }} 

699 """ 

700 elif get_dataset_is_quadstore(): 

701 # For non-virtuoso quadstore, we need to query all graphs 

702 query = f""" 

703 SELECT ?predicate ?object ?g WHERE {{ 

704 GRAPH ?g {{ 

705 <{subject}> ?predicate ?object. 

706 }} 

707 }} 

708 """ 

709 else: 

710 # For regular triplestore 

711 query = f""" 

712 SELECT ?predicate ?object WHERE {{ 

713 <{subject}> ?predicate ?object. 

714 }} 

715 """ 

716 

717 sparql.setQuery(query) 

718 sparql.setReturnFormat(JSON) 

719 bindings = get_sparql_bindings(sparql.query().convert()) 

720 

721 for result in bindings: 

722 value = _binding_to_node(result["object"]) 

723 

724 # Add triple/quad based on store type 

725 if get_dataset_is_quadstore(): 

726 graph_uri = URIRef(result["g"]["value"]) 

727 g.add( 

728 ( # type: ignore[arg-type] 

729 subject, 

730 URIRef(result["predicate"]["value"]), 

731 value, 

732 graph_uri, 

733 ) 

734 ) 

735 else: 

736 g.add((subject, URIRef(result["predicate"]["value"]), value)) 

737 

738 return g 

739 

740 

741def parse_sparql_update(query: str) -> dict[str, list[tuple[Node, Node, Node]]]: 

742 parsed = parseUpdate(query) 

743 translated = translateUpdate(parsed).algebra 

744 modifications = {} 

745 

746 def extract_quads( 

747 quads: defaultdict[Node, list[tuple[Node, Node, Node]]], 

748 ) -> list[tuple[Node, Node, Node]]: 

749 return [ 

750 (triple[0], triple[1], triple[2]) 

751 for triples in quads.values() 

752 for triple in triples 

753 ] 

754 

755 for operation in translated: 

756 if operation.name == "DeleteData": 

757 if hasattr(operation, "quads") and operation.quads: 

758 deletions = extract_quads(operation.quads) 

759 else: 

760 deletions = operation.triples 

761 if deletions: 

762 modifications.setdefault("Deletions", []).extend(deletions) 

763 elif operation.name == "InsertData": 

764 if hasattr(operation, "quads") and operation.quads: 

765 additions = extract_quads(operation.quads) 

766 else: 

767 additions = operation.triples 

768 if additions: 

769 modifications.setdefault("Additions", []).extend(additions) 

770 

771 return modifications 

772 

773 

774def fetch_current_state_with_related_entities( 

775 provenance: dict, 

776) -> Graph | Dataset: 

777 """ 

778 Fetch the current state of an entity and all its related entities known from 

779 provenance. 

780 

781 Args: 

782 provenance (dict): Dictionary containing provenance metadata for main entity and 

783 related entities 

784 

785 Returns: 

786 Dataset: A graph containing the current state of all entities 

787 """ 

788 combined_graph = Dataset() if get_dataset_is_quadstore() else Graph() 

789 

790 # Fetch state for all entities mentioned in provenance 

791 for entity_uri in provenance: 

792 current_graph = fetch_data_graph_for_subject(URIRef(entity_uri)) 

793 

794 if get_dataset_is_quadstore(): 

795 for quad in current_graph.quads(): # type: ignore[union-attr] 

796 combined_graph.add(quad) # type: ignore[call-overload] 

797 else: 

798 for triple in current_graph: 

799 combined_graph.add(triple) # type: ignore[call-overload] 

800 

801 return combined_graph 

802 

803 

804@dataclass(frozen=True, slots=True) 

805class DeletedEntitiesQuery: 

806 page: int = 1 

807 per_page: int = 50 

808 sort_direction: str = "DESC" 

809 selected_class: str | None = None 

810 selected_shape: str | None = None 

811 

812 

813PROV = "http://www.w3.org/ns/prov#" 

814OCO_HAS_UPDATE_QUERY = "https://w3id.org/oc/ontology/hasUpdateQuery" 

815RDF_TYPE_STATEMENT = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>" 

816DELETION_TIME_PROPERTY = { 

817 "property": "deletionTime", 

818 "displayName": "Deletion Time", 

819 "sortType": "date", 

820} 

821RELATED_STATE_DEPTH = 5 

822 

823 

824def _deletion_snapshots_pattern(selected_class: str | None) -> str: 

825 """ 

826 Match the snapshots that deleted their entity. 

827 

828 A deletion snapshot is generated and invalidated at the same instant, and no 

829 later snapshot derives from it. Restoring an entity adds a snapshot derived 

830 from the deletion one, which is what excludes restored entities here. 

831 """ 

832 class_pattern = "" 

833 if selected_class: 

834 class_pattern = f""" 

835 ?snapshot <{OCO_HAS_UPDATE_QUERY}> ?candidateUpdateQuery . 

836 FILTER(CONTAINS( 

837 ?candidateUpdateQuery, 

838 "{RDF_TYPE_STATEMENT} <{selected_class}>" 

839 ))""" 

840 return f""" 

841 ?snapshot <{PROV}invalidatedAtTime> ?invalidationTime ; 

842 <{PROV}generatedAtTime> ?deletionTime . 

843 FILTER(?deletionTime = ?invalidationTime) 

844 FILTER NOT EXISTS {{ 

845 ?laterSnapshot <{PROV}wasDerivedFrom> ?snapshot . 

846 }}{class_pattern}""" 

847 

848 

849def _deleted_entity_types(entity_uri: str, update_query: str) -> list[str]: 

850 """Read the entity types out of the DELETE DATA recorded by the snapshot.""" 

851 statement = re.escape(f"<{entity_uri}> {RDF_TYPE_STATEMENT} ") 

852 return re.findall(f"{statement}<([^>]+)>", update_query) 

853 

854 

855def _count_deleted_class_instances( 

856 class_uri: str, limit: int = COUNT_LIMIT 

857) -> tuple[str, int]: 

858 """ 

859 Count deleted entities of a class up to a limit. 

860 

861 Returns: 

862 tuple: (display_count, numeric_count) where display_count may be "LIMIT+" 

863 """ 

864 provenance_sparql = get_provenance_sparql() 

865 provenance_sparql.setQuery(f""" 

866 SELECT (COUNT(?snapshot) AS ?count) 

867 WHERE {{ 

868 {{ 

869 SELECT ?snapshot 

870 WHERE {{{_deletion_snapshots_pattern(class_uri)} 

871 }} 

872 LIMIT {limit + 1} 

873 }} 

874 }} 

875 """)