Coverage for heritrace/utils/sparql_utils.py: 94%

1import os 

2from collections import defaultdict 

3from concurrent.futures import ProcessPoolExecutor, as_completed 

4from typing import List 

5 

6from rdflib import RDF, Dataset, Graph, Literal, URIRef 

7from rdflib.plugins.sparql.algebra import translateUpdate 

8from rdflib.plugins.sparql.parser import parseUpdate 

9from SPARQLWrapper import JSON 

10from time_agnostic_library.agnostic_entity import AgnosticEntity 

11 

12from heritrace.editor import Editor 

13from heritrace.extensions import (get_change_tracking_config, 

14 get_classes_with_multiple_shapes, 

15 get_custom_filter, get_dataset_is_quadstore, 

16 get_display_rules, get_provenance_sparql, 

17 get_shacl_graph, get_sparql) 

18from heritrace.utils.converters import convert_to_datetime 

19from heritrace.utils.display_rules_utils import (find_matching_rule, 

20 get_highest_priority_class, 

21 get_sortable_properties, 

22 is_entity_type_visible) 

23from heritrace.utils.shacl_utils import (determine_shape_for_classes, 

24 determine_shape_for_entity_triples) 

25from heritrace.utils.virtuoso_utils import (VIRTUOSO_EXCLUDED_GRAPHS, 

26 is_virtuoso) 

27 

28_AVAILABLE_CLASSES_CACHE = None 

29 

30 

31def get_triples_from_graph(graph_or_dataset, pattern): 

32 """ 

33 Get triples from a Graph or Dataset, handling both cases correctly. 

34 

35 For Dataset (quadstore), converts quads to triples by extracting (s, p, o). 

36 For Graph (triplestore), uses triples() directly. 

37 

38 Args: 

39 graph_or_dataset: Graph or Dataset instance 

40 pattern: Triple pattern tuple (s, p, o) where each can be None 

41 

42 Returns: 

43 Generator of triples (s, p, o) 

44 """ 

45 if isinstance(graph_or_dataset, Dataset): 

46 # For Dataset, use quads() and extract only (s, p, o) 

47 for s, p, o, g in graph_or_dataset.quads(pattern): 

48 yield (s, p, o) 

49 else: 

50 # For Graph, use triples() directly 

51 yield from graph_or_dataset.triples(pattern) 

52COUNT_LIMIT = int(os.getenv("COUNT_LIMIT", "10000")) 

53 

54 

55def precompute_available_classes_cache(): 

56 """Pre-compute available classes cache at application startup.""" 

57 global _AVAILABLE_CLASSES_CACHE 

58 _AVAILABLE_CLASSES_CACHE = get_available_classes() 

59 return _AVAILABLE_CLASSES_CACHE 

60 

61 

62def _wrap_virtuoso_graph_pattern(pattern: str) -> str: 

63 """Wrap a SPARQL pattern with Virtuoso GRAPH clause if needed.""" 

64 if is_virtuoso(): 

65 return f""" 

66 GRAPH ?g {{ 

67 {pattern} 

68 }} 

69 FILTER(?g NOT IN (<{'>, <'.join(VIRTUOSO_EXCLUDED_GRAPHS)}>)) 

70 """ 

71 return pattern 

72 

73 

74def _build_count_query_with_limit(class_uri: str, limit: int) -> str: 

75 """Build a COUNT query with LIMIT for a specific class.""" 

76 

77 return f""" 

78 SELECT (COUNT(?subject) as ?count) 

79 WHERE {{ 

80 {{ 

81 SELECT DISTINCT ?subject 

82 WHERE {{ 

83 ?subject a <{class_uri}> . 

84 }} 

85 LIMIT {limit} 

86 }} 

87 }} 

88 """ 

89 

90 

91def _count_class_instances(class_uri: str, limit: int = COUNT_LIMIT) -> tuple: 

92 """ 

93 Count instances of a class up to a limit. 

94 

95 Returns: 

96 tuple: (display_count, numeric_count) where display_count may be "LIMIT+" 

97 """ 

98 sparql = get_sparql() 

99 query = _build_count_query_with_limit(class_uri, limit + 1) 

100 

101 sparql.setQuery(query) 

102 sparql.setReturnFormat(JSON) 

103 result = sparql.query().convert() 

104 

105 count = int(result["results"]["bindings"][0]["count"]["value"]) 

106 

107 if count > limit: 

108 return f"{limit}+", limit 

109 return str(count), count 

110 

111 

112def _get_entities_with_enhanced_shape_detection(class_uri: str, classes_with_multiple_shapes: set, limit: int = COUNT_LIMIT): 

113 """ 

114 Get entities for a class using enhanced shape detection for classes with multiple shapes. 

115 Uses LIMIT to avoid loading all entities. 

116 """ 

117 # Early exit if no classes have multiple shapes 

118 if not classes_with_multiple_shapes or class_uri not in classes_with_multiple_shapes: 

119 return defaultdict(list) 

120 

121 sparql = get_sparql() 

122 

123 subjects_query = f""" 

124 SELECT DISTINCT ?subject 

125 WHERE {{ 

126 ?subject a <{class_uri}> . 

127 }} 

128 LIMIT {limit} 

129 """ 

130 

131 sparql.setQuery(subjects_query) 

132 sparql.setReturnFormat(JSON) 

133 subjects_results = sparql.query().convert() 

134 

135 subjects = [r["subject"]["value"] for r in subjects_results["results"]["bindings"]] 

136 

137 if not subjects: 

138 return defaultdict(list) 

139 

140 # Fetch triples only for these specific subjects 

141 subjects_filter = " ".join([f"(<{s}>)" for s in subjects]) 

142 pattern_with_filter = f"?subject a <{class_uri}> . ?subject ?p ?o . VALUES (?subject) {{ {subjects_filter} }}" 

143 

144 triples_query = f""" 

145 SELECT ?subject ?p ?o 

146 WHERE {{ 

147 {pattern_with_filter} 

148 }} 

149 """ 

150 

151 sparql.setQuery(triples_query) 

152 sparql.setReturnFormat(JSON) 

153 results = sparql.query().convert() 

154 

155 entities_triples = defaultdict(list) 

156 for binding in results["results"]["bindings"]: 

157 subject = binding["subject"]["value"] 

158 predicate = binding["p"]["value"] 

159 obj = binding["o"]["value"] 

160 entities_triples[subject].append((subject, predicate, obj)) 

161 

162 shape_to_entities = defaultdict(list) 

163 for subject_uri, triples in entities_triples.items(): 

164 shape_uri = determine_shape_for_entity_triples(triples) 

165 if shape_uri: 

166 entity_key = (class_uri, shape_uri) 

167 if is_entity_type_visible(entity_key): 

168 shape_to_entities[shape_uri].append({ 

169 "uri": subject_uri, 

170 "class": class_uri, 

171 "shape": shape_uri 

172 }) 

173 

174 return shape_to_entities 

175 

176 

177def get_classes_from_shacl_or_display_rules(): 

178 """Extract classes from SHACL shapes or display_rules configuration.""" 

179 SH_TARGET_CLASS = URIRef("http://www.w3.org/ns/shacl#targetClass") 

180 classes = set() 

181 

182 shacl_graph = get_shacl_graph() 

183 if shacl_graph: 

184 for shape in shacl_graph.subjects(SH_TARGET_CLASS, None, unique=True): 

185 for target_class in shacl_graph.objects(shape, SH_TARGET_CLASS, unique=True): 

186 classes.add(str(target_class)) 

187 

188 if not classes: 

189 display_rules = get_display_rules() 

190 if display_rules: 

191 for rule in display_rules: 

192 if "target" in rule and "class" in rule["target"]: 

193 classes.add(rule["target"]["class"]) 

194 

195 return list(classes) 

196 

197 

198def get_available_classes(): 

199 """ 

200 Fetch and format all available entity classes. 

201 Returns cached result if available (computed at startup). 

202 For small datasets (< COUNT_LIMIT), cache is invalidated to keep counts accurate. 

203 """ 

204 global _AVAILABLE_CLASSES_CACHE 

205 

206 if _AVAILABLE_CLASSES_CACHE is not None: 

207 total_count = sum(cls.get('count_numeric', 0) for cls in _AVAILABLE_CLASSES_CACHE) 

208 if total_count < COUNT_LIMIT: 

209 _AVAILABLE_CLASSES_CACHE = None 

210 

211 if _AVAILABLE_CLASSES_CACHE is not None: 

212 return _AVAILABLE_CLASSES_CACHE 

213 

214 custom_filter = get_custom_filter() 

215 classes_from_config = get_classes_from_shacl_or_display_rules() 

216 

217 if classes_from_config: 

218 class_uris = classes_from_config 

219 else: 

220 sparql = get_sparql() 

221 pattern = "?subject a ?class ." 

222 wrapped_pattern = _wrap_virtuoso_graph_pattern(pattern) 

223 

224 query = f""" 

225 SELECT DISTINCT ?class 

226 WHERE {{ 

227 {wrapped_pattern} 

228 }} 

229 """ 

230 

231 sparql.setQuery(query) 

232 sparql.setReturnFormat(JSON) 

233 results = sparql.query().convert() 

234 class_uris = [r["class"]["value"] for r in results["results"]["bindings"]] 

235 

236 # Count instances for each class 

237 classes_with_counts = [] 

238 for class_uri in class_uris: 

239 display_count, numeric_count = _count_class_instances(class_uri) 

240 classes_with_counts.append({ 

241 "uri": class_uri, 

242 "display_count": display_count, 

243 "numeric_count": numeric_count 

244 }) 

245 

246 # Sort by count descending 

247 classes_with_counts.sort(key=lambda x: x["numeric_count"], reverse=True) 

248 

249 available_classes = [] 

250 classes_with_multiple_shapes = get_classes_with_multiple_shapes() 

251 

252 for class_data in classes_with_counts: 

253 class_uri = class_data["uri"] 

254 

255 if classes_with_multiple_shapes and class_uri in classes_with_multiple_shapes: 

256 shape_to_entities = _get_entities_with_enhanced_shape_detection( 

257 class_uri, classes_with_multiple_shapes, limit=COUNT_LIMIT 

258 ) 

259 

260 for shape_uri, entities in shape_to_entities.items(): 

261 if entities: 

262 entity_key = (class_uri, shape_uri) 

263 available_classes.append({ 

264 "uri": class_uri, 

265 "label": custom_filter.human_readable_class(entity_key), 

266 "count": f"{len(entities)}+" if len(entities) >= COUNT_LIMIT else str(len(entities)), 

267 "count_numeric": len(entities), 

268 "shape": shape_uri 

269 }) 

270 else: 

271 shape_uri = determine_shape_for_classes([class_uri]) 

272 entity_key = (class_uri, shape_uri) 

273 

274 if is_entity_type_visible(entity_key): 

275 available_classes.append({ 

276 "uri": class_uri, 

277 "label": custom_filter.human_readable_class(entity_key), 

278 "count": class_data["display_count"], 

279 "count_numeric": class_data["numeric_count"], 

280 "shape": shape_uri 

281 }) 

282 

283 available_classes.sort(key=lambda x: x["label"].lower()) 

284 return available_classes 

285 

286 

287def build_sort_clause(sort_property: str, entity_type: str, shape_uri: str = None) -> str: 

288 """ 

289 Build a SPARQL sort clause based on the sortableBy configuration. 

290 

291 Args: 

292 sort_property: The property to sort by 

293 entity_type: The entity type URI 

294 shape_uri: Optional shape URI for more specific sorting rules 

295 

296 Returns: 

297 SPARQL sort clause or empty string 

298 """ 

299 if not sort_property or not entity_type: 

300 return "" 

301 

302 rule = find_matching_rule(entity_type, shape_uri) 

303 

304 if not rule or "sortableBy" not in rule: 

305 return "" 

306 

307 sort_config = next( 

308 (s for s in rule["sortableBy"] if s.get("property") == sort_property), 

309 None 

310 ) 

311 

312 if not sort_config: 

313 return "" 

314 

315 return f"OPTIONAL {{ ?subject <{sort_property}> ?sortValue }}" 

316 

317 

318def get_entities_for_class( 

319 selected_class, page, per_page, sort_property=None, sort_direction="ASC", selected_shape=None 

320): 

321 """ 

322 Retrieve entities for a specific class with pagination and sorting. 

323 

324 Args: 

325 selected_class (str): URI of the class to retrieve entities for 

326 page (int): Page number (1-indexed) 

327 per_page (int): Number of entities per page 

328 sort_property (str, optional): Property URI to sort by. Defaults to None. 

329 sort_direction (str, optional): Sort direction ("ASC" or "DESC"). Defaults to "ASC". 

330 selected_shape (str, optional): Shape URI for filtering entities. Defaults to None. 

331 

332 Returns: 

333 tuple: (list of entities, total count) 

334 

335 Performance Notes: 

336 - If sort_property is None, NO ORDER BY clause is applied to the SPARQL query. 

337 This significantly improves performance for large datasets by avoiding expensive 

338 sorting operations on URIs. 

339 - Without explicit ordering, the triplestore returns results in its natural order, 

340 which is deterministic within a session but may vary after database reloads. 

341 - For optimal performance with large datasets, configure display_rules.yaml without 

342 sortableBy properties to prevent users from triggering expensive sort operations. 

343 """ 

344 sparql = get_sparql() 

345 custom_filter = get_custom_filter() 

346 classes_with_multiple_shapes = get_classes_with_multiple_shapes() 

347 

348 use_shape_filtering = (selected_shape and selected_class in classes_with_multiple_shapes) 

349 

350 if use_shape_filtering: 

351 # For shape filtering, we need to fetch entities and check their shape 

352 # Use a larger LIMIT to ensure we get enough entities after filtering 

353 offset = (page - 1) * per_page 

354 fetch_limit = per_page * 5 # Safety margin for filtering 

355 

356 subjects_query = f""" 

357 SELECT DISTINCT ?subject 

358 WHERE {{ 

359 ?subject a <{selected_class}> . 

360 }} 

361 LIMIT {fetch_limit} 

362 OFFSET {offset} 

363 """ 

364 

365 sparql.setQuery(subjects_query) 

366 sparql.setReturnFormat(JSON) 

367 subjects_results = sparql.query().convert() 

368 

369 subjects = [r["subject"]["value"] for r in subjects_results["results"]["bindings"]] 

370 

371 if not subjects: 

372 return [], 0 

373 

374 # Now fetch triples for these specific subjects 

375 subjects_filter = " ".join([f"(<{s}>)" for s in subjects]) 

376 

377 triples_query = f""" 

378 SELECT ?subject ?p ?o 

379 WHERE {{ 

380 ?subject a <{selected_class}> . ?subject ?p ?o . VALUES (?subject) {{ {subjects_filter} }} 

381 }} 

382 """ 

383 

384 sparql.setQuery(triples_query) 

385 sparql.setReturnFormat(JSON) 

386 results = sparql.query().convert() 

387 

388 entities_triples = defaultdict(list) 

389 for binding in results["results"]["bindings"]: 

390 subject = binding["subject"]["value"] 

391 predicate = binding["p"]["value"] 

392 obj = binding["o"]["value"] 

393 entities_triples[subject].append((subject, predicate, obj)) 

394 

395 filtered_entities = [] 

396 for subject_uri, triples in entities_triples.items(): 

397 entity_shape = determine_shape_for_entity_triples(list(triples)) 

398 if entity_shape == selected_shape: 

399 entity_label = custom_filter.human_readable_entity( 

400 subject_uri, (selected_class, selected_shape), None 

401 ) 

402 filtered_entities.append({"uri": subject_uri, "label": entity_label}) 

403 

404 if sort_property and sort_direction: 

405 reverse_sort = sort_direction.upper() == "DESC" 

406 filtered_entities.sort(key=lambda x: x["label"].lower(), reverse=reverse_sort) 

407 

408 # For shape-filtered results, we can't accurately determine total_count without scanning all entities 

409 # Return the number of filtered entities as an approximation 

410 total_count = len(filtered_entities) 

411 return filtered_entities[:per_page], total_count 

412 

413 # Standard pagination path 

414 offset = (page - 1) * per_page 

415 sort_clause = "" 

416 order_clause = "" 

417 

418 if sort_property: 

419 sort_clause = build_sort_clause(sort_property, selected_class, selected_shape) 

420 if sort_clause: 

421 order_clause = f"ORDER BY {sort_direction}(?sortValue)" 

422 

423 entities_query = f""" 

424 SELECT ?subject {f"?sortValue" if sort_property else ""} 

425 WHERE {{ 

426 ?subject a <{selected_class}> . {sort_clause} 

427 }} 

428 {order_clause} 

429 LIMIT {per_page} 

430 OFFSET {offset} 

431 """ 

432 

433 available_classes = get_available_classes() 

434 

435 class_info = next( 

436 (c for c in available_classes 

437 if c["uri"] == selected_class and c.get("shape") == selected_shape), 

438 None 

439 ) 

440 total_count = class_info.get("count_numeric", 0) if class_info else 0 

441 

442 sparql.setQuery(entities_query) 

443 sparql.setReturnFormat(JSON) 

444 entities_results = sparql.query().convert() 

445 

446 entities = [] 

447 shape = selected_shape if selected_shape else determine_shape_for_classes([selected_class]) 

448 

449 for result in entities_results["results"]["bindings"]: 

450 subject_uri = result["subject"]["value"] 

451 entity_label = custom_filter.human_readable_entity( 

452 subject_uri, (selected_class, shape), None 

453 ) 

454 entities.append({"uri": subject_uri, "label": entity_label}) 

455 

456 return entities, total_count 

457 

458 

459def get_catalog_data( 

460 selected_class: str, 

461 page: int, 

462 per_page: int, 

463 sort_property: str = None, 

464 sort_direction: str = "ASC", 

465 selected_shape: str = None 

466) -> dict: 

467 """ 

468 Get catalog data with pagination and sorting. 

469 

470 Args: 

471 selected_class (str): Selected class URI 

472 page (int): Current page number 

473 per_page (int): Items per page 

474 sort_property (str, optional): Property to sort by 

475 sort_direction (str, optional): Sort direction ('ASC' or 'DESC') 

476 selected_shape (str, optional): URI of the shape to use for sorting rules 

477 

478 Returns: 

479 dict: Catalog data including entities, pagination info, and sort settings 

480 """ 

481 

482 entities = [] 

483 total_count = 0 

484 sortable_properties = [] 

485 

486 if selected_class: 

487 sortable_properties = get_sortable_properties( 

488 (selected_class, selected_shape) 

489 ) 

490 

491 if not sort_property and sortable_properties: 

492 sort_property = sortable_properties[0]["property"] 

493 

494 entities, total_count = get_entities_for_class( 

495 selected_class, page, per_page, sort_property, sort_direction, selected_shape 

496 ) 

497 

498 return { 

499 "entities": entities, 

500 "total_pages": ( 

501 (total_count + per_page - 1) // per_page if total_count > 0 else 0 

502 ), 

503 "current_page": page, 

504 "per_page": per_page, 

505 "total_count": total_count, 

506 "sort_property": sort_property, 

507 "sort_direction": sort_direction, 

508 "sortable_properties": sortable_properties, 

509 "selected_class": selected_class, 

510 "selected_shape": selected_shape, 

511 } 

512 

513 

514def fetch_data_graph_for_subject(subject: str) -> Graph | Dataset: 

515 """ 

516 Fetch all triples/quads associated with a subject from the dataset. 

517 Handles both triplestore and quadstore cases appropriately. 

518 

519 Args: 

520 subject (str): The URI of the subject to fetch data for 

521 

522 Returns: 

523 Graph|Dataset: A graph containing all triples/quads for the subject 

524 """ 

525 g = Dataset() if get_dataset_is_quadstore() else Graph() 

526 sparql = get_sparql() 

527 

528 if is_virtuoso(): 

529 # For virtuoso we need to explicitly query the graph 

530 query = f""" 

531 SELECT ?predicate ?object ?g WHERE {{ 

532 GRAPH ?g {{ 

533 <{subject}> ?predicate ?object. 

534 }} 

535 FILTER(?g NOT IN (<{'>, <'.join(VIRTUOSO_EXCLUDED_GRAPHS)}>)) 

536 }} 

537 """ 

538 else: 

539 if get_dataset_is_quadstore(): 

540 # For non-virtuoso quadstore, we need to query all graphs 

541 query = f""" 

542 SELECT ?predicate ?object ?g WHERE {{ 

543 GRAPH ?g {{ 

544 <{subject}> ?predicate ?object. 

545 }} 

546 }} 

547 """ 

548 else: 

549 # For regular triplestore 

550 query = f""" 

551 SELECT ?predicate ?object WHERE {{ 

552 <{subject}> ?predicate ?object. 

553 }} 

554 """ 

555 

556 sparql.setQuery(query) 

557 sparql.setReturnFormat(JSON) 

558 query_results = sparql.query().convert() 

559 results = query_results.get("results", {}).get("bindings", []) 

560 

561 for result in results: 

562 # Create the appropriate value (Literal or URIRef) 

563 obj_data = result["object"] 

564 if obj_data["type"] in {"literal", "typed-literal"}: 

565 if "datatype" in obj_data: 

566 value = Literal( 

567 obj_data["value"], datatype=URIRef(obj_data["datatype"]) 

568 ) 

569 else: 

570 # Create literal without explicit datatype to match Reader.import_entities_from_triplestore 

571 value = Literal(obj_data["value"]) 

572 else: 

573 value = URIRef(obj_data["value"]) 

574 

575 # Add triple/quad based on store type 

576 if get_dataset_is_quadstore(): 

577 graph_uri = URIRef(result["g"]["value"]) 

578 g.add( 

579 ( 

580 URIRef(subject), 

581 URIRef(result["predicate"]["value"]), 

582 value, 

583 graph_uri, 

584 ) 

585 ) 

586 else: 

587 g.add((URIRef(subject), URIRef(result["predicate"]["value"]), value)) 

588 

589 return g 

590 

591 

592def parse_sparql_update(query) -> dict: 

593 parsed = parseUpdate(query) 

594 translated = translateUpdate(parsed).algebra 

595 modifications = {} 

596 

597 def extract_quads(quads): 

598 result = [] 

599 for graph, triples in quads.items(): 

600 for triple in triples: 

601 result.append((triple[0], triple[1], triple[2])) 

602 return result 

603 

604 for operation in translated: 

605 if operation.name == "DeleteData": 

606 if hasattr(operation, "quads") and operation.quads: 

607 deletions = extract_quads(operation.quads) 

608 else: 

609 deletions = operation.triples 

610 if deletions: 

611 modifications.setdefault("Deletions", list()).extend(deletions) 

612 elif operation.name == "InsertData": 

613 if hasattr(operation, "quads") and operation.quads: 

614 additions = extract_quads(operation.quads) 

615 else: 

616 additions = operation.triples 

617 if additions: 

618 modifications.setdefault("Additions", list()).extend(additions) 

619 

620 return modifications 

621 

622 

623def fetch_current_state_with_related_entities( 

624 provenance: dict, 

625) -> Graph | Dataset: 

626 """ 

627 Fetch the current state of an entity and all its related entities known from provenance. 

628 

629 Args: 

630 provenance (dict): Dictionary containing provenance metadata for main entity and related entities 

631 

632 Returns: 

633 Dataset: A graph containing the current state of all entities 

634 """ 

635 combined_graph = Dataset() if get_dataset_is_quadstore() else Graph() 

636 

637 # Fetch state for all entities mentioned in provenance 

638 for entity_uri in provenance.keys(): 

639 current_graph = fetch_data_graph_for_subject(entity_uri) 

640 

641 if get_dataset_is_quadstore(): 

642 for quad in current_graph.quads(): 

643 combined_graph.add(quad) 

644 else: 

645 for triple in current_graph: 

646 combined_graph.add(triple) 

647 

648 return combined_graph 

649 

650 

651def get_deleted_entities_with_filtering( 

652 page=1, 

653 per_page=50, 

654 sort_property="deletionTime", 

655 sort_direction="DESC", 

656 selected_class=None, 

657 selected_shape=None, 

658): 

659 """ 

660 Fetch and process deleted entities from the provenance graph, with filtering and sorting. 

661 """ 

662 sortable_properties = [ 

663 {"property": "deletionTime", "displayName": "Deletion Time", "sortType": "date"} 

664 ] 

665 provenance_sparql = get_provenance_sparql() 

666 custom_filter = get_custom_filter() 

667 

668 prov_query = """ 

669 SELECT DISTINCT ?entity ?lastSnapshot ?deletionTime ?agent ?lastValidSnapshotTime 

670 WHERE { 

671 ?lastSnapshot a <http://www.w3.org/ns/prov#Entity> ; 

672 <http://www.w3.org/ns/prov#specializationOf> ?entity ; 

673 <http://www.w3.org/ns/prov#generatedAtTime> ?deletionTime ; 

674 <http://www.w3.org/ns/prov#invalidatedAtTime> ?invalidationTime ; 

675 <http://www.w3.org/ns/prov#wasDerivedFrom> ?lastValidSnapshot. 

676 

677 ?lastValidSnapshot <http://www.w3.org/ns/prov#generatedAtTime> ?lastValidSnapshotTime . 

678 

679 OPTIONAL { ?lastSnapshot <http://www.w3.org/ns/prov#wasAttributedTo> ?agent . } 

680 

681 FILTER NOT EXISTS { 

682 ?laterSnapshot <http://www.w3.org/ns/prov#wasDerivedFrom> ?lastSnapshot . 

683 } 

684 } 

685 """ 

686 provenance_sparql.setQuery(prov_query) 

687 provenance_sparql.setReturnFormat(JSON) 

688 prov_results = provenance_sparql.query().convert() 

689 

690 results_bindings = prov_results["results"]["bindings"] 

691 if not results_bindings: 

692 return [], [], None, None, [], 0 

693 

694 deleted_entities = [] 

695 max_workers = max(1, min(os.cpu_count() or 4, len(results_bindings))) 

696 with ProcessPoolExecutor(max_workers=max_workers) as executor: 

697 future_to_entity = { 

698 executor.submit(process_deleted_entity, result, sortable_properties): result 

699 for result in results_bindings 

700 } 

701 for future in as_completed(future_to_entity): 

702 entity_info = future.result() 

703 if entity_info is not None: 

704 deleted_entities.append(entity_info) 

705 

706 class_counts = {} 

707 for entity in deleted_entities: 

708 for type_uri in entity["entity_types"]: 

709 class_counts[type_uri] = class_counts.get(type_uri, 0) + 1 

710 

711 available_classes = [ 

712 { 

713 "uri": class_uri, 

714 "label": custom_filter.human_readable_class((class_uri, determine_shape_for_classes([class_uri]))), 

715 "count": count, 

716 } 

717 for class_uri, count in class_counts.items() 

718 ] 

719 

720 reverse_sort = sort_direction.upper() == "DESC" 

721 if sort_property == "deletionTime": 

722 deleted_entities.sort(key=lambda e: e["deletionTime"], reverse=reverse_sort) 

723 else: 

724 deleted_entities.sort( 

725 key=lambda e: e["sort_values"].get(sort_property, "").lower(), 

726 reverse=reverse_sort, 

727 ) 

728 

729 available_classes.sort(key=lambda x: x["label"].lower()) 

730 if not selected_class and available_classes: 

731 selected_class = available_classes[0]["uri"] 

732 

733 if selected_class: 

734 if selected_shape is None: 

735 selected_shape = determine_shape_for_classes([selected_class]) 

736 entity_key = (selected_class, selected_shape) 

737 sortable_properties.extend( 

738 get_sortable_properties(entity_key) 

739 ) 

740 

741 if selected_class: 

742 filtered_entities = [ 

743 entity 

744 for entity in deleted_entities 

745 if selected_class in entity["entity_types"] 

746 ] 

747 else: 

748 filtered_entities = deleted_entities 

749 

750 total_count = len(filtered_entities) 

751 offset = (page - 1) * per_page 

752 paginated_entities = filtered_entities[offset : offset + per_page] 

753 

754 return paginated_entities, available_classes, selected_class, selected_shape, sortable_properties, total_count 

755 

756 

757def process_deleted_entity(result: dict, sortable_properties: list) -> dict | None: 

758 """ 

759 Process a single deleted entity, filtering by visible classes. 

760 """ 

761 change_tracking_config = get_change_tracking_config() 

762 custom_filter = get_custom_filter() 

763 

764 entity_uri = result["entity"]["value"] 

765 last_valid_snapshot_time = result["lastValidSnapshotTime"]["value"] 

766 

767 agnostic_entity = AgnosticEntity( 

768 res=entity_uri, config=change_tracking_config, include_related_objects=True, include_merged_entities=True, include_reverse_relations=True 

769 ) 

770 state, _, _ = agnostic_entity.get_state_at_time( 

771 (last_valid_snapshot_time, last_valid_snapshot_time) 

772 ) 

773 

774 if entity_uri not in state: 

775 return None 

776 

777 last_valid_time = convert_to_datetime(last_valid_snapshot_time, stringify=True) 

778 last_valid_state: Dataset = state[entity_uri][last_valid_time] 

779 

780 entity_types = [ 

781 str(o) 

782 for s, p, o in get_triples_from_graph(last_valid_state, (URIRef(entity_uri), RDF.type, None)) 

783 ] 

784 highest_priority_type = get_highest_priority_class(entity_types) 

785 shape = determine_shape_for_classes([highest_priority_type]) 

786 visible_types = [t for t in entity_types if is_entity_type_visible((t, determine_shape_for_classes([t])))] 

787 if not visible_types: 

788 return None 

789 

790 sort_values = {} 

791 for prop in sortable_properties: 

792 prop_uri = prop["property"] 

793 values = [ 

794 str(o) 

795 for s, p, o in get_triples_from_graph( 

796 last_valid_state, (URIRef(entity_uri), URIRef(prop_uri), None) 

797 ) 

798 ] 

799 sort_values[prop_uri] = values[0] if values else "" 

800 

801 return { 

802 "uri": entity_uri, 

803 "deletionTime": result["deletionTime"]["value"], 

804 "deletedBy": custom_filter.format_agent_reference( 

805 result.get("agent", {}).get("value", "") 

806 ), 

807 "lastValidSnapshotTime": last_valid_snapshot_time, 

808 "type": custom_filter.human_readable_predicate( 

809 highest_priority_type, (highest_priority_type, shape) 

810 ), 

811 "label": custom_filter.human_readable_entity( 

812 entity_uri, (highest_priority_type, shape), last_valid_state 

813 ), 

814 "entity_types": visible_types, 

815 "sort_values": sort_values, 

816 } 

817 

818 

819def find_orphaned_entities(subject, entity_type, predicate=None, object_value=None): 

820 """ 

821 Find entities that would become orphaned after deleting a triple or an entire entity, 

822 including intermediate relation entities. 

823 

824 An entity is considered orphaned if: 

825 1. It has no incoming references from other entities (except from the entity being deleted) 

826 2. It does not reference any entities that are subjects of other triples 

827 

828 For intermediate relations, an entity is also considered orphaned if: 

829 1. It connects to the entity being deleted 

830 2. It has no other valid connections after the deletion 

831 3. It is directly involved in the deletion operation (if predicate and object_value are specified) 

832 

833 Args: 

834 subject (str): The URI of the subject being deleted 

835 entity_type (str): The type of the entity being deleted 

836 predicate (str, optional): The predicate being deleted 

837 object_value (str, optional): The object value being deleted 

838 

839 Returns: 

840 tuple: Lists of (orphaned_entities, intermediate_orphans) 

841 """ 

842 sparql = get_sparql() 

843 display_rules = get_display_rules() 

844 

845 intermediate_classes = set() 

846 

847 for rule in display_rules: 

848 if "target" in rule and "class" in rule["target"] and rule["target"]["class"] == entity_type: 

849 for prop in rule.get("displayProperties", []): 

850 if "intermediateRelation" in prop: 

851 intermediate_classes.add(prop["intermediateRelation"]["class"]) 

852 

853 orphan_query = f""" 

854 SELECT DISTINCT ?entity ?type 

855 WHERE {{ 

856 {f"<{subject}> <{predicate}> ?entity ." if predicate and object_value else ""} 

857 {f"FILTER(?entity = <{object_value}>)" if predicate and object_value else ""} 

858  

859 # If no specific predicate, get all connected entities 

860 {f"<{subject}> ?p ?entity ." if not predicate else ""} 

861  

862 FILTER(isIRI(?entity)) 

863 ?entity a ?type . 

864  

865 # No incoming references from other entities 

866 FILTER NOT EXISTS {{ 

867 ?other ?anyPredicate ?entity . 

868 FILTER(?other != <{subject}>) 

869 }} 

870  

871 # No outgoing references to active entities 

872 FILTER NOT EXISTS {{ 

873 ?entity ?outgoingPredicate ?connectedEntity . 

874 ?connectedEntity ?furtherPredicate ?furtherObject . 

875 {f"FILTER(?connectedEntity != <{subject}>)" if not predicate else ""} 

876 }} 

877  

878 # Exclude intermediate relation entities 

879 FILTER(?type NOT IN (<{f">, <".join(intermediate_classes)}>)) 

880 }} 

881 """ 

882 

883 # Query to find orphaned intermediate relations 

884 if predicate and object_value: 

885 intermediate_query = f""" 

886 SELECT DISTINCT ?entity ?type 

887 WHERE {{ 

888 <{object_value}> a ?type . 

889 FILTER(?type IN (<{f">, <".join(intermediate_classes)}>))  

890 BIND(<{object_value}> AS ?entity) 

891 }} 

892 """ 

893 else: 

894 # Se stiamo cancellando l'intera entità, trova tutte le entità intermedie collegate 

895 intermediate_query = f""" 

896 SELECT DISTINCT ?entity ?type 

897 WHERE {{ 

898 # Find intermediate relations connected to the entity being deleted 

899 {{ 

900 <{subject}> ?p ?entity . 

901 ?entity a ?type . 

902 FILTER(?type IN (<{f">, <".join(intermediate_classes)}>)) 

903 }} UNION {{ 

904 ?entity ?p <{subject}> . 

905 ?entity a ?type . 

906 FILTER(?type IN (<{f">, <".join(intermediate_classes)}>))