Coverage for heritrace/extensions.py: 99%

1# SPDX-FileCopyrightText: 2024-2026 Arcangelo Massari <arcangelo.massari@unibo.it> 

2# 

3# SPDX-License-Identifier: ISC 

4 

5import json 

6import os 

7from collections import defaultdict 

8from dataclasses import dataclass 

9from datetime import datetime, timedelta, timezone 

10from pathlib import Path 

11from typing import cast 

12from urllib.parse import urlparse, urlunparse 

13 

14import yaml 

15from flask import Flask, current_app, g, redirect, session, url_for 

16from flask_babel import Babel 

17from flask_login import LoginManager 

18from flask_login.signals import user_loaded_from_cookie 

19from rdflib import Graph 

20from rdflib_ocdm.counter_handler.counter_handler import CounterHandler 

21from redis import Redis 

22from redis.exceptions import RedisError 

23from SPARQLWrapper import JSON 

24from time_agnostic_library.support import generate_config_file 

25 

26from heritrace.counter_handler import CounterInitializationPolicy 

27from heritrace.models import User 

28from heritrace.services.resource_lock_manager import ResourceLockManager 

29from heritrace.sparql import SPARQLWrapperWithRetry, get_sparql_bindings, select_results 

30from heritrace.uri_generator.uri_generator import CounterBasedURIGenerator 

31from heritrace.utils.filters import Filter, split_namespace 

32 

33 

34@dataclass(frozen=True) 

35class AppState: 

36 dataset_endpoint: str 

37 provenance_endpoint: str 

38 sparql: SPARQLWrapperWithRetry 

39 provenance_sparql: SPARQLWrapperWithRetry 

40 change_tracking_config: dict 

41 custom_filter: Filter 

42 display_rules: list[dict] 

43 form_fields_cache: dict 

44 dataset_is_quadstore: bool 

45 shacl_graph: Graph 

46 classes_with_multiple_shapes: set[str] 

47 display_rules_use_inverse_relations: bool 

48 

49 

50def get_app_state() -> AppState: 

51 return current_app.extensions["heritrace"] 

52 

53 

54def init_extensions( 

55 app: Flask, babel: Babel, login_manager: LoginManager, redis: Redis 

56) -> None: 

57 from heritrace.utils.display_rules_utils import ( # noqa: PLC0415 

58 uses_inverse_relations, 

59 ) 

60 

61 babel.init_app( 

62 app=app, 

63 locale_selector=lambda: session.get("lang", "en"), 

64 default_translation_directories=str( 

65 Path(__file__).resolve().parent.parent / "babel" / "translations" 

66 ), 

67 ) 

68 

69 init_login_manager(app, login_manager) 

70 

71 ( 

72 dataset_endpoint, 

73 provenance_endpoint, 

74 sparql, 

75 provenance_sparql, 

76 change_tracking_config, 

77 ) = init_sparql_services(app) 

78 initialize_counter_handler(app, redis, sparql, provenance_sparql) 

79 

80 app.extensions["heritrace"] = AppState( 

81 dataset_endpoint=dataset_endpoint, 

82 provenance_endpoint=provenance_endpoint, 

83 sparql=sparql, 

84 provenance_sparql=provenance_sparql, 

85 change_tracking_config=change_tracking_config, 

86 custom_filter=cast("Filter", None), 

87 display_rules=[], 

88 form_fields_cache={}, 

89 dataset_is_quadstore=False, 

90 shacl_graph=Graph(), 

91 classes_with_multiple_shapes=set(), 

92 display_rules_use_inverse_relations=False, 

93 ) 

94 

95 ( 

96 display_rules, 

97 form_fields_cache, 

98 dataset_is_quadstore, 

99 shacl_graph, 

100 classes_with_multiple_shapes, 

101 ) = initialize_global_variables(app) 

102 custom_filter = init_filters(app, display_rules, dataset_endpoint) 

103 init_request_handlers(app, redis) 

104 

105 app.extensions["heritrace"] = AppState( 

106 dataset_endpoint=dataset_endpoint, 

107 provenance_endpoint=provenance_endpoint, 

108 sparql=sparql, 

109 provenance_sparql=provenance_sparql, 

110 change_tracking_config=change_tracking_config, 

111 custom_filter=custom_filter, 

112 display_rules=display_rules, 

113 form_fields_cache=form_fields_cache, 

114 dataset_is_quadstore=dataset_is_quadstore, 

115 shacl_graph=shacl_graph, 

116 classes_with_multiple_shapes=classes_with_multiple_shapes, 

117 display_rules_use_inverse_relations=uses_inverse_relations(display_rules), 

118 ) 

119 app.extensions["login_manager"] = login_manager 

120 app.extensions["redis_client"] = redis 

121 

122 

123def init_login_manager(app: Flask, login_manager: LoginManager) -> None: 

124 login_manager.init_app(app) 

125 login_manager.login_view = "auth.login" # type: ignore[reportAttributeAccessIssue] 

126 login_manager.unauthorized_handler(lambda: redirect(url_for("auth.login"))) 

127 

128 @login_manager.user_loader 

129 def load_user(user_id: str) -> User: 

130 user_name = session.get("user_name", "Unknown User") 

131 return User(user_id=user_id, name=user_name, orcid=user_id) 

132 

133 @user_loaded_from_cookie.connect 

134 def rotate_session_token(_sender: object, _user: object) -> None: 

135 session.modified = True 

136 

137 

138def initialize_change_tracking_config( 

139 app: Flask, 

140 adjusted_dataset_endpoint: str | None = None, 

141 adjusted_provenance_endpoint: str | None = None, 

142) -> dict: 

143 config_needs_generation = False 

144 config_path = None 

145 config = None 

146 

147 if "CHANGE_TRACKING_CONFIG" in app.config: 

148 config_path = app.config["CHANGE_TRACKING_CONFIG"] 

149 if not Path(config_path).exists(): 

150 app.logger.warning( 

151 "Change tracking configuration file not found at specified path: %s", 

152 config_path, 

153 ) 

154 config_needs_generation = True 

155 else: 

156 config_needs_generation = True 

157 config_path = str(Path(app.instance_path) / "change_tracking_config.json") 

158 Path(app.instance_path).mkdir(parents=True, exist_ok=True) 

159 

160 if config_needs_generation: 

161 dataset_urls = [adjusted_dataset_endpoint] if adjusted_dataset_endpoint else [] 

162 provenance_urls = ( 

163 [adjusted_provenance_endpoint] if adjusted_provenance_endpoint else [] 

164 ) 

165 

166 db_triplestore = app.config.get("DATASET_DB_TRIPLESTORE", "").lower() 

167 text_index_enabled = app.config.get("DATASET_DB_TEXT_INDEX_ENABLED", False) 

168 

169 blazegraph_search = db_triplestore == "blazegraph" and text_index_enabled 

170 fuseki_search = db_triplestore == "fuseki" and text_index_enabled 

171 virtuoso_search = db_triplestore == "virtuoso" and text_index_enabled 

172 

173 graphdb_connector = "" # TODO(@arcangelo-massari): Add graphdb support 

174 # https://github.com/opencitations/heritrace/issues/1 

175 

176 try: 

177 config = generate_config_file( 

178 config_path=config_path, 

179 dataset_urls=dataset_urls, 

180 dataset_dirs=app.config.get("DATASET_DIRS", []), 

181 dataset_is_quadstore=app.config.get("DATASET_IS_QUADSTORE", False), 

182 provenance_urls=provenance_urls, 

183 provenance_is_quadstore=app.config.get( 

184 "PROVENANCE_IS_QUADSTORE", False 

185 ), 

186 provenance_dirs=app.config.get("PROVENANCE_DIRS", []), 

187 blazegraph_full_text_search=blazegraph_search, 

188 fuseki_full_text_search=fuseki_search, 

189 virtuoso_full_text_search=virtuoso_search, 

190 graphdb_connector_name=graphdb_connector, 

191 ) 

192 app.logger.info( 

193 "Generated new change tracking configuration at: %s", config_path 

194 ) 

195 except OSError as e: 

196 msg = f"Failed to generate change tracking configuration: {e!s}" 

197 raise RuntimeError(msg) from e 

198 

199 try: 

200 if not config: 

201 with Path(config_path).open(encoding="utf8") as f: 

202 config = json.load(f) 

203 

204 except json.JSONDecodeError as e: 

205 msg = f"Invalid change tracking configuration JSON at {config_path}: {e!s}" 

206 raise RuntimeError(msg) from e 

207 except OSError as e: 

208 msg = f"Error reading change tracking configuration at {config_path}: {e!s}" 

209 raise RuntimeError(msg) from e 

210 

211 app.config["CHANGE_TRACKING_CONFIG"] = config_path 

212 return config 

213 

214 

215def need_initialization(app: Flask, redis: Redis) -> bool: 

216 redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0") 

217 is_external_redis = redis_url != "redis://localhost:6379/0" 

218 

219 if is_external_redis: 

220 app.logger.info( 

221 "Using external Redis at %s - skipping counter initialization", redis_url 

222 ) 

223 return False 

224 

225 cache_validity_days = app.config["CACHE_VALIDITY_DAYS"] 

226 

227 try: 

228 last_init_raw: str | None = redis.get("heritrace:last_initialization") # type: ignore[assignment] 

229 if not last_init_raw: 

230 return True 

231 

232 last_init = datetime.fromisoformat(last_init_raw) 

233 return datetime.now(tz=timezone.utc) - last_init > timedelta( 

234 days=cache_validity_days 

235 ) 

236 except (RedisError, ValueError): 

237 return True 

238 

239 

240def update_cache(_app: Flask, redis: Redis) -> None: 

241 current_time = datetime.now(tz=timezone.utc).isoformat() 

242 redis.set("heritrace:last_initialization", current_time) 

243 redis.set("heritrace:cache_version", "1.0") 

244 

245 

246def initialize_counter_handler( 

247 app: Flask, 

248 redis: Redis, 

249 sparql: SPARQLWrapperWithRetry, 

250 provenance_sparql: SPARQLWrapperWithRetry, 

251) -> None: 

252 counter_handler = cast("CounterHandler", app.config["COUNTER_HANDLER"]) 

253 if ( 

254 isinstance(counter_handler, CounterInitializationPolicy) 

255 and not counter_handler.should_initialize_from_triplestore() 

256 ): 

257 return 

258 

259 if not need_initialization(app, redis): 

260 return 

261 

262 uri_generator = app.config["URI_GENERATOR"] 

263 if isinstance(uri_generator, CounterBasedURIGenerator): 

264 uri_generator.initialize_counters(sparql) 

265 

266 prov_query = """ 

267 SELECT ?entity (COUNT(DISTINCT ?snapshot) as ?count) 

268 WHERE { 

269 ?snapshot a <http://www.w3.org/ns/prov#Entity> ; 

270 <http://www.w3.org/ns/prov#specializationOf> ?entity . 

271 OPTIONAL { 

272 ?snapshot <http://www.w3.org/ns/prov#wasDerivedFrom> ?prev . 

273 } 

274 } 

275 GROUP BY ?entity 

276 """ 

277 

278 provenance_sparql.setQuery(prov_query) 

279 provenance_sparql.setReturnFormat(JSON) 

280 prov_bindings = get_sparql_bindings(provenance_sparql.query().convert()) 

281 

282 for result in prov_bindings: 

283 entity = result["entity"]["value"] 

284 count = int(result["count"]["value"]) 

285 counter_handler.set_counter(count, entity) 

286 

287 update_cache(app, redis) 

288 

289 

290def identify_classes_with_multiple_shapes( 

291 display_rules: list[dict], shacl_graph: Graph 

292) -> set[str]: 

293 if not display_rules or not shacl_graph: 

294 return set() 

295 

296 from heritrace.utils.display_rules_utils import ( # noqa: PLC0415 

297 is_entity_type_visible, 

298 ) 

299 

300 class_to_shapes: defaultdict[str, set[str]] = defaultdict(set) 

301 

302 for rule in display_rules: 

303 target = rule.get("target", {}) 

304 

305 if "class" in target: 

306 class_uri = target["class"] 

307 query_string = f""" 

308 SELECT DISTINCT ?shape WHERE {{ 

309 ?shape <http://www.w3.org/ns/shacl#targetClass> <{class_uri}> . 

310 }} 

311 """ 

312 results = shacl_graph.query(query_string) 

313 for row in select_results(results): 

314 shape_uri = str(row.shape) 

315 entity_key = (class_uri, shape_uri) 

316 if is_entity_type_visible(entity_key): 

317 class_to_shapes[class_uri].add(shape_uri) 

318 

319 elif "shape" in target: 

320 shape_uri = target["shape"] 

321 query_string = f""" 

322 SELECT DISTINCT ?class WHERE {{ 

323 <{shape_uri}> <http://www.w3.org/ns/shacl#targetClass> ?class . 

324 }} 

325 """ 

326 results = shacl_graph.query(query_string) 

327 for row in select_results(results): 

328 class_uri = str(row[0]) 

329 entity_key = (class_uri, shape_uri) 

330 if is_entity_type_visible(entity_key): 

331 class_to_shapes[class_uri].add(shape_uri) 

332 

333 return { 

334 class_uri for class_uri, shapes in class_to_shapes.items() if len(shapes) > 1 

335 } 

336 

337 

338def initialize_global_variables( 

339 app: Flask, 

340) -> tuple[list[dict], dict, bool, Graph, set[str]]: 

341 try: 

342 dataset_is_quadstore = app.config.get("DATASET_IS_QUADSTORE", False) 

343 

344 display_rules: list[dict] = [] 

345 if app.config.get("DISPLAY_RULES_PATH"): 

346 if not app.config["DISPLAY_RULES_PATH"].exists(): 

347 app.logger.warning( 

348 "Display rules file not found at: %s", 

349 app.config["DISPLAY_RULES_PATH"], 

350 ) 

351 else: 

352 try: 

353 with app.config["DISPLAY_RULES_PATH"].open() as f: 

354 yaml_content = yaml.safe_load(f) 

355 display_rules = yaml_content["rules"] 

356 except yaml.YAMLError as e: 

357 app.logger.exception("Error loading display rules") 

358 msg = f"Failed to load display rules: {e!s}" 

359 raise RuntimeError(msg) from e 

360 

361 shacl_graph = Graph() 

362 form_fields_cache: dict = {} 

363 if app.config.get("SHACL_PATH"): 

364 if not app.config["SHACL_PATH"].exists(): 

365 app.logger.warning( 

366 "SHACL file not found at: %s", app.config["SHACL_PATH"] 

367 ) 

368 else: 

369 try: 

370 shacl_graph.parse(source=app.config["SHACL_PATH"], format="turtle") 

371 

372 from heritrace.utils.shacl_utils import ( # noqa: PLC0415 

373 get_form_fields_from_shacl, 

374 ) 

375 

376 form_fields_cache = get_form_fields_from_shacl( 

377 shacl_graph, display_rules, app=app 

378 ) 

379 except (OSError, ValueError) as e: 

380 app.logger.exception("Error initializing form fields from SHACL") 

381 msg = f"Failed to initialize form fields: {e!s}" 

382 raise RuntimeError(msg) from e 

383 

384 classes_with_multiple_shapes = identify_classes_with_multiple_shapes( 

385 display_rules, shacl_graph 

386 ) 

387 

388 app.logger.info("Global variables initialized successfully") 

389 

390 except RuntimeError: 

391 raise 

392 except (OSError, yaml.YAMLError, ValueError) as e: 

393 app.logger.exception("Error during global variables initialization") 

394 msg = f"Global variables initialization failed: {e!s}" 

395 raise RuntimeError(msg) from e 

396 else: 

397 return ( 

398 display_rules, 

399 form_fields_cache, 

400 dataset_is_quadstore, 

401 shacl_graph, 

402 classes_with_multiple_shapes, 

403 ) 

404 

405 

406def init_sparql_services( 

407 app: Flask, 

408) -> tuple[str, str, SPARQLWrapperWithRetry, SPARQLWrapperWithRetry, dict]: 

409 dataset_endpoint = adjust_endpoint_url(app.config["DATASET_DB_URL"]) 

410 provenance_endpoint = adjust_endpoint_url(app.config["PROVENANCE_DB_URL"]) 

411 

412 sparql = SPARQLWrapperWithRetry(dataset_endpoint, timeout=30.0) 

413 provenance_sparql = SPARQLWrapperWithRetry(provenance_endpoint, timeout=30.0) 

414 

415 change_tracking_config = initialize_change_tracking_config( 

416 app, 

417 adjusted_dataset_endpoint=dataset_endpoint, 

418 adjusted_provenance_endpoint=provenance_endpoint, 

419 ) 

420 

421 return ( 

422 dataset_endpoint, 

423 provenance_endpoint, 

424 sparql, 

425 provenance_sparql, 

426 change_tracking_config, 

427 ) 

428 

429 

430def init_filters( 

431 app: Flask, display_rules: list[dict], dataset_endpoint: str 

432) -> Filter: 

433 with (Path(__file__).parent / "utils" / "context.json").open() as config_file: 

434 context = json.load(config_file)["@context"] 

435 

436 custom_filter = Filter(context, display_rules or None, dataset_endpoint) 

437 

438 app.jinja_env.filters["human_readable_predicate"] = ( 

439 custom_filter.human_readable_predicate 

440 ) 

441 app.jinja_env.filters["human_readable_class"] = custom_filter.human_readable_class 

442 app.jinja_env.filters["human_readable_entity"] = custom_filter.human_readable_entity 

443 app.jinja_env.filters["human_readable_primary_source"] = ( 

444 custom_filter.human_readable_primary_source 

445 ) 

446 app.jinja_env.filters["format_datetime"] = custom_filter.human_readable_datetime 

447 app.jinja_env.filters["split_ns"] = split_namespace 

448 app.jinja_env.filters["format_source_reference"] = ( 

449 custom_filter.format_source_reference 

450 ) 

451 app.jinja_env.filters["format_agent_reference"] = ( 

452 custom_filter.format_agent_reference 

453 ) 

454 return custom_filter 

455 

456 

457def init_request_handlers(app: Flask, redis: Redis) -> None: 

458 @app.before_request 

459 def initialize_lock_manager() -> None: 

460 if not hasattr(g, "resource_lock_manager"): 

461 g.resource_lock_manager = ResourceLockManager(redis) 

462 

463 @app.teardown_appcontext 

464 def close_redis_connection(_error: BaseException | None) -> None: 

465 if hasattr(g, "resource_lock_manager"): 

466 del g.resource_lock_manager 

467 

468 

469def adjust_endpoint_url(url: str) -> str: 

470 if not running_in_docker(): 

471 return url 

472 

473 local_patterns = ["localhost", "127.0.0.1", "0.0.0.0"] 

474 parsed_url = urlparse(url) 

475 

476 if any(pattern in parsed_url.netloc for pattern in local_patterns): 

477 netloc_parts = parsed_url.netloc.split(":") 

478 new_netloc = ( 

479 f"host.docker.internal:{netloc_parts[1]}" 

480 if len(netloc_parts) > 1 

481 else "host.docker.internal" 

482 ) 

483 url_parts = list(parsed_url) 

484 url_parts[1] = new_netloc 

485 return urlunparse(url_parts) 

486 

487 return url 

488 

489 

490def running_in_docker() -> bool: 

491 return Path("/.dockerenv").exists() 

492 

493 

494def get_dataset_endpoint() -> str: 

495 return get_app_state().dataset_endpoint 

496 

497 

498def get_sparql() -> SPARQLWrapperWithRetry: 

499 return get_app_state().sparql 

500 

501 

502def get_provenance_endpoint() -> str: 

503 return get_app_state().provenance_endpoint 

504 

505 

506def get_provenance_sparql() -> SPARQLWrapperWithRetry: 

507 return get_app_state().provenance_sparql 

508 

509 

510def get_counter_handler() -> CounterHandler: 

511 uri_generator = current_app.config.get("URI_GENERATOR") 

512 if not isinstance(uri_generator, CounterBasedURIGenerator): 

513 current_app.logger.error("CounterHandler not found in URIGenerator config.") 

514 msg = "CounterHandler is not available. Initialization might have failed." 

515 raise TypeError(msg) 

516 return uri_generator.counter_handler 

517 

518 

519def get_custom_filter() -> Filter: 

520 return get_app_state().custom_filter 

521 

522 

523def get_change_tracking_config() -> dict: 

524 return get_app_state().change_tracking_config 

525 

526 

527def get_display_rules() -> list[dict]: 

528 return get_app_state().display_rules 

529 

530 

531def get_form_fields() -> dict: 

532 return get_app_state().form_fields_cache 

533 

534 

535def get_dataset_is_quadstore() -> bool: 

536 return get_app_state().dataset_is_quadstore 

537 

538 

539def get_display_rules_use_inverse_relations() -> bool: 

540 return get_app_state().display_rules_use_inverse_relations 

541 

542 

543def get_shacl_graph() -> Graph: 

544 return get_app_state().shacl_graph 

545 

546 

547def get_classes_with_multiple_shapes() -> set[str]: 

548 return get_app_state().classes_with_multiple_shapes