Coverage for test/curator_test.py: 99%

1import csv 

2import shutil 

3import unittest 

4 

5from oc_ocdm import Storer 

6from SPARQLWrapper import POST, SPARQLWrapper 

7from rdflib import Graph, ConjunctiveGraph 

8import redis 

9from oc_ocdm.counter_handler.redis_counter_handler import RedisCounterHandler 

10 

11from oc_meta.core.creator import Creator 

12from oc_meta.core.curator import * 

13from oc_meta.lib.file_manager import get_csv_data 

14from oc_meta.lib.finder import ResourceFinder 

15from oc_meta.plugins.multiprocess.resp_agents_curator import RespAgentsCurator 

16 

17SERVER = 'http://127.0.0.1:8805/sparql' 

18BASE_DIR = os.path.join('test') 

19MANUAL_DATA_CSV = f'{BASE_DIR}/manual_data.csv' 

20MANUAL_DATA_RDF = f'{BASE_DIR}/testcases/ts/testcase_ts-13.ttl' 

21REAL_DATA_CSV = os.path.join(BASE_DIR, 'real_data.csv') 

22REAL_DATA_RDF = f'{BASE_DIR}/testcases/ts/real_data.nt' 

23REAL_DATA_RDF_WITH_PROV = f'{BASE_DIR}/testcases/ts/real_data_with_prov.nq' 

24BASE_IRI = 'https://w3id.org/oc/meta/' 

25CURATOR_COUNTER_DIR = f'{BASE_DIR}/curator_counter' 

26OUTPUT_DIR = f'{BASE_DIR}/output' 

27PROV_CONFIG = f'{BASE_DIR}/prov_config.json' 

28 

29# Redis configuration 

30REDIS_HOST = 'localhost' 

31REDIS_PORT = 6379 

32REDIS_DB = 5 

33 

34def get_path(path:str) -> str: 

35 # absolute_path:str = os.path.abspath(path) 

36 universal_path = path.replace('\\', '/') 

37 return universal_path 

38 

39def reset_redis_counters(): 

40 redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB) 

41 redis_client.flushdb() 

42 

43def get_counter_handler(): 

44 return RedisCounterHandler(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB) 

45 

46def reset(): 

47 reset_redis_counters() 

48 

49def reset_server(server:str=SERVER) -> None: 

50 ts = SPARQLWrapper(server) 

51 for graph in {'https://w3id.org/oc/meta/br/', 'https://w3id.org/oc/meta/ra/', 'https://w3id.org/oc/meta/re/', 'https://w3id.org/oc/meta/id/', 'https://w3id.org/oc/meta/ar/', 'http://default.graph/'}: 

52 ts.setQuery(f'CLEAR GRAPH <{graph}>') 

53 ts.setMethod(POST) 

54 ts.query() 

55 

56def add_data_ts(server:str=SERVER, data_path:str=os.path.abspath(os.path.join('test', 'testcases', 'ts', 'real_data.nt')).replace('\\', '/'), batch_size:int=100, default_graph_uri=URIRef("http://default.graph/")): 

57 reset_server(server) 

58 f_path = get_path(data_path) 

59 

60 # Determina il formato del file 

61 file_extension = os.path.splitext(f_path)[1].lower() 

62 if file_extension == '.nt': 

63 g = Graph() 

64 g.parse(location=f_path, format='nt') 

65 elif file_extension == '.nq': 

66 g = ConjunctiveGraph() 

67 g.parse(location=f_path, format='nquads') 

68 elif file_extension == '.ttl': 

69 g = Graph() 

70 g.parse(location=f_path, format='turtle') 

71 else: 

72 raise ValueError(f"Unsupported file extension: {file_extension}") 

73 

74 triples_list = [] 

75 if file_extension in {'.nt', '.ttl'}: 

76 for subj, pred, obj in g: 

77 triples_list.append((subj, pred, obj, default_graph_uri)) 

78 elif file_extension == '.nq': 

79 for subj, pred, obj, ctx in g.quads((None, None, None, None)): 

80 triples_list.append((subj, pred, obj, ctx)) 

81 

82 for i in range(0, len(triples_list), batch_size): 

83 batch_triples = triples_list[i:i + batch_size] 

84 

85 triples_str = "" 

86 for subj, pred, obj, ctx in batch_triples: 

87 if ctx: 

88 triples_str += f"GRAPH {ctx.n3().replace('[', '').replace(']', '')} {{ {subj.n3()} {pred.n3()} {obj.n3()} }} " 

89 else: 

90 triples_str += f"{subj.n3()} {pred.n3()} {obj.n3()} . " 

91 

92 query = f"INSERT DATA {{ {triples_str} }}" 

93 

94 ts = SPARQLWrapper(server) 

95 ts.setQuery(query) 

96 ts.setMethod(POST) 

97 ts.query() 

98 

99def store_curated_data(curator_obj:Curator, server:str) -> None: 

100 creator_obj = Creator(curator_obj.data, SERVER, BASE_IRI, None, None, 'https://orcid.org/0000-0002-8420-0696', 

101 curator_obj.index_id_ra, curator_obj.index_id_br, curator_obj.re_index, 

102 curator_obj.ar_index, curator_obj.VolIss, preexisting_entities=set(), everything_everywhere_allatonce=Graph()) 

103 creator = creator_obj.creator(source=None) 

104 res_storer = Storer(creator) 

105 res_storer.upload_all(server, base_dir=None, batch_size=100) 

106 

107def prepare_to_test(data, name): 

108 reset_redis_counters() 

109 

110 reset_server(SERVER) 

111 if float(name) > 12: 

112 add_data_ts(SERVER, os.path.abspath(os.path.join('test', 'testcases', 'ts', 'testcase_ts-13.ttl')).replace('\\', '/')) 

113 

114 testcase_csv = get_path('test/testcases/testcase_data/testcase_' + name + '_data.csv') 

115 testcase_id_br = get_path('test/testcases/testcase_data/indices/' + name + '/index_id_br_' + name + '.csv') 

116 testcase_id_ra = get_path('test/testcases/testcase_data/indices/' + name + '/index_id_ra_' + name + '.csv') 

117 testcase_ar = get_path('test/testcases/testcase_data/indices/' + name + '/index_ar_' + name + '.csv') 

118 testcase_re = get_path('test/testcases/testcase_data/indices/' + name + '/index_re_' + name + '.csv') 

119 testcase_vi = get_path('test/testcases/testcase_data/indices/' + name + '/index_vi_' + name + '.json') 

120 

121 counter_handler = get_counter_handler() 

122 settings = {'normalize_titles': True} 

123 curator_obj = Curator(data, SERVER, prov_config=PROV_CONFIG, counter_handler=counter_handler, settings=settings) 

124 curator_obj.curator() 

125 testcase_csv = get_csv_data(testcase_csv) 

126 for csv in [testcase_csv, curator_obj.data]: 

127 for row in csv: 

128 row['id'] = sorted(row['id'].split()) 

129 testcase_id_br = get_csv_data(testcase_id_br) 

130 testcase_id_ra = get_csv_data(testcase_id_ra) 

131 testcase_ar = get_csv_data(testcase_ar) 

132 testcase_re = get_csv_data(testcase_re) 

133 for csv in [testcase_id_br, testcase_id_ra, testcase_ar, testcase_re, curator_obj.index_id_br, curator_obj.index_id_ra, curator_obj.ar_index, curator_obj.re_index]: 

134 try: 

135 csv.sort(key=lambda x:x['id']) 

136 except KeyError: 

137 try: 

138 csv.sort(key=lambda x:x['meta']) 

139 except KeyError: 

140 csv.sort(key=lambda x:x['br']) 

141 with open(testcase_vi) as json_file: 

142 testcase_vi = json.load(json_file) 

143 testcase = [testcase_csv, testcase_id_br, testcase_id_ra, testcase_ar, testcase_re, testcase_vi] 

144 data_curated = [curator_obj.data, curator_obj.index_id_br, curator_obj.index_id_ra, curator_obj.ar_index, 

145 curator_obj.re_index, curator_obj.VolIss] 

146 return data_curated, testcase 

147 

148def prepareCurator(data:list, server:str=SERVER, resp_agents_only:bool=False) -> Curator: 

149 settings = {'normalize_titles': True} 

150 reset_redis_counters() 

151 counter_handler = get_counter_handler() 

152 if resp_agents_only: 

153 curator = RespAgentsCurator(data, server, prov_config=PROV_CONFIG, counter_handler=counter_handler) 

154 else: 

155 curator = Curator(data, server, prov_config=PROV_CONFIG, counter_handler=counter_handler, settings=settings) 

156 return curator 

157 

158 

159class test_Curator(unittest.TestCase): 

160 @classmethod 

161 def setUpClass(cls): 

162 add_data_ts() 

163 

164 def setUp(self): 

165 reset_redis_counters() 

166 

167 def tearDown(self): 

168 reset_redis_counters() 

169 

170 def test_merge_entities_in_csv(self): 

171 curator = prepareCurator(list()) 

172 curator.counter_handler.set_counter(4, 'id', supplier_prefix='060') 

173 entity_dict = {'0601': {'ids': [], 'title': 'Money Growth, Interest Rates, Inflation And Raw Materials Prices: China', 'others': []}} 

174 id_dict = dict() 

175 curator.merge_entities_in_csv(['doi:10.1787/eco_outlook-v2011-2-graph138-en'], '0601', 'Money Growth, Interest Rates, Inflation And Raw Materials Prices: China', entity_dict, id_dict) 

176 expected_output = ( 

177 {'0601': {'ids': ['doi:10.1787/eco_outlook-v2011-2-graph138-en'], 'title': 'Money Growth, Interest Rates, Inflation And Raw Materials Prices: China', 'others': []}}, 

178 {'doi:10.1787/eco_outlook-v2011-2-graph138-en': '0605'} 

179 ) 

180 self.assertEqual((entity_dict, id_dict), expected_output) 

181 

182 def test_clean_id_list(self): 

183 input = ['doi:10.001/B-1', 'wikidata:B1111111', 'OMID:br/060101'] 

184 output = Curator.clean_id_list(input, br=True) 

185 expected_output = (['doi:10.001/b-1', 'wikidata:B1111111'], '060101') 

186 self.assertEqual(output, expected_output) 

187 

188 def test_equalizer(self): 

189 # Test equalizer with a row that contains an ID that can be resolved to an existing entity 

190 row = {'id': 'doi:10.1001/archderm.104.1.106', 'title': '', 'author': '', 'pub_date': '1972-12-01', 'venue': '', 'volume': '', 'issue': '', 'page': '', 'type': '', 'publisher': '', 'editor': ''} 

191 curator = prepareCurator(list()) 

192 curator.finder = ResourceFinder(ts_url=SERVER, base_iri=BASE_IRI) 

193 

194 metavals, identifiers, vvis = curator.extract_identifiers_and_metavals(row, valid_dois_cache=set()) 

195 curator.finder.get_everything_about_res(metavals=metavals, identifiers=identifiers, vvis=vvis) 

196 

197 curator.log[0] = {'id': {}} 

198 curator.clean_id(row) 

199 extracted_metaval = row['id'] 

200 self.assertEqual(extracted_metaval, '3757') 

201 

202 # Reset the row to test equalizer 

203 row = {'id': '', 'title': '', 'author': '', 'pub_date': '1972-12-01', 'venue': '', 'volume': '', 'issue': '', 'page': '', 'type': '', 'publisher': '', 'editor': ''} 

204 

205 curator.rowcnt = 0 

206 curator.log[0] = { 

207 'id': {}, 

208 'author': {}, 

209 'venue': {}, 

210 'editor': {}, 

211 'publisher': {}, 

212 'page': {}, 

213 'volume': {}, 

214 'issue': {}, 

215 'pub_date': {}, 

216 'type': {}, 

217 'title': {} 

218 } 

219 curator.equalizer(row, extracted_metaval) 

220 output = (curator.log, row) 

221 

222 expected_output = ( 

223 {0: {'id': {'status': 'Entity already exists'}, 'author': {}, 'venue': {}, 'editor': {}, 'publisher': {}, 'page': {}, 'volume': {}, 'issue': {}, 'pub_date': {'status': 'New value proposed'}, 'type': {}, 'title': {}}}, 

224 {'id': '', 'title': '', 'author': 'Curth, W. [omid:ra/6033]', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [omid:br/4416 issn:0003-987X]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': 'American Medical Association (ama) [omid:ra/3309 crossref:10]', 'editor': ''} 

225 ) 

226 self.assertEqual(output, expected_output) 

227 

228 def test_clean_id_metaid_not_in_ts(self): 

229 # A MetaId was specified, but it is not on ts. Therefore, it is invalid 

230 curator = prepareCurator(list()) 

231 row = {'id': 'omid:br/131313', 'title': 'Multiple Keloids', 'author': '', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''} 

232 curator.log[0] = {'id': {}} 

233 curator.clean_id(row) 

234 expected_output = {'id': 'wannabe_0', 'title': 'Multiple Keloids', 'author': '', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''} 

235 self.assertEqual(row, expected_output) 

236 

237 def test_clean_id(self): 

238 curator = prepareCurator(list()) 

239 row = {'id': 'doi:10.1001/archderm.104.1.106', 'title': 'Multiple Blasto', 'author': '', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [omid:br/4416]', 'volume': '', 'issue': '', 'page': '', 'type': '', 'publisher': '', 'editor': ''} 

240 curator.log[0] = {'id': {}} 

241 curator.finder.get_everything_about_res(metavals=set(), identifiers={'doi:10.1001/archderm.104.1.106'}, vvis=set()) 

242 curator.clean_id(row) 

243 expected_output = {'id': '3757', 'title': 'Multiple Keloids', 'author': '', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [omid:br/4416]', 'volume': '', 'issue': '', 'page': '', 'type': '', 'publisher': '', 'editor': ''} 

244 self.assertEqual(row, expected_output) 

245 

246 def test_merge_duplicate_entities(self): 

247 # Test merge_duplicate_entities with realistic data that includes an ID that resolves to an existing entity 

248 data = [ 

249 {'id': 'doi:10.1001/archderm.104.1.106', 'title': 'Multiple Keloids', 'author': '', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''}, 

250 {'id': '', 'title': 'Multiple Keloids', 'author': '', 'pub_date': '1971-07-02', 'venue': 'Archives Of Dermatology [omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''}, 

251 {'id': '', 'title': 'Multiple Keloids', 'author': '', 'pub_date': '1971-07-03', 'venue': 'Archives Of Blast [omid:br/4416]', 'volume': '105', 'issue': '2', 'page': '106-108', 'type': 'journal volume', 'publisher': '', 'editor': ''}, 

252 ] 

253 curator = prepareCurator(list()) 

254 curator.data = data 

255 curator.finder = ResourceFinder(ts_url=SERVER, base_iri=BASE_IRI) 

256 

257 # Extract metavals and identifiers from each row 

258 all_metavals = set() 

259 all_identifiers = set() 

260 all_vvis = set() 

261 

262 for row in data: 

263 metavals, identifiers, vvis = curator.extract_identifiers_and_metavals(row, valid_dois_cache=set()) 

264 all_metavals.update(metavals) 

265 all_identifiers.update(identifiers) 

266 all_vvis.update(vvis) 

267 

268 curator.finder.get_everything_about_res(metavals=all_metavals, identifiers=all_identifiers, vvis=all_vvis) 

269 

270 # Process each row with clean_id to get the actual metavals 

271 for i, row in enumerate(data): 

272 curator.log[i] = {'id': {}} 

273 curator.rowcnt = i 

274 curator.clean_id(row) 

275 

276 # Initialize log for merge_duplicate_entities 

277 for i in range(3): 

278 curator.log[i] = { 

279 'id': {}, 

280 'author': {}, 

281 'venue': {}, 

282 'editor': {}, 

283 'publisher': {}, 

284 'page': {}, 

285 'volume': {}, 

286 'issue': {}, 

287 'pub_date': {}, 

288 'type': {} 

289 } 

290 

291 # The brdict should be populated by clean_id, but we need to set up the "others" relationship 

292 # The first row should have resolved to '3757', and the other rows should be wannabes 

293 first_row_metaval = curator.data[0]['id'] # Should be '3757' 

294 self.assertEqual(first_row_metaval, '3757') 

295 

296 # Set up the relationship between the existing entity and the wannabes 

297 if first_row_metaval in curator.brdict: 

298 curator.brdict[first_row_metaval]['others'].extend(['wannabe_0', 'wannabe_1']) 

299 

300 curator.merge_duplicate_entities() 

301 output = (curator.data, curator.log) 

302 

303 expected_output = ( 

304 [ 

305 {'id': '3757', 'title': 'Multiple Keloids', 'author': 'Curth, W. [omid:ra/6033]', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [issn:0003-987X omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': 'American Medical Association (ama) [omid:ra/3309 crossref:10]', 'editor': ''}, 

306 {'id': '3757', 'title': 'Multiple Keloids', 'author': 'Curth, W. [omid:ra/6033]', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [issn:0003-987X omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': 'American Medical Association (ama) [omid:ra/3309 crossref:10]', 'editor': ''}, 

307 {'id': '3757', 'title': 'Multiple Keloids', 'author': 'Curth, W. [omid:ra/6033]', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [issn:0003-987X omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': 'American Medical Association (ama) [omid:ra/3309 crossref:10]', 'editor': ''} 

308 ], 

309 { 

310 0: {'id': {'status': 'Entity already exists'}, 'author': {}, 'venue': {}, 'editor': {}, 'publisher': {}, 'page': {}, 'volume': {}, 'issue': {}, 'pub_date': {}, 'type': {}}, 

311 1: {'id': {'status': 'Entity already exists'}, 'author': {}, 'venue': {'status': 'New value proposed'}, 'editor': {}, 'publisher': {}, 'page': {}, 'volume': {}, 'issue': {}, 'pub_date': {'status': 'New value proposed'}, 'type': {}}, 

312 2: {'id': {'status': 'Entity already exists'}, 'author': {}, 'venue': {'status': 'New value proposed'}, 'editor': {}, 'publisher': {}, 'page': {'status': 'New value proposed'}, 'volume': {'status': 'New value proposed'}, 'issue': {'status': 'New value proposed'}, 'pub_date': {'status': 'New value proposed'}, 'type': {'status': 'New value proposed'}} 

313 } 

314 ) 

315 self.assertEqual(output, expected_output) 

316 

317 def test_clean_vvi_all_data_on_ts(self): 

318 # All data are already on the triplestore. They need to be retrieved and organized correctly 

319 row = {'id': 'doi:10.1001/archderm.104.1.106', 'title': 'Multiple Keloids', 'author': '', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''} 

320 curator = prepareCurator(list()) 

321 curator.finder = ResourceFinder(ts_url=SERVER, base_iri=BASE_IRI) 

322 

323 metavals, identifiers, vvis = curator.extract_identifiers_and_metavals(row, valid_dois_cache=set()) 

324 curator.finder.get_everything_about_res(metavals=metavals, identifiers=identifiers, vvis=vvis) 

325 

326 curator.log[0] = {'id': {}} 

327 curator.clean_id(row) 

328 

329 curator.clean_vvi(row) 

330 expected_output = { 

331 "4416": { 

332 "issue": {}, 

333 "volume": { 

334 "104": { 

335 "id": "4712", 

336 "issue": { 

337 "1": { 

338 "id": "4713" 

339 } 

340 } 

341 } 

342 } 

343 } 

344 } 

345 self.assertEqual(curator.vvi, expected_output) 

346 

347 def test_clean_vvi_new_venue(self): 

348 # It is a new venue 

349 row = {'id': 'wannabe_1', 'title': 'Money growth, interest rates, inflation and raw materials prices: China', 'author': '', 'pub_date': '2011-11-28', 'venue': 'OECD Economic Outlook', 'volume': '2011', 'issue': '2', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''} 

350 curator = prepareCurator(list()) 

351 curator.clean_vvi(row) 

352 expected_output = {'wannabe_0': {'volume': {'2011': {'id': 'wannabe_1', 'issue': {'2': {'id': 'wannabe_2'}}}}, 'issue': {}}} 

353 self.assertEqual(curator.vvi, expected_output) 

354 

355 def test_clean_vvi_volume_with_title(self): 

356 # A journal volume having a title 

357 row = [{'id': '', 'title': 'The volume title', 'author': '', 'pub_date': '', 'venue': 'OECD Economic Outlook', 'volume': '2011', 'issue': '2', 'page': '', 'type': 'journal volume', 'publisher': '', 'editor': ''}] 

358 curator = prepareCurator(row) 

359 curator.curator() 

360 expected_output = [{'id': 'omid:br/0601', 'title': 'The Volume Title', 'author': '', 'pub_date': '', 'venue': 'OECD Economic Outlook [omid:br/0602]', 'volume': '', 'issue': '', 'page': '', 'type': 'journal volume', 'publisher': '', 'editor': ''}] 

361 self.assertEqual(curator.data, expected_output) 

362 

363 def test_clean_vvi_invalid_volume(self): 

364 # The data must be invalidated, because the resource is journal volume but an issue has also been specified 

365 row = {'id': 'wannabe_1', 'title': '', 'author': '', 'pub_date': '', 'venue': 'OECD Economic Outlook', 'volume': '2011', 'issue': '2', 'page': '', 'type': 'journal volume', 'publisher': '', 'editor': ''} 

366 curator = prepareCurator(list()) 

367 curator.clean_vvi(row) 

368 expected_output = {'wannabe_0': {'volume': {}, 'issue': {}}} 

369 self.assertEqual(curator.vvi, expected_output) 

370 

371 def test_clean_vvi_invalid_venue(self): 

372 # The data must be invalidated, because the resource is journal but a volume has also been specified 

373 row = {'id': 'wannabe_1', 'title': '', 'author': '', 'pub_date': '', 'venue': 'OECD Economic Outlook', 'volume': '2011', 'issue': '', 'page': '', 'type': 'journal', 'publisher': '', 'editor': ''} 

374 curator = prepareCurator(list()) 

375 curator.clean_vvi(row) 

376 expected_output = {'wannabe_0': {'volume': {}, 'issue': {}}} 

377 self.assertEqual(curator.vvi, expected_output) 

378 

379 def test_clean_vvi_new_volume_and_issue(self): 

380 # There is a row with vvi and no ids 

381 row = {'id': '', 'title': '', 'author': '', 'pub_date': '', 'venue': 'Archives Of Surgery [omid:br/4480]', 'volume': '147', 'issue': '11', 'page': '', 'type': 'journal article', 'publisher': '', 'editor': ''} 

382 curator = prepareCurator(list()) 

383 curator.finder = ResourceFinder(ts_url=SERVER, base_iri=BASE_IRI) 

384 

385 metavals, identifiers, vvis = curator.extract_identifiers_and_metavals(row, valid_dois_cache=set()) 

386 curator.finder.get_everything_about_res(metavals=metavals, identifiers=identifiers, vvis=vvis) 

387 curator.clean_id(row) 

388 curator.clean_vvi(row) 

389 expected_output = { 

390 "4480": { 

391 "issue": {}, 

392 "volume": { 

393 "147": { 

394 "id": "4481", 

395 "issue": { 

396 "11": { 

397 "id": "4482" 

398 } 

399 } 

400 } 

401 } 

402 } 

403 } 

404 self.assertEqual(curator.vvi, expected_output) 

405 

406 def test_clean_ra_overlapping_surnames(self): 

407 # The surname of one author is included in the surname of another. 

408 row = {'id': 'wannabe_0', 'title': 'Giant Oyster Mushroom Pleurotus giganteus (Agaricomycetes) Enhances Adipocyte Differentiation and Glucose Uptake via Activation of PPARγ and Glucose Transporters 1 and 4 in 3T3-L1 Cells', 'author': 'Paravamsivam, Puvaneswari; Heng, Chua Kek; Malek, Sri Nurestri Abdul [orcid:0000-0001-6278-8559]; Sabaratnam, Vikineswary; M, Ravishankar Ram; Kuppusamy, Umah Rani', 'pub_date': '2016', 'venue': 'International Journal of Medicinal Mushrooms [issn:1521-9437]', 'volume': '18', 'issue': '9', 'page': '821-831', 'type': 'journal article', 'publisher': 'Begell House [crossref:613]', 'editor': ''} 

409 curator = prepareCurator(list()) 

410 curator.brdict = {'wannabe_0': {'ids': ['doi:10.1615/intjmedmushrooms.v18.i9.60'], 'title': 'Giant Oyster Mushroom Pleurotus giganteus (Agaricomycetes) Enhances Adipocyte Differentiation and Glucose Uptake via Activation of PPARγ and Glucose Transporters 1 and 4 in 3T3-L1 Cells', 'others': []}} 

411 curator.clean_ra(row, 'author') 

412 output = (curator.ardict, curator.radict, curator.idra) 

413 expected_output = ( 

414 {'wannabe_0': {'author': [('0601', 'wannabe_0'), ('0602', 'wannabe_1'), ('0603', 'wannabe_2'), ('0604', 'wannabe_3'), ('0605', 'wannabe_4'), ('0606', 'wannabe_5')], 'editor': [], 'publisher': []}}, 

415 {'wannabe_0': {'ids': [], 'others': [], 'title': 'Paravamsivam, Puvaneswari'}, 'wannabe_1': {'ids': [], 'others': [], 'title': 'Heng, Chua Kek'}, 'wannabe_2': {'ids': ['orcid:0000-0001-6278-8559'], 'others': [], 'title': 'Malek, Sri Nurestri Abdul'}, 'wannabe_3': {'ids': [], 'others': [], 'title': 'Sabaratnam, Vikineswary'}, 'wannabe_4': {'ids': [], 'others': [], 'title': 'M, Ravishankar Ram'}, 'wannabe_5': {'ids': [], 'others': [], 'title': 'Kuppusamy, Umah Rani'}}, 

416 {'orcid:0000-0001-6278-8559': '0601'} 

417 ) 

418 self.assertEqual(output, expected_output) 

419 

420 def test_clean_ra_with_br_metaid(self): 

421 # One author is in the triplestore, the other is not.  

422 # br_metaval is a MetaID 

423 # There are two ids for one author 

424 row = {'id': 'doi:10.1001/archderm.104.1.106', 'title': 'Multiple Keloids', 'author': 'Curth, W.; McSorley, J. [orcid:0000-0003-0530-4305 schema:12345]', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''} 

425 curator = prepareCurator(list()) 

426 curator.finder = ResourceFinder(ts_url=SERVER, base_iri=BASE_IRI) 

427 metavals, identifiers, vvis = curator.extract_identifiers_and_metavals(row, valid_dois_cache=set()) 

428 curator.finder.get_everything_about_res(metavals=metavals, identifiers=identifiers, vvis=vvis) 

429 

430 curator.log[0] = {'id': {}} 

431 curator.clean_id(row) 

432 

433 resolved_metaval = row['id'] 

434 self.assertEqual(resolved_metaval, '3757') 

435 curator.brdict = {resolved_metaval: {'ids': ['doi:10.1001/archderm.104.1.106'], 'title': 'Multiple Keloids', 'others': []}} 

436 

437 curator.clean_ra(row, 'author') 

438 output = (curator.ardict, curator.radict, curator.idra) 

439 expected_output = ( 

440 {'3757': {'author': [('9445', '6033'), ('0601', 'wannabe_0')], 'editor': [], 'publisher': []}}, 

441 {'6033': {'ids': [], 'others': [], 'title': 'Curth, W.'}, 'wannabe_0': {'ids': ['orcid:0000-0003-0530-4305', 'schema:12345'], 'others': [], 'title': 'McSorley, J.'}}, 

442 {'orcid:0000-0003-0530-4305': '0601', 'schema:12345': '0602'} 

443 ) 

444 self.assertEqual(output, expected_output) 

445 

446 def test_clean_ra_with_br_wannabe(self): 

447 # Authors not on the triplestore.  

448 # br_metaval is a wannabe 

449 row = {'id': 'wannabe_0', 'title': 'Multiple Keloids', 'author': 'Curth, W. [orcid:0000-0002-8420-0696] ; McSorley, J. [orcid:0000-0003-0530-4305]', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''} 

450 curator = prepareCurator(list()) 

451 curator.brdict = {'wannabe_0': {'ids': ['doi:10.1001/archderm.104.1.106'], 'title': 'Multiple Keloids', 'others': []}} 

452 curator.wnb_cnt = 1 

453 curator.clean_ra(row, 'author') 

454 output = (curator.ardict, curator.radict, curator.idra) 

455 expected_output = ( 

456 {'wannabe_0': {'author': [('0601', 'wannabe_1'), ('0602', 'wannabe_2')], 'editor': [], 'publisher': []}}, 

457 {'wannabe_1': {'ids': ['orcid:0000-0002-8420-0696'], 'others': [], 'title': 'Curth, W.'}, 'wannabe_2': {'ids': ['orcid:0000-0003-0530-4305'], 'others': [], 'title': 'McSorley, J.'}}, 

458 {'orcid:0000-0002-8420-0696': '0601', 'orcid:0000-0003-0530-4305': '0602'} 

459 ) 

460 self.assertEqual(output, expected_output) 

461 

462 def test_clean_ra_with_empty_square_brackets(self): 

463 # One author's name contains a closed square bracket. 

464 row = {'id': 'doi:10.1001/archderm.104.1.106', 'title': 'Multiple Keloids', 'author': 'Bernacki, Edward J. [ ]', 'pub_date': '1971-07-01', 'venue': 'Archives Of Dermatology [omid:br/4416]', 'volume': '104', 'issue': '1', 'page': '106-107', 'type': 'journal article', 'publisher': '', 'editor': ''} 

465 curator = prepareCurator(list()) 

466 curator.finder = ResourceFinder(ts_url=SERVER, base_iri=BASE_IRI) 

467 

468 metavals, identifiers, vvis = curator.extract_identifiers_and_metavals(row, valid_dois_cache=set()) 

469 curator.finder.get_everything_about_res(metavals=metavals, identifiers=identifiers, vvis=vvis) 

470 

471 curator.log[0] = {'id': {}} 

472 curator.clean_id(row) 

473 

474 resolved_metaval = row['id'] 

475 self.assertEqual(resolved_metaval, '3757') 

476 curator.brdict = {resolved_metaval: {'ids': ['doi:10.1001/archderm.104.1.106'], 'title': 'Multiple Keloids', 'others': []}} 

477 

478 curator.clean_ra(row, 'author') 

479 output = (curator.ardict, curator.radict, curator.idra) 

480 expected_output = ( 

481 {'3757': {'author': [('9445', '6033'), ('0601', 'wannabe_0')], 'editor': [], 'publisher': []}}, 

482 {'6033': {'ids': [], 'others': [], 'title': 'Curth, W.'}, 'wannabe_0': {'ids': [], 'others': [], 'title': 'Bernacki, Edward J.'}}, 

483 {} 

484 ) 

485 self.assertEqual(output, expected_output) 

486 

487 def test_meta_maker(self): 

488 curator = prepareCurator(list()) 

489 curator.brdict = {'3757': {'ids': ['doi:10.1001/archderm.104.1.106', 'pmid:29098884'], 'title': 'Multiple Keloids', 'others': []}, '4416': {'ids': ['issn:0003-987X'], 'title': 'Archives Of Dermatology', 'others': []}} 

490 curator.radict = {'6033': {'ids': [], 'others': [], 'title': 'Curth, W.'}, 'wannabe_0': {'ids': ['orcid:0000-0003-0530-4305', 'schema:12345'], 'others': [], 'title': 'Mcsorley, J.'}} 

491 curator.ardict = {'3757': {'author': [('9445', '6033'), ('0601', 'wannabe_0')], 'editor': [], 'publisher': []}} 

492 curator.vvi = {'4416': {'issue': {}, 'volume': {'107': {'id': '4733', 'issue': {'1': {'id': '4734'}, '2': {'id': '4735'}, '3': {'id': '4736'}, '4': {'id': '4737'}, '5': {'id': '4738'}, '6': {'id': '4739'}}}, '108': {'id': '4740', 'issue': {'1': {'id': '4741'}, '2': {'id': '4742'}, '3': {'id': '4743'}, '4': {'id': '4744'}}}, '104': {'id': '4712', 'issue': {'1': {'id': '4713'}, '2': {'id': '4714'}, '3': {'id': '4715'}, '4': {'id': '4716'}, '5': {'id': '4717'}, '6': {'id': '4718'}}}, '148': {'id': '4417', 'issue': {'12': {'id': '4418'}, '11': {'id': '4419'}}}, '100': {'id': '4684', 'issue': {'1': {'id': '4685'}, '2': {'id': '4686'}, '3': {'id': '4687'}, '4': {'id': '4688'}, '5': {'id': '4689'}, '6': {'id': '4690'}}}, '101': {'id': '4691', 'issue': {'1': {'id': '4692'}, '2': {'id': '4693'}, '3': {'id': '4694'}, '4': {'id': '4695'}, '5': {'id': '4696'}, '6': {'id': '4697'}}}, '102': {'id': '4698', 'issue': {'1': {'id': '4699'}, '2': {'id': '4700'}, '3': {'id': '4701'}, '4': {'id': '4702'}, '5': {'id': '4703'}, '6': {'id': '4704'}}}, '103': {'id': '4705', 'issue': {'1': {'id': '4706'}, '2': {'id': '4707'}, '3': {'id': '4708'}, '4': {'id': '4709'}, '5': {'id': '4710'}, '6': {'id': '4711'}}}, '105': {'id': '4719', 'issue': {'1': {'id': '4720'}, '2': {'id': '4721'}, '3': {'id': '4722'}, '4': {'id': '4723'}, '5': {'id': '4724'}, '6': {'id': '4725'}}}, '106': {'id': '4726', 'issue': {'6': {'id': '4732'}, '1': {'id': '4727'}, '2': {'id': '4728'}, '3': {'id': '4729'}, '4': {'id': '4730'}, '5': {'id': '4731'}}}}}} 

493 curator.meta_maker() 

494 output = (curator.brmeta, curator.rameta, curator.armeta) 

495 expected_output = ( 

496 {'3757': {'ids': ['doi:10.1001/archderm.104.1.106', 'pmid:29098884', 'omid:br/3757'], 'title': 'Multiple Keloids', 'others': []}, '4416': {'ids': ['issn:0003-987X', 'omid:br/4416'], 'title': 'Archives Of Dermatology', 'others': []}}, 

497 {'6033': {'ids': ['omid:ra/6033'], 'others': [], 'title': 'Curth, W.'}, '0601': {'ids': ['orcid:0000-0003-0530-4305', 'schema:12345', 'omid:ra/0601'], 'others': ['wannabe_0'], 'title': 'Mcsorley, J.'}}, 

498 {'3757': {'author': [('9445', '6033'), ('0601', '0601')], 'editor': [], 'publisher': []}} 

499 ) 

500 self.assertEqual(output, expected_output) 

501