Coverage for src / crawler / abstract_crawlers / base_crawler.py: 64%

1import logging 

2import time 

3from collections.abc import Iterable 

4from datetime import datetime, timedelta 

5from email.policy import EmailPolicy 

6from typing import TYPE_CHECKING, Literal 

7 

8import aiohttp 

9import regex 

10import requests 

11from bs4 import BeautifulSoup 

12from django.conf import settings 

13from django.contrib.auth.models import User 

14from django.db.utils import IntegrityError 

15from django.utils import timezone 

16from langcodes import standardize_tag 

17from lingua import LanguageDetector, LanguageDetectorBuilder 

18from opentelemetry import trace 

19from ptf.cmds.xml.ckeditor.utils import ( 

20 build_jats_data_from_html_field, 

21) 

22from ptf.cmds.xml.jats.builder.references import ( 

23 get_article_title_xml, 

24 get_author_xml, 

25 get_fpage_xml, 

26 get_lpage_xml, 

27 get_source_xml, 

28 get_year_xml, 

29) 

30from ptf.cmds.xml.jats.jats_parser import JatsBase 

31from ptf.model_data import ( 

32 ArticleData, 

33 ContributorDict, 

34 IssueData, 

35 ResourceData, 

36 TitleDict, 

37 create_abstract, 

38 create_contributor, 

39 create_extlink, 

40 create_issuedata, 

41 create_publisherdata, 

42 create_subj, 

43 create_titledata, 

44) 

45from ptf.model_data_converter import update_data_for_jats 

46from ptf.models import ExtLink 

47from pylatexenc.latex2text import LatexNodes2Text 

48from pysolr import SolrError 

49from requests.adapters import HTTPAdapter 

50from requests_cache import CachedSession 

51from urllib3 import Retry 

52 

53from crawler.cmds.xml_cmds import addOrUpdateGDMLIssueXmlCmd 

54from crawler.models import Source 

55from crawler.models.extlink_checked import ExtlinkChecked 

56from crawler.types import CitationLiteral 

57from crawler.utils import ( 

58 add_pdf_link_to_xarticle, 

59 cleanup_str, 

60 get_all_cols, 

61 get_or_create_collection, 

62 get_session, 

63) 

64 

65if TYPE_CHECKING: 

66 from typing import Callable 

67 

68 from bs4 import Tag 

69 

70 

71class CrawlerTitleDict(TitleDict): 

72 title_tex: str | None 

73 

74 

75class BaseCollectionCrawler: 

76 """ 

77 Base collection for the crawlers. 

78 To create a crawler: 

79 1) derive a class from BaseCollectionCrawler and name it XXXCrawler 

80 2) override the functions parse_collection_content, parse_issue_content and parse_article_content 

81 3) update factory.py so that crawler_factory can return your new crawler 

82 """ 

83 

84 logger = logging.getLogger(__name__) 

85 tracer = trace.get_tracer(__name__) 

86 

87 source_name = "" 

88 source_domain = "" 

89 source_website = "" 

90 

91 issue_href = "" 

92 

93 collection = None 

94 source = None 

95 user = None 

96 session: requests.Session | CachedSession 

97 async_session: aiohttp.ClientSession 

98 is_checkable = True 

99 verify = True 

100 headers = { 

101 "accept_encoding": "utf-8", 

102 "User-Agent": getattr(settings, "REQUESTS_USER_AGENT", "Mathdoc/1.0.0"), 

103 "From": getattr(settings, "REQUESTS_EMAIL", "accueil@listes.mathdoc.fr"), 

104 } 

105 

106 requests_interval = getattr(settings, "REQUESTS_INTERVAL", 90) 

107 "seconds to wait between two http requests" 

108 requests_timeout = 60 

109 "seconds to wait before aborting the connection (if no bytes are recieved)" 

110 

111 latext_parser = LatexNodes2Text() 

112 

113 # Override the values in your concrete crawler if the formulas in text (titles, abstracts) 

114 # do not use the "$" to surround tex formulas 

115 delimiter_inline_formula = "$" 

116 delimiter_disp_formula = "$" 

117 

118 # HACK : Workaround for tests (monkeypatching) 

119 # We store the class here, so we can monkeypatch it when running tests 

120 # subCrawlers = { 

121 # LofplCrawler: None 

122 # } 

123 subCrawlers: dict[type["BaseCollectionCrawler"], "BaseCollectionCrawler | None"] = {} 

124 

125 _language_detector: LanguageDetector | None = None 

126 _language_detector_builder = LanguageDetectorBuilder.from_all_languages() 

127 

128 force_refresh = False 

129 

130 match_headers = False 

131 "Whereas to include headers in requests cache key" 

132 orcid_re = r"https\:\/\/orcid\.org\/(?P<orcid>\d{4}-\d{4}-\d{4}-\d{4})" 

133 

134 ignore_missing_pdf = True 

135 "Set this to False on a Crawler-basis to allow inserting articles without PDFs" 

136 pid_year_restrictions: dict[str, int] = {} 

137 "pid -> excluded years count" 

138 

139 pause_function: "Callable[[int], None]" 

140 "Overridable the pause function (used in celery tasks to speedup aborting)" 

141 

142 @classmethod 

143 def get_view_id(cls): 

144 return cls.source_domain 

145 

146 @property 

147 def language_detector(self): 

148 """Crawler Instance singleton for language builder. 

149 Late init of LanguageDetector to save on memory""" 

150 if not self._language_detector: 

151 self._language_detector = self._language_detector_builder.build() 

152 return self._language_detector 

153 

154 def __init__( 

155 self, 

156 *args, 

157 username: str, 

158 collection_id: str, 

159 dry: bool = False, 

160 publisher: str = "", 

161 force_refresh=False, 

162 collection_url: str | None = None, 

163 backend=None, 

164 pause_function=staticmethod(time.sleep), 

165 ): 

166 if not collection_url: 166 ↛ 167line 166 didn't jump to line 167 because the condition on line 166 was never true

167 all_cols = get_all_cols() 

168 col = all_cols[collection_id] 

169 

170 collection_url = col["sources"].get(self.source_domain, None) 

171 if collection_url is None: 

172 raise ValueError( 

173 f"Source {self.source_domain} not found for collection {collection_id}" 

174 ) 

175 self.collection_url = collection_url 

176 for CrawlerClass in self.subCrawlers: 176 ↛ 177line 176 didn't jump to line 177 because the loop on line 176 never started

177 self.subCrawlers[CrawlerClass] = CrawlerClass( 

178 *args, 

179 username=username, 

180 collection_id=collection_id, 

181 dry=dry, 

182 publisher=publisher, 

183 collection_url=collection_url, 

184 ) 

185 self.logger = logging.getLogger(__name__ + "." + self.source_domain) 

186 # self.logger = logging.getLogger(__name__) 

187 

188 self.username = username 

189 

190 self.collection_id = collection_id 

191 

192 self.dry = dry 

193 self.publisher = publisher 

194 

195 # Classproperty : We sometimes want to use the session without initializing the class (rot monitoring) 

196 BaseCollectionCrawler.session = requests.Session() 

197 

198 self.pause_function = pause_function 

199 

200 # Skipped when running tests 

201 self.initialize() 

202 

203 self.force_refresh = force_refresh 

204 self.backend = backend 

205 

206 def initialize(self): 

207 """ 

208 Acts as a "second" init function to skip model accesses during test data generation 

209 """ 

210 self.collection = get_or_create_collection(self.collection_id) 

211 self.source = self.get_or_create_source() 

212 self.user = User.objects.get(username=self.username) 

213 BaseCollectionCrawler.session = get_session() 

214 BaseCollectionCrawler.session.verify = self.verify 

215 self.session.pause_function = self.pause_function 

216 self.session.delay = self.requests_interval 

217 retries = Retry( 

218 total=0, 

219 ) 

220 self.session.mount("https://", HTTPAdapter(max_retries=retries)) 

221 self.session.mount("http://", HTTPAdapter(max_retries=retries)) 

222 

223 @classmethod 

224 def can_crawl(cls, pid: str) -> bool: 

225 return True 

226 

227 def parse_collection_content(self, content: str) -> list[IssueData]: 

228 """ 

229 Parse the HTML content with BeautifulSoup 

230 returns a list of xissue. 

231 Override this function in a derived class 

232 """ 

233 return [] 

234 

235 def parse_issue_content(self, content: str, xissue: IssueData): 

236 """ 

237 Parse the HTML content with BeautifulSoup 

238 Fills the xissue.articles 

239 Override this function in a derived class. 

240 

241 CAV : You are supposed to create articles there. Please assign a PID to each article. 

242 The PID can be `a + article_index`, like this : `a0` `a21` 

243 """ 

244 

245 def parse_article_content( 

246 self, content: str, xissue: IssueData, xarticle: ArticleData, url: str 

247 ) -> ArticleData | None: 

248 """ 

249 Parse the HTML content with BeautifulSoup 

250 returns the xarticle. 

251 Override this function in a derived class. 

252 The xissue is passed to the function in case the article page has issue information (ex: publisher) 

253 The article url is also passed as a parameter 

254 

255 CAV : You are supposed to assign articles pid again here 

256 """ 

257 return xarticle 

258 

259 @tracer.start_as_current_span("crawl_collection") 

260 def crawl_collection(self): 

261 # TODO: Comments, filter 

262 """ 

263 Crawl an entire collection. ptf.models.Container objects are created. 

264 - get the HTML content of the collection_url 

265 - parse the HTML content with beautifulsoup to extract the list of issues 

266 - merge the xissues (some Source can have multiple pages for 1 volume/issue. We create only 1 container) 

267 - crawl each issue if col_only is False 

268 - Returns the list of merged issues. 

269 It is an OrderedDict {pid: {"issues": xissues}} 

270 The key is the pid of the merged issues. 

271 Ex: The source may have Ex: Volume 6 (2000) and Volume 6 (1999) 

272 the pid is then made with 1999-2000__6_ 

273 """ 

274 

275 if self.source is None: 

276 raise RuntimeError("ERROR: the source is not set") 

277 

278 content = self.download_file(self.collection_url) 

279 if content: 

280 xissues = self.parse_collection_content(content) 

281 else: 

282 # download_file returns None (404) 

283 return None 

284 

285 """ 

286 Some collections split the same volumes in different pages 

287 Ex: Volume 6 (2000) and Volume 6 (1999) 

288 We merge the 2 xissues with the same volume number => Volume 6 (1999-2000) 

289 """ 

290 # merged_xissues = self.merge_xissues(xissues) 

291 

292 xissues_dict = {str(i.pid): i for i in xissues} 

293 

294 return xissues_dict 

295 

296 def start_process_issue(self, xissue: IssueData): 

297 # Some source, like EuDML do not have a separate HTML pages for an issue's table of content. 

298 # The list of articles directly come from the collection HTML page: the xissue has no url attribute 

299 issue_url = xissue.url 

300 if issue_url is not None: 

301 if issue_url.endswith(".pdf"): 

302 add_pdf_link_to_xarticle(xissue, issue_url) 

303 xissue.url = None 

304 else: 

305 content = self.download_file(issue_url) 

306 with self.tracer.start_as_current_span("parse_issue_content"): 

307 self.parse_issue_content(content, xissue) 

308 

309 @tracer.start_as_current_span("crawl_issue") 

310 def crawl_issue(self, xissue: IssueData): 

311 """ 

312 Crawl 1 wag page of an issue. 

313 - get the HTML content of the issue 

314 - parse the HTML content with beautifulsoup to extract the list of articles and/or the issue metadata 

315 - crawl each article 

316 """ 

317 

318 self.start_process_issue(xissue) 

319 

320 xarticles = xissue.articles 

321 

322 parsed_xarticles = [] 

323 

324 for xarticle in xarticles: 

325 parsed_xarticle = self.crawl_article(xarticle, xissue) 

326 if parsed_xarticle is not None: 

327 parsed_xarticles.append(parsed_xarticle) 

328 

329 xissue.articles = parsed_xarticles 

330 

331 issue_has_pdf = self.article_has_pdf(xissue) 

332 

333 if self.ignore_missing_pdf: 

334 xissue.articles = [a for a in xissue.articles if self.article_has_pdf(a)] 

335 if self.dry: 

336 return 

337 if len(xissue.articles) == 0 and not issue_has_pdf: 

338 return 

339 self.process_resource_metadata(xissue, resource_type="issue") 

340 

341 self.add_xissue_into_database(xissue) 

342 

343 @staticmethod 

344 def article_has_source(art: ArticleData | IssueData): 

345 return ( 

346 next( 

347 (e_link for e_link in art.ext_links if e_link["rel"] == "source"), 

348 None, 

349 ) 

350 is not None 

351 ) 

352 

353 @staticmethod 

354 def article_has_pdf(art: ArticleData | IssueData): 

355 return ( 

356 next( 

357 (link for link in art.ext_links if link["rel"] in ["article-pdf", "article-html"]), 

358 None, 

359 ) 

360 is not None 

361 ) 

362 

363 def crawl_article(self, xarticle: ArticleData, xissue: IssueData): 

364 # ARTICLE URL as en ExtLink (to display the link in the article page) 

365 if xarticle.url is None: 

366 if not self.article_has_source(xarticle): 366 ↛ 376line 366 didn't jump to line 376 because the condition on line 366 was always true

367 if xissue.url: 

368 article_source = xissue.url 

369 else: 

370 article_source = self.collection_url 

371 ext_link = create_extlink() 

372 ext_link["rel"] = "source" 

373 ext_link["location"] = article_source 

374 ext_link["metadata"] = self.source_domain 

375 xarticle.ext_links.append(ext_link) 

376 return self.process_article_metadata(xarticle) 

377 

378 parsed_xarticle = xarticle 

379 if self.parse_article_content.__func__ != BaseCollectionCrawler.parse_article_content: 

380 content = self.download_file(xarticle.url) 

381 xarticle.pid = f"{xissue.pid}_{xarticle.pid}" 

382 

383 try: 

384 with self.tracer.start_as_current_span("parse_article_content"): 

385 parsed_xarticle = self.parse_article_content( 

386 content, xissue, xarticle, xarticle.url 

387 ) 

388 except ValueError as e: 

389 self.logger.warning(e) 

390 self.logger.warning("Retrying in 5 mins while invalidating cache") 

391 self.pause_function(5 * 60) 

392 content = self.download_file(xarticle.url, force_refresh=True) 

393 with self.tracer.start_as_current_span("parse_article_content"): 

394 parsed_xarticle = self.parse_article_content( 

395 content, xissue, xarticle, xarticle.url 

396 ) 

397 

398 if parsed_xarticle is None: 398 ↛ 399line 398 didn't jump to line 399 because the condition on line 398 was never true

399 return None 

400 

401 if parsed_xarticle.doi: 

402 parsed_xarticle.pid = ( 

403 parsed_xarticle.doi.replace("/", "_").replace(".", "_").replace("-", "_") 

404 ) 

405 

406 if not self.article_has_source(parsed_xarticle) and parsed_xarticle.url: 

407 ext_link = create_extlink() 

408 ext_link["rel"] = "source" 

409 ext_link["location"] = parsed_xarticle.url 

410 ext_link["metadata"] = self.source_domain 

411 parsed_xarticle.ext_links.append(ext_link) 

412 

413 # The article title may have formulas surrounded with '$' 

414 return self.process_article_metadata(parsed_xarticle) 

415 

416 def process_resource_metadata(self, xresource: ResourceData, resource_type="article"): 

417 tag = "article-title" if resource_type == "article" else "issue-title" 

418 

419 # Process title tex 

420 ckeditor_data = build_jats_data_from_html_field( 

421 xresource.title_tex, 

422 tag=tag, 

423 text_lang=xresource.lang, 

424 delimiter_inline=self.delimiter_inline_formula, 

425 delimiter_disp=self.delimiter_disp_formula, 

426 ) 

427 

428 xresource.title_html = ckeditor_data["value_html"] 

429 # xresource.title_tex = ckeditor_data["value_tex"] 

430 xresource.title_xml = ckeditor_data["value_xml"] 

431 

432 abstracts_to_parse = [ 

433 xabstract for xabstract in xresource.abstracts if xabstract["tag"] == "abstract" 

434 ] 

435 # abstract may have formulas surrounded with '$' 

436 if len(abstracts_to_parse) > 0: 

437 for xabstract in abstracts_to_parse: 

438 ckeditor_data = build_jats_data_from_html_field( 

439 xabstract["value_tex"], 

440 tag="abstract", 

441 text_lang=xabstract["lang"], 

442 resource_lang=xresource.lang, 

443 field_type="abstract", 

444 delimiter_inline=self.delimiter_inline_formula, 

445 delimiter_disp=self.delimiter_disp_formula, 

446 ) 

447 

448 xabstract["value_html"] = ckeditor_data["value_html"] 

449 # xabstract["value_tex"] = ckeditor_data["value_tex"] 

450 xabstract["value_xml"] = ckeditor_data["value_xml"] 

451 

452 return xresource 

453 

454 def process_article_metadata(self, xarticle: ArticleData): 

455 self.process_resource_metadata(xarticle) 

456 for bibitem in xarticle.bibitems: 

457 bibitem.type = "unknown" 

458 update_data_for_jats(xarticle, with_label=False) 

459 

460 return xarticle 

461 

462 def download_file(self, url: str, force_refresh=False, headers={}): 

463 """ 

464 Downloads a page and returns its content (decoded string). 

465 """ 

466 

467 for attempt in range(3): 

468 response = self.get( 

469 url, 

470 force_refresh=force_refresh, 

471 headers=headers, 

472 pause_function=self.pause_function, 

473 ) 

474 

475 content = self.decode_response(response) 

476 if content == "" or not content: 

477 self.logger.debug("Got empty content while fetching ! ") 

478 # 15 mins, 30 mins, 45 mins 

479 delay_minutes = attempt * 15 

480 self.logger.debug( 

481 f"Retrying in {delay_minutes}mins ({(datetime.now() + timedelta(minutes=delay_minutes)).time()})", 

482 extra={"url": url}, 

483 ) 

484 self.pause_function(delay_minutes * 60) 

485 continue 

486 return content 

487 raise ValueError(f"Could not decode content at {url}") 

488 

489 @classmethod 

490 def get(cls, url, *args, headers={}, force_refresh=False, pause_function=time.sleep, **kwargs): 

491 current_exception = Exception(f"Could fetch url {url}") 

492 for attempt in range(3): 

493 try: 

494 kwargs = { 

495 "url": url, 

496 "headers": {**cls.headers, **headers}, 

497 "timeout": cls.requests_timeout, 

498 **kwargs, 

499 } 

500 if isinstance(cls.session, CachedSession): 

501 kwargs["force_refresh"] = force_refresh 

502 if attempt > 0: 

503 kwargs["force_refresh"] = True 

504 response = cls.session.get(*args, **kwargs) 

505 return response 

506 except ( 

507 requests.ConnectionError, 

508 requests.ConnectTimeout, 

509 requests.exceptions.HTTPError, 

510 ) as e: 

511 current_exception = e 

512 cls.logger.debug(f"Caught error : {e}", extra={"url": url}) 

513 # 15 mins, 30 mins, 45 mins 

514 delay_minutes = attempt * 15 

515 cls.logger.debug( 

516 f"Retrying in {delay_minutes}mins ({(datetime.now() + timedelta(minutes=delay_minutes)).time()})", 

517 extra={"url": url}, 

518 ) 

519 pause_function(delay_minutes * 60) 

520 

521 raise current_exception 

522 

523 def decode_response(self, response: requests.Response, encoding: str | None = None): 

524 """Override this if the content-type headers from the sources are advertising something else than the actual content 

525 SASA needs this""" 

526 # Force 

527 if encoding: 

528 response.encoding = encoding 

529 return response.text 

530 

531 # Attempt to get encoding using HTTP headers 

532 content_type_tag = response.headers.get("Content-Type", None) 

533 

534 if content_type_tag: 534 ↛ 541line 534 didn't jump to line 541 because the condition on line 534 was always true

535 charset = self.parse_content_type_charset(content_type_tag) 

536 if charset: 536 ↛ 537line 536 didn't jump to line 537 because the condition on line 536 was never true

537 response.encoding = charset 

538 return response.text 

539 

540 # Attempt to get encoding using HTML meta charset tag 

541 soup = BeautifulSoup(response.text, "html5lib") 

542 charset = soup.select_one("meta[charset]") 

543 if charset: 

544 htmlencoding = charset.get("charset") 

545 if isinstance(htmlencoding, str): 545 ↛ 550line 545 didn't jump to line 550 because the condition on line 545 was always true

546 response.encoding = htmlencoding 

547 return response.text 

548 

549 # Attempt to get encoding using HTML meta content type tag 

550 content_type_tag = soup.select_one( 

551 'meta[http-equiv="Content-Type"],meta[http-equiv="content-type"]' 

552 ) 

553 if content_type_tag: 

554 content_type = content_type_tag.get("content") 

555 if isinstance(content_type, str): 555 ↛ 561line 555 didn't jump to line 561 because the condition on line 555 was always true

556 charset = self.parse_content_type_charset(content_type) 

557 if charset: 557 ↛ 561line 557 didn't jump to line 561 because the condition on line 557 was always true

558 response.encoding = charset 

559 return response.text 

560 

561 return response.text 

562 

563 @staticmethod 

564 def parse_content_type_charset(content_type: str): 

565 header = EmailPolicy.header_factory("content-type", content_type) 

566 if "charset" in header.params: 

567 return header.params.get("charset") 

568 

569 @tracer.start_as_current_span("add_xissue_to_database") 

570 def add_xissue_into_database(self, xissue: IssueData) -> IssueData: 

571 xissue.journal = self.collection 

572 xissue.source = self.source_domain 

573 

574 if xissue.fyear == 0: 

575 raise ValueError("Failsafe : Cannot insert issue without a year") 

576 

577 xpub = create_publisherdata() 

578 xpub.name = self.publisher 

579 xissue.publisher = xpub 

580 xissue.last_modified_iso_8601_date_str = timezone.now().isoformat() 

581 

582 attempt = 1 

583 success = False 

584 

585 while not success and attempt < 4: 

586 try: 

587 params = {"xissue": xissue, "use_body": False} 

588 cmd = addOrUpdateGDMLIssueXmlCmd(params) 

589 cmd.do() 

590 success = True 

591 self.logger.debug(f"Issue {xissue.pid} inserted in database") 

592 return xissue 

593 except SolrError: 

594 self.logger.warning( 

595 f"Encoutered SolrError while inserting issue {xissue.pid} in database" 

596 ) 

597 attempt += 1 

598 self.logger.debug(f"Attempt {attempt}. sleeping 10 seconds.") 

599 self.pause_function(10) 

600 except Exception as e: 

601 self.logger.error( 

602 f"Got exception while attempting to insert {xissue.pid} in database : {e}" 

603 ) 

604 raise e 

605 

606 if success is False: 

607 raise ConnectionRefusedError("Cannot connect to SolR") 

608 

609 assert False, "Unreachable" 

610 

611 def get_metadata_using_citation_meta( 

612 self, 

613 xarticle: ArticleData, 

614 xissue: IssueData, 

615 soup: BeautifulSoup, 

616 what: list[CitationLiteral] = [], 

617 ): 

618 """ 

619 :param xarticle: the xarticle that will collect the metadata 

620 :param xissue: the xissue that will collect the publisher 

621 :param soup: the BeautifulSoup object of tha article page 

622 :param what: list of citation_ items to collect. 

623 :return: None. The given article is modified 

624 """ 

625 

626 if "title" in what: 

627 # TITLE 

628 citation_title_node = soup.select_one("meta[name='citation_title']") 

629 if citation_title_node: 629 ↛ 634line 629 didn't jump to line 634 because the condition on line 629 was always true

630 title = citation_title_node.get("content") 

631 if isinstance(title, str): 631 ↛ 634line 631 didn't jump to line 634 because the condition on line 631 was always true

632 xarticle.title_tex = title 

633 

634 if "author" in what: 634 ↛ 663line 634 didn't jump to line 663 because the condition on line 634 was always true

635 # AUTHORS 

636 citation_author_nodes = soup.select("meta[name^='citation_author']") 

637 current_author: ContributorDict | None = None 

638 for citation_author_node in citation_author_nodes: 

639 if citation_author_node.get("name") == "citation_author": 

640 text_author = citation_author_node.get("content") 

641 if not isinstance(text_author, str): 641 ↛ 642line 641 didn't jump to line 642 because the condition on line 641 was never true

642 raise ValueError("Cannot parse author") 

643 if text_author == "": 643 ↛ 644line 643 didn't jump to line 644 because the condition on line 643 was never true

644 current_author = None 

645 continue 

646 current_author = create_contributor(role="author", string_name=text_author) 

647 xarticle.contributors.append(current_author) 

648 continue 

649 if current_author is None: 649 ↛ 650line 649 didn't jump to line 650 because the condition on line 649 was never true

650 self.logger.warning("Couldn't parse citation author") 

651 continue 

652 if citation_author_node.get("name") == "citation_author_institution": 

653 text_institution = citation_author_node.get("content") 

654 if not isinstance(text_institution, str): 654 ↛ 655line 654 didn't jump to line 655 because the condition on line 654 was never true

655 continue 

656 current_author["addresses"].append(text_institution) 

657 if citation_author_node.get("name") == "citation_author_ocrid": 657 ↛ 658line 657 didn't jump to line 658 because the condition on line 657 was never true

658 text_orcid = citation_author_node.get("content") 

659 if not isinstance(text_orcid, str): 

660 continue 

661 current_author["orcid"] = text_orcid 

662 

663 if "pdf" in what: 

664 # PDF 

665 citation_pdf_node = soup.select_one('meta[name="citation_pdf_url"]') 

666 if citation_pdf_node: 

667 pdf_url = citation_pdf_node.get("content") 

668 if isinstance(pdf_url, str): 668 ↛ 671line 668 didn't jump to line 671 because the condition on line 668 was always true

669 add_pdf_link_to_xarticle(xarticle, pdf_url) 

670 

671 if "lang" in what: 

672 # LANG 

673 citation_lang_node = soup.select_one("meta[name='citation_language']") 

674 if citation_lang_node: 674 ↛ 680line 674 didn't jump to line 680 because the condition on line 674 was always true

675 # TODO: check other language code 

676 content_text = citation_lang_node.get("content") 

677 if isinstance(content_text, str): 677 ↛ 680line 677 didn't jump to line 680 because the condition on line 677 was always true

678 xarticle.lang = standardize_tag(content_text) 

679 

680 if "abstract" in what: 

681 # ABSTRACT 

682 abstract_node = soup.select_one("meta[name='citation_abstract']") 

683 if abstract_node is not None: 

684 abstract = abstract_node.get("content") 

685 if not isinstance(abstract, str): 685 ↛ 686line 685 didn't jump to line 686 because the condition on line 685 was never true

686 raise ValueError("Couldn't parse abstract from meta") 

687 abstract = BeautifulSoup(abstract, "html.parser").text 

688 lang = abstract_node.get("lang") 

689 if not isinstance(lang, str): 

690 lang = self.detect_language(abstract, xarticle) 

691 xarticle.abstracts.append(create_abstract(lang=lang, value_tex=abstract)) 

692 

693 if "page" in what: 

694 # PAGES 

695 citation_fpage_node = soup.select_one("meta[name='citation_firstpage']") 

696 if citation_fpage_node: 

697 page = citation_fpage_node.get("content") 

698 if isinstance(page, str): 698 ↛ 703line 698 didn't jump to line 703 because the condition on line 698 was always true

699 page = page.split("(")[0] 

700 if len(page) < 32: 700 ↛ 703line 700 didn't jump to line 703 because the condition on line 700 was always true

701 xarticle.fpage = page 

702 

703 citation_lpage_node = soup.select_one("meta[name='citation_lastpage']") 

704 if citation_lpage_node: 

705 page = citation_lpage_node.get("content") 

706 if isinstance(page, str): 706 ↛ 711line 706 didn't jump to line 711 because the condition on line 706 was always true

707 page = page.split("(")[0] 

708 if len(page) < 32: 708 ↛ 711line 708 didn't jump to line 711 because the condition on line 708 was always true

709 xarticle.lpage = page 

710 

711 if "doi" in what: 

712 # DOI 

713 citation_doi_node = soup.select_one("meta[name='citation_doi']") 

714 if citation_doi_node: 714 ↛ 723line 714 didn't jump to line 723 because the condition on line 714 was always true

715 doi = citation_doi_node.get("content") 

716 if isinstance(doi, str): 716 ↛ 723line 716 didn't jump to line 723 because the condition on line 716 was always true

717 doi = doi.strip() 

718 pos = doi.find("10.") 

719 if pos > 0: 719 ↛ 720line 719 didn't jump to line 720 because the condition on line 719 was never true

720 doi = doi[pos:] 

721 xarticle.doi = doi 

722 

723 if "mr" in what: 

724 # MR 

725 citation_mr_node = soup.select_one("meta[name='citation_mr']") 

726 if citation_mr_node: 726 ↛ 734line 726 didn't jump to line 734 because the condition on line 726 was always true

727 mr = citation_mr_node.get("content") 

728 if isinstance(mr, str): 728 ↛ 734line 728 didn't jump to line 734 because the condition on line 728 was always true

729 mr = mr.strip() 

730 if mr.find("MR") == 0: 730 ↛ 734line 730 didn't jump to line 734 because the condition on line 730 was always true

731 mr = mr[2:] 

732 xarticle.extids.append(("mr-item-id", mr)) 

733 

734 if "zbl" in what: 

735 # ZBL 

736 citation_zbl_node = soup.select_one("meta[name='citation_zbl']") 

737 if citation_zbl_node: 737 ↛ 745line 737 didn't jump to line 745 because the condition on line 737 was always true

738 zbl = citation_zbl_node.get("content") 

739 if isinstance(zbl, str): 739 ↛ 745line 739 didn't jump to line 745 because the condition on line 739 was always true

740 zbl = zbl.strip() 

741 if zbl.find("Zbl") == 0: 741 ↛ 745line 741 didn't jump to line 745 because the condition on line 741 was always true

742 zbl = zbl[3:].strip() 

743 xarticle.extids.append(("zbl-item-id", zbl)) 

744 

745 if "publisher" in what: 

746 # PUBLISHER 

747 citation_publisher_node = soup.select_one("meta[name='citation_publisher']") 

748 if citation_publisher_node: 748 ↛ 757line 748 didn't jump to line 757 because the condition on line 748 was always true

749 pub = citation_publisher_node.get("content") 

750 if isinstance(pub, str): 750 ↛ 757line 750 didn't jump to line 757 because the condition on line 750 was always true

751 pub = pub.strip() 

752 if pub != "": 752 ↛ 757line 752 didn't jump to line 757 because the condition on line 752 was always true

753 xpub = create_publisherdata() 

754 xpub.name = pub 

755 xissue.publisher = xpub 

756 

757 if "keywords" in what: 

758 # KEYWORDS 

759 citation_kwd_nodes = soup.select("meta[name='citation_keywords']") 

760 for kwd_node in citation_kwd_nodes: 

761 kwds = kwd_node.get("content") 

762 if isinstance(kwds, str): 762 ↛ 760line 762 didn't jump to line 760 because the condition on line 762 was always true

763 kwds = kwds.split(",") 

764 for kwd in kwds: 

765 if kwd == "": 

766 continue 

767 kwd = kwd.strip() 

768 xarticle.kwds.append({"type": "", "lang": xarticle.lang, "value": kwd}) 

769 

770 if "references" in what: 

771 citation_references = soup.select("meta[name='citation_reference']") 

772 for index, tag in enumerate(citation_references): 

773 content = tag.get("content") 

774 if not isinstance(content, str): 774 ↛ 775line 774 didn't jump to line 775 because the condition on line 774 was never true

775 raise ValueError("Cannot parse citation_reference meta") 

776 label = str(index + 1) 

777 if regex.match(r"^\[\d+\].*", content): 777 ↛ 778line 777 didn't jump to line 778 because the condition on line 777 was never true

778 label = None 

779 xarticle.bibitems.append(self.__parse_meta_citation_reference(content, label)) 

780 

781 def get_metadata_using_dcterms( 

782 self, 

783 xarticle: ArticleData, 

784 soup: "Tag", 

785 what: "Iterable[Literal['abstract', 'keywords', 'date_published', 'article_type']]", 

786 ): 

787 if "abstract" in what: 787 ↛ 795line 787 didn't jump to line 795 because the condition on line 787 was always true

788 abstract_tag = soup.select_one("meta[name='DCTERMS.abstract']") 

789 if abstract_tag: 789 ↛ 795line 789 didn't jump to line 795 because the condition on line 789 was always true

790 abstract_text = self.get_str_attr(abstract_tag, "content") 

791 xarticle.abstracts.append( 

792 create_abstract(lang="en", value_tex=cleanup_str(abstract_text)) 

793 ) 

794 

795 if "keywords" in what: 795 ↛ 804line 795 didn't jump to line 804 because the condition on line 795 was always true

796 keyword_tags = soup.select("meta[name='DC.subject']") 

797 for tag in keyword_tags: 

798 kwd_text = tag.get("content") 

799 if not isinstance(kwd_text, str) or len(kwd_text) == 0: 799 ↛ 800line 799 didn't jump to line 800 because the condition on line 799 was never true

800 continue 

801 kwd = create_subj(value=kwd_text) 

802 xarticle.kwds.append(kwd) 

803 

804 if "date_published" in what: 804 ↛ 805line 804 didn't jump to line 805 because the condition on line 804 was never true

805 published_tag = soup.select_one("meta[name='DC.Date.created']") 

806 if published_tag: 

807 published_text = self.get_str_attr(published_tag, "content") 

808 xarticle.date_published = published_text 

809 

810 if "article_type" in what: 810 ↛ 811line 810 didn't jump to line 811 because the condition on line 810 was never true

811 type_tag = soup.select_one("meta[name='DC.Type.articleType']") 

812 if type_tag: 

813 type_text = self.get_str_attr(type_tag, "content") 

814 xarticle.atype = type_text 

815 

816 def create_xissue( 

817 self, 

818 url: str | None, 

819 year: int, 

820 volume_number: str | None, 

821 issue_number: str | None = None, 

822 vseries: str | None = None, 

823 ): 

824 if url is not None and url.endswith("/"): 824 ↛ 825line 824 didn't jump to line 825 because the condition on line 824 was never true

825 url = url[:-1] 

826 xissue = create_issuedata() 

827 xissue.url = url 

828 

829 xissue.pid = self.get_issue_pid( 

830 self.collection_id, year, volume_number, issue_number, vseries 

831 )