Coverage for src/crawler/abstract_crawlers/base_crawler.py: 62%
624 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
1import logging
2import time
3from collections.abc import Iterable
4from datetime import datetime, timedelta
5from email.policy import EmailPolicy
6from typing import TYPE_CHECKING, Literal
8import aiohttp
9import regex
10import requests
11from aiohttp.client_exceptions import ClientResponseError
12from bs4 import BeautifulSoup
13from django.conf import settings
14from django.contrib.auth.models import User
15from django.db.utils import IntegrityError
16from django.utils import timezone
17from langcodes import standardize_tag
18from lingua import LanguageDetector, LanguageDetectorBuilder
19from opentelemetry import trace
20from ptf.cmds.xml.ckeditor.utils import (
21 build_jats_data_from_html_field,
22)
23from ptf.cmds.xml.jats.builder.references import (
24 get_article_title_xml,
25 get_author_xml,
26 get_fpage_xml,
27 get_lpage_xml,
28 get_source_xml,
29 get_year_xml,
30)
31from ptf.cmds.xml.jats.jats_parser import parse_mixed_citation_into_ref
32from ptf.model_data import (
33 ArticleData,
34 ContributorDict,
35 IssueData,
36 ResourceData,
37 TitleDict,
38 create_abstract,
39 create_contributor,
40 create_extid,
41 create_issuedata,
42 create_publisherdata,
43 create_subj,
44 create_titledata,
45)
46from ptf.model_data_converter import update_data_for_jats
47from ptf.models import ExtLink
48from ptf_external.session import get_session
49from pylatexenc.latex2text import LatexNodes2Text
50from pysolr import SolrError
51from requests_cache import CachedSession
53from crawler.cmds.xml_cmds import addOrUpdateGDMLIssueXmlCmd
54from crawler.models import Source
55from crawler.models.extlink_checked import ExtlinkChecked
56from crawler.types import CitationLiteral
57from crawler.utils import (
58 add_pdf_link_to_xarticle,
59 add_source_link_to_xarticle,
60 cleanup_str,
61 get_all_cols,
62 get_or_create_collection,
63)
65if TYPE_CHECKING:
66 from collections.abc import Callable
68 from bs4 import Tag
71class CrawlerTitleDict(TitleDict):
72 title_tex: str | None
75class BaseCollectionCrawler:
76 """
77 Base collection for the crawlers.
78 To create a crawler:
79 1) derive a class from BaseCollectionCrawler and name it XXXCrawler
80 2) override the functions parse_collection_content, parse_issue_content and parse_article_content
81 3) update factory.py so that crawler_factory can return your new crawler
82 """
84 logger = logging.getLogger(__name__)
85 tracer = trace.get_tracer(__name__)
87 source_name = ""
88 source_domain = ""
89 source_website = ""
91 issue_href = ""
93 collection = None
94 source = None
95 user = None
96 session: requests.Session | CachedSession
97 async_session: aiohttp.ClientSession
98 is_checkable = True
99 verify = True
100 headers = {}
102 requests_interval = getattr(settings, "REQUESTS_INTERVAL", 90)
103 "seconds to wait between two http requests"
104 requests_timeout = 60
105 "seconds to wait before aborting the connection (if no bytes are recieved)"
107 latext_parser = LatexNodes2Text()
109 # Override the values in your concrete crawler if the formulas in text (titles, abstracts)
110 # do not use the "$" to surround tex formulas
111 delimiter_inline_formula = "$"
112 delimiter_disp_formula = "$"
114 # HACK : Workaround for tests (monkeypatching)
115 # We store the class here, so we can monkeypatch it when running tests
116 # subCrawlers = {
117 # LofplCrawler: None
118 # }
119 subCrawlers: dict[type["BaseCollectionCrawler"], "BaseCollectionCrawler | None"] = {}
121 _language_detector: LanguageDetector | None = None
122 _language_detector_builder = LanguageDetectorBuilder.from_all_languages()
124 match_headers = False
125 "Whereas to include headers in requests cache key"
126 orcid_re = r"https\:\/\/orcid\.org\/(?P<orcid>\d{4}-\d{4}-\d{4}-\d{4})"
128 ignore_missing_pdf = True
129 "Set this to False on a Crawler-basis to allow inserting articles without PDFs"
130 pid_year_restrictions: dict[str, int] = {}
131 "pid -> excluded years count"
133 pause_function: "Callable[[int], None]"
134 "Overridable the pause function (used in celery tasks to speedup aborting)"
136 @classmethod
137 def get_view_id(cls):
138 return cls.source_domain
140 @property
141 def language_detector(self):
142 """Crawler Instance singleton for language builder.
143 Late init of LanguageDetector to save on memory"""
144 if not self._language_detector:
145 self._language_detector = self._language_detector_builder.build()
146 return self._language_detector
148 def __init__(
149 self,
150 *args,
151 username: str,
152 collection_id: str,
153 dry: bool = False,
154 publisher: str = "",
155 collection_url: str | None = None,
156 backend=None,
157 pause_function=staticmethod(time.sleep), # noqa: B008
158 ):
159 if not collection_url: 159 ↛ 160line 159 didn't jump to line 160 because the condition on line 159 was never true
160 all_cols = get_all_cols()
161 col = all_cols[collection_id]
163 collection_url = col["sources"].get(self.source_domain, None)
164 if collection_url is None:
165 raise ValueError(
166 f"Source {self.source_domain} not found for collection {collection_id}"
167 )
168 self.collection_url = collection_url
169 for CrawlerClass in self.subCrawlers: 169 ↛ 170line 169 didn't jump to line 170 because the loop on line 169 never started
170 self.subCrawlers[CrawlerClass] = CrawlerClass(
171 *args,
172 username=username,
173 collection_id=collection_id,
174 dry=dry,
175 publisher=publisher,
176 collection_url=collection_url,
177 )
178 self.logger = logging.getLogger(__name__ + "." + self.source_domain)
179 # self.logger = logging.getLogger(__name__)
181 self.username = username
183 self.collection_id = collection_id
185 self.dry = dry
186 self.publisher = publisher
188 # Classproperty : We sometimes want to use the session without initializing the class (rot monitoring)
189 BaseCollectionCrawler.session = requests.Session()
191 self.pause_function = pause_function
193 # Skipped when running tests
194 self.initialize()
196 self.backend = backend
198 def initialize(self):
199 """
200 Acts as a "second" init function to skip model accesses during test data generation
201 """
202 self.collection = get_or_create_collection(self.collection_id)
203 self.source = self.get_or_create_source()
204 self.user = User.objects.get(username=self.username)
205 BaseCollectionCrawler.session = get_session()
206 BaseCollectionCrawler.session.verify = self.verify
207 self.session.pause_function = self.pause_function
208 self.session.delay = self.requests_interval
210 @classmethod
211 def can_crawl(cls, pid: str) -> bool:
212 return True
214 def parse_collection_content(self, content: str) -> list[IssueData]:
215 """
216 Parse the HTML content with BeautifulSoup
217 returns a list of xissue.
218 Override this function in a derived class
219 """
220 return []
222 def parse_issue_content(self, content: str, xissue: IssueData):
223 """
224 Parse the HTML content with BeautifulSoup
225 Fills the xissue.articles
226 Override this function in a derived class.
228 CAV : You are supposed to create articles there. Please assign a PID to each article.
229 The PID can be `a + article_index`, like this : `a0` `a21`
230 """
232 def parse_article_content(
233 self, content: str, xissue: IssueData, xarticle: ArticleData, url: str
234 ) -> ArticleData | None:
235 """
236 Parse the HTML content with BeautifulSoup
237 returns the xarticle.
238 Override this function in a derived class.
239 The xissue is passed to the function in case the article page has issue information (ex: publisher)
240 The article url is also passed as a parameter
241 CkeditorParser must not be called here; the abstract is created with the value_tex,
242 even if the string is not a tex
244 CAV : You are supposed to assign articles pid again here
245 """
246 return xarticle
248 @tracer.start_as_current_span("crawl_collection")
249 def crawl_collection(self):
250 # TODO: Comments, filter
251 """
252 Crawl an entire collection. ptf.models.Container objects are created.
253 - get the HTML content of the collection_url
254 - parse the HTML content with beautifulsoup to extract the list of issues
255 - merge the xissues (some Source can have multiple pages for 1 volume/issue. We create only 1 container)
256 - crawl each issue if col_only is False
257 - Returns the list of merged issues.
258 It is an OrderedDict {pid: {"issues": xissues}}
259 The key is the pid of the merged issues.
260 Ex: The source may have Ex: Volume 6 (2000) and Volume 6 (1999)
261 the pid is then made with 1999-2000__6_
262 """
264 if self.source is None:
265 raise RuntimeError("ERROR: the source is not set")
267 content = self.download_file(self.collection_url)
268 if content:
269 xissues = self.parse_collection_content(content)
270 else:
271 # download_file returns None (404)
272 return None
274 """
275 Some collections split the same volumes in different pages
276 Ex: Volume 6 (2000) and Volume 6 (1999)
277 We merge the 2 xissues with the same volume number => Volume 6 (1999-2000)
278 """
279 # merged_xissues = self.merge_xissues(xissues)
281 xissues_dict = {str(i.pid): i for i in xissues}
283 return xissues_dict
285 def start_process_issue(self, xissue: IssueData):
286 # Some source, like EuDML do not have a separate HTML pages for an issue's table of content.
287 # The list of articles directly come from the collection HTML page: the xissue has no url attribute
288 issue_url = xissue.url
289 if issue_url is not None:
290 if issue_url.endswith(".pdf"):
291 add_pdf_link_to_xarticle(xissue, issue_url)
292 xissue.url = None
293 else:
294 content = self.download_file(issue_url)
295 with self.tracer.start_as_current_span("parse_issue_content"):
296 self.parse_issue_content(content, xissue)
298 @tracer.start_as_current_span("crawl_issue")
299 def crawl_issue(self, xissue: IssueData):
300 """
301 Crawl 1 wag page of an issue.
302 - get the HTML content of the issue
303 - parse the HTML content with beautifulsoup to extract the list of articles and/or the issue metadata
304 - crawl each article
305 """
307 self.start_process_issue(xissue)
309 xarticles = xissue.articles
311 parsed_xarticles = []
313 for xarticle in xarticles:
314 parsed_xarticle = self.crawl_article(xarticle, xissue)
315 if parsed_xarticle is not None:
316 parsed_xarticles.append(parsed_xarticle)
318 xissue.articles = parsed_xarticles
320 issue_has_pdf = self.article_has_pdf(xissue)
322 if self.ignore_missing_pdf:
323 xissue.articles = [a for a in xissue.articles if self.article_has_pdf(a)]
324 if self.dry:
325 return
326 if len(xissue.articles) == 0 and not issue_has_pdf:
327 return
328 self.process_resource_metadata(xissue, resource_type="issue")
330 self.add_xissue_into_database(xissue)
332 @staticmethod
333 def article_has_source(art: ArticleData | IssueData):
334 return (
335 next(
336 (e_link for e_link in art.ext_links if e_link["rel"] == "source"),
337 None,
338 )
339 is not None
340 )
342 @staticmethod
343 def article_has_pdf(art: ArticleData | IssueData):
344 return (
345 next(
346 (link for link in art.ext_links if link["rel"] in ["article-pdf", "article-html"]),
347 None,
348 )
349 is not None
350 )
352 def crawl_article(self, xarticle: ArticleData, xissue: IssueData):
353 # ARTICLE URL as en ExtLink (to display the link in the article page)
354 if xarticle.url is None:
355 if not self.article_has_source(xarticle): 355 ↛ 361line 355 didn't jump to line 361 because the condition on line 355 was always true
356 if xissue.url:
357 article_source = xissue.url
358 else:
359 article_source = self.collection_url
360 add_source_link_to_xarticle(xarticle, article_source, self.source_domain)
361 return self.process_article_metadata(xarticle)
363 if self.parse_article_content.__func__ != BaseCollectionCrawler.parse_article_content:
364 content = self.download_file(xarticle.url)
365 xarticle.pid = f"{xissue.pid}_{xarticle.pid}"
367 try:
368 with self.tracer.start_as_current_span("parse_article_content"):
369 parsed_xarticle = self.parse_article_content(
370 content, xissue, xarticle, xarticle.url
371 )
372 except ValueError as e:
373 self.logger.warning(e)
374 self.logger.warning("Retrying in 5 mins while invalidating cache")
375 self.pause_function(5 * 60)
376 content = self.download_file(xarticle.url, headers={"Cache-Control": "no-cache"})
377 with self.tracer.start_as_current_span("parse_article_content"):
378 parsed_xarticle = self.parse_article_content(
379 content, xissue, xarticle, xarticle.url
380 )
382 if not parsed_xarticle: 382 ↛ 383line 382 didn't jump to line 383 because the condition on line 382 was never true
383 return None
385 xarticle = parsed_xarticle
387 if xarticle.doi:
388 xarticle.pid = xarticle.doi.replace("/", "_").replace(".", "_").replace("-", "_")
390 if not self.article_has_source(xarticle) and xarticle.url:
391 add_source_link_to_xarticle(xarticle, xarticle.url, self.source_domain)
393 # The article title may have formulas surrounded with '$'
394 return self.process_article_metadata(xarticle)
396 def process_resource_metadata(self, xresource: ResourceData, resource_type="article"):
397 tag = "article-title" if resource_type == "article" else "issue-title"
399 # Process title tex
400 ckeditor_data = build_jats_data_from_html_field(
401 xresource.title_tex,
402 tag=tag,
403 text_lang=xresource.lang,
404 delimiter_inline=self.delimiter_inline_formula,
405 delimiter_disp=self.delimiter_disp_formula,
406 )
408 xresource.title_html = ckeditor_data["value_html"]
409 # xresource.title_tex = ckeditor_data["value_tex"]
410 xresource.title_xml = ckeditor_data["value_xml"]
412 abstracts_to_parse = [
413 xabstract for xabstract in xresource.abstracts if xabstract["tag"] == "abstract"
414 ]
415 # abstract may have formulas surrounded with '$'
416 if len(abstracts_to_parse) > 0:
417 for xabstract in abstracts_to_parse:
418 ckeditor_data = build_jats_data_from_html_field(
419 xabstract["value_tex"],
420 tag="abstract",
421 text_lang=xabstract["lang"],
422 resource_lang=xresource.lang,
423 field_type="abstract",
424 delimiter_inline=self.delimiter_inline_formula,
425 delimiter_disp=self.delimiter_disp_formula,
426 )
428 xabstract["value_html"] = ckeditor_data["value_html"]
429 # xabstract["value_tex"] = ckeditor_data["value_tex"]
430 xabstract["value_xml"] = ckeditor_data["value_xml"]
432 # Remove useless titles (duplicates)
433 titles = []
434 for trans_title in xresource.titles:
435 if trans_title["title_html"] != xresource.title_html: 435 ↛ 434line 435 didn't jump to line 434 because the condition on line 435 was always true
436 # trans_title["title_html"] = self.latex_converter.latex_to_text(
437 # trans_title["title_html"]
438 # )
439 titles.append(trans_title)
441 xresource.titles = titles
443 # remove duplicated keywords and add "msc" type to msc keywords :
444 xresource_keywords = []
445 for resource_keyword_dict in xresource.kwds:
446 resource_keyword = resource_keyword_dict["value"]
447 if resource_keyword == "" or resource_keyword in [ 447 ↛ 450line 447 didn't jump to line 450 because the condition on line 447 was never true
448 xresource_keyword["value"] for xresource_keyword in xresource_keywords
449 ]:
450 continue
452 if any(char.isdigit() for char in resource_keyword):
453 resource_keyword_dict["type"] = "msc"
455 xresource_keywords.append(resource_keyword_dict)
456 xresource.kwds = xresource_keywords
458 return xresource
460 def process_article_metadata(self, xarticle: ArticleData):
461 self.process_resource_metadata(xarticle)
462 for bibitem in xarticle.bibitems:
463 bibitem.type = "unknown"
464 update_data_for_jats(xarticle, with_label=False)
466 return xarticle
468 def download_file(self, url: str, headers=None):
469 """
470 Downloads a page and returns its content (decoded string).
471 """
473 if headers is None:
474 headers = {}
475 for attempt in range(3):
476 session: CachedSession = get_session() # type:ignore
477 response = session.get(
478 url,
479 headers=headers,
480 )
482 content = self.decode_response(response)
483 if content == "" or not content:
484 self.logger.debug("Got empty content while fetching ! ")
485 # 15 mins, 30 mins, 45 mins
486 delay_minutes = attempt * 15
487 self.logger.debug(
488 f"Retrying in {delay_minutes}mins ({(datetime.now() + timedelta(minutes=delay_minutes)).time()})",
489 extra={"url": url},
490 )
491 self.pause_function(delay_minutes * 60)
492 continue
493 return content
494 raise ValueError(f"Could not decode content at {url}")
496 def decode_response(self, response: requests.Response, encoding: str | None = None):
497 """Override this if the content-type headers from the sources are advertising something else than the actual content
498 SASA needs this"""
499 # Force
500 if encoding:
501 response.encoding = encoding
502 return response.text
504 # Attempt to get encoding using HTTP headers
505 content_type_tag = response.headers.get("Content-Type", None)
507 if content_type_tag: 507 ↛ 514line 507 didn't jump to line 514 because the condition on line 507 was always true
508 charset = self.parse_content_type_charset(content_type_tag)
509 if charset: 509 ↛ 510line 509 didn't jump to line 510 because the condition on line 509 was never true
510 response.encoding = charset
511 return response.text
513 # Attempt to get encoding using HTML meta charset tag
514 soup = BeautifulSoup(response.text, "html5lib")
515 charset = soup.select_one("meta[charset]")
516 if charset:
517 htmlencoding = charset.get("charset")
518 if isinstance(htmlencoding, str): 518 ↛ 523line 518 didn't jump to line 523 because the condition on line 518 was always true
519 response.encoding = htmlencoding
520 return response.text
522 # Attempt to get encoding using HTML meta content type tag
523 content_type_tag = soup.select_one(
524 'meta[http-equiv="Content-Type"],meta[http-equiv="content-type"]'
525 )
526 if content_type_tag:
527 content_type = content_type_tag.get("content")
528 if isinstance(content_type, str): 528 ↛ 534line 528 didn't jump to line 534 because the condition on line 528 was always true
529 charset = self.parse_content_type_charset(content_type)
530 if charset: 530 ↛ 534line 530 didn't jump to line 534 because the condition on line 530 was always true
531 response.encoding = charset
532 return response.text
534 return response.text
536 @staticmethod
537 def parse_content_type_charset(content_type: str):
538 header = EmailPolicy.header_factory("content-type", content_type)
539 if "charset" in header.params:
540 return header.params.get("charset")
542 @tracer.start_as_current_span("add_xissue_to_database")
543 def add_xissue_into_database(self, xissue: IssueData) -> IssueData:
544 xissue.journal = self.collection
545 xissue.source = self.source_domain
547 if xissue.fyear == 0:
548 raise ValueError("Failsafe : Cannot insert issue without a year")
550 xpub = create_publisherdata()
551 xpub.name = self.publisher
552 xissue.publisher = xpub
553 xissue.last_modified_iso_8601_date_str = timezone.now().isoformat()
555 attempt = 1
556 success = False
558 while not success and attempt < 4:
559 try:
560 params = {"xissue": xissue, "use_body": False}
561 cmd = addOrUpdateGDMLIssueXmlCmd(params)
562 cmd.do()
563 success = True
564 self.logger.debug(f"Issue {xissue.pid} inserted in database")
565 return xissue
566 except SolrError:
567 self.logger.warning(
568 f"Encoutered SolrError while inserting issue {xissue.pid} in database"
569 )
570 attempt += 1
571 self.logger.debug(f"Attempt {attempt}. sleeping 10 seconds.")
572 self.pause_function(10)
573 except Exception as e:
574 self.logger.error(
575 f"Got exception while attempting to insert {xissue.pid} in database : {e}"
576 )
577 raise
579 if success is False:
580 raise ConnectionRefusedError("Cannot connect to SolR")
582 assert False, "Unreachable"
584 def get_metadata_using_citation_meta(
585 self,
586 xarticle: ArticleData,
587 xissue: IssueData,
588 soup: BeautifulSoup,
589 what: list[CitationLiteral] | None = None,
590 ):
591 """
592 :param xarticle: the xarticle that will collect the metadata
593 :param xissue: the xissue that will collect the publisher
594 :param soup: the BeautifulSoup object of tha article page
595 :param what: list of citation_ items to collect.
596 :return: None. The given article is modified
597 """
599 if what is None: 599 ↛ 600line 599 didn't jump to line 600 because the condition on line 599 was never true
600 what = []
601 if "title" in what:
602 # TITLE
603 citation_title_node = soup.select_one("meta[name='citation_title']")
604 if citation_title_node: 604 ↛ 609line 604 didn't jump to line 609 because the condition on line 604 was always true
605 title = citation_title_node.get("content")
606 if isinstance(title, str): 606 ↛ 609line 606 didn't jump to line 609 because the condition on line 606 was always true
607 xarticle.title_tex = title
609 if "author" in what: 609 ↛ 638line 609 didn't jump to line 638 because the condition on line 609 was always true
610 # AUTHORS
611 citation_author_nodes = soup.select("meta[name^='citation_author']")
612 current_author: ContributorDict | None = None
613 for citation_author_node in citation_author_nodes:
614 if citation_author_node.get("name") == "citation_author":
615 text_author = citation_author_node.get("content")
616 if not isinstance(text_author, str): 616 ↛ 617line 616 didn't jump to line 617 because the condition on line 616 was never true
617 raise ValueError("Cannot parse author")
618 if text_author == "": 618 ↛ 619line 618 didn't jump to line 619 because the condition on line 618 was never true
619 current_author = None
620 continue
621 current_author = create_contributor(role="author", string_name=text_author)
622 xarticle.contributors.append(current_author)
623 continue
624 if current_author is None: 624 ↛ 625line 624 didn't jump to line 625 because the condition on line 624 was never true
625 self.logger.warning("Couldn't parse citation author")
626 continue
627 if citation_author_node.get("name") == "citation_author_institution":
628 text_institution = citation_author_node.get("content")
629 if not isinstance(text_institution, str): 629 ↛ 630line 629 didn't jump to line 630 because the condition on line 629 was never true
630 continue
631 current_author["addresses"].append(text_institution)
632 if citation_author_node.get("name") == "citation_author_ocrid": 632 ↛ 633line 632 didn't jump to line 633 because the condition on line 632 was never true
633 text_orcid = citation_author_node.get("content")
634 if not isinstance(text_orcid, str):
635 continue
636 current_author["orcid"] = text_orcid
638 if "pdf" in what:
639 # PDF
640 citation_pdf_node = soup.select_one('meta[name="citation_pdf_url"]')
641 if citation_pdf_node:
642 pdf_url = citation_pdf_node.get("content")
643 if isinstance(pdf_url, str): 643 ↛ 646line 643 didn't jump to line 646 because the condition on line 643 was always true
644 add_pdf_link_to_xarticle(xarticle, pdf_url)
646 if "lang" in what:
647 # LANG
648 citation_lang_node = soup.select_one("meta[name='citation_language']")
649 if citation_lang_node: 649 ↛ 655line 649 didn't jump to line 655 because the condition on line 649 was always true
650 # TODO: check other language code
651 content_text = citation_lang_node.get("content")
652 if isinstance(content_text, str): 652 ↛ 655line 652 didn't jump to line 655 because the condition on line 652 was always true
653 xarticle.lang = standardize_tag(content_text)
655 if "abstract" in what:
656 # ABSTRACT
657 abstract_node = soup.select_one("meta[name='citation_abstract']")
658 if abstract_node is not None:
659 abstract = abstract_node.get("content")
660 if not isinstance(abstract, str): 660 ↛ 661line 660 didn't jump to line 661 because the condition on line 660 was never true
661 raise ValueError("Couldn't parse abstract from meta")
662 abstract = BeautifulSoup(abstract, "html.parser").text
663 lang = abstract_node.get("lang")
664 if not isinstance(lang, str):
665 lang = self.detect_language(abstract, xarticle)
666 xarticle.abstracts.append(create_abstract(lang=lang, value_tex=abstract))
668 if "page" in what:
669 # PAGES
670 citation_fpage_node = soup.select_one("meta[name='citation_firstpage']")
671 if citation_fpage_node:
672 page = citation_fpage_node.get("content")
673 if isinstance(page, str): 673 ↛ 678line 673 didn't jump to line 678 because the condition on line 673 was always true
674 page = page.split("(")[0]
675 if len(page) < 32: 675 ↛ 678line 675 didn't jump to line 678 because the condition on line 675 was always true
676 xarticle.fpage = page
678 citation_lpage_node = soup.select_one("meta[name='citation_lastpage']")
679 if citation_lpage_node:
680 page = citation_lpage_node.get("content")
681 if isinstance(page, str): 681 ↛ 686line 681 didn't jump to line 686 because the condition on line 681 was always true
682 page = page.split("(")[0]
683 if len(page) < 32: 683 ↛ 686line 683 didn't jump to line 686 because the condition on line 683 was always true
684 xarticle.lpage = page
686 if "doi" in what:
687 # DOI
688 citation_doi_node = soup.select_one("meta[name='citation_doi']")
689 if citation_doi_node: 689 ↛ 698line 689 didn't jump to line 698 because the condition on line 689 was always true
690 doi = citation_doi_node.get("content")
691 if isinstance(doi, str): 691 ↛ 698line 691 didn't jump to line 698 because the condition on line 691 was always true
692 doi = doi.strip()
693 pos = doi.find("10.")
694 if pos > 0: 694 ↛ 695line 694 didn't jump to line 695 because the condition on line 694 was never true
695 doi = doi[pos:]
696 xarticle.doi = doi
698 if "mr" in what:
699 # MR
700 citation_mr_node = soup.select_one("meta[name='citation_mr']")
701 if citation_mr_node: 701 ↛ 710line 701 didn't jump to line 710 because the condition on line 701 was always true
702 mr = citation_mr_node.get("content")
703 if isinstance(mr, str): 703 ↛ 710line 703 didn't jump to line 710 because the condition on line 703 was always true
704 mr = mr.strip()
705 if mr.find("MR") == 0: 705 ↛ 710line 705 didn't jump to line 710 because the condition on line 705 was always true
706 mr = mr[2:]
707 extid = create_extid("mr-item-id", mr)
708 xarticle.extids.append(extid)
710 if "zbl" in what:
711 # ZBL
712 citation_zbl_node = soup.select_one("meta[name='citation_zbl']")
713 if citation_zbl_node: 713 ↛ 722line 713 didn't jump to line 722 because the condition on line 713 was always true
714 zbl = citation_zbl_node.get("content")
715 if isinstance(zbl, str): 715 ↛ 722line 715 didn't jump to line 722 because the condition on line 715 was always true
716 zbl = zbl.strip()
717 if zbl.find("Zbl") == 0: 717 ↛ 722line 717 didn't jump to line 722 because the condition on line 717 was always true
718 zbl = zbl[3:].strip()
719 extid = create_extid("zbl-item-id", zbl)
720 xarticle.extids.append(extid)
722 if "publisher" in what:
723 # PUBLISHER
724 citation_publisher_node = soup.select_one("meta[name='citation_publisher']")
725 if citation_publisher_node: 725 ↛ 734line 725 didn't jump to line 734 because the condition on line 725 was always true
726 pub = citation_publisher_node.get("content")
727 if isinstance(pub, str): 727 ↛ 734line 727 didn't jump to line 734 because the condition on line 727 was always true
728 pub = pub.strip()
729 if pub != "": 729 ↛ 734line 729 didn't jump to line 734 because the condition on line 729 was always true
730 xpub = create_publisherdata()
731 xpub.name = pub
732 xissue.publisher = xpub
734 if "keywords" in what:
735 # KEYWORDS
736 citation_kwd_nodes = soup.select("meta[name='citation_keywords']")
737 for kwd_node in citation_kwd_nodes:
738 kwds = kwd_node.get("content")
739 if isinstance(kwds, str): 739 ↛ 737line 739 didn't jump to line 737 because the condition on line 739 was always true
740 kwds = kwds.split(",")
741 for kwd in kwds:
742 if kwd == "": 742 ↛ 743line 742 didn't jump to line 743 because the condition on line 742 was never true
743 continue
744 kwd = kwd.strip()
745 xarticle.kwds.append({"type": "", "lang": xarticle.lang, "value": kwd})
747 if "references" in what:
748 citation_references = soup.select("meta[name='citation_reference']")
749 for index, tag in enumerate(citation_references):
750 content = tag.get("content")
751 if not isinstance(content, str): 751 ↛ 752line 751 didn't jump to line 752 because the condition on line 751 was never true
752 raise ValueError("Cannot parse citation_reference meta")
753 label = str(index + 1)
754 if regex.match(r"^\[\d+\].*", content): 754 ↛ 755line 754 didn't jump to line 755 because the condition on line 754 was never true
755 label = None
756 xarticle.bibitems.append(self.__parse_meta_citation_reference(content, label))
758 def get_metadata_using_dcterms(
759 self,
760 xarticle: ArticleData,
761 soup: "Tag",
762 what: "Iterable[Literal['abstract', 'keywords', 'date_published', 'article_type']]",
763 ):
764 if "abstract" in what: 764 ↛ 772line 764 didn't jump to line 772 because the condition on line 764 was always true
765 abstract_tag = soup.select_one("meta[name='DCTERMS.abstract']")
766 if abstract_tag: 766 ↛ 772line 766 didn't jump to line 772 because the condition on line 766 was always true
767 abstract_text = self.get_str_attr(abstract_tag, "content")
768 xarticle.abstracts.append(
769 create_abstract(lang="en", value_tex=cleanup_str(abstract_text))
770 )
772 if "keywords" in what: 772 ↛ 781line 772 didn't jump to line 781 because the condition on line 772 was always true
773 keyword_tags = soup.select("meta[name='DC.subject']")
774 for tag in keyword_tags:
775 kwd_text = tag.get("content")
776 if not isinstance(kwd_text, str) or len(kwd_text) == 0: 776 ↛ 777line 776 didn't jump to line 777 because the condition on line 776 was never true
777 continue
778 kwd = create_subj(value=kwd_text)
779 xarticle.kwds.append(kwd)
781 if "date_published" in what: 781 ↛ 782line 781 didn't jump to line 782 because the condition on line 781 was never true
782 published_tag = soup.select_one("meta[name='DC.Date.created']")
783 if published_tag:
784 published_text = self.get_str_attr(published_tag, "content")
785 xarticle.date_published = published_text
787 if "article_type" in what: 787 ↛ 788line 787 didn't jump to line 788 because the condition on line 787 was never true
788 type_tag = soup.select_one("meta[name='DC.Type.articleType']")
789 if type_tag:
790 type_text = self.get_str_attr(type_tag, "content")
791 xarticle.atype = type_text
793 def create_xissue(
794 self,
795 url: str | None,
796 year: int,
797 volume_number: str | None,
798 issue_number: str | None = None,
799 vseries: str | None = None,
800 lyear: int | None = None,
801 ):
802 if url is not None and url.endswith("/"): 802 ↛ 803line 802 didn't jump to line 803 because the condition on line 802 was never true
803 url = url[:-1]
804 xissue = create_issuedata()
805 xissue.url = url
807 year_str = year
808 xissue.fyear = year
809 if lyear: 809 ↛ 810line 809 didn't jump to line 810 because the condition on line 809 was never true
810 year_str = f"{year}_{lyear}"
811 xissue.lyear = lyear
812 xissue.pid = self.get_issue_pid(
813 self.collection_id, year_str, volume_number, issue_number, vseries
814 )
816 if volume_number is not None:
817 xissue.volume = regex.sub(r"[^\w-]+", "_", volume_number)
819 if issue_number is not None:
820 xissue.number = issue_number.replace(",", "-")
822 if vseries is not None: 822 ↛ 823line 822 didn't jump to line 823 because the condition on line 822 was never true
823 xissue.vseries = vseries
824 return xissue
826 def detect_language(self, text: str, article: ArticleData | None = None):
827 if article and article.lang is not None and article.lang != "und":
828 return article.lang
830 language = self.language_detector.detect_language_of(text)
832 if not language: 832 ↛ 833line 832 didn't jump to line 833 because the condition on line 832 was never true
833 return "und"
834 return language.iso_code_639_1.name.lower()
836 def get_str_attr(self, tag: "Tag", attr: str):
837 """Equivalent of `tag.get(attr)`, but ensures the return value is a string"""
838 node_attr = tag.get(attr)
839 if isinstance(node_attr, list): 839 ↛ 840line 839 didn't jump to line 840 because the condition on line 839 was never true
840 raise ValueError(
841 f"[{self.source_domain}] {self.collection_id} : html tag has multiple {attr} attributes."
842 )
843 if node_attr is None: 843 ↛ 844line 843 didn't jump to line 844 because the condition on line 843 was never true
844 raise ValueError(
845 f"[{self.source_domain}] {self.collection_id} : html tag doesn't have any {attr} attributes"
846 )
847 return node_attr
849 def create_trans_title(
850 self,
851 resource_type: str,
852 title_str: str,
853 lang: str,
854 xresource_lang: str,
855 title_type: str = "main",
856 ):
857 tag = "trans-title" if resource_type == "article" else "issue-title"
859 ckeditor_data = build_jats_data_from_html_field(
860 title_str,
861 tag=tag,
862 text_lang=lang,
863 resource_lang=xresource_lang,
864 delimiter_inline=self.delimiter_inline_formula,
865 delimiter_disp=self.delimiter_disp_formula,
866 )
868 titledata = create_titledata(
869 lang=lang,
870 type="main",
871 title_html=ckeditor_data["value_html"],
872 title_xml=ckeditor_data["value_xml"],
873 )
875 return titledata
877 references_mapping = {
878 "citation_title": get_article_title_xml,
879 "citation_journal_title": get_source_xml,
880 "citation_publication_date": get_year_xml,
881 "citation_firstpage": get_fpage_xml,
882 "citation_lastpage": get_lpage_xml,
883 }
885 @classmethod
886 def __parse_meta_citation_reference(cls, content: str, label=None):
887 categories = content.split(";")
889 if len(categories) == 1:
890 return parse_mixed_citation_into_ref(content, label=label)
892 citation_data = [c.split("=") for c in categories if "=" in c]
893 del categories
895 xml_string = ""
896 authors_parsed = False
897 authors_strings = []
898 for data in citation_data:
899 key = data[0].strip()
900 citation_content = data[1]
901 if key == "citation_author":
902 authors_strings.append(get_author_xml(template_str=citation_content))
903 continue
904 elif not authors_parsed:
905 xml_string += ", ".join(authors_strings)
906 authors_parsed = True
908 if key in cls.references_mapping:
909 xml_string += " " + cls.references_mapping[key](citation_content)
911 return parse_mixed_citation_into_ref(xml_string, label=label)
913 @classmethod
914 def get_or_create_source(cls):
915 source, created = Source.objects.get_or_create(
916 domain=cls.source_domain,
917 defaults={
918 "name": cls.source_name,
919 "website": cls.source_website,
920 "view_id": cls.get_view_id(),
921 },
922 )
923 if created:
924 source.save()
925 return source
927 @staticmethod
928 def get_issue_pid(
929 collection_id: str,
930 year: int | str,
931 volume_number: str | None = None,
932 issue_number: str | None = None,
933 series: str | None = None,
934 ):
935 # Replace any non-word character with an underscore
936 pid = f"{collection_id}_{year}"
937 if series is not None: 937 ↛ 938line 937 didn't jump to line 938 because the condition on line 937 was never true
938 pid += f"_{series}"
939 if volume_number is not None:
940 pid += f"_{volume_number}"
941 if issue_number is not None:
942 pid += f"_{issue_number}"
943 pid = regex.sub(r"[^\w-]+", "_", cleanup_str(pid))
944 return pid
946 @staticmethod
947 def set_pages(article: ArticleData, pages: str, separator: str = "-"):
948 pages_split = pages.split(separator)
949 if len(pages_split) == 0: 949 ↛ 950line 949 didn't jump to line 950 because the condition on line 949 was never true
950 article.page_range = pages
951 if len(pages_split) > 0 and pages[0].isnumeric(): 951 ↛ exitline 951 didn't return from function 'set_pages' because the condition on line 951 was always true
952 article.fpage = pages_split[0]
953 if ( 953 ↛ 958line 953 didn't jump to line 958 because the condition on line 953 was never true
954 len(pages_split) > 1
955 and pages_split[0] != pages_split[1]
956 and pages_split[1].isnumeric()
957 ):
958 article.lpage = pages_split[1]
960 @staticmethod
961 def _process_pdf_header(chunk: str, response: requests.Response | aiohttp.ClientResponse):
962 content_type = response.headers.get("Content-Type")
963 if regex.match(rb"^%PDF-\d\.\d", chunk):
964 if content_type and "application/pdf" in content_type:
965 # The file is unmistakably a pdf
966 return [
967 True,
968 response,
969 {
970 "status": ExtlinkChecked.Status.OK,
971 "message": "",
972 },
973 ]
974 # The file is a pdf, but the content type advertised by the server is wrong
975 return [
976 True,
977 response,
978 {
979 "status": ExtlinkChecked.Status.WARNING,
980 "message": f"Content-Type header: {content_type}",
981 },
982 ]
984 # Reaching here means we couldn't find the pdf.
985 if not content_type or "application/pdf" not in content_type:
986 return [
987 False,
988 response,
989 {
990 "status": ExtlinkChecked.Status.ERROR,
991 "message": f"Content-Type header: {content_type}; PDF Header not found: got {chunk}",
992 },
993 ]
995 return [
996 False,
997 response,
998 {
999 "status": ExtlinkChecked.Status.ERROR,
1000 "message": f"PDF Header not found: got {chunk}",
1001 },
1002 ]
1004 @classmethod
1005 async def a_check_pdf_link_validity(
1006 cls, url: str, verify=True
1007 ) -> list[bool | aiohttp.ClientResponse | dict]:
1008 """
1009 Check the validity of the PDF links.
1010 """
1011 CHUNK_SIZE = 10 # Nombre de caractères à récupérer
1012 headers = {
1013 "Range": f"bytes=0-{CHUNK_SIZE}",
1014 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0",
1015 }
1016 try:
1017 async with cls.async_session.get(
1018 url, headers=headers, allow_redirects=True, ssl=verify
1019 ) as response:
1020 try:
1021 chunk = await response.content.read(CHUNK_SIZE)
1022 return BaseCollectionCrawler._process_pdf_header(chunk, response)
1023 except StopIteration:
1024 return [
1025 False,
1026 response,
1027 {
1028 "status": ExtlinkChecked.Status.ERROR,
1029 "message": "Error reading PDF header",
1030 },
1031 ]
1032 except ClientResponseError:
1033 return [
1034 False,
1035 None,
1036 {
1037 "status": ExtlinkChecked.Status.ERROR,
1038 "message": "The response is not http formated",
1039 },
1040 ]
1042 @classmethod
1043 def check_pdf_link_validity(
1044 cls, url: str, verify=True
1045 ) -> list[bool | requests.Response | None | dict]:
1046 """
1047 Check the validity of the PDF links.
1048 """
1049 CHUNK_SIZE = 10 # Nombre de caractères à récupérer
1050 header = {
1051 "Range": f"bytes=0-{CHUNK_SIZE}",
1052 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0",
1053 }
1054 with get_session().get(
1055 url, headers=header, allow_redirects=True, verify=verify, stream=True
1056 ) as response:
1057 try:
1058 chunk = next(response.iter_content(CHUNK_SIZE))
1059 return BaseCollectionCrawler._process_pdf_header(chunk, response)
1060 except StopIteration:
1061 return [
1062 False,
1063 response,
1064 {
1065 "status": ExtlinkChecked.Status.ERROR,
1066 "message": "Error reading PDF header",
1067 },
1068 ]
1070 @classmethod
1071 async def check_extlink_validity(cls, extlink: "ExtLink"):
1072 """
1073 Method used by rot_monitoring to check if links have expired
1074 """
1075 defaults: dict = {"date": datetime.now(), "status": ExtlinkChecked.Status.OK}
1076 header = {
1077 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0"
1078 }
1079 verify = True
1080 if not cls.verify:
1081 verify = False
1082 try:
1083 # For the GDZ links, we just check if the http response is 200 or 206
1084 if (
1085 extlink.rel == "article-pdf"
1086 and "gdz.sub.uni-goettingen.de" not in extlink.location
1087 ):
1088 _isok, response, message = await cls.a_check_pdf_link_validity(
1089 extlink.location, verify
1090 )
1091 defaults.update(message)
1092 try:
1093 defaults["http_status"] = response.status
1094 except AttributeError:
1095 # No html response
1096 defaults["http_status"] = 400
1097 else:
1098 try:
1099 async with cls.async_session.get(
1100 url=extlink.location,
1101 headers=header,
1102 allow_redirects=True,
1103 ssl=verify,
1104 ) as response:
1105 defaults["http_status"] = response.status
1106 if response.status not in (200, 206):
1107 defaults["status"] = ExtlinkChecked.Status.ERROR
1108 except ClientResponseError:
1109 defaults["http_status"] = 400
1110 defaults["status"] = ExtlinkChecked.Status.ERROR
1112 except aiohttp.ClientSSLError:
1113 cls.logger.error("SSL error for the url: %s", extlink.location)
1114 defaults["status"] = ExtlinkChecked.Status.ERROR
1115 defaults["message"] = "SSL error"
1116 except aiohttp.ClientConnectionError:
1117 cls.logger.error("Connection error for the url: %s", extlink.location)
1118 defaults["status"] = ExtlinkChecked.Status.ERROR
1119 defaults["message"] = "Connection error"
1120 except TimeoutError:
1121 cls.logger.error("Timeout error for the url: %s", extlink.location)
1122 defaults["status"] = ExtlinkChecked.Status.ERROR
1123 defaults["message"] = "Timeout error"
1124 finally:
1125 try:
1126 await ExtlinkChecked.objects.aupdate_or_create(extlink=extlink, defaults=defaults)
1127 cls.logger.info(
1128 "DB Update, source: %s, url: %s", cls.source_domain, extlink.location
1129 )
1130 except IntegrityError:
1131 cls.logger.error(
1132 "Extlink was deleted, source: %s, url: %s", cls.source_domain, extlink.location
1133 )
1135 def resolve_year_end(self, pid: str, default: int) -> int:
1136 if pid in self.pid_year_restrictions:
1137 return datetime.now().year - self.pid_year_restrictions[pid]
1138 return default
1141def get_first_last_years(years_str: str):
1142 years_splitted = years_str.split("-")
1143 fyear = int(years_splitted[0])
1144 lyear = None
1145 if len(years_splitted) > 1:
1146 lyear = int(years_splitted[1])
1147 return fyear, lyear