Coverage for src/crawler/by_source/ems_crawler.py: 91%
62 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
1from urllib.parse import urljoin
3from bs4 import BeautifulSoup
4from ptf.model_data import create_abstract, create_articledata, create_subj
6from crawler.abstract_crawlers.matching_crawler import MatchingCrawler
7from crawler.utils import cleanup_str, regex_to_dict
10class EmsCrawler(MatchingCrawler):
11 source_name = "EMS Press"
12 source_domain = "EMS"
13 source_website = "https://ems.press/"
15 issue_re_1 = (
16 r"Vol\. (?P<volume>\d+),(?: No\. (?P<number>[^,]+),)?pp\. (?P<fpage>\d+)–(?P<lpage>\d+)"
17 )
19 def parse_collection_content(self, content):
20 xissues = []
21 soup = BeautifulSoup(content, "html.parser")
22 issues = soup.select("a.issue-title")
23 for issue in issues:
24 previous_tag = issue.parent.find_previous_sibling("h2", {"class": "volume-title"})
25 volume_tag = previous_tag.select_one(".volume-year")
26 volume_year = int(volume_tag.text)
28 issue_group = regex_to_dict(
29 self.issue_re_1, issue.text, error_msg="Couldn't parse issue data"
30 )
32 issue_href = issue.get("href")
33 if not isinstance(issue_href, str): 33 ↛ 34line 33 didn't jump to line 34 because the condition on line 33 was never true
34 raise ValueError("Couldn't parse issue url")
36 xissues.append(
37 self.create_xissue(
38 urljoin(self.source_website, issue_href),
39 volume_year,
40 issue_group["volume"],
41 # issue_number=issue_group.get("number_1")+ "-" + issue_group.get("number_2"),
42 issue_group.get("number"),
43 )
44 )
46 return xissues
48 def parse_issue_content(self, content, xissue):
49 soup = BeautifulSoup(content, "html.parser")
50 articles = soup.select("article > a.unstyled")
51 for index, article_tag in enumerate(articles):
52 xarticle = create_articledata()
53 xarticle.pid = "a" + str(index)
54 article_href = article_tag.get("href")
55 if not isinstance(article_href, str): 55 ↛ 56line 55 didn't jump to line 56 because the condition on line 55 was never true
56 raise ValueError("Couldn't parse article href")
57 xarticle.url = urljoin(self.source_website, article_href)
58 xissue.articles.append(xarticle)
60 def parse_article_content(self, content, xissue, xarticle, url):
61 soup = BeautifulSoup(content, "html.parser")
63 self.get_metadata_using_citation_meta(
64 xarticle, xissue, soup, ["author", "doi", "title", "pdf", "page", "title"]
65 )
67 # Abstract
68 abstract_tag = soup.select_one("div.formatted-text > p")
70 if abstract_tag: 70 ↛ 75line 70 didn't jump to line 75 because the condition on line 70 was always true
71 abstract_text = cleanup_str(abstract_tag.text)
72 xarticle.abstracts.append(create_abstract(lang="en", value_tex=abstract_text))
74 # Keywords
75 keywords_tag = soup.select("ul.keywords > li")
76 for k_tag in keywords_tag:
77 kwd_type = ""
78 if k_tag.parent.previous_sibling.text == "Mathematics Subject Classification":
79 kwd_type = "msc"
80 kwd_text = cleanup_str(k_tag.text)
81 if kwd_text != "": 81 ↛ 76line 81 didn't jump to line 76 because the condition on line 81 was always true
82 keyword = create_subj(value=kwd_text, type=kwd_type)
83 xarticle.kwds.append(keyword)
85 # Contributors ORCID
86 contributors_tag = soup.find("div", class_="person-group")
87 if contributors_tag: 87 ↛ 96line 87 didn't jump to line 96 because the condition on line 87 was always true
88 contributor_list = contributors_tag.find_all("section", class_="person")
89 if len(contributor_list) == len(xarticle.contributors): 89 ↛ 96line 89 didn't jump to line 96 because the condition on line 89 was always true
90 for i in range(len(contributor_list)):
91 orcid_tag = contributor_list[i].find("a", string="ORCID")
92 orcid_url = orcid_tag["href"] if orcid_tag else None
93 if orcid_url:
94 orcid = orcid_url.split("/")[-1]
95 xarticle.contributors[i]["orcid"] = orcid
96 return xarticle