Coverage for src/crawler/by_source/msp_crawler.py: 8%
152 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
1import os
2from urllib.parse import urljoin, urlparse
4from bs4 import BeautifulSoup, Tag
5from ptf.cmds.xml.jats.builder.references import get_article_title_xml, get_ext_link_xml
6from ptf.cmds.xml.jats.jats_parser import parse_mixed_citation_into_ref
7from ptf.model_data import (
8 ArticleData,
9 create_abstract,
10 create_articledata,
11 create_subj,
12)
14from crawler.abstract_crawlers.threaded_crawler import ThreadedCrawler
15from crawler.utils import cleanup_str, regex_to_dict
18class MspCrawler(ThreadedCrawler):
19 source_name = "Mathematical Sciences Publishers"
20 source_domain = "MSP"
21 source_website = "https://msp.org/"
23 issue_re = r"\/\w+\/(?P<year>\d+)\/(?P<volume>\d+)(?:\-(?P<number>\d+))?"
25 def parse_collection_content(self, content):
26 xissues = []
27 soup = BeautifulSoup(content, "html.parser")
28 issues = soup.select("td.issues-area a.about[href]")
29 for issue in issues:
30 issue_href = issue.get("href")
31 if not isinstance(issue_href, str):
32 raise TypeError("Couldn't parse issue href")
34 issue_dict = regex_to_dict(
35 self.issue_re, issue_href, error_msg="Couldn't parse issue data"
36 )
37 if issue_dict["volume"] == "6":
38 issue_dict["volume"] = "6+7"
40 xissues.append(
41 self.create_xissue(
42 urljoin(self.source_website, issue_href),
43 int(issue_dict["year"]),
44 issue_dict["volume"],
45 issue_dict.get("number", None),
46 )
47 )
48 return xissues
50 def parse_issue_content(self, content, xissue):
51 soup = BeautifulSoup(content, "html.parser")
52 if xissue.url is None:
53 raise ValueError("Cannot parse article : issue url is None")
55 incomplete = soup.select_one(".incomplete")
56 if (
57 incomplete
58 and cleanup_str(incomplete.text) != "Publication of this issue is now complete."
59 ):
60 self.logger.debug(
61 "Ignoring : Issue is not available due to S2O policy",
62 extra={"pid": xissue.pid},
63 )
64 return
66 title_tag = soup.select_one("#navigation-area .page-titles-area")
67 if title_tag:
68 title_tex = title_tag.get_text(" ", strip=True)
69 xissue.title_tex = title_tex
71 issue_doi_tag = soup.select_one("div.issue-doi a")
72 if issue_doi_tag and issue_doi_tag != "10.2140/ant.2023.17.261":
73 xissue.doi = cleanup_str(issue_doi_tag.text)
74 else:
75 xissue.doi = None
77 articles = soup.select("#toc-area .title")
78 for index, article_tag in enumerate(articles):
79 xarticle = create_articledata()
80 article_href = article_tag.get("href")
81 if not isinstance(article_href, str):
82 raise TypeError("Couldn't parse article url")
83 xarticle.url = urljoin(xissue.url, article_href)
84 xarticle.pid = "a" + str(index)
85 xissue.articles.append(xarticle)
87 def parse_article_content(self, content, xissue, xarticle, url):
88 soup = BeautifulSoup(content, "html.parser")
90 # Warn : meta doi is sometimes incorrect
91 # the metadata title is not correct: no $ delimiter
92 self.get_metadata_using_citation_meta(
93 xarticle, xissue, soup, ["title", "author", "page", "pdf", "publisher"]
94 )
96 if not self.is_article_openaccess(xarticle):
97 return
99 doi_tag = soup.select_one(".paper-doi > a")
100 if doi_tag:
101 xarticle.doi = doi_tag.text
103 # here we over write the title because no formula delimiter in the metadata title
104 title_tex = soup.select_one("#content-area .title-area a.title").get_text(strip=True)
105 if title_tex:
106 xarticle.title_tex = title_tex
108 article_data: dict[str, Tag] = {}
109 article_sections = soup.select("#content-area > .article")
110 # this loop to catch the Abtract section
111 for section in article_sections:
112 if section.select_one(".copyright-license"):
113 continue
115 tabs = section.select("tr")
116 section_title_tag = tabs[0].select_one("h5")
117 if not section_title_tag:
118 self.logger.debug(f"Skipping {tabs[0].name} section at {xarticle.pid}")
119 continue
120 section_title = section_title_tag.text
121 section_title_tag.decompose()
122 del section_title_tag
124 section_content = tabs[0]
125 if len(tabs) > 1:
126 section_content = tabs[1]
127 section_tag = section_content.select_one("tr > td.article-area")
128 if section_tag:
129 article_data[section_title] = section_tag
130 del article_sections
132 if "Keywords" in article_data and article_data["Keywords"] != "":
133 for kwd in article_data["Keywords"].text.split(", "):
134 xarticle.kwds.append(create_subj(lang="en", type="kwd", value=kwd))
136 if (
137 "Mathematical Subject Classification 2010" in article_data
138 and article_data["Mathematical Subject Classification 2010"] != ""
139 ):
140 msc_long_text = (
141 cleanup_str(article_data["Mathematical Subject Classification 2010"].text)
142 .replace("Primary: ", "")
143 .replace(" Secondary: ", ", ")
144 )
145 for kwd in msc_long_text.split(", "):
146 xarticle.kwds.append(create_subj(lang="en", type="msc", value=kwd))
148 if (
149 "Mathematical Subject Classification 2000" in article_data
150 and article_data["Mathematical Subject Classification 2000"] != ""
151 ):
152 msc_long_text = (
153 cleanup_str(article_data["Mathematical Subject Classification 2000"].text)
154 .replace("Primary: ", "")
155 .replace(" Secondary: ", ", ")
156 )
157 for kwd in msc_long_text.split(", "):
158 xarticle.kwds.append(create_subj(lang="en", type="msc", value=kwd))
160 if "Abstract" in article_data and article_data["Abstract"] != "":
161 for img in article_data["Abstract"].select("img"):
162 img.decompose()
163 abstract_str = "".join(str(e) for e in article_data["Abstract"].select("p"))
165 abstract = create_abstract(
166 lang="en",
167 value_tex=abstract_str,
168 )
169 xarticle.abstracts.append(abstract)
171 self.parse_msp_references(xarticle)
172 if xarticle.doi == "10.2140/ant.2023.17.261":
173 # this article is a correction of the article pid=10_2140_ant_2023_17_261
174 # and it has the same doi.
175 # As the article pid is built with the doi, same doi > same pid > unicity constraint violation
176 xarticle.doi = None
177 return xarticle
179 def is_article_openaccess(self, xarticle: ArticleData):
180 stream = next(
181 (stream for stream in xarticle.streams if stream["rel"] == "full-text"), None
182 )
183 if stream is None:
184 self.logger.error(f"Could not find pdf for article {xarticle.url}")
185 return False
186 pdf_url = stream["location"]
188 isok, *_ = self.check_pdf_link_validity(url=pdf_url, verify=True)
190 return isok
192 def parse_msp_references(self, xarticle: ArticleData):
193 url = urlparse(xarticle.url)
194 dirname = os.path.dirname(url.path)
195 filename = os.path.basename(url.path)
196 url = url._replace(path=urljoin(str(dirname) + "/", str(filename).replace("p", "b")))
198 content = self.download_file(str(url.geturl()))
199 soup = BeautifulSoup(content, "html.parser")
200 references = soup.select("#content-area table.article:last-of-type tr")
202 # TODO : extensive parsing (authors, title etc...)
203 # Currently, only the text is inserted
204 for ref_node in references:
205 td = ref_node.select("td")
206 value_xml = self.parse_single_ref(td[1])
207 label = cleanup_str(td[0].text)
208 ref = parse_mixed_citation_into_ref(value_xml, label=label)
209 xarticle.bibitems.append(ref)
211 def parse_single_ref(self, tag: Tag):
212 xml_list = []
213 ext_links = []
214 authors_closed = False
216 for element in tag.contents:
217 if isinstance(element, str):
218 xml_list.append(element)
219 continue
220 if isinstance(element, Tag):
221 if element.name == "b" and not authors_closed:
222 xml_list.append(f"<string-name>{element.text}</string-name>")
223 elif element.name == "i" and not authors_closed:
224 temp_element = xml_list.pop()
225 xml_list = [
226 f'<person-group person-group-type="author">{cleanup_str("".join(xml_list))}</person-group>',
227 temp_element,
228 ]
229 xml_list.append(get_article_title_xml(element.text))
230 del temp_element
232 link = element.select_one("a")
233 if link:
234 link_href = link.get("href")
235 if isinstance(link_href, str) and link_href.startswith("https://doi.org/"):
236 link_href = link_href.removeprefix("https://doi.org/")
237 ext_links.append(get_ext_link_xml(link_href, link_href, "doi"))
239 authors_closed = True
240 elif element.name == "a":
241 pass
242 continue
244 return cleanup_str("".join(xml_list) + "".join(ext_links))