Coverage for src/crawler/by_source/msp_crawler.py: 8%

152 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-19 16:36 +0000

1import os 

2from urllib.parse import urljoin, urlparse 

3 

4from bs4 import BeautifulSoup, Tag 

5from ptf.cmds.xml.jats.builder.references import get_article_title_xml, get_ext_link_xml 

6from ptf.cmds.xml.jats.jats_parser import parse_mixed_citation_into_ref 

7from ptf.model_data import ( 

8 ArticleData, 

9 create_abstract, 

10 create_articledata, 

11 create_subj, 

12) 

13 

14from crawler.abstract_crawlers.threaded_crawler import ThreadedCrawler 

15from crawler.utils import cleanup_str, regex_to_dict 

16 

17 

18class MspCrawler(ThreadedCrawler): 

19 source_name = "Mathematical Sciences Publishers" 

20 source_domain = "MSP" 

21 source_website = "https://msp.org/" 

22 

23 issue_re = r"\/\w+\/(?P<year>\d+)\/(?P<volume>\d+)(?:\-(?P<number>\d+))?" 

24 

25 def parse_collection_content(self, content): 

26 xissues = [] 

27 soup = BeautifulSoup(content, "html.parser") 

28 issues = soup.select("td.issues-area a.about[href]") 

29 for issue in issues: 

30 issue_href = issue.get("href") 

31 if not isinstance(issue_href, str): 

32 raise TypeError("Couldn't parse issue href") 

33 

34 issue_dict = regex_to_dict( 

35 self.issue_re, issue_href, error_msg="Couldn't parse issue data" 

36 ) 

37 if issue_dict["volume"] == "6": 

38 issue_dict["volume"] = "6+7" 

39 

40 xissues.append( 

41 self.create_xissue( 

42 urljoin(self.source_website, issue_href), 

43 int(issue_dict["year"]), 

44 issue_dict["volume"], 

45 issue_dict.get("number", None), 

46 ) 

47 ) 

48 return xissues 

49 

50 def parse_issue_content(self, content, xissue): 

51 soup = BeautifulSoup(content, "html.parser") 

52 if xissue.url is None: 

53 raise ValueError("Cannot parse article : issue url is None") 

54 

55 incomplete = soup.select_one(".incomplete") 

56 if ( 

57 incomplete 

58 and cleanup_str(incomplete.text) != "Publication of this issue is now complete." 

59 ): 

60 self.logger.debug( 

61 "Ignoring : Issue is not available due to S2O policy", 

62 extra={"pid": xissue.pid}, 

63 ) 

64 return 

65 

66 title_tag = soup.select_one("#navigation-area .page-titles-area") 

67 if title_tag: 

68 title_tex = title_tag.get_text(" ", strip=True) 

69 xissue.title_tex = title_tex 

70 

71 issue_doi_tag = soup.select_one("div.issue-doi a") 

72 if issue_doi_tag and issue_doi_tag != "10.2140/ant.2023.17.261": 

73 xissue.doi = cleanup_str(issue_doi_tag.text) 

74 else: 

75 xissue.doi = None 

76 

77 articles = soup.select("#toc-area .title") 

78 for index, article_tag in enumerate(articles): 

79 xarticle = create_articledata() 

80 article_href = article_tag.get("href") 

81 if not isinstance(article_href, str): 

82 raise TypeError("Couldn't parse article url") 

83 xarticle.url = urljoin(xissue.url, article_href) 

84 xarticle.pid = "a" + str(index) 

85 xissue.articles.append(xarticle) 

86 

87 def parse_article_content(self, content, xissue, xarticle, url): 

88 soup = BeautifulSoup(content, "html.parser") 

89 

90 # Warn : meta doi is sometimes incorrect 

91 # the metadata title is not correct: no $ delimiter 

92 self.get_metadata_using_citation_meta( 

93 xarticle, xissue, soup, ["title", "author", "page", "pdf", "publisher"] 

94 ) 

95 

96 if not self.is_article_openaccess(xarticle): 

97 return 

98 

99 doi_tag = soup.select_one(".paper-doi > a") 

100 if doi_tag: 

101 xarticle.doi = doi_tag.text 

102 

103 # here we over write the title because no formula delimiter in the metadata title 

104 title_tex = soup.select_one("#content-area .title-area a.title").get_text(strip=True) 

105 if title_tex: 

106 xarticle.title_tex = title_tex 

107 

108 article_data: dict[str, Tag] = {} 

109 article_sections = soup.select("#content-area > .article") 

110 # this loop to catch the Abtract section 

111 for section in article_sections: 

112 if section.select_one(".copyright-license"): 

113 continue 

114 

115 tabs = section.select("tr") 

116 section_title_tag = tabs[0].select_one("h5") 

117 if not section_title_tag: 

118 self.logger.debug(f"Skipping {tabs[0].name} section at {xarticle.pid}") 

119 continue 

120 section_title = section_title_tag.text 

121 section_title_tag.decompose() 

122 del section_title_tag 

123 

124 section_content = tabs[0] 

125 if len(tabs) > 1: 

126 section_content = tabs[1] 

127 section_tag = section_content.select_one("tr > td.article-area") 

128 if section_tag: 

129 article_data[section_title] = section_tag 

130 del article_sections 

131 

132 if "Keywords" in article_data and article_data["Keywords"] != "": 

133 for kwd in article_data["Keywords"].text.split(", "): 

134 xarticle.kwds.append(create_subj(lang="en", type="kwd", value=kwd)) 

135 

136 if ( 

137 "Mathematical Subject Classification 2010" in article_data 

138 and article_data["Mathematical Subject Classification 2010"] != "" 

139 ): 

140 msc_long_text = ( 

141 cleanup_str(article_data["Mathematical Subject Classification 2010"].text) 

142 .replace("Primary: ", "") 

143 .replace(" Secondary: ", ", ") 

144 ) 

145 for kwd in msc_long_text.split(", "): 

146 xarticle.kwds.append(create_subj(lang="en", type="msc", value=kwd)) 

147 

148 if ( 

149 "Mathematical Subject Classification 2000" in article_data 

150 and article_data["Mathematical Subject Classification 2000"] != "" 

151 ): 

152 msc_long_text = ( 

153 cleanup_str(article_data["Mathematical Subject Classification 2000"].text) 

154 .replace("Primary: ", "") 

155 .replace(" Secondary: ", ", ") 

156 ) 

157 for kwd in msc_long_text.split(", "): 

158 xarticle.kwds.append(create_subj(lang="en", type="msc", value=kwd)) 

159 

160 if "Abstract" in article_data and article_data["Abstract"] != "": 

161 for img in article_data["Abstract"].select("img"): 

162 img.decompose() 

163 abstract_str = "".join(str(e) for e in article_data["Abstract"].select("p")) 

164 

165 abstract = create_abstract( 

166 lang="en", 

167 value_tex=abstract_str, 

168 ) 

169 xarticle.abstracts.append(abstract) 

170 

171 self.parse_msp_references(xarticle) 

172 if xarticle.doi == "10.2140/ant.2023.17.261": 

173 # this article is a correction of the article pid=10_2140_ant_2023_17_261 

174 # and it has the same doi. 

175 # As the article pid is built with the doi, same doi > same pid > unicity constraint violation 

176 xarticle.doi = None 

177 return xarticle 

178 

179 def is_article_openaccess(self, xarticle: ArticleData): 

180 stream = next( 

181 (stream for stream in xarticle.streams if stream["rel"] == "full-text"), None 

182 ) 

183 if stream is None: 

184 self.logger.error(f"Could not find pdf for article {xarticle.url}") 

185 return False 

186 pdf_url = stream["location"] 

187 

188 isok, *_ = self.check_pdf_link_validity(url=pdf_url, verify=True) 

189 

190 return isok 

191 

192 def parse_msp_references(self, xarticle: ArticleData): 

193 url = urlparse(xarticle.url) 

194 dirname = os.path.dirname(url.path) 

195 filename = os.path.basename(url.path) 

196 url = url._replace(path=urljoin(str(dirname) + "/", str(filename).replace("p", "b"))) 

197 

198 content = self.download_file(str(url.geturl())) 

199 soup = BeautifulSoup(content, "html.parser") 

200 references = soup.select("#content-area table.article:last-of-type tr") 

201 

202 # TODO : extensive parsing (authors, title etc...) 

203 # Currently, only the text is inserted 

204 for ref_node in references: 

205 td = ref_node.select("td") 

206 value_xml = self.parse_single_ref(td[1]) 

207 label = cleanup_str(td[0].text) 

208 ref = parse_mixed_citation_into_ref(value_xml, label=label) 

209 xarticle.bibitems.append(ref) 

210 

211 def parse_single_ref(self, tag: Tag): 

212 xml_list = [] 

213 ext_links = [] 

214 authors_closed = False 

215 

216 for element in tag.contents: 

217 if isinstance(element, str): 

218 xml_list.append(element) 

219 continue 

220 if isinstance(element, Tag): 

221 if element.name == "b" and not authors_closed: 

222 xml_list.append(f"<string-name>{element.text}</string-name>") 

223 elif element.name == "i" and not authors_closed: 

224 temp_element = xml_list.pop() 

225 xml_list = [ 

226 f'<person-group person-group-type="author">{cleanup_str("".join(xml_list))}</person-group>', 

227 temp_element, 

228 ] 

229 xml_list.append(get_article_title_xml(element.text)) 

230 del temp_element 

231 

232 link = element.select_one("a") 

233 if link: 

234 link_href = link.get("href") 

235 if isinstance(link_href, str) and link_href.startswith("https://doi.org/"): 

236 link_href = link_href.removeprefix("https://doi.org/") 

237 ext_links.append(get_ext_link_xml(link_href, link_href, "doi")) 

238 

239 authors_closed = True 

240 elif element.name == "a": 

241 pass 

242 continue 

243 

244 return cleanup_str("".join(xml_list) + "".join(ext_links))