Coverage for src/crawler/by_source/eudml

1import base64

2import json

3import os

4import re

5import subprocess

7import regex

8import requests

9from bs4 import BeautifulSoup

10from ptf.model_data import (

11 ArticleData,

12 create_abstract,

13 create_articledata,

14 create_contributor,

15 create_extid,

16 create_extlink,

17 create_issuedata,

18 create_subj,

19)

20from ptf.utils import execute_cmd

22from crawler.abstract_crawlers.matching_crawler import MatchingCrawler

23from crawler.utils import add_pdf_link_to_xarticle

26class EudmlCrawler(MatchingCrawler):

27 source_name = "European Digital Mathematics Library"

28 source_domain = "EUDML"

29 source_website = "https://eudml.org"

31 def parse_collection_content(self, content):

32 """

33 Parse the HTML page of a EuDml journal and returns a list of xissue.

34 Each xissue has a list of articles with just an url.

35 """

36 data = json.loads(content)

37 soup = BeautifulSoup(base64.b64decode(data["page"]), "html.parser")

38 xissues = []

39 volume_year_re = regex.compile(r".*\(<strong>(\d+).*<\/strong>\)")

40 # Extract the list of volumes

41 volume_count = 0

42 issue_count = 0

43 for v in data["volumes"]:

44 volume_count += 1

45 volume_number = v["name"]

47 year_re_groups = volume_year_re.search(v["desc"])

48 if year_re_groups is None:

49 self.logger.debug("skipping volume : no year found")

50 continue

51 year = year_re_groups.group(1)

52 if year == "":

53 self.logger.debug("volume year is an empty string... Skipping")

54 continue

55 if len(v["issues"]) > 0:

56 # Extract all the issues

57 for i in v["issues"]:

58 issue_count += 1

59 xissue = self.create_eudml_xissue(i, year, i["name"], volume_number)

60 xissues.append(xissue)

61 else:

62 # No issues, articles are directly in the volumeF

63 xissue = self.create_eudml_xissue(v, year, None, volume_number)

64 xissues.append(xissue)

66 # EuDML stores the total of issues and articles in the <ul class="article-details unit unit-list">

67 # This info is used to check the number of articles/issues parsed in the page

68 volumes_to_find = 0

69 issues_to_find = 0

70 articles_to_find = 0

71 article_details_nodes = soup.find_all("ul", {"class": "article-details unit unit-list"})

72 for article_detail_node in article_details_nodes:

73 unit_nodes = article_detail_node.find_all("li")

74 for unit_node in unit_nodes:

75 strong_node = unit_node.find("strong")

76 if strong_node is not None:

77 text = strong_node.get_text()

78 if text == "Issue count:":

79 value = unit_node.get_text()[13:]

80 issues_to_find += int(value)

81 elif text == "Volume count:":

82 value = unit_node.get_text()[14:]

83 volumes_to_find += int(value)

84 elif text == "Number of articles:":

85 value = unit_node.get_text()[20:]

86 articles_to_find += int(value)

88 if volume_count != volumes_to_find:

89 txt = f"EuDML declares {volumes_to_find} volumes for {self.collection_id}. We parsed {volume_count}"

90 self.logger.debug(txt)

92 if issue_count != issues_to_find:

93 txt = f"EuDML declares {issues_to_find} issues for {self.collection_id}. We parsed {issue_count}"

94 self.logger.debug(txt)

96 article_count = sum([len(xissue.articles) for xissue in xissues])

97 if article_count != articles_to_find:

98 txt = f"EuDML declares {articles_to_find} articles for {self.collection_id}. We parsed {article_count}"

99 self.logger.debug(txt)

100

101 return xissues

102

103 def create_eudml_xissue(

104 self, issue_data: dict, year_str, issue_number: str | None, volume_number

105 ):

106 """

107 EuDML does not have a separate HTML page for an issue.

108 The list of issues/articles is directly found in the collection page.

109

110 create_xissue creates an IssueData (see ptf/model_data.py) and sets its year/volume

111 The PID is temporary and will be updated with the issue number (if any)

112 create_xissue directly creates articles, but with just a pid and an url.

113 """

114 xissue = create_issuedata()

115 xissue.pid = self.collection_id + "_" + year_str + "__" + volume_number

116 if issue_number:

117 xissue.pid = xissue.pid + "_" + issue_number

118 xissue.fyear = int(year_str)

119 xissue.volume = volume_number

120 if issue_number:

121 xissue.number = issue_number

122

123 issue_data["articles"].sort(key=lambda a: a["sortKey"])

124 for index_article, article_data in enumerate(issue_data["articles"]):

125 xarticle = create_articledata()

126 xarticle.pid = "a" + str(index_article)

127 xarticle.url = article_data["url"]

128 xissue.articles.append(xarticle)

129 return xissue

130

131 def parse_article_content(self, content, xissue, xarticle, url):

132 """

133 Parse the content with Beautifulsoup and returns an ArticleData

134 """

135 soup = BeautifulSoup(content, "xml")

136

137 self.get_metadata_using_citation_meta(

138 xarticle,

139 xissue,

140 soup,

141 [

142 "lang",

143 "title",

144 "author",

145 "pdf",

146 "abstract",

147 "page",

148 "doi",

149 "mr",

150 "zbl",

151 "publisher",

152 "keywords",

153 ],

154 )

155

156 # LINK to SOURCE

157 url_full_text_node = soup.find("a", text="Access to full text")

158 if url_full_text_node is not None:

159 url_full_text = url_full_text_node.get("href")

160 if isinstance(url_full_text, str):

161 ext_link = create_extlink(rel="primary-source", location=url_full_text)

162 xarticle.ext_links.append(ext_link)

163

164 # MSC KEYWORDS

165 subj_part = soup.select_one("article#unit-subject-areas")

166 if subj_part is not None:

167 reg_msc = re.compile("/subject/MSC/[a-zA-Z0-9.]+")

168 subjs = [a for a in subj_part.select("a") if reg_msc.search(a.get("href"))]

169 for subj in subjs:

170 type_class = subj.get("href").split("/")

171 subject = create_subj(type="msc", lang=xarticle.lang)

172 subject["value"] = type_class[3]

173 xarticle.kwds.append(subject)

174

175 # FALLBACK

176 if not xarticle.title_tex:

177 try:

178 title = soup.select_one("h1").get_text(strip=True).replace("\xa0", " ")

179 txt = f"{url} Fallback for title"

180 self.logger.debug(txt, extra={"pid": xarticle.pid})

181 xarticle.title_tex = title.replace("\xa0", " ").replace("\n", "")

182 # FIXME

183 except: # noqa: E722

184 pass

185

186 if len(xarticle.contributors) == 0:

187 # AUTHORS

188 authors_bloc = soup.select_one("p.sub-title-1")

189 if authors_bloc:

190 authors_node = authors_bloc.find_all("a")

191 if len(authors_node) > 0:

192 txt = f"{url} Fallback for authors"

193 self.logger.debug(txt, extra={"pid": xarticle.pid})

194 for author_node in authors_node:

195 text_author = author_node.get_text()

196 text_author = text_author.replace(",", "")

197

198 author = create_contributor()

199 author["role"] = "author"

200 author["string_name"] = text_author

201

202 xarticle.contributors.append(author)

203

204 if len(xarticle.streams) == 0:

205 # PDF

206 pdf_node = soup.find("a", text="Full (PDF)")

207 if pdf_node is not None:

208 pdf_url = pdf_node.get("href")

209 if pdf_url:

210 add_pdf_link_to_xarticle(xarticle, pdf_url)

211

212 if len(xarticle.streams) == 0:

213 if not url_full_text_node:

214 self.logger.debug("Couldn't find pdf", extra={"pid": xarticle.pid})

215 else:

216 add_pdf_link_to_xarticle(xarticle, url_full_text_node.get("href"))

217

218 if len(xarticle.abstracts) == 0:

219 # ABSTRACT

220 abstract_node = soup.find("article", {"id": "unit-article-abstract"})

221 if abstract_node is not None:

222 abstract_section_node = abstract_node.find("section")

223 if abstract_section_node:

224 abstract = str(abstract_section_node)

225

226 xarticle.abstracts.append(

227 create_abstract(value_tex=abstract, lang=xarticle.lang)

228 )

229

230 if len(xarticle.contributors) == 0 or not xarticle.fpage:

231 # LANG, PAGES, (AUTHORS)

232 # EuDML has an export BibTex section with some information (lang, pages, authors)

233 self.parse_bibtex(soup, xarticle, url)

234

235 if xarticle.doi is None:

236 # DOI

237 doi_link = soup.find("article", {"id": "unit-other-ids"})

238 if doi_link is not None:

239 # Simplify ?

240 # See https://eudml.org/doc/54683 with http://dx.doi.org/10.1155/2007/10368%E2%80%89

241 try:

242 reg_doi = re.compile("doi.org")

243 doi_array = [

244 d.get("href")

245 for d in doi_link.find_all("a")

246 if reg_doi.search(str(d.get("href")))

247 ]

248 if doi_array:

249 if len(doi_array) > 1:

250 start_dois = len(doi_array) - 1

251 doi = doi_array[start_dois:][0]

252 else:

253 doi = doi_array[0]

254

255 doi_array = doi.split("doi.org/")

256 # strip unwanted chars present

257 if len(doi_array) > 1:

258 doi = doi_array[1].encode("ascii", "ignore")

259 doi = str(doi.decode())

260 doi_array = doi.split("\\u")

261 doi = str(doi_array[0])

262

263 doi = re.sub("}", "", doi)

264 doi = re.sub("\t", "", doi)

265 doi = doi.encode("ascii", "ignore")

266 doi = doi.decode()

267

268 doi = bytes(r"{}".format(r"" + doi + ""), "utf-8")

269 doi = doi.decode()

270 doi_array = doi.split("\\u")

271 doi = str(doi_array[0]).strip()

272 doi = doi.replace(" ", "")

273

274 xarticle.doi = doi

275 except TypeError as e:

276 self.logger.debug(e, {"pid": xarticle.pid})

277

278 has_zblid = (

279 len([extid for extid in xarticle.extids if extid["type"] == "zbl-item-id"]) == 1

280 )

281 if not has_zblid:

282 zb_tag = soup.select_one("article#unit-other-ids a:-soup-contains('ZBMath')")

283 if zb_tag:

284 zb_href = zb_tag.get("href")

285 if not isinstance(zb_href, str):

286 raise ValueError("Couldn't parse zbmath href")

287 zblid = zb_href.removeprefix(

288 "http://www.zentralblatt-math.org/zmath/en/advanced/?q=an:"

289 )

290 extid = create_extid("zbl-item-id", zblid)

291 xarticle.extids.append(extid)

292

293 # In Other Databases is not (always ?) the publisher

294 # if not xissue.publisher:

295 # # PUBLISHER

296 # section_oai = soup.find("h3", text="In Other Databases")

297 # if section_oai is not None:

298 # section_oai_array = section_oai.parent.find_all("dd")

299 # if section_oai is not None:

300 # pub = [

301 # d.text

302 # for d in section_oai_array

303 # if d.text.strip() not in ["DOI", "ZBMath", "MathSciNet", "PUBLISHER"]

304 # ]

305 # if pub != "":

306 # print(f"{url} Fallback for publisher")

307 # xpub = create_publisherdata()

308 # xpub.name = pub[0].strip()

309 # xissue.publisher = xpub

310 return xarticle

311

312 def parse_bibtex(self, soup, xarticle: ArticleData, url):

313 """

314 Parse the BibTeX section of a EuDML article page.

315 Extract

316 - the authors (if no author was already found in the page)

317 - the article language

318 - the article pages

319 """

320 bib_div = [p for p in soup.find_all("p") if "@article" in p.text]

321

322 if len(bib_div) > 0:

323 bib_tex = bib_div[0].get_text()

324 text = bib_tex.split("\t")

325

326 for text_part in text:

327 # AUTHORS (only if no authors were already found in the page)

328 if len(xarticle.contributors) == 0:

329 reg_author = re.compile("author =")

330 if reg_author.search(text_part):

331 txt = f"{url} Fallback for authors with the bibtex"

332 self.logger.debug(txt, extra={"pid": xarticle.pid})

333

334 authors_text = (

335 text_part.replace("{", "").replace("}", "").replace("author = ", "")

336 )

337 authors_bib = authors_text.split(",")

338 for index, name in enumerate(authors_bib):

339 if index % 2 == 1:

340 author_name = authors_bib[index - 1] + " " + authors_bib[index]

341 author_name = self.latext_parser.latex_to_text(author_name)

342 author_name = author_name.replace("\xa0", "")

343

344 author = create_contributor()

345 author["role"] = "author"

346 author["string_name"] = author_name

347 xarticle.contributors.append(author)

348

349 # LANG

350 reg_lang = re.compile("language = ")

351 if reg_lang.search(text_part):

352 xarticle.lang = (

353 text_part.replace("{", "")

354 .replace("}", "")

355 .replace("language = ", "")

356 .replace(",", "")

357 )

358 if len(xarticle.lang) >= 3:

359 xarticle.lang = xarticle.lang[:-1]

360

361 if len(xarticle.lang) > 0 and len(xarticle.abstracts) > 0:

362 xarticle.abstracts[0]["lang"] = xarticle.lang

363

364 if not xarticle.fpage:

365 # PAGES

366 reg_pages = re.compile("pages =")

367 if reg_pages.search(text_part):

368 pages = (

369 text_part.replace("{", "")

370 .replace("}", "")

371 .replace("(", "")

372 .replace(")", "")

373 .replace("[", "")

374 .replace("]", "")

375 .replace("pages = ", "")

376 )

377 if len(pages) > 0 and pages != "null":

378 pages = pages.split(",")

379 if re.compile(r"\d+-\d+").search(pages[0]):

380 txt = f"{url} Fallback for pages with the bibtex"

381 self.logger.debug(txt, {"pid", xarticle.pid})

382

383 pages = pages[0].split("-")

384 xarticle.fpage = pages[0]

385 if len(pages) > 1:

386 reg_digit = re.compile(r"\d+")

387 if re.search(reg_digit, str(pages[1])):

388 pages[1] = re.search(reg_digit, str(pages[1]))[0]

389 xarticle.lpage = pages[1]

390 # FIXME : wrong page_range format... Maybe this can be deleted ?

391 xarticle.page_range = pages[0] + "-" + pages[1]

392

393 # reg_title = re.compile("title")

394 # if reg_title.search(text_part):

395 # if (

396 # xarticle.title_html is None

397 # or xarticle.title_html == ""

398 # or xarticle.title_html == "Contents"

399 # ):

400 # xarticle.title_html = (

401 # text_part.replace("{", "")

402 # .replace("}", "")

403 # .replace("title = ", "")

404 # .replace(",", "")

405 # )

406 # xarticle.title_tex = xarticle.title_html

407 # xarticle.title_xml = f"<title-group><article-title>{xarticle.title_html}</article-title></title-group>"

408

409 def download_file(self, url: str, headers={}):

410 if url.startswith("https://eudml.org/doc"):

411 return super().download_file(url, headers=headers)

412

413 content = ""

414 filename = "/tmp/crawler/puppeteer/" + str(base64.b64encode(url.encode("utf-8")), "utf-8")

415 attempt = 0

416 while not content and attempt < 3:

417 attempt += 1

418 try:

419 cmd = f"{os.path.dirname(os.path.realpath(__file__))}/crawl_eudml_col.js -u {url} -o {filename}"

420 execute_cmd(cmd, force_execute=True)

421

422 if os.path.isfile(filename):

423 with open(filename) as file_:

424 content = file_.read()

425 # TODO : Mock an HTTP requests to inject the data into the cache

426 # if not isinstance(self.session, CachedSession):

427 # continue

428

429 except subprocess.CalledProcessError:

430 pass

431

432 if not content:

433 raise requests.exceptions.HTTPError(f"Unable to download {url}")

434

435 return content

436

437 # @classmethod

438 # def check_extlink_validity(cls, extlink):

439 # TODO if we restore this method, it has to be async

440 # """

441 # Method used by rot_monitoring to check if links have expired

442 # """

443 # defaults: dict = {"date": time.time(), "status": ExtlinkChecked.Status.OK}

444 #

445 # if not extlink.location.startswith("http://gdz.sub.uni-goettingen.de"):

446 # return super().check_extlink_validity(extlink)

447 #

448 # response = requests.get(extlink.location)

449 #

450 # defaults["http_status"] = response.status_code

451 #

452 # if response.status_code not in (200, 206):

453 # defaults["status"] = ExtlinkChecked.Status.ERROR

454 #

455 # ExtlinkChecked.objects.update_or_create(extlink=extlink, defaults=defaults)

Coverage for src/crawler/by_source/eudml_crawler.py: 6%

251 statements