Coverage for src/crawler/by_source/episciences_crawlers/episciences_dmtcs_crawler.py: 29%

1import json 

2import logging 

3import math 

4from urllib.parse import urljoin 

5 

6from ptf.model_data import ( 

7 create_issuedata, 

8) 

9 

10from crawler.abstract_crawlers.base_crawler import get_first_last_years 

11from crawler.by_source.episciences_crawlers.episciences_crawler import EpisciencesCrawler 

12from crawler.crawler_utils import get_issue_pid 

13from crawler.utils import regex_to_dict 

14 

15# TODO : Find another system to save the title information with 

16# Maybe a toggle attribute on collection ? 

17 

18HAS_TITLES = {"COMPOSIT": False, "ARIMA": True} 

19 

20logger = logging.getLogger(__name__) 

21 

22 

23# We could improve our data further by augmenting the articles using arxiv 

24# (references) 

25class EpisciencesDmtcsCrawler(EpisciencesCrawler): 

26 source_domain = "EPISCIENCES_DMTCS" 

27 

28 def prefetch_episciences_issue(self, issue: dict): 

29 if issue.get("vol_year", None): 

30 year = str(issue["vol_year"]) 

31 else: 

32 mid_art = issue["papers"][math.floor(len(issue["papers"]) / 2)] 

33 mid_art_url = urljoin(self.collection_url, mid_art["@id"]) 

34 mid_art_content = self.download_file(mid_art_url) 

35 mid_art_data = json.loads(mid_art_content) 

36 year = mid_art_data["document"]["journal"]["journal_issue"]["publication_date"]["year"] 

37 issue_title = issue["titles"]["en"] 

38 

39 # parsed_url = urlparse(self.collection_url) 

40 # issue_view_url = f"https://{parse_qs(parsed_url.query)['rvcode'][0]}.episciences.org/volume/view/id/{issue['vid']}" 

41 

42 if "DMTCS Proceedings" in issue_title: 

43 xissue = create_issuedata() 

44 xissue.url = None 

45 xissue.fyear = int(year) 

46 xissue.pid = get_issue_pid( 

47 self.collection_id, xissue.fyear, "special_" + str(issue["vid"]) 

48 ) 

49 # HACK : handle this elsewhere ? transform title_tex into title_xml 

50 # Is title_xml here even valid ? 

51 xissue.title_tex = issue_title 

52 xissue.title_html = issue_title 

53 xissue.title_xml = issue_title 

54 xissue.volume = issue_title 

55 

56 else: 

57 # Vol. 1 

58 # Vol. 3 no. 2 

59 # Vol. 17 no.2 

60 # vol. 24, no 2 

61 # Vol. 18 no. 2, Permutation Patterns 2015 

62 # Vol. 19 no. 4, FCT '15 

63 # vol. 27:2 

64 # vol. 25:3 special issue ICGT'22 

65 # vol. 26:1, Permutation Patterns 2023 

66 # vol. 19:3 

67 title_dict = regex_to_dict( 

68 r"[Vv]ol. +(?P<volume>\d+)(?:(?:(?:,? no\.? ?)|(?:\:))?(?P<number>\d+))?(?:,? (?P<title>.+))?", 

69 issue_title, 

70 error_msg="Couldn't parse issue title", 

71 ) 

72 fyear, lyear = get_first_last_years(year) 

73 xissue = self.create_xissue( 

74 None, fyear, title_dict["volume"], title_dict.get("number", None), lyear=lyear 

75 ) 

76 if title_dict["title"] is not None: 

77 # HACK : handle this elsewhere ? transform title_tex into title_xml 

78 # Is title_xml here even valid ? 

79 xissue.title_tex = title_dict["title"] 

80 xissue.title_html = title_dict["title"] 

81 xissue.title_xml = title_dict["title"] 

82 if issue["papers"] is not None: 

83 self.prefetch_issue_content(xissue, issue_data=issue) 

84 return xissue