Coverage for src/crawler/by_source/episciences_crawlers/episciences_rdm_crawler.py: 43%

1import logging 

2 

3from crawler.by_source.episciences_crawlers.episciences_crawler import EpisciencesCrawler 

4from crawler.utils import regex_to_dict 

5 

6# TODO : Find another system to save the title information with 

7# Maybe a toggle attribute on collection ? 

8 

9HAS_TITLES = {"COMPOSIT": False, "ARIMA": True} 

10 

11logger = logging.getLogger(__name__) 

12 

13 

14# We could improve our data further by augmenting the articles using arxiv 

15# (references) 

16class EpisciencesRdmCrawler(EpisciencesCrawler): 

17 source_domain = "EPISCIENCES_RDM" 

18 volume_title_re = r"(?P<volume>\d+)\s*\((?P<number>\d+)\)" 

19 article_volume_match_re = r"(?P<volume>\d+)\s*\((?P<number>\d+)\)" 

20 

21 def process_issue_title(self, xissue: dict, issue_title: str): 

22 title_dict = regex_to_dict( 

23 self.volume_title_re, 

24 issue_title, 

25 error_msg="Couldn't parse issue title", 

26 ) 

27 

28 xissue.volume = title_dict["volume"] 

29 xissue.number = title_dict["number"] 

30 xissue.title_tex = f"Volume {xissue.volume}" 

31 if xissue.title_tex == "Synthesis and perspectives in didactics of mathematics": 

32 print() 

33 if not title_dict["volume"].isdigit(): 

34 xissue.title_tex = f"{xissue.volume}" 

35 

36 return xissue