Coverage for src/crawler/by_source/journalfi_crawler.py: 81%

1import regex 

2from bs4 import BeautifulSoup 

3from ptf.model_data import IssueData, create_abstract, create_articledata 

4 

5from crawler.abstract_crawlers.matching_crawler import MatchingCrawler 

6from crawler.utils import cleanup_str 

7 

8 

9class JournalfiCrawler(MatchingCrawler): 

10 source_name = "Journal.fi" 

11 source_domain = "JOURNALFI" 

12 source_website = "https://journal.fi/" 

13 

14 delimiter_inline_formula = "\\(" 

15 

16 issue_re = r"Vol\. (?P<volume>\d+) No\. (?P<number>\d+) \((?P<year>\d+)\)" 

17 issue_re_2 = r"Volume (?P<volume>[\d\-]+), (?P<year>\d+)" 

18 issue_re_3 = r"Numbers (?P<volume>\d+\-\d+), (?P<year>\d+)" 

19 

20 def parse_collection_content(self, content): 

21 self.logger.info("Collection parsing may take some time, please wait") 

22 xissues = self.parse_journalfi_collection_content(content) 

23 return list(xissues.values()) 

24 

25 def parse_journalfi_collection_content(self, content): 

26 soup = BeautifulSoup(content, "html.parser") 

27 issues = soup.select("a.title") 

28 xissues: dict[str, IssueData] = {} 

29 for issue in issues: 

30 issue_title_str = cleanup_str(issue.text) 

31 issue_search = regex.search(self.issue_re, issue_title_str) 

32 

33 # try issue_re_2 

34 if not issue_search: 

35 issue_search = regex.search(self.issue_re_2, issue_title_str) 

36 if issue_search: 

37 issue_title_tag = issue.parent.select_one("div.series") 

38 title_str = cleanup_str(issue_title_tag.text) 

39 issue_search = regex.search(self.issue_re, title_str) 

40 

41 # try issue_re_3 

42 if not issue_search: 

43 issue_search = regex.search(self.issue_re_3, issue_title_str) 

44 

45 if not issue_search: 

46 self.logger.debug(f"Couldn't parse issue regex for issue : {issue_title_str}") 

47 continue 

48 

49 issue_url = issue.get("href") 

50 issue_dict = issue_search.groupdict() 

51 if not isinstance(issue_url, str): 51 ↛ 52line 51 didn't jump to line 52 because the condition on line 51 was never true

52 raise ValueError("Cannot parse issue url") 

53 xissue = self.create_xissue( 

54 issue_url, 

55 int(issue_dict["year"]), 

56 issue_dict["volume"], 

57 issue_dict.get("number", None), 

58 ) 

59 if not xissue.pid: 59 ↛ 60line 59 didn't jump to line 60 because the condition on line 59 was never true

60 raise ValueError("xissue.pid is None") 

61 

62 # HACK 

63 # Create article data in advance because we won't be able to access article urls ( 

64 # https://afm.journal.fi/issue/view/10338) 

65 if issue_search.re.pattern == self.issue_re_3: 

66 issue_content = self.download_file(issue_url) 

67 self.parse_issue_content(issue_content, xissue) 

68 xissue.url = None 

69 if xissue.pid not in xissues: 

70 xissues[xissue.pid] = xissue 

71 else: 

72 xissues[xissue.pid].articles.extend(xissue.articles) 

73 

74 next_page = soup.select_one("a.next") 

75 if not next_page: 

76 return xissues 

77 

78 next_url = next_page.get("href") 

79 if not isinstance(next_url, str): 79 ↛ 80line 79 didn't jump to line 80 because the condition on line 79 was never true

80 raise ValueError("Couldn't parse issue pagination") 

81 next_content = self.download_file(next_url) 

82 other_xissues = self.parse_journalfi_collection_content(next_content) 

83 for issue_pid in other_xissues: 

84 if issue_pid in xissues: 

85 xissues[issue_pid].articles.extend(other_xissues[issue_pid].articles) 

86 else: 

87 xissues[issue_pid] = other_xissues[issue_pid] 

88 

89 for xissue in xissues.values(): 

90 for index, article in enumerate(xissue.articles): 

91 article.pid = "a" + str(index) 

92 return xissues 

93 

94 def parse_issue_content(self, content, xissue): 

95 soup = BeautifulSoup(content, "html.parser") 

96 articles = soup.select("a[id^='article']") 

97 for index, article_tag in enumerate(articles): 

98 article_url = article_tag.get("href") 

99 if not isinstance(article_url, str): 99 ↛ 100line 99 didn't jump to line 100 because the condition on line 99 was never true

100 raise ValueError("Couldn't parse Article url") 

101 xarticle = create_articledata() 

102 xarticle.pid = "a" + str(index) 

103 xarticle.url = article_url 

104 xissue.articles.append(xarticle) 

105 

106 def parse_article_content(self, content, xissue, xarticle, url): 

107 soup = BeautifulSoup(content, "html.parser") 

108 self.get_metadata_using_citation_meta( 

109 xarticle, xissue, soup, ["author", "title", "lang", "page", "doi", "keywords", "pdf"] 

110 ) 

111 abstract_tag = soup.select_one("meta[name='DC.Description']") 

112 if abstract_tag: 112 ↛ 113line 112 didn't jump to line 113 because the condition on line 112 was never true

113 abstarct_lang = xarticle.lang 

114 

115 abstract_lang_str = abstract_tag.get("xml:lang") 

116 if isinstance(abstract_lang_str, str): 

117 abstarct_lang = abstract_lang_str 

118 del abstract_lang_str 

119 

120 abstract_content = abstract_tag.get("content") 

121 if isinstance(abstract_content, str): 

122 xarticle.abstracts.append( 

123 create_abstract(lang=abstarct_lang, value_tex=abstract_content) 

124 ) 

125 

126 if xarticle.title_tex == "": 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 if xissue.pid == "AFM_2003_28_1" and xarticle.pid == "a10": 

128 xarticle.title_tex = "$\\delta$-stable Fuchsian groups" 

129 return xarticle