Coverage for src/crawler/by_source/amp_crawler.py: 12%

1import lingua 

2import requests 

3from bs4 import BeautifulSoup 

4from lingua import LanguageDetectorBuilder 

5from ptf.model_data import IssueData, create_abstract, create_articledata, create_extid 

6 

7from crawler.abstract_crawlers.matching_crawler import MatchingCrawler 

8from crawler.types import CitationLiteral 

9 

10 

11class AmpCrawler(MatchingCrawler): 

12 source_name = "Annals of Mathematics website" 

13 source_domain = "AMP" 

14 source_website = "https://annals.math.princeton.edu" 

15 

16 _language_detector_builder = LanguageDetectorBuilder.from_languages( 

17 lingua.Language.ENGLISH, 

18 lingua.Language.FRENCH, 

19 ) 

20 

21 def crawl_issue(self, xissue: IssueData): 

22 try: 

23 super().crawl_issue(xissue) 

24 except requests.exceptions.HTTPError: 

25 self.logger.warning("Got HTTPError while crawling issue. Skipping") 

26 

27 def parse_collection_content(self, content): 

28 """ 

29 Parse the HTML page of Annals of Math and returns a list of xissue. 

30 Each xissue has its volume/number/year metadata + its url 

31 """ 

32 soup = BeautifulSoup(content, "html.parser") 

33 xissues = [] 

34 

35 # Extract the list of issues 

36 issue_nodes = soup.find_all("div", {"class": "cat-item-2"}) 

37 

38 for issue_node in issue_nodes: 

39 issue_link_node = issue_node.find("a") 

40 if issue_link_node: 

41 url = issue_link_node.get("href") 

42 xissue = self.create_amp_xissue(url) 

43 if xissue: 

44 xissues.append(xissue) 

45 

46 return xissues 

47 

48 def create_amp_xissue(self, url): 

49 url = url.removesuffix("/") 

50 parts = url.split("/") 

51 

52 last_part = parts[-1] 

53 exceptions = last_part.split("-") 

54 if len(exceptions) > 2: 

55 year = exceptions[0] 

56 volume = exceptions[0] 

57 number = exceptions[1] 

58 else: 

59 year = parts[-2] 

60 if len(year) < 4: 

61 # The links are different with volumes before 2015 

62 year = parts[-3] 

63 

64 volume_number = parts[-1] 

65 volume_number_parts = volume_number.split("-") 

66 volume = volume_number_parts[0] 

67 number = volume_number_parts[1] 

68 

69 return self.create_xissue(url, year, volume, number) 

70 

71 def parse_issue_content(self, content, xissue): 

72 soup = BeautifulSoup(content, "html.parser") 

73 article_nodes = soup.select("h2.entry-title") 

74 

75 for index_article, article_node in enumerate(article_nodes): 

76 article_link_node = article_node.select_one("a") 

77 if article_link_node: 

78 href = article_link_node.get("href") 

79 if not isinstance(href, str): 

80 raise ValueError("Couldn't parse issue: href is not a string") 

81 xarticle = create_articledata() 

82 xarticle.pid = "a" + str(index_article) 

83 xarticle.url = href 

84 xissue.articles.append(xarticle) 

85 

86 def parse_article_content(self, content, xissue, xarticle, url): 

87 """ 

88 Parse the content with Beautifulsoup and returns an ArticleData 

89 """ 

90 xarticle.lang = "en" 

91 

92 soup = BeautifulSoup(content, "html.parser") 

93 

94 what: list[CitationLiteral] = ["author", "abstract", "page"] 

95 

96 doi_tag = soup.select_one("meta[name='citation_doi']") 

97 if doi_tag and not doi_tag.text.startswith("https://doi.org/"): 

98 what.append("pdf") 

99 

100 title_node = soup.find("h1", {"class": "entry-title"}) 

101 if title_node: 

102 what.append("title") 

103 

104 if url != "https://annals.math.princeton.edu/2010/172-3/p06": 

105 # Exception with Annals of Math: 2 articles have the same DOI ! 

106 # https://annals.math.princeton.edu/2010/172-3/p06 and https://annals.math.princeton.edu/2011/173-1/p14 

107 # we ignore DOI/ZBMATH/MR for the first one 

108 what.append("doi") 

109 

110 self.get_metadata_using_citation_meta(xarticle, xissue, soup, what) 

111 

112 abstract_node = soup.select_one("div.entry-content") 

113 if abstract_node is not None: 

114 abstract_section_node = abstract_node.select_one("p") 

115 if abstract_section_node: 

116 abstract = str(abstract_section_node) 

117 xarticle.abstracts.append( 

118 create_abstract( 

119 value_tex=abstract, lang=self.detect_language(abstract, xarticle) 

120 ) 

121 ) 

122 

123 # ZBMATH 

124 metadata_header_nodes = soup.find_all("div", {"class": "metadata-headers"}) 

125 for metadata_header_node in metadata_header_nodes: 

126 text = metadata_header_node.get_text() 

127 

128 if text == "zbMATH": 

129 link_node = metadata_header_node.parent.find("a") 

130 if link_node: 

131 zblid = link_node.get("href") 

132 pos = zblid.find("?q=an:") 

133 if pos > 0: 

134 zblid = zblid[pos + 6 :] 

135 extid = create_extid("zbl-item-id", zblid) 

136 xarticle.extids.append(extid) 

137 elif text == "MR": 

138 link_node = metadata_header_node.parent.find("a") 

139 if link_node: 

140 mrid = link_node.get("href") 

141 pos = mrid.find("?mr=") 

142 if pos > 0: 

143 mrid = mrid[pos + 4 :] 

144 extid = create_extid("mr-item-id", mrid) 

145 xarticle.extids.append(extid) 

146 

147 return xarticle