Coverage for src/crawler/by_source/jsig_crawler.py: 10%

1from urllib.parse import urljoin 

2 

3from bs4 import BeautifulSoup, Tag 

4from django.conf import settings 

5from ptf.model_data import create_abstract, create_articledata, create_contributor, create_subj 

6 

7from crawler.abstract_crawlers.matching_crawler import MatchingCrawler 

8from crawler.utils import ( 

9 add_pdf_link_to_xarticle, 

10 add_source_link_to_xarticle, 

11 cleanup_str, 

12 regex_to_dict, 

13) 

14 

15 

16class JsigCrawler(MatchingCrawler): 

17 source_name = "Journal of Singularities website" 

18 source_domain = "JSIG" 

19 source_website = "https://journalofsing.org" 

20 

21 issue_re = r"volume (?P<volume>\d+), (?P<year>\d{4})" 

22 requests_interval = max(getattr(settings, "REQUESTS_INTERVAL", 90), 35) 

23 

24 def parse_collection_content(self, content): 

25 xissues = [] 

26 soup = BeautifulSoup(content, "html5lib") 

27 issues_tags = soup.select("#navcontainer a[href^='volume']") 

28 

29 # issues 1-10 are listed in a dedicated html page 

30 nested = soup.select_one("#navcontainer a[href^='volume1-10']") 

31 if isinstance(nested, Tag): 

32 issues_tags.remove(nested) 

33 nested_href = nested.get("href") 

34 if isinstance(nested_href, str): 

35 additional_content = self.download_file(urljoin(self.collection_url, nested_href)) 

36 more_soup = BeautifulSoup(additional_content, "html5lib") 

37 more_issues = more_soup.select("#col-text-content2 a") 

38 issues_tags.extend(more_issues) 

39 

40 for tag in issues_tags: 

41 issue_dict = regex_to_dict( 

42 self.issue_re, tag.text, error_msg="Couldn't parse issue data" 

43 ) 

44 

45 issue_href = tag.get("href") 

46 if not isinstance(issue_href, str): 

47 raise ValueError("Couldn't parse issue url") 

48 

49 xissues.append( 

50 self.create_xissue( 

51 urljoin(self.collection_url, issue_href), 

52 int(issue_dict["year"]), 

53 issue_dict["volume"], 

54 None, 

55 ) 

56 ) 

57 return xissues 

58 

59 def parse_issue_content(self, content, xissue): 

60 if not xissue.url: 

61 raise ValueError("xissue must have an url") 

62 

63 soup = BeautifulSoup(content, "html5lib") 

64 articles_tags = soup.select( 

65 "#col-text-content2 a[href$='.html'], #col-text-content2 a[href$='.pdf']" 

66 ) 

67 for index, tag in enumerate(articles_tags): 

68 article_url = tag.get("href") 

69 if not isinstance(article_url, str): 

70 raise ValueError("Couldn't parse article data") 

71 xarticle = create_articledata() 

72 xarticle.pid = "a" + str(index) 

73 

74 if article_url.endswith(".html"): 

75 xarticle.url = urljoin(xissue.url, article_url) 

76 else: 

77 # branch here if article does not have any dedicated html page 

78 add_source_link_to_xarticle(xarticle, xissue.url, self.source_domain) 

79 add_pdf_link_to_xarticle(xarticle, urljoin(xissue.url, article_url)) 

80 xarticle.title_tex = tag.text 

81 

82 article_table = tag.parent.parent.parent 

83 if article_table: 

84 authors_tag = article_table.select_one("td[colspan='2'][width='525']") 

85 authors_list = authors_tag.text.replace(" and ", ", ").split(", ") 

86 for author_str in authors_list: 

87 xarticle.contributors.append( 

88 create_contributor(string_name=cleanup_str(author_str), role="author") 

89 ) 

90 xissue.articles.append(xarticle) 

91 

92 def parse_article_content(self, content, xissue, xarticle, url): 

93 if not xarticle.url: 

94 raise ValueError("xarticle must have an url") 

95 

96 soup = BeautifulSoup(content, "html5lib") 

97 soup = soup.select_one("#col-text-content") 

98 

99 if not soup: 

100 raise ValueError("Couldn't parse article page") 

101 

102 # Title and pdf 

103 title_tag = soup.select_one("h2") 

104 if not title_tag: 

105 raise ValueError("Couldn't parse article title") 

106 xarticle.title_tex = cleanup_str(title_tag.text) 

107 pdf_tag = title_tag.select_one("a") 

108 if not pdf_tag: 

109 raise ValueError("Couldn't find article pdf") 

110 pdf_href = pdf_tag.get("href") 

111 if not isinstance(pdf_href, str): 

112 raise ValueError("Couldn't parse article pdf") 

113 pdf_href = urljoin(xarticle.url, pdf_href) 

114 add_pdf_link_to_xarticle(xarticle, pdf_href) 

115 title_tag.decompose() 

116 

117 # Authors 

118 authors_tag = soup.select_one("div > span") 

119 if authors_tag: 

120 authors_tag.decompose() 

121 

122 # MSC 

123 msc_header = soup.select_one("p:-soup-contains('Mathematical Subject Classification')") 

124 if msc_header: 

125 msc_tag = msc_header.findNext("p") 

126 if isinstance(msc_tag, Tag): 

127 for msc_str in ( 

128 msc_tag.text.replace(";", ",") 

129 .replace("Primary", "") 

130 .replace("Secondary", "") 

131 .replace(" ", "") 

132 .split(",") 

133 ): 

134 xarticle.kwds.append(create_subj(type="msc", value=cleanup_str(msc_str))) 

135 msc_tag.decompose() 

136 msc_header.decompose() 

137 

138 # Abstract 

139 abstract_header = soup.select_one("p:-soup-contains('Abstract')") 

140 if abstract_header: 

141 abstract_tag = abstract_header.findNext("p") 

142 if isinstance(abstract_tag, Tag): 

143 xarticle.abstracts.append( 

144 create_abstract(value_tex=cleanup_str(abstract_tag.text), lang="en") 

145 ) 

146 abstract_tag.decompose() 

147 abstract_header.decompose() 

148 

149 # Pages 

150 pages_tag = soup.select_one("p.style3") 

151 if isinstance(pages_tag, Tag): 

152 pages_text = cleanup_str(pages_tag.text) 

153 pages_dict = regex_to_dict( 

154 r"volume \d+ \(\d+\), (?P<fpage>\d+)\-(?P<lpage>\d+)", pages_text 

155 ) 

156 xarticle.fpage = pages_dict["fpage"] 

157 xarticle.lpage = pages_dict["lpage"] 

158 pages_tag.decompose() 

159 

160 # DOI 

161 doi_tag = soup.select_one("p:-soup-contains('DOI:')") 

162 if isinstance(doi_tag, Tag): 

163 xarticle.doi = cleanup_str( 

164 doi_tag.select_one("a").get("href").removeprefix("http://dx.doi.org/") 

165 ) 

166 doi_tag.decompose() 

167 if xarticle.url == "https://journalofsing.org/volume22/article2.html": 

168 xarticle.doi = "10.5427/jsing.2020.22b" 

169 

170 return xarticle