Coverage for src / crawler / by_source / nsjom / nsjom_2010_crawler.py: 79%

1import re 

2import typing 

3from datetime import datetime 

4 

5from bs4 import BeautifulSoup, Tag 

6from ptf.model_data import ( 

7 IssueData, 

8 create_articledata, 

9 create_contributor, 

10 create_extlink, 

11 create_issuedata, 

12) 

13 

14from crawler.crawler_utils import set_pages 

15from crawler.utils import add_pdf_link_to_xarticle, cleanup_str 

16 

17if typing.TYPE_CHECKING: 

18 from .nsjom_crawler import NsjomCrawler 

19 

20source_domain = "NSJOM" 

21 

22 

23def parse_collection_content( 

24 self: "NsjomCrawler", 

25 content: str, 

26 source_domain: str = "NSJOM", 

27): 

28 """ 

29 Parses all articles from year-specific webpages : https://sites.dmi.uns.ac.rs/nsjom/ns2010.html 

30 From 2010 to 2014 (included) 

31 """ 

32 xissues: list[IssueData] = [] 

33 year_start = 2010 

34 year_end = min(2014, datetime.now().year) 

35 

36 for year in range(year_start, year_end + 1): 

37 url = f"https://sites.dmi.uns.ac.rs/nsjom/ns{year}.html" 

38 year_content = self.download_file(url) 

39 xissues = xissues + parse_year(self, year_content, year, url) 

40 

41 return xissues 

42 

43 

44def is_heading(element: Tag): 

45 if element.select_one(".HeadingNSJOM"): 

46 return True 

47 classname = element.get("class") 

48 if not classname: 

49 return False 

50 

51 if isinstance(classname, str): 51 ↛ 52line 51 didn't jump to line 52 because the condition on line 51 was never true

52 if classname == "HeadingNSJOM": 

53 return True 

54 return False 

55 

56 if "HeadingNSJOM" in classname: 

57 return True 

58 return False 

59 

60 

61def parse_issue_content(self, content: str, xissue: IssueData): 

62 if not xissue.year: 

63 raise ValueError("Issue year is not set") 

64 parse_year(self, content, int(xissue.year), xissue.url, xissue.pid) 

65 

66 

67def parse_year( 

68 self: "NsjomCrawler", 

69 content: str, 

70 year: int, 

71 url: str | None = None, 

72 pid_to_crawl: str | None = None, 

73): 

74 soup = BeautifulSoup(content, "html.parser") 

75 page_elements = soup.select("p.HeadingNSJOM, p.style1, p.style1+blockquote a") 

76 issues: list[list[Tag]] = [] 

77 # Sort tags into issues 

78 for current_element in page_elements: 

79 if is_heading(current_element): 

80 issues.append([current_element]) 

81 continue 

82 if current_element.text == "\xa0": 

83 continue 

84 issues[-1].append(current_element) 

85 

86 xissues: list[IssueData] = [] 

87 for issue_elements in issues: 

88 xissue = parse_issue_tags(self, issue_elements, year, pid_to_crawl) 

89 if not xissue: 89 ↛ 90line 89 didn't jump to line 90 because the condition on line 89 was never true

90 continue 

91 xissue.url = url 

92 xissues.append(xissue) 

93 

94 return xissues 

95 

96 

97def parse_issue_tags( 

98 self: "NsjomCrawler", tags: list[Tag], year: int, pid_to_crawl: str | None = None 

99): 

100 xissue = create_issuedata() 

101 xissue.year = str(year) 

102 ext_link = create_extlink( 

103 rel="source", 

104 location=f"https://sites.dmi.uns.ac.rs/nsjom/issue.html?year={year}", 

105 metadata=source_domain, 

106 ) 

107 xissue.ext_links.append(ext_link) 

108 issue_title_tag = tags.pop(0) 

109 match = re.search( 

110 r"NSJOM Vol\. (?P<Volume>\d+)(?:, No. (?P<Issue>\d+))?", issue_title_tag.text 

111 ) 

112 if match is None: 112 ↛ 113line 112 didn't jump to line 113 because the condition on line 112 was never true

113 raise ValueError("Cannot find volume number") 

114 volume_number = match.group("Volume") 

115 issue_number = match.group("Issue") 

116 if volume_number: 116 ↛ 119line 116 didn't jump to line 119 because the condition on line 116 was always true

117 xissue.volume = volume_number 

118 else: 

119 raise ValueError("Cannot read volume number") 

120 if issue_number: 120 ↛ 122line 120 didn't jump to line 122 because the condition on line 120 was always true

121 xissue.number = issue_number 

122 xissue.pid = f"{source_domain}_{year}_{volume_number}_{issue_number}" 

123 

124 if pid_to_crawl and xissue.pid != pid_to_crawl: 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true

125 return 

126 

127 while len(tags) > 0: 

128 article_meta = tags.pop(0) 

129 article_meta_text = cleanup_str(article_meta.text) 

130 if article_meta_text == "": 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true

131 continue 

132 

133 article_title = tags.pop(0) 

134 article = parse_article( 

135 self, 

136 article_meta_text, 

137 article_title, 

138 xissue.pid, 

139 len(xissue.articles), 

140 f"https://sites.dmi.uns.ac.rs/nsjom/issue.html?year={year}", 

141 ) 

142 if article is None: 142 ↛ 143line 142 didn't jump to line 143 because the condition on line 142 was never true

143 continue 

144 xissue.articles.append(article) 

145 return xissue 

146 

147 

148def parse_article( 

149 self: "NsjomCrawler", 

150 meta_text: str, 

151 title_tag: Tag, 

152 issue_pid: str, 

153 index: int, 

154 source: str | None = None, 

155): 

156 xarticle = create_articledata() 

157 xarticle.pid = f"{issue_pid}a_{index}" 

158 if source: 158 ↛ 161line 158 didn't jump to line 161 because the condition on line 158 was always true

159 ext_link = create_extlink(rel="source", location=source, metadata=self.source_domain) 

160 xarticle.ext_links.append(ext_link) 

161 match = re.search(r"\d+ \/ \d+ \/ \d+ \/ (?P<Pages>\d+(?:-\d+)?): (?P<Authors>.+)", meta_text) 

162 if match is None: 162 ↛ 163line 162 didn't jump to line 163 because the condition on line 162 was never true

163 raise ValueError("Cannot parse authors or page number") 

164 set_pages(xarticle, match.group("Pages")) 

165 authors = re.findall(r"(?: and )?((?:(?<!,)(?<! and).(?!and ))+)", match.group("Authors")) 

166 for a in authors: 

167 author = create_contributor(role="author", string_name=a) 

168 xarticle.contributors.append(author) 

169 

170 article_pdf_link = title_tag.get("href") 

171 if article_pdf_link is None: 171 ↛ 172line 171 didn't jump to line 172 because the condition on line 171 was never true

172 self.logger.debug("Article does not have a pdf", extra={"pid": xarticle.pid}) 

173 return None 

174 if isinstance(article_pdf_link, list): 174 ↛ 175line 174 didn't jump to line 175 because the condition on line 174 was never true

175 raise ValueError("Article link is a list") 

176 pdf_link = self.source_website + article_pdf_link 

177 add_pdf_link_to_xarticle(xarticle, pdf_link) 

178 

179 title = cleanup_str(title_tag.text) 

180 xarticle.title_tex = title 

181 

182 return xarticle