Coverage for src/crawler/by_source/nsjom/nsjom_2010_crawler.py: 83%
99 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-29 13:39 +0000
1import re
2import typing
3from datetime import datetime
5from bs4 import BeautifulSoup, Tag
6from ptf.model_data import (
7 IssueData,
8 create_articledata,
9 create_contributor,
10 create_extlink,
11 create_issuedata,
12)
14from crawler.crawler_utils import set_pages
15from crawler.utils import add_pdf_link_to_xarticle, cleanup_str
17if typing.TYPE_CHECKING:
18 from .nsjom_crawler import NsjomCrawler
20source_domain = "NSJOM"
23def parse_collection_content(
24 self: "NsjomCrawler",
25 content: str,
26 source_domain: str = "NSJOM",
27):
28 """
29 Parses all articles from year-specific webpages : https://sites.dmi.uns.ac.rs/nsjom/ns2010.html
30 From 2010 to 2014 (included)
31 """
32 xissues: list[IssueData] = []
33 year_start = 2010
34 year_end = min(2014, datetime.now().year)
36 for year in range(year_start, year_end + 1):
37 url = f"https://sites.dmi.uns.ac.rs/nsjom/ns{year}.html"
38 year_content = self.download_file(url)
39 xissues = xissues + parse_year(self, year_content, year, url)
41 return xissues
44def is_heading(element: Tag):
45 if element.select_one(".HeadingNSJOM"):
46 return True
47 classname = element.get("class")
48 if not classname:
49 return False
51 if isinstance(classname, str): 51 ↛ 52line 51 didn't jump to line 52 because the condition on line 51 was never true
52 return classname == "HeadingNSJOM"
54 return "HeadingNSJOM" in classname
57def parse_issue_content(self, content: str, xissue: IssueData):
58 if xissue.fyear == 0:
59 raise ValueError("Issue fyear is not set")
60 parse_year(self, content, int(xissue.fyear), xissue.url, xissue.pid)
63def parse_year(
64 self: "NsjomCrawler",
65 content: str,
66 year: int,
67 url: str | None = None,
68 pid_to_crawl: str | None = None,
69):
70 soup = BeautifulSoup(content, "html.parser")
71 page_elements = soup.select("p.HeadingNSJOM, p.style1, p.style1+blockquote a")
72 issues: list[list[Tag]] = []
73 # Sort tags into issues
74 for current_element in page_elements:
75 if is_heading(current_element):
76 issues.append([current_element])
77 continue
78 if current_element.text == "\xa0":
79 continue
80 issues[-1].append(current_element)
82 xissues: list[IssueData] = []
83 for issue_elements in issues:
84 xissue = parse_issue_tags(self, issue_elements, year, pid_to_crawl)
85 if not xissue: 85 ↛ 86line 85 didn't jump to line 86 because the condition on line 85 was never true
86 continue
87 xissue.url = url
88 xissues.append(xissue)
90 return xissues
93def parse_issue_tags(
94 self: "NsjomCrawler", tags: list[Tag], year: int, pid_to_crawl: str | None = None
95):
96 xissue = create_issuedata()
97 xissue.fyear = year
98 ext_link = create_extlink(
99 rel="source",
100 location=f"https://sites.dmi.uns.ac.rs/nsjom/issue.html?year={year}",
101 metadata=source_domain,
102 )
103 xissue.ext_links.append(ext_link)
104 issue_title_tag = tags.pop(0)
105 match = re.search(
106 r"NSJOM Vol\. (?P<Volume>\d+)(?:, No. (?P<Issue>\d+))?", issue_title_tag.text
107 )
108 if match is None: 108 ↛ 109line 108 didn't jump to line 109 because the condition on line 108 was never true
109 raise ValueError("Cannot find volume number")
110 volume_number = match.group("Volume")
111 issue_number = match.group("Issue")
112 if volume_number: 112 ↛ 115line 112 didn't jump to line 115 because the condition on line 112 was always true
113 xissue.volume = volume_number
114 else:
115 raise ValueError("Cannot read volume number")
116 if issue_number: 116 ↛ 118line 116 didn't jump to line 118 because the condition on line 116 was always true
117 xissue.number = issue_number
118 xissue.pid = f"{source_domain}_{year}_{volume_number}_{issue_number}"
120 if pid_to_crawl and xissue.pid != pid_to_crawl: 120 ↛ 121line 120 didn't jump to line 121 because the condition on line 120 was never true
121 return
123 while len(tags) > 0:
124 article_meta = tags.pop(0)
125 article_meta_text = cleanup_str(article_meta.text)
126 if article_meta_text == "": 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true
127 continue
129 article_title = tags.pop(0)
130 article = parse_article(
131 self,
132 article_meta_text,
133 article_title,
134 xissue.pid,
135 len(xissue.articles),
136 f"https://sites.dmi.uns.ac.rs/nsjom/issue.html?year={year}",
137 )
138 if article is None: 138 ↛ 139line 138 didn't jump to line 139 because the condition on line 138 was never true
139 continue
140 xissue.articles.append(article)
141 return xissue
144def parse_article(
145 self: "NsjomCrawler",
146 meta_text: str,
147 title_tag: Tag,
148 issue_pid: str,
149 index: int,
150 source: str | None = None,
151):
152 xarticle = create_articledata()
153 xarticle.pid = f"{issue_pid}a_{index}"
154 if source: 154 ↛ 157line 154 didn't jump to line 157 because the condition on line 154 was always true
155 ext_link = create_extlink(rel="source", location=source, metadata=self.source_domain)
156 xarticle.ext_links.append(ext_link)
157 match = re.search(r"\d+ \/ \d+ \/ \d+ \/ (?P<Pages>\d+(?:-\d+)?): (?P<Authors>.+)", meta_text)
158 if match is None: 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true
159 raise ValueError("Cannot parse authors or page number")
160 set_pages(xarticle, match.group("Pages"))
161 authors = re.findall(r"(?: and )?((?:(?<!,)(?<! and).(?!and ))+)", match.group("Authors"))
162 for a in authors:
163 author = create_contributor(role="author", string_name=a)
164 xarticle.contributors.append(author)
166 article_pdf_link = self.get_str_attr(title_tag, "href")
168 pdf_link = self.source_website + article_pdf_link
169 add_pdf_link_to_xarticle(xarticle, pdf_link)
171 title = cleanup_str(title_tag.text)
172 xarticle.title_tex = title
174 return xarticle