Coverage for src/crawler/by_source/amuc_crawler.py: 9%
203 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-29 13:39 +0000
1import re
3from bs4 import BeautifulSoup
4from ptf.model_data import create_abstract, create_articledata, create_contributor
6from crawler.abstract_crawlers.matching_crawler import MatchingCrawler
7from crawler.utils import add_pdf_link_to_xarticle, cleanup_str, regex_to_dict
10class AmucCrawler(MatchingCrawler):
11 source_name = "Comenius University"
12 source_domain = "AMUC"
13 source_website = "http://www.iam.fmph.uniba.sk"
15 issue_re = r"Vol (?P<volume>\d+) No (?P<issue>\S+) \((?P<year>\S+)\)"
16 archive_issue_re = r"Number (?P<issue>\S+)"
17 archive_volume_re = r"Volume (?P<volume>\S+) \((?P<year>\S+)\)"
18 archive_abstract_re = r"Abstract (?P<abstract>\S+) AMS"
20 def parse_collection_content(self, content):
21 xissues = []
22 soup = BeautifulSoup(content, "html.parser")
23 issues = soup.select("div.issue-summary")
24 for issue in issues:
25 issue_group = regex_to_dict(
26 self.issue_re, issue.text, error_msg="Couldn't parse issue data"
27 )
29 issue_href = issue.select("a.title")[0].get("href")
30 if not isinstance(issue_href, str):
31 raise ValueError("Couldn't parse issue url")
33 if issue_group["volume"] == "0":
34 xissues += self.parse_archived_collection_content(issue_href)
36 else:
37 xissues.append(
38 self.create_xissue(
39 url=issue_href,
40 year=int(issue_group["year"]),
41 volume_number=issue_group["volume"],
42 issue_number=issue_group["issue"],
43 )
44 )
46 # If next page exist, get next page issues
47 pagination_tag = soup.select_one("ul.pagination")
48 next_pages_tag = pagination_tag.select("a")
49 for next_page_tag in next_pages_tag:
50 if next_page_tag.text == ">":
51 next_page_url = next_page_tag.get("href")
52 next_page_content = self.download_file(next_page_url)
53 xissues += self.parse_collection_content(next_page_content)
54 return xissues
56 def parse_archived_collection_content(self, url):
57 """
58 For volumes from 60 to 80
59 """
60 content = self.download_file(url)
61 soup = BeautifulSoup(content, "html.parser")
62 issues_tag = soup.select("ul > li.show > ul > li.show > a")
63 artived_xissues = []
64 for issue_tag in issues_tag:
65 issue_href = issue_tag.get("href")
66 if not isinstance(issue_href, str):
67 raise ValueError("Couldn't parse issue url")
69 issue_nb = regex_to_dict(
70 self.archive_issue_re, issue_tag.text, error_msg="Couldn't parse issue data"
71 ).get("issue")
73 volume_tag = issue_tag.parent.parent.parent.select_one("span > strong")
74 volume_group = regex_to_dict(
75 self.archive_volume_re, volume_tag.text, error_msg="Couldn't parse volume data"
76 )
78 artived_xissues.append(
79 self.create_xissue(
80 url=issue_href,
81 year=volume_group["year"],
82 volume_number=volume_group["volume"],
83 issue_number=issue_nb,
84 )
85 )
86 return artived_xissues
88 def parse_issue_content(self, content, xissue):
89 soup = BeautifulSoup(content, "html.parser")
90 articles = soup.select("div.article-summary")
91 if len(articles) == 0:
92 self.parse_archived_issue_content(soup, xissue)
93 else:
94 for article_number, article in enumerate(articles):
95 xarticle = create_articledata()
96 article_href = article.select("h3.media-heading")[0].select("a")[0].get("href")
97 if not isinstance(article_href, str):
98 raise ValueError("Couldn't parse article href")
99 xarticle.url = article_href
100 xarticle.pid = "a" + str(article_number)
101 xissue.articles.append(xarticle)
103 def parse_archived_issue_content(self, soup, xissue):
104 """
105 For issues from volumes 60 to 80
106 """
107 article_number = 0
108 body_tag = soup.find("body")
109 if not body_tag:
110 raise ValueError("Couldn't find body tag in archived issue")
111 if body_tag and body_tag.get("bgcolor") == "#FFFFF0":
112 # On cible les liens "Abstract" qui sont présents dans tous les formats
113 abstract_links = soup.find_all(
114 "a", string=lambda t: t and "abstract" in t.strip().lower()
115 )
116 if not abstract_links:
117 raise ValueError(
118 "Couldn't find abstract links in archived issue with white background"
119 )
120 for abstract_link in abstract_links:
121 href = abstract_link.get("href")
122 if not href:
123 continue
125 xarticle = create_articledata()
126 abstract_url = "/".join(xissue.url.split("/")[0:-1]) + "/" + href
127 xarticle.pid = "a" + str(article_number)
128 article_number += 1
129 xarticle.title_tex = "Archived article white background"
130 xarticle.url = abstract_url
131 xissue.articles.append(xarticle)
132 if body_tag and body_tag.get("bgcolor") == "#CCE6FF":
133 articles_abstract_tags = soup.findAll("a", href=True, text="Abstract")
134 if not articles_abstract_tags:
135 raise ValueError(
136 "Couldn't find abstract links in archived issue with blue background"
137 )
139 for article_abstract_tag in articles_abstract_tags:
140 xarticle = create_articledata()
141 abstract_url = (
142 "/".join(xissue.url.split("/")[0:-1]) + "/" + article_abstract_tag.get("href")
143 )
144 xarticle.pid = "a" + str(article_number)
145 article_number += 1
146 xarticle.title_tex = "Archived article blue background"
147 xarticle.url = abstract_url
148 xissue.articles.append(xarticle)
150 def parse_article_content(self, content, xissue, xarticle, url):
151 soup = BeautifulSoup(content, "html.parser")
153 # If archived/old article :
154 if xarticle.title_tex in [
155 "Archived article white background",
156 "Archived article blue background",
157 ]:
158 return self.parse_archived_article_content(soup, xissue, xarticle)
160 self.get_metadata_using_citation_meta(
161 xarticle, xissue, soup, ["author", "doi", "title", "pdf", "page", "title"]
162 )
164 # Contributors
165 contributors = soup.select_one("div.authors").select("strong")
166 for contributor in contributors:
167 xarticle.contributors.append(
168 create_contributor(role="author", string_name=contributor.text)
169 )
171 # pdf link
172 pdf_url = soup.select_one("div.download").select_one("a").get("href")
173 if isinstance(pdf_url, str):
174 add_pdf_link_to_xarticle(xarticle, pdf_url)
176 # Abstract
177 abstract_tag = soup.select_one("div.article-abstract")
178 if abstract_tag:
179 xarticle.abstracts.append(create_abstract(value_tex=cleanup_str(abstract_tag.text)))
180 return xarticle
182 def parse_archived_article_content(self, soup, xissue, xarticle):
183 """
184 Parse content of archived articles (from volumes 60 to 80)
185 """
186 try:
187 extract_metadata = self.extract_archived_metadata(soup, xarticle)
188 except ValueError as e:
189 self.logger.error(f"Error extracting metadata for archived article: {e}")
190 xarticle = self.parse_archived_article_content(soup, xissue, xarticle)
192 xarticle.title_tex = extract_metadata["title"]
193 if extract_metadata.get("authors"):
194 for author in extract_metadata["authors"]:
195 xarticle.contributors.append(create_contributor(role="author", string_name=author))
196 if extract_metadata.get("abstract"):
197 xarticle.abstracts.append(
198 create_abstract(value_tex=cleanup_str(extract_metadata["abstract"]))
199 )
200 if extract_metadata.get("keywords"):
201 xarticle.keywords = extract_metadata["keywords"]
202 if extract_metadata.get("pdf_url"):
203 pdf_url = "/".join(xarticle.url.split("/")[0:-1]) + "/" + extract_metadata["pdf_url"]
204 add_pdf_link_to_xarticle(xarticle, pdf_url)
205 return xarticle
207 def extract_archived_metadata_blue_bg(self, soup):
208 """
209 Extract metadata for articles with blue background
210 """
211 title_tag = soup.find("font", {"color": "#A52A2A"})
212 if not title_tag:
213 raise ValueError("Couldn't find title in archived article with blue background")
214 title = title_tag.get_text(separator=" ", strip=True) if title_tag else None
216 author_tag = soup.find("font", {"color": "#008B8B"})
217 if not author_tag:
218 raise ValueError("Couldn't find authors in archived article with blue background")
219 authors = author_tag.get_text(strip=True) if author_tag else None
220 authors = re.split(", | and ", authors) if authors else []
222 pdf_url_tag = soup.select_one("a", href=True, text="PDF")
223 if not pdf_url_tag:
224 raise ValueError("Couldn't find pdf url")
225 pdf_url = pdf_url_tag.get("href")
227 return title, authors, pdf_url
229 def extract_archived_metadata_white_bg(self, soup):
230 """
231 Extract metadata for articles with white background
232 """
233 title_tag = soup.find("font", {"color": "#A52A2A"})
234 if not title_tag:
235 title_tag = soup.select_one('span[style*="color: brown"]')
236 if not title_tag:
237 title_tag = soup.select("font", {"color": "#a52a2a"})
238 if not title_tag:
239 raise ValueError(
240 "Couldn't find title in archived article with white background"
241 )
242 title_tag = title_tag[4]
243 title = title_tag.get_text(separator=" ", strip=True) if title_tag else None
245 author_tag = soup.find("font", {"color": "#008B8B"})
246 if not author_tag:
247 author_tag = soup.select_one('span[style*="color: darkcyan"]')
248 if not author_tag:
249 author_tag = soup.select("font")
250 if not author_tag:
251 raise ValueError(
252 "Couldn't find authors in archived article with white background"
253 )
254 author_tag = author_tag[5]
255 authors = author_tag.get_text(strip=True) if author_tag else None
256 authors = re.split(", | and ", authors) if authors else []
257 authors = self.parse_authors_caps_names(authors)
258 pdf_url_tag = soup.select_one("a", href=True, text="Adobe PDF")
260 if not pdf_url_tag:
261 raise ValueError("Couldn't find pdf url")
262 pdf_url = pdf_url_tag.get("href")
264 return title, authors, pdf_url
266 def get_text_until_next_section(self, tag):
267 """
268 For archived articles, get the text content of a section (abstract or keywords) until the next section (keywords or AMS) or the end of the document.
269 """
270 SECTION_KEYWORDS = ["abstract", "keyword", "ams"]
271 content = []
272 for sibling in tag.next_siblings:
273 if sibling.name == "b" and any(
274 sibling.get_text(strip=True).lower().startswith(k) for k in SECTION_KEYWORDS
275 ):
276 break
277 content.append(sibling if isinstance(sibling, str) else sibling.get_text())
278 return " ".join(content).strip().lstrip(":. \xa0")
280 def extract_abstract_and_keywords(self, soup):
281 """
282 Extract abstract and keywords for archived articles, which can be in different formats and places depending on the article. The method looks for the "Abstract" section and the "Keywords" section, and extracts their content until the next section or the end of the document.
283 """
284 abstract = None
285 keywords = []
287 for tag in soup.find_all("b"):
288 text = tag.get_text(strip=True).lower()
289 if text.startswith("abstract"):
290 abstract = self.get_text_until_next_section(tag)
291 elif text.startswith("keyword"):
292 raw = self.get_text_until_next_section(tag)
293 keywords = [kw.strip() for kw in raw.split(";") if kw.strip()]
295 return abstract, keywords
297 def extract_archived_metadata(self, soup, xarticle):
298 """
299 Extract metadata for archived articles.
300 """
301 background_color = xarticle.title_tex
302 if background_color == "Archived article blue background":
303 title, authors, pdf_url = self.extract_archived_metadata_blue_bg(soup)
304 elif background_color == "Archived article white background":
305 title, authors, pdf_url = self.extract_archived_metadata_white_bg(soup)
306 else:
307 raise ValueError("Unrecognized archived article backgroud color")
309 abstract, keywords = self.extract_abstract_and_keywords(soup)
311 return {
312 "title": title,
313 "authors": authors,
314 "pdf_url": pdf_url,
315 "abstract": abstract,
316 "keywords": keywords,
317 }
319 def parse_authors_caps_names(self, string_name_list):
320 final_string_name_list = []
321 for string_name in string_name_list:
322 string_name_split = string_name.split()
323 family_name = string_name_split[-1]
324 family_name = family_name[0].upper() + family_name[1:].lower()
325 string_name = " ".join(string_name_split[:-1] + [family_name])
326 final_string_name_list.append(string_name)
327 return final_string_name_list