Coverage for src/crawler/by_source/slc_crawler.py: 9%
160 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-29 13:39 +0000
1import re
2from time import strftime, strptime
3from urllib.parse import urljoin
5import requests
6from bs4 import BeautifulSoup, Comment, PageElement, Tag
7from ptf.model_data import (
8 ArticleData,
9 IssueData,
10 create_abstract,
11 create_articledata,
12 create_contributor,
13)
15from crawler.abstract_crawlers.base_crawler import BaseCollectionCrawler
16from crawler.utils import add_pdf_link_to_xarticle, cleanup_str, regex_to_dict
19def is_relevant_tag(tag: PageElement):
20 if isinstance(tag, str):
21 return cleanup_str(tag) != ""
23 return cleanup_str(tag.text) != ""
26class Slc_Crawler(BaseCollectionCrawler):
27 source_name = "Séminaire Lotharingien de Combinatoire website"
28 source_domain = "SLC"
29 source_website = "https://www.mat.univie.ac.at/~slc/"
31 year_regex = r"Vol\. (?P<volume>\w+).+\((?P<year>\d+)(?:[\-\/](?P<year_end>\d+))?\)"
32 abstract_regex = (
33 r"<b>(?:Résumé.<\/b>(?P<resume>.+))?(?:(?:English )?Abstract.<\/b>(?P<abstract>.+))?"
34 )
35 dates_regex = r"(?:Received: (?P<received>[\w ]+,? \d{1,2},? \d+), )?(?:Revised(?: Version)?:? (?P<revised>[\w ]+,? \d{1,2},? \d+), )?Accepted:? (?P<accepted>[\w ]+,? \d{1,2},? \d+),(?: Final Version: (?P<final>[\w ]+,? \d{1,2},? \d+))?"
37 title_corrections = {
38 "SLC_1989_22_a15": "On the Growth Rate of Certain Combinatorial Functions",
39 }
41 def parse_collection_content(self, content):
42 xissues: list[IssueData] = []
44 soup = BeautifulSoup(content, "html5lib")
45 issue_tags = soup.select("table[border='1'] > tbody > tr > td")
46 for i_tag in issue_tags:
47 a_tag = i_tag.select_one("a")
48 if not a_tag:
49 continue
50 href = a_tag.get("href")
51 if not isinstance(href, str):
52 raise (TypeError("Couldn't parse issue href"))
53 href = urljoin(self.collection_url, href)
54 issue_dict = regex_to_dict(
55 self.year_regex, cleanup_str(i_tag.text), error_msg="Couldn't parse issue year"
56 )
58 year = issue_dict["year"]
59 if issue_dict.get("year_end") is not None:
60 year += "-"
61 if len(issue_dict["year_end"]) < 3:
62 year += year[0 : len(issue_dict["year_end"])]
63 year += issue_dict["year_end"]
65 issue_data = self.create_xissue(href, year, issue_dict["volume"], issue_number=None)
66 xissues.append(issue_data)
68 return xissues
70 def parse_issue_content(self, content: str, xissue: IssueData):
71 if not xissue.url:
72 raise ValueError("xissue must have an url")
73 if not xissue.url.endswith(".html") and not xissue.url.endswith("/"):
74 xissue.url += "/"
76 soup = BeautifulSoup(content, "html5lib")
78 # Preface
79 preface_tag = soup.select_one("a:-soup-contains-own('Preface')")
80 if preface_tag:
81 preface_href = preface_tag.get("href")
82 if isinstance(preface_href, str):
83 preface_href = urljoin(xissue.url, preface_href)
84 try:
85 preface_content = self.download_file(preface_href)
86 self.parse_slc_preface(preface_content)
87 except requests.exceptions.HTTPError:
88 self.logger.debug(
89 "Couldn't download file", extra={"url": preface_href, "pid": xissue.pid}
90 )
92 # Articles
93 articles_tags = soup.select("dl a")
94 for index, a_tag in enumerate(articles_tags):
95 href = a_tag.get("href", None)
96 if not isinstance(href, str):
97 continue
98 href = urljoin(xissue.url, href)
99 if a_tag.text == "Scanned copy":
100 add_pdf_link_to_xarticle(xissue, href)
101 continue
102 a_href = a_tag.get("href")
103 if not isinstance(a_href, str):
104 continue
105 xarticle = create_articledata()
106 xarticle.pid = f"a{index}"
107 xarticle.url = urljoin(xissue.url, a_href)
108 xissue.articles.append(xarticle)
110 pdf_tag = soup.select_one("a[href]:-soup-contains-own('Scanned copy')")
111 if pdf_tag:
112 pdf_href = pdf_tag.get("href")
113 if isinstance(pdf_href, str):
114 add_pdf_link_to_xarticle(xissue, urljoin(xissue.url, pdf_href))
116 def parse_article_content(
117 self, content: str, xissue: IssueData, xarticle: ArticleData, url: str
118 ) -> ArticleData | None:
119 if not xarticle.url:
120 raise ValueError("Article must have an url")
122 soup = BeautifulSoup(content, "html5lib")
123 body = soup.select_one("body")
124 if not body:
125 raise ValueError("Couldn't parse article body")
127 # PDF
128 pdf_tag = body.select_one(
129 "a[href]:-soup-contains-own('PDF'), a[href]:-soup-contains-own('Scan of original article')"
130 )
131 if not pdf_tag:
132 self.logger.debug(
133 "Couldn't find article pdf", extra={"pid": xarticle.pid, "url": xarticle.url}
134 )
135 return
136 pdf_href = pdf_tag.get("href")
137 if not isinstance(pdf_href, str):
138 raise TypeError("Couldn't parse pdf href")
139 add_pdf_link_to_xarticle(xarticle, urljoin(xarticle.url, pdf_href))
141 # Author
142 author_tag = body.select_one("h1:nth-of-type(1)")
143 if not isinstance(author_tag, Tag):
144 raise TypeError("Couldn't find article authors")
145 authors_str = cleanup_str(author_tag.text).replace(" and ", ", ")
146 if authors_str != "":
147 for author in authors_str.split(", "):
148 xarticle.contributors.append(create_contributor(role="author", string_name=author))
150 # Title
151 title_tag = body.select_one("h1:nth-of-type(2)")
152 if not isinstance(title_tag, Tag):
153 raise TypeError("Couldn't find article title")
154 xarticle.title_tex = cleanup_str(title_tag.text)
156 if xarticle.pid in self.title_corrections:
157 xarticle.title_tex = self.title_corrections[xarticle.pid]
159 # TEX
160 tex_tag = body.select_one("a[href]:-soup-contains-own('Tex version')")
161 if tex_tag:
162 tex_href = tex_tag.get("href")
163 if isinstance(tex_href, str):
164 add_pdf_link_to_xarticle(
165 xarticle, urljoin(xarticle.url, tex_href), mimetype="application/x-tex"
166 )
168 # Here we decompose/extract every element to keep only the abstracts
169 author_tag.decompose()
170 title_tag.decompose()
171 body.select_one("h5").decompose()
172 form = body.select_one("form")
173 if form:
174 form.decompose()
175 links = body.select_one(
176 "ul:-soup-contains('Scan of original article'), ul:-soup-contains('PDF')"
177 )
178 if links is not None:
179 link_header = links.find_previous_sibling("p")
180 if isinstance(link_header, Tag):
181 link_header.decompose()
182 links.decompose()
183 body.select_one("body")
184 for child in body.children:
185 if isinstance(child, Comment):
186 child.extract()
188 # Dates
189 dates = next(
190 (c for c in body.children if cleanup_str(str(c)).startswith("Received")), None
191 )
192 if dates:
193 dates_str = cleanup_str(dates.text)
194 dates_str = re.sub(r"[\.;]", ",", dates_str)
195 dates_str = re.sub(r"Oct,? ", "October ", dates_str)
196 dates_str = re.sub(r"Sept,? ", "September ", dates_str)
198 dates_dict = regex_to_dict(
199 self.dates_regex, dates_str, error_msg="Couldn't parse dates"
200 )
201 xarticle.date_accepted = strftime(
202 "%Y-%m-%d", strptime(dates_dict["accepted"].replace(",", ""), "%B %d %Y")
203 )
204 if dates_dict["received"] is not None:
205 xarticle.date_received = strftime(
206 "%Y-%m-%d", strptime(dates_dict["received"].replace(",", ""), "%B %d %Y")
207 )
208 if dates_dict["revised"] is not None:
209 xarticle.date_revised = strftime(
210 "%Y-%m-%d", strptime(dates_dict["revised"].replace(",", ""), "%B %d %Y")
211 )
212 if dates_dict["final"] is not None:
213 xarticle.date_published = strftime(
214 "%Y-%m-%d", strptime(dates_dict["final"].replace(",", ""), "%B %d %Y")
215 )
216 dates.extract()
218 # Abstract
219 for img in body.select("img[alt]"):
220 alt = img.get("alt")
221 if not isinstance(alt, str):
222 raise TypeError("Couldn't parse abstract : invalid img alt")
223 img.replace_with(alt)
225 abstract_text = cleanup_str("".join([str(c) for c in body.contents]))
226 try:
227 abstract_dict = regex_to_dict(
228 self.abstract_regex, abstract_text, error_msg="Couldn't parse article abstract"
229 )
230 except ValueError:
231 abstract_dict = {"abstract": abstract_text}
233 if abstract_dict["abstract"] is not None:
234 xarticle.abstracts.append(
235 create_abstract(value_tex=cleanup_str(abstract_dict["abstract"]), lang="en")
236 )
238 if abstract_dict.get("resume", None) is not None:
239 xarticle.abstracts.append(
240 create_abstract(value_tex=cleanup_str(abstract_dict["resume"]), lang="fr")
241 )
243 return xarticle
245 def parse_slc_preface(self, content: str):
246 pass