Coverage for src/crawler/by_source/ptm_crawler.py: 12%
122 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-29 13:39 +0000
1import lingua
2import regex
3from bs4 import BeautifulSoup, Tag
4from lingua import LanguageDetectorBuilder
5from ptf.cmds.xml.jats.jats_parser import parse_mixed_citation_into_ref
6from ptf.model_data import (
7 ArticleData,
8 create_abstract,
9 create_articledata,
10 create_contributor,
11 create_subj,
12)
14from crawler.abstract_crawlers.base_crawler import BaseCollectionCrawler
15from crawler.utils import add_pdf_link_to_xarticle, cleanup_str, regex_to_dict
18class PtmCrawler(BaseCollectionCrawler):
19 source_name = "Annales Societatis Mathematicae Polonae Series "
20 source_domain = "PTM"
21 source_website = "https://wydawnictwa.ptm.org.pl/"
22 is_checkable = False
24 issue_re = (
25 r"(?:Vol|Tom) (?P<volume>\d+)(?:, (?:No|Nr) (?P<number>[\d\-\/]+))? \((?P<year>\d{4})\)"
26 )
28 _language_detector_builder = LanguageDetectorBuilder.from_languages(
29 lingua.Language.ENGLISH, lingua.Language.POLISH, lingua.Language.RUSSIAN
30 )
32 def __init__(self, *args, **kwargs):
33 super().__init__(*args, **kwargs)
34 self.update_cookies()
36 def download_file(self, url, headers=None):
37 if headers is None:
38 headers = {}
39 content = super().download_file(url, headers=headers)
40 if (
41 "Access to this website is possible only using browser with JavaScript and Cookies enabled."
42 in content
43 ):
44 self.update_cookies()
45 return self.download_file(url, headers=headers)
46 return content
48 def update_cookies(self):
49 script_content = super().download_file(self.source_website)
50 cookie_search = regex.search(r"createCookie\('vjs','(?P<cookie>\d+)',60\)", script_content)
51 if not cookie_search:
52 raise ValueError("Couldn't set cookie for ptm")
53 self.headers.update(
54 {
55 "Cookie": f"vjs={cookie_search.group(1)}",
56 # "Accept-Language": "en-US,en;q=0.5",
57 "Accept-Encoding": "gzip, deflate, br, zstd",
58 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
59 "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0",
60 }
61 )
63 def parse_collection_content(self, content):
64 soup = BeautifulSoup(content, "html.parser")
65 xissues = []
66 for issue_tag in soup.select("#issue h4 a"):
67 issue_url = issue_tag.get("href")
68 if not isinstance(issue_url, str):
69 raise TypeError("Couldn't parse issue url")
70 issue_data = regex_to_dict(
71 self.issue_re,
72 issue_tag.text,
73 error_msg="Couldn't parse issue data",
74 )
75 xissues.append(
76 self.create_xissue(
77 issue_url, issue_data["year"], issue_data["volume"], issue_data["number"]
78 )
79 )
80 return xissues
82 def parse_issue_content(self, content, xissue):
83 soup = BeautifulSoup(content, "html.parser")
84 for index, article in enumerate(soup.select(".tocTitle a")):
85 xarticle = create_articledata()
86 xarticle.pid = f"a{index}"
87 article_url = article.get("href")
88 if not isinstance(article_url, str):
89 raise TypeError("Couldn't parse article url")
90 xarticle.url = article_url
91 xissue.articles.append(xarticle)
93 def parse_article_content(self, content, xissue, xarticle, url):
94 soup = BeautifulSoup(content, "html.parser")
95 frame = soup.select_one("frameset frame:first-child")
96 if not frame:
97 raise ValueError("Couldn't parse article")
98 real_url = frame.get("src")
99 if not isinstance(real_url, str):
100 raise TypeError("Couldn't find article url")
102 content = self.download_file(real_url)
103 soup = BeautifulSoup(content, "html.parser")
105 # Title
106 title_tag = soup.select_one("#articleTitle")
107 if not title_tag:
108 self.logger.debug(
109 "Couldn't parse article. Skipping", extra={"pid": xarticle.pid, "url": real_url}
110 )
111 return None
112 raise ValueError("Couldn't parse title")
113 xarticle.title_tex = cleanup_str(title_tag.text)
115 # DOI
116 doi_header = soup.select_one(
117 "strong:-soup-contains-own('Digital Object Identifier (DOI):')"
118 )
119 if doi_header:
120 doi_tag = doi_header.parent
121 doi_header.decompose()
122 xarticle.doi = cleanup_str(doi_tag.text)
124 # Abstract
125 abstract_tag = soup.select_one("#articleAbstract div")
126 if abstract_tag:
127 abstract_text = cleanup_str(abstract_tag.text)
128 if len(abstract_text) > 0 and abstract_text not in (
129 "Artykuł nie zawiera streszczenia",
130 "-",
131 ):
132 xarticle.abstracts.append(
133 create_abstract(
134 value_tex=abstract_text,
135 lang=self.detect_language(abstract_text),
136 )
137 )
139 # Pages
140 pages_header = soup.select_one("strong:-soup-contains-own('Pages:')") or soup.select_one(
141 "strong:-soup-contains-own('Strony:')"
142 )
143 if pages_header:
144 pages_tag = pages_header.parent
145 pages_header.decompose()
146 pages_splitted = pages_tag.text.split("-")
147 if len(pages_splitted) > 0:
148 xarticle.fpage = pages_splitted[0]
149 if len(pages_splitted) > 1:
150 xarticle.lpage = pages_splitted[1]
152 # pdf
153 pdf_tag = soup.select_one("a.file")
154 if pdf_tag:
155 pdf_url = pdf_tag.get("href")
156 if not isinstance(pdf_url, str):
157 raise ValueError("Couldn't parse pdf url")
159 pdf_url = pdf_url.replace("/view/", "/download/")
160 add_pdf_link_to_xarticle(xarticle, pdf_url)
162 if "(ENGLISH)" in pdf_tag.text:
163 xarticle.lang = "en"
164 elif "(POLSKI)" in pdf_tag.text:
165 xarticle.lang = "pl"
166 else:
167 xarticle.lang = "pl"
168 else:
169 self.logger.debug("Couldn't find article pdf", extra={"pid": xarticle.pid})
170 return None
172 # Authors
173 authors_tag = soup.select_one("#authorString")
174 if authors_tag:
175 for author in cleanup_str(authors_tag.text).split(", "):
176 xarticle.contributors.append(create_contributor(string_name=author, role="author"))
178 # msc
179 msc_header = soup.select_one(
180 "strong:-soup-contains-own('Subject classification:')"
181 ) or soup.select_one("strong:-soup-contains-own('Kklasyfikacja tematyczna:')")
182 if msc_header:
183 msc_tag = msc_header.parent
184 msc_header.decompose()
185 for msc in cleanup_str(msc_tag.text).split("; "):
186 xarticle.kwds.append(create_subj(type="msc", value=msc))
188 # Keywords
189 kwd_header = soup.select_one("strong:-soup-contains-own('Keywords:')") or soup.select_one(
190 "strong:-soup-contains-own('Słowa kluczowe:')"
191 )
192 if kwd_header:
193 kwd_tag = kwd_header.parent
194 kwd_header.decompose()
195 for kwd in cleanup_str(kwd_tag.text).split("; "):
196 xarticle.kwds.append(create_subj(value=kwd))
198 # References
199 # Disabling references for now : PTM doesn't have a "clean" way to display references (eg : https://wydawnictwa.ptm.org.pl/index.php/antiquitates-mathematicae/article/view/7321)
201 # refs_header = soup.select_one("h4:-soup-contains-own('References')") or soup.select_one(
202 # "h4:-soup-contains-own('Cytowania')"
203 # )
204 # if refs_header:
205 # refs_tag = refs_header.next_sibling.next_sibling
206 # if refs_tag and isinstance(refs_tag, Tag):
207 # self.parse_references(xarticle, refs_tag)
209 return xarticle
211 def parse_references(self, xarticle: ArticleData, references: Tag):
212 # TODO : extensive parsing (authors, title etc...)
213 # Currently, only the text is inserted
214 for ref_node in references.get_text(strip=True, separator="\n").splitlines():
215 value_xml = cleanup_str(ref_node)
216 ref = parse_mixed_citation_into_ref(value_xml)
217 xarticle.bibitems.append(ref)