Coverage for src/crawler/by_source/j_stage_crawler.py: 19%
60 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
1import logging
3from bs4 import BeautifulSoup
4from ptf.model_data import create_abstract, create_articledata
6from crawler.abstract_crawlers.matching_crawler import MatchingCrawler
7from crawler.utils import regex_to_dict
9logger = logging.getLogger(__name__)
12class JStageCrawler(MatchingCrawler):
13 source_name = "J-STAGE"
14 source_domain = "J-STAGE"
15 source_website = "https://www.jstage.jst.go.jp/browse/fesi/list/-char/en"
17 volume_re = r"Volume (?P<volume>\d+) \((?P<year>\d+)\)"
18 issue_re = r"Issue (?P<issue>\d+) Pages (?P<page>[^-]+)"
20 def parse_collection_content(self, content):
21 xissues = []
22 soup = BeautifulSoup(content, "html.parser")
23 volumes_tag = soup.select(
24 "div.facetsearch-content-wrap div.facetsearch-subheader div.facetsearch-content-innerwrap"
25 )
26 for volume_tag in volumes_tag:
27 volume_title_tag = volume_tag.parent.select_one("span.facetsearch-subheader-link a")
28 if not volume_title_tag:
29 continue
30 volume_text = volume_title_tag.text
31 if volume_text is None:
32 raise ValueError("Couldn't parse volume tag")
33 volume_group = regex_to_dict(
34 self.volume_re, volume_text, error_msg="Couldn't parse volume number & year"
35 )
37 issues_tag = volume_tag.select("ul li a")
38 for issue_tag in issues_tag:
39 issue_url = issue_tag.get("href")
40 if not issue_url:
41 raise ValueError("Couldn't find issue url")
43 issue_text = issue_tag.text
44 if not issue_text:
45 raise ValueError("Couldn't parse issue number")
46 try:
47 issue_group = regex_to_dict(
48 self.issue_re, issue_text, error_msg="Couldn't parse volume number & year"
49 )
50 except ValueError:
51 raise ValueError("Couldn't parse issue number")
53 try:
54 xissues.append(
55 self.create_xissue(
56 issue_url,
57 int(volume_group.get("year")),
58 volume_number=volume_group.get("volume"),
59 issue_number=str(issue_group.get("issue")),
60 )
61 )
62 except TypeError:
63 print("can't create issue")
64 return xissues
66 def parse_issue_content(self, content, xissue):
67 soup = BeautifulSoup(content, "html.parser")
68 # On 12/06/2026 no managing of pagination required
69 articles_tag = soup.select("ul.search-resultslisting li")
70 for article_tag in articles_tag:
71 xarticle = create_articledata()
72 article_url_tag = article_tag.select_one("div.searchlist-title a")
73 if not article_url_tag:
74 raise ValueError(f"Couldn't parse article url for issue {xissue.url}")
75 article_url = article_url_tag.get("href")
77 xarticle.url = article_url
78 xissue.articles.append(xarticle)
80 def parse_article_content(self, content, xissue, xarticle, url):
81 soup = BeautifulSoup(content, "html.parser")
83 self.get_metadata_using_citation_meta(
84 xarticle,
85 xissue,
86 soup,
87 [
88 "pdf",
89 "title",
90 "author",
91 "doi",
92 "publisher",
93 "lang",
94 "keywords",
95 "reference",
96 ],
97 )
99 abstract_tag = soup(text="Abstract")
100 if abstract_tag and len(abstract_tag) > 0:
101 abstract = abstract_tag[1].parent.parent.select_one("div.global-para-14").text
102 xarticle.abstracts.append(create_abstract(lang=xarticle.lang, value_tex=abstract))
103 return xarticle