Coverage for src/crawler/by_source/j_stage_crawler.py: 19%

60 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-19 16:36 +0000

1import logging 

2 

3from bs4 import BeautifulSoup 

4from ptf.model_data import create_abstract, create_articledata 

5 

6from crawler.abstract_crawlers.matching_crawler import MatchingCrawler 

7from crawler.utils import regex_to_dict 

8 

9logger = logging.getLogger(__name__) 

10 

11 

12class JStageCrawler(MatchingCrawler): 

13 source_name = "J-STAGE" 

14 source_domain = "J-STAGE" 

15 source_website = "https://www.jstage.jst.go.jp/browse/fesi/list/-char/en" 

16 

17 volume_re = r"Volume (?P<volume>\d+) \((?P<year>\d+)\)" 

18 issue_re = r"Issue (?P<issue>\d+) Pages (?P<page>[^-]+)" 

19 

20 def parse_collection_content(self, content): 

21 xissues = [] 

22 soup = BeautifulSoup(content, "html.parser") 

23 volumes_tag = soup.select( 

24 "div.facetsearch-content-wrap div.facetsearch-subheader div.facetsearch-content-innerwrap" 

25 ) 

26 for volume_tag in volumes_tag: 

27 volume_title_tag = volume_tag.parent.select_one("span.facetsearch-subheader-link a") 

28 if not volume_title_tag: 

29 continue 

30 volume_text = volume_title_tag.text 

31 if volume_text is None: 

32 raise ValueError("Couldn't parse volume tag") 

33 volume_group = regex_to_dict( 

34 self.volume_re, volume_text, error_msg="Couldn't parse volume number & year" 

35 ) 

36 

37 issues_tag = volume_tag.select("ul li a") 

38 for issue_tag in issues_tag: 

39 issue_url = issue_tag.get("href") 

40 if not issue_url: 

41 raise ValueError("Couldn't find issue url") 

42 

43 issue_text = issue_tag.text 

44 if not issue_text: 

45 raise ValueError("Couldn't parse issue number") 

46 try: 

47 issue_group = regex_to_dict( 

48 self.issue_re, issue_text, error_msg="Couldn't parse volume number & year" 

49 ) 

50 except ValueError: 

51 raise ValueError("Couldn't parse issue number") 

52 

53 try: 

54 xissues.append( 

55 self.create_xissue( 

56 issue_url, 

57 int(volume_group.get("year")), 

58 volume_number=volume_group.get("volume"), 

59 issue_number=str(issue_group.get("issue")), 

60 ) 

61 ) 

62 except TypeError: 

63 print("can't create issue") 

64 return xissues 

65 

66 def parse_issue_content(self, content, xissue): 

67 soup = BeautifulSoup(content, "html.parser") 

68 # On 12/06/2026 no managing of pagination required 

69 articles_tag = soup.select("ul.search-resultslisting li") 

70 for article_tag in articles_tag: 

71 xarticle = create_articledata() 

72 article_url_tag = article_tag.select_one("div.searchlist-title a") 

73 if not article_url_tag: 

74 raise ValueError(f"Couldn't parse article url for issue {xissue.url}") 

75 article_url = article_url_tag.get("href") 

76 

77 xarticle.url = article_url 

78 xissue.articles.append(xarticle) 

79 

80 def parse_article_content(self, content, xissue, xarticle, url): 

81 soup = BeautifulSoup(content, "html.parser") 

82 

83 self.get_metadata_using_citation_meta( 

84 xarticle, 

85 xissue, 

86 soup, 

87 [ 

88 "pdf", 

89 "title", 

90 "author", 

91 "doi", 

92 "publisher", 

93 "lang", 

94 "keywords", 

95 "reference", 

96 ], 

97 ) 

98 

99 abstract_tag = soup(text="Abstract") 

100 if abstract_tag and len(abstract_tag) > 0: 

101 abstract = abstract_tag[1].parent.parent.select_one("div.global-para-14").text 

102 xarticle.abstracts.append(create_abstract(lang=xarticle.lang, value_tex=abstract)) 

103 return xarticle