Coverage for src/crawler/by_source/scholastica_crawler.py: 26%

1import json 

2from urllib.parse import parse_qs, urlparse, urlunparse 

3 

4from bs4 import BeautifulSoup 

5from dateutil import parser 

6from django.conf import settings 

7from ptf.model_data import IssueData, create_articledata 

8from ptf_external.backends.arxiv_backend import ArxivExternalBackend 

9from pysolr import urlencode 

10 

11from crawler.abstract_crawlers.base_crawler import BaseCollectionCrawler 

12from crawler.utils import add_pdf_link_to_xarticle 

13 

14 

15class GDMLArxivExternalBackend(ArxivExternalBackend): 

16 @classmethod 

17 def parse_article(cls, data): 

18 article_data = super().parse_article(data) 

19 

20 pdf_url = next( 

21 (link["href"] for link in data.links if link["type"] == "application/pdf"), None 

22 ) 

23 if pdf_url: 

24 add_pdf_link_to_xarticle(article_data, pdf_url) 

25 return article_data 

26 

27 

28class ScholasticaCrawler(BaseCollectionCrawler): 

29 source_name = "Scholastica" 

30 source_domain = "SCHOLASTICA" 

31 source_website = "https://scholasticahq.com" 

32 

33 requests_interval = max(getattr(settings, "REQUESTS_INTERVAL", 90), 5) 

34 

35 def parse_collection_content(self, content): 

36 xissues_years: dict[int, IssueData] = {} 

37 articles_dicts = [] 

38 parsed_url = urlparse(self.collection_url) 

39 query = parse_qs(parsed_url.query) 

40 

41 while True: 

42 data = json.loads(content) 

43 if len(data["articles"]) == 0: 

44 break 

45 articles_dicts.extend(data["articles"]) 

46 query["offset"] = [str(int(query["offset"][0]) + int(query["per_page"][0]))] 

47 parsed_url = parsed_url._replace(query=urlencode(query, True)) 

48 content = self.download_file(urlunparse(parsed_url)) 

49 

50 for a in articles_dicts: 

51 xarticle = create_articledata() 

52 xarticle.url = a["url"] 

53 xarticle.date_published_iso_8601_date_str = a["published_at"] 

54 

55 year = parser.parse(a["published_at"]).year 

56 xissues_years.setdefault(year, self.create_xissue(None, str(year), None, None)) 

57 

58 xarticle.pid = f"a{len(xissues_years[year].articles)}" 

59 

60 xissues_years[year].articles.append(xarticle) 

61 

62 return list(xissues_years.values()) 

63 

64 def parse_article_content(self, content, xissue, xarticle, url): 

65 """ 

66 Parse the content with Beautifulsoup and returns an ArticleData 

67 """ 

68 

69 # We only parse the arXiv id in the Discrete Analysis article page 

70 soup = BeautifulSoup(content, "html.parser") 

71 

72 a_node = soup.select_one("div#article-show-page a:-soup-contains-own('Read article')") 

73 if a_node is None: 

74 raise ValueError("a_node is None") 

75 

76 href = a_node.get("href") 

77 if not isinstance(href, str): 

78 raise TypeError("href is not a string") 

79 id = href.split("/")[-1] 

80 

81 new_xarticle = GDMLArxivExternalBackend.get_xarticle_by_id(id) 

82 if new_xarticle is None: 

83 raise ValueError("new_xarticle is None") 

84 new_xarticle.pid = xarticle.pid 

85 new_xarticle.doi = xarticle.doi 

86 new_xarticle.ext_links = xarticle.ext_links 

87 new_xarticle.url = url 

88 new_xarticle.lang = "en" 

89 new_xarticle.date_published_iso_8601_date_str = xarticle.date_published_iso_8601_date_str 

90 

91 add_pdf_link_to_xarticle(new_xarticle, new_xarticle.pdf_url) 

92 

93 return new_xarticle