Coverage for src/crawler/augment.py: 0%

65 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-19 16:36 +0000

1import requests 

2from ptf import model_data_converter 

3from ptf.cmds import xml_cmds 

4from ptf.model_data import ArticleData 

5from ptf.models import Article 

6from ptf_external.backends.zbmath_backend import ZbMathExternalBackend 

7from ptf_external.fetch_metadata import fetch_article 

8 

9 

10def augment_article(article: Article, source, what="all", doi_to_fetch=None): 

11 """ 

12 Fetch metadata from external sources and augment an article metadata. 

13 

14 what is the database to query: crossref/zbmath/all 

15 doi_to_fetch let's you specify an alternate DOI to use, for example in the case of a translation 

16 

17 """ 

18 

19 # An article in Mathnet.ru may have a DOI and a DOI for the translation 

20 # Metadata in Crossref for the DOI are in Russian: use the translated version. 

21 

22 databases = ["crossref", "zbl-item-id"] 

23 

24 if source == "EUDML": 

25 # Ignore some databases of some sources 

26 # TODO: Move the check in a concrete crawler ? 

27 databases = ["zbl-item-id"] 

28 

29 if what != "all": 

30 databases = [what] 

31 

32 for what_single in databases: 

33 article_data = fetch_article_data(article, what_single, doi_to_fetch) 

34 if article_data is not None: 

35 update_article(article, article_data, what_single) 

36 

37 

38def update_article(article: Article, article_data: ArticleData, what): 

39 """ 

40 Update the article with external metadata stored in article_data 

41 """ 

42 collection = article.get_collection() 

43 existing_article_data = model_data_converter.db_to_article_data(article) 

44 

45 model_data_converter.update_data_for_jats(article_data) 

46 

47 # Protection to make sure the database returned something valid. 

48 if article_data.title_xml: 

49 # Preserve existing values not set by the Crossref/zbMATH API 

50 article_data.pid = article.pid 

51 article_data.doi = article.doi 

52 article_data.seq = existing_article_data.seq 

53 article_data.extids = existing_article_data.extids 

54 article_data.ext_links = existing_article_data.ext_links 

55 if not article_data.fpage: 

56 article_data.fpage = existing_article_data.fpage 

57 if not article_data.lpage: 

58 article_data.lpage = existing_article_data.lpage 

59 article_data.streams = existing_article_data.streams 

60 

61 # Replace with metadata from the database (if it is found) 

62 

63 # TITLE 

64 if not article_data.title_html or ( 

65 what == "crossref" and existing_article_data.title_html 

66 ): 

67 # TODO: handle Crossref titles with formulas. 

68 # The quality depends on the publisher so we might not be able to know in general if we have to replace 

69 article_data.title_html = existing_article_data.title_html 

70 article_data.title_tex = existing_article_data.title_tex 

71 article_data.title_xml = existing_article_data.title_xml 

72 

73 # AUTHOR 

74 if not article_data.contributors: 

75 article_data.contributors = existing_article_data.contributors 

76 

77 # ABSTRACTS 

78 if not article_data.abstracts: 

79 article_data.abstracts = existing_article_data.abstracts 

80 

81 params = { 

82 "xarticle": article_data, 

83 "use_body": False, 

84 "issue": article.my_container, 

85 "standalone": True, 

86 } 

87 cmd = xml_cmds.addArticleXmlCmd(params) 

88 cmd.set_collection(collection) 

89 article = cmd.do() 

90 if not article_data.title_html: 

91 print(f" Warning: {article.pid} has no title_html after XmlCmd") 

92 

93 

94def fetch_article_data(article, what, doi_to_fetch=None): 

95 article_data = None 

96 

97 if what == "crossref": 

98 doi_to_fetch = doi_to_fetch if doi_to_fetch is not None else article.doi 

99 

100 # qs = article.extid_set.filter(id_type="doi-translation") 

101 # if qs: 

102 # doi_translation = qs.first().id_value 

103 # print(f" DOI Translation {doi_translation}") 

104 

105 article_data = fetch_article(doi_to_fetch) 

106 elif what == "zbmath": 

107 qs = article.extid_set.filter(id_type="zbl-item-id") 

108 if qs: 

109 zblid = qs.first().id_value 

110 if "|" not in zblid and "%7" not in zblid: 

111 attempt = 0 

112 done = False 

113 while not done and attempt < 3: 

114 try: 

115 article_data = ZbMathExternalBackend.fetch_article(zblid) 

116 done = True 

117 except ( 

118 requests.exceptions.ConnectionError, 

119 requests.exceptions.ReadTimeout, 

120 ): 

121 attempt += 1 

122 

123 return article_data