Coverage for src/crawler/augment.py: 0%
65 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
1import requests
2from ptf import model_data_converter
3from ptf.cmds import xml_cmds
4from ptf.model_data import ArticleData
5from ptf.models import Article
6from ptf_external.backends.zbmath_backend import ZbMathExternalBackend
7from ptf_external.fetch_metadata import fetch_article
10def augment_article(article: Article, source, what="all", doi_to_fetch=None):
11 """
12 Fetch metadata from external sources and augment an article metadata.
14 what is the database to query: crossref/zbmath/all
15 doi_to_fetch let's you specify an alternate DOI to use, for example in the case of a translation
17 """
19 # An article in Mathnet.ru may have a DOI and a DOI for the translation
20 # Metadata in Crossref for the DOI are in Russian: use the translated version.
22 databases = ["crossref", "zbl-item-id"]
24 if source == "EUDML":
25 # Ignore some databases of some sources
26 # TODO: Move the check in a concrete crawler ?
27 databases = ["zbl-item-id"]
29 if what != "all":
30 databases = [what]
32 for what_single in databases:
33 article_data = fetch_article_data(article, what_single, doi_to_fetch)
34 if article_data is not None:
35 update_article(article, article_data, what_single)
38def update_article(article: Article, article_data: ArticleData, what):
39 """
40 Update the article with external metadata stored in article_data
41 """
42 collection = article.get_collection()
43 existing_article_data = model_data_converter.db_to_article_data(article)
45 model_data_converter.update_data_for_jats(article_data)
47 # Protection to make sure the database returned something valid.
48 if article_data.title_xml:
49 # Preserve existing values not set by the Crossref/zbMATH API
50 article_data.pid = article.pid
51 article_data.doi = article.doi
52 article_data.seq = existing_article_data.seq
53 article_data.extids = existing_article_data.extids
54 article_data.ext_links = existing_article_data.ext_links
55 if not article_data.fpage:
56 article_data.fpage = existing_article_data.fpage
57 if not article_data.lpage:
58 article_data.lpage = existing_article_data.lpage
59 article_data.streams = existing_article_data.streams
61 # Replace with metadata from the database (if it is found)
63 # TITLE
64 if not article_data.title_html or (
65 what == "crossref" and existing_article_data.title_html
66 ):
67 # TODO: handle Crossref titles with formulas.
68 # The quality depends on the publisher so we might not be able to know in general if we have to replace
69 article_data.title_html = existing_article_data.title_html
70 article_data.title_tex = existing_article_data.title_tex
71 article_data.title_xml = existing_article_data.title_xml
73 # AUTHOR
74 if not article_data.contributors:
75 article_data.contributors = existing_article_data.contributors
77 # ABSTRACTS
78 if not article_data.abstracts:
79 article_data.abstracts = existing_article_data.abstracts
81 params = {
82 "xarticle": article_data,
83 "use_body": False,
84 "issue": article.my_container,
85 "standalone": True,
86 }
87 cmd = xml_cmds.addArticleXmlCmd(params)
88 cmd.set_collection(collection)
89 article = cmd.do()
90 if not article_data.title_html:
91 print(f" Warning: {article.pid} has no title_html after XmlCmd")
94def fetch_article_data(article, what, doi_to_fetch=None):
95 article_data = None
97 if what == "crossref":
98 doi_to_fetch = doi_to_fetch if doi_to_fetch is not None else article.doi
100 # qs = article.extid_set.filter(id_type="doi-translation")
101 # if qs:
102 # doi_translation = qs.first().id_value
103 # print(f" DOI Translation {doi_translation}")
105 article_data = fetch_article(doi_to_fetch)
106 elif what == "zbmath":
107 qs = article.extid_set.filter(id_type="zbl-item-id")
108 if qs:
109 zblid = qs.first().id_value
110 if "|" not in zblid and "%7" not in zblid:
111 attempt = 0
112 done = False
113 while not done and attempt < 3:
114 try:
115 article_data = ZbMathExternalBackend.fetch_article(zblid)
116 done = True
117 except (
118 requests.exceptions.ConnectionError,
119 requests.exceptions.ReadTimeout,
120 ):
121 attempt += 1
123 return article_data