Coverage for src/crawler/cmds/augment/augment_cmd.py: 21%
109 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
1import logging
2from concurrent.futures import Future
3from typing import TYPE_CHECKING, NotRequired, TypedDict
5from history.model_data import HistoryChildDict, MatchingEventResult
6from history.models import HistoryEventStatus
7from history.utils import insert_history_event
8from ptf import model_helpers
9from ptf.cmds.base_cmds import baseCmd
10from ptf.display.resolver import ExtIdName
11from ptf.models import Article, Collection
12from ptf.utils import DjangoConnectionThreadPoolExecutor
13from ptf_external.matching import get_matching_backend
15from crawler.cmds.augment import update_article_with_xarticle
16from crawler.models.source import Source
18if TYPE_CHECKING:
19 from collections.abc import Callable
21 from ptf.models import ArticleQuerySet
22 from ptf_external.matching.base_matching import MatchingBackend
24 class AugmentArticlesParams(TypedDict):
25 source_id: str
26 collection_pid: NotRequired[str]
27 issue_pids: NotRequired[list[str]]
28 update_pdf_link: NotRequired[bool]
30 from typing import NotRequired, TypedDict
32 class MatchingOperationMessage(TypedDict):
33 pid: str
34 zbl_id: "NotRequired[str]"
35 score: "NotRequired[int]"
36 status: HistoryEventStatus
37 message: "NotRequired[str]"
38 status_message: "NotRequired[str]"
41_logger = logging.getLogger(__name__)
44def compile_history_events(event_messages: "list[MatchingOperationMessage]"):
45 children = []
46 for msg in event_messages:
47 resource = model_helpers.get_resource(msg["pid"])
48 if not resource:
49 raise ValueError("Resource not found")
50 data: HistoryChildDict = {
51 "resource": resource,
52 "type": "zbl-id",
53 "status": msg.get("status"),
54 }
56 if "zbl_id" in msg and msg["zbl_id"] != "":
57 data["url"] = f"https://zbmath.org/{msg['zbl_id']}"
59 if "score" in msg:
60 data["score"] = msg["score"]
62 if "message" in msg:
63 data["message"] = msg["message"]
65 if "status_message" in msg:
66 data["status_message"] = msg["status_message"]
68 children.append(data)
69 return children
72class AugmentArticlesCmd(baseCmd[None]):
73 """
74 Metadata augment command.
75 Calls the desired Matching Backend defined in ptf-back.
76 """
78 required_params = ["backend", "source_id"]
80 backend: "type[MatchingBackend]"
81 source_id: str
82 collection_pid: str | None = None
83 issue_pids: list[str] = []
84 update_pdf_link: bool = False
85 collection: "Collection | None" = None
86 queryset: "ArticleQuerySet"
87 event_messages: list[dict]
88 callback: "Callable" = lambda _: _
90 def __init__(self, params: "AugmentArticlesParams", backend: ExtIdName):
91 super().__init__(params)
93 self.backend = get_matching_backend(backend)
95 self.event_messages = []
96 self.queryset = Article.objects.prefetch_related("extid_set").all()
98 if self.collection_pid:
99 self.queryset = self.queryset.filter(
100 my_container__my_collection__pid=self.collection_pid
101 )
102 self.collection = Collection.objects.get(pid=self.collection_pid)
104 if self.source_id:
105 source = Source.objects.get(domain=self.source_id)
106 self.queryset = self.queryset.filter(my_container__origin__source=source)
108 if self.issue_pids:
109 self.queryset = self.queryset.filter(my_container__pid__in=self.issue_pids)
111 def internal_do(self) -> None:
112 _logger.info(
113 f"Start augment [{self.backend.name}] "
114 f"source={self.source_id} collection={self.collection_pid} "
115 f"({self.queryset.count()} articles)"
116 )
117 promises: set[Future] = set()
118 with DjangoConnectionThreadPoolExecutor(max_workers=1) as executor:
119 for article in self.queryset.iterator(chunk_size=2000):
120 xarticle, event = self._find_article(article)
122 if event:
123 self.event_messages.append(event)
125 if not xarticle:
126 self.callback()
127 continue
129 # Handle asyncronous results and exceptions as soon as possible
130 completed: set[Future] = set()
131 for f in promises:
132 if f.done():
133 completed.add(f)
134 exc = f.exception()
135 if exc:
136 executor.shutdown(wait=False, cancel_futures=True)
137 raise exc
138 promises -= completed
139 # Runs the database insertion in parallel
140 promise = executor.submit(
141 update_article_with_xarticle,
142 article,
143 xarticle,
144 merge_titles=False,
145 update_pdf_link=self.update_pdf_link,
146 )
147 promise.add_done_callback(self.callback)
148 promises.add(promise)
149 _logger.info(f"Augment [{self.backend.name}] over")
151 def _find_article(self, article: Article) -> tuple:
152 """
153 Calls ArticleData from backend.
154 """
155 # Init matching backend instance
156 self.matcher = self.backend.init_from_db_article(resource=article)
157 # With external ID
158 xarticle = self.matcher.get_xarticle_by_extids()
159 if xarticle:
160 _logger.debug(f"[{self.matcher.name}] {article.pid}: found via External ID")
161 return xarticle, self._make_event(article, MatchingEventResult.ALREADY_PRESENT)
163 # Matching fuzzy
164 xarticle = self.matcher.get_xarticle_by_matching()
165 if xarticle:
166 _logger.debug(f"[{self.matcher.name}] {article.pid}: found via Matching")
167 return xarticle, self._make_event(article, MatchingEventResult.ADDED)
169 _logger.info(
170 f"[{self.matcher.name}] {article.pid}: not found with External ID nor Matching"
171 )
172 return None, self._make_event(
173 article, MatchingEventResult.NOT_FOUND, status=HistoryEventStatus.WARNING
174 )
176 def _make_event(
177 self,
178 article: Article,
179 status_message: str,
180 status: HistoryEventStatus = HistoryEventStatus.OK,
181 ):
182 return {
183 "pid": article.pid,
184 "status": status,
185 "status_message": status_message,
186 "backend": self.matcher.name,
187 }
189 def insert_history_event(self):
190 children = []
191 for msg in self.event_messages:
192 resource = model_helpers.get_resource(msg["pid"])
193 if not resource:
194 raise ValueError(f"Resource not found: {msg['pid']}")
195 data: HistoryChildDict = {
196 "resource": resource,
197 "type": msg.get("backend", self.backend.name),
198 "status": msg.get("status"),
199 }
200 if "score" in msg:
201 data["score"] = msg["score"]
202 if "message" in msg:
203 data["message"] = msg["message"]
204 if "status_message" in msg:
205 data["status_message"] = msg["status_message"]
206 children.append(data)
208 insert_history_event(
209 {
210 "pid": f"{self.source_id}_{self.collection_pid or ''}_{self.backend.name}_matching",
211 "col": self.collection,
212 "source": self.source_id,
213 "status": HistoryEventStatus.OK,
214 "type": "matching",
215 "children": children,
216 }
217 )