Coverage for src/crawler/abstract_crawlers/matching_crawler.py: 56%
30 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
1from typing import TYPE_CHECKING
3from ptf.display.resolver import ExtIdName
4from ptf.utils import DjangoConnectionThreadPoolExecutor
6from crawler.abstract_crawlers.threaded_crawler import ThreadedCrawler
7from crawler.by_source.slc_crawler import IssueData
8from crawler.cmds.augment.augment_cmd import AugmentArticlesCmd
10if TYPE_CHECKING:
11 from concurrent.futures import (
12 Executor,
13 Future,
14 )
16 from crawler.cmds.augment.zbmath import MatchingOperationMessage
19class MatchingCrawler(ThreadedCrawler):
20 zbmath_executor: "Executor"
21 zbmath_futures: "list[Future]"
22 exception: BaseException | None = None
24 matching_event_messages: "list[MatchingOperationMessage]"
26 def __init__(
27 self,
28 # backend=None,
29 *args,
30 **kwargs,
31 ):
32 super().__init__(*args, **kwargs)
33 self.zbmath_executor = DjangoConnectionThreadPoolExecutor(
34 max_workers=1, thread_name_prefix="crawler_zbmath_thread"
35 )
36 self.zbmath_futures = []
37 self.matching_event_messages = []
39 def _issue_added_callback(self, future: "Future[IssueData]"):
40 # Runs in database thread
41 container = future.result()
42 if not container.pid:
43 raise ValueError("Issue doesn't have any pid")
44 matching_future = self.zbmath_executor.submit(self.match_zbmath_issue, container.pid)
45 self.zbmath_futures.append(matching_future)
46 matching_future.add_done_callback(self._pop_matching_future)
48 def match_zbmath_issue(self, issue_pid: str):
49 cmd = AugmentArticlesCmd(
50 {
51 "collection_pid": self.collection_id,
52 "source_id": self.source_domain,
53 "issue_pids": [issue_pid],
54 },
55 backend=ExtIdName.ZBMATH,
56 )
57 cmd.do()
58 self.matching_event_messages.extend(cmd.event_messages)
60 def _pop_matching_future(self, future: "Future[None]"):
61 if future in self.database_futures:
62 self.database_futures.remove(future)