Coverage for src/crawler/abstract_crawlers/matching_crawler.py: 56%

30 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-19 16:36 +0000

1from typing import TYPE_CHECKING 

2 

3from ptf.display.resolver import ExtIdName 

4from ptf.utils import DjangoConnectionThreadPoolExecutor 

5 

6from crawler.abstract_crawlers.threaded_crawler import ThreadedCrawler 

7from crawler.by_source.slc_crawler import IssueData 

8from crawler.cmds.augment.augment_cmd import AugmentArticlesCmd 

9 

10if TYPE_CHECKING: 

11 from concurrent.futures import ( 

12 Executor, 

13 Future, 

14 ) 

15 

16 from crawler.cmds.augment.zbmath import MatchingOperationMessage 

17 

18 

19class MatchingCrawler(ThreadedCrawler): 

20 zbmath_executor: "Executor" 

21 zbmath_futures: "list[Future]" 

22 exception: BaseException | None = None 

23 

24 matching_event_messages: "list[MatchingOperationMessage]" 

25 

26 def __init__( 

27 self, 

28 # backend=None, 

29 *args, 

30 **kwargs, 

31 ): 

32 super().__init__(*args, **kwargs) 

33 self.zbmath_executor = DjangoConnectionThreadPoolExecutor( 

34 max_workers=1, thread_name_prefix="crawler_zbmath_thread" 

35 ) 

36 self.zbmath_futures = [] 

37 self.matching_event_messages = [] 

38 

39 def _issue_added_callback(self, future: "Future[IssueData]"): 

40 # Runs in database thread 

41 container = future.result() 

42 if not container.pid: 

43 raise ValueError("Issue doesn't have any pid") 

44 matching_future = self.zbmath_executor.submit(self.match_zbmath_issue, container.pid) 

45 self.zbmath_futures.append(matching_future) 

46 matching_future.add_done_callback(self._pop_matching_future) 

47 

48 def match_zbmath_issue(self, issue_pid: str): 

49 cmd = AugmentArticlesCmd( 

50 { 

51 "collection_pid": self.collection_id, 

52 "source_id": self.source_domain, 

53 "issue_pids": [issue_pid], 

54 }, 

55 backend=ExtIdName.ZBMATH, 

56 ) 

57 cmd.do() 

58 self.matching_event_messages.extend(cmd.event_messages) 

59 

60 def _pop_matching_future(self, future: "Future[None]"): 

61 if future in self.database_futures: 

62 self.database_futures.remove(future)