Coverage for src/crawler/abstract_crawlers/threaded_crawler.py: 50%

34 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-19 16:36 +0000

1from abc import ABC 

2from concurrent.futures import ( 

3 Executor, 

4 wait, 

5) 

6from typing import TYPE_CHECKING 

7 

8from celery import current_app 

9from ptf.model_data import ( 

10 IssueData, 

11 TitleDict, 

12) 

13from ptf.utils import DjangoConnectionThreadPoolExecutor 

14 

15from crawler.abstract_crawlers.base_crawler import BaseCollectionCrawler 

16 

17if TYPE_CHECKING: 

18 from concurrent.futures import Future 

19 

20 

21class CrawlerTitleDict(TitleDict): 

22 title_tex: str | None 

23 

24 

25class ThreadedCrawler(BaseCollectionCrawler, ABC): 

26 """ 

27 Main thread Database Thread Matching Thread 

28 ----------------------------------------------------------------------------------------------------------------------------------------- 

29 BaseCollectionCrawler.crawl_issue 

30 | 

31 v 

32 ThreadedCrawler.add_xissue_into_database --> BaseCollectionCrawler.add_xissue_into_database 

33 | 

34 v 

35 ThreadedCrawler._handle_future_exceptions 

36 MatchingCrawler._issue_added_callback --> MatchingCrawler.match_zbmath_issue 

37 

38 Wait database threads 

39 Insert crawl history event 

40 

41 Wait matching threads 

42 Insert matching history event 

43 

44 """ 

45 

46 database_executor: Executor 

47 database_futures: "list[Future]" 

48 exception: BaseException | None = None 

49 

50 def __init__(self, *args, **kwargs): 

51 super().__init__(*args, **kwargs) 

52 self.database_executor = DjangoConnectionThreadPoolExecutor( 

53 max_workers=1, thread_name_prefix="crawler_database_thread" 

54 ) 

55 self.database_futures = [] 

56 

57 def add_xissue_into_database(self, xissue): 

58 # Runs in main thread 

59 if self.exception: 

60 raise self.exception 

61 # Do not parallelize if inside a celery worker 

62 if current_app.current_worker_task: 

63 super().add_xissue_into_database(xissue) 

64 return xissue 

65 

66 # Avoid launching too many threaded tasks as they take quite some RAM 

67 if len(self.database_futures) > 127: 

68 wait(self.database_futures, return_when="FIRST_COMPLETED") 

69 

70 future = self.database_executor.submit(super().add_xissue_into_database, xissue) 

71 future.add_done_callback(self._issue_added_callback) 

72 future.add_done_callback(self._pop_database_future) 

73 

74 self.database_futures.append(future) 

75 return xissue 

76 

77 def _issue_added_callback(self, future: "Future[IssueData]"): 

78 # Runs in database thread 

79 pass 

80 

81 def _pop_database_future(self, future: "Future[IssueData]"): 

82 self.database_futures.remove(future)