Coverage for src/crawler/abstract_crawlers/threaded_crawler.py: 50%
34 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-11-19 16:36 +0000
1from abc import ABC
2from concurrent.futures import (
3 Executor,
4 wait,
5)
6from typing import TYPE_CHECKING
8from celery import current_app
9from ptf.model_data import (
10 IssueData,
11 TitleDict,
12)
13from ptf.utils import DjangoConnectionThreadPoolExecutor
15from crawler.abstract_crawlers.base_crawler import BaseCollectionCrawler
17if TYPE_CHECKING:
18 from concurrent.futures import Future
21class CrawlerTitleDict(TitleDict):
22 title_tex: str | None
25class ThreadedCrawler(BaseCollectionCrawler, ABC):
26 """
27 Main thread Database Thread Matching Thread
28 -----------------------------------------------------------------------------------------------------------------------------------------
29 BaseCollectionCrawler.crawl_issue
30 |
31 v
32 ThreadedCrawler.add_xissue_into_database --> BaseCollectionCrawler.add_xissue_into_database
33 |
34 v
35 ThreadedCrawler._handle_future_exceptions
36 MatchingCrawler._issue_added_callback --> MatchingCrawler.match_zbmath_issue
38 Wait database threads
39 Insert crawl history event
41 Wait matching threads
42 Insert matching history event
44 """
46 database_executor: Executor
47 database_futures: "list[Future]"
48 exception: BaseException | None = None
50 def __init__(self, *args, **kwargs):
51 super().__init__(*args, **kwargs)
52 self.database_executor = DjangoConnectionThreadPoolExecutor(
53 max_workers=1, thread_name_prefix="crawler_database_thread"
54 )
55 self.database_futures = []
57 def add_xissue_into_database(self, xissue):
58 # Runs in main thread
59 if self.exception:
60 raise self.exception
61 # Do not parallelize if inside a celery worker
62 if current_app.current_worker_task:
63 super().add_xissue_into_database(xissue)
64 return xissue
66 # Avoid launching too many threaded tasks as they take quite some RAM
67 if len(self.database_futures) > 127:
68 wait(self.database_futures, return_when="FIRST_COMPLETED")
70 future = self.database_executor.submit(super().add_xissue_into_database, xissue)
71 future.add_done_callback(self._issue_added_callback)
72 future.add_done_callback(self._pop_database_future)
74 self.database_futures.append(future)
75 return xissue
77 def _issue_added_callback(self, future: "Future[IssueData]"):
78 # Runs in database thread
79 pass
81 def _pop_database_future(self, future: "Future[IssueData]"):
82 self.database_futures.remove(future)