Coverage for src/crawler/factory.py: 97%
68 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-09-23 14:47 +0000
1import time
2from typing import TYPE_CHECKING
4from crawler.abstract_crawlers.base_crawler import BaseCollectionCrawler
5from crawler.by_source.amc_crawler import AmcCrawler
6from crawler.by_source.ami_crawler import AmiCrawler
7from crawler.by_source.amp_crawler import AmpCrawler
8from crawler.by_source.ams_crawler import AmsCrawler
9from crawler.by_source.amuc_crawler import AmucCrawler
10from crawler.by_source.arsia_crawler import ArsiaCrawler
11from crawler.by_source.asuo_crawler import AsuoCrawler
12from crawler.by_source.aulfm_crawler import AulfmCrawler
13from crawler.by_source.bdim_crawler import BdimCrawler
14from crawler.by_source.bmms_crawler import BmmsCrawler
15from crawler.by_source.csis_crawler import CsisCrawler
16from crawler.by_source.cup_crawler import CupCrawler
17from crawler.by_source.dml_e_crawler import Dml_eCrawler
18from crawler.by_source.dmlbul_crawler import DmlbulCrawler
19from crawler.by_source.dmlcz_crawler import DmlczCrawler
20from crawler.by_source.dmlpl_crawler import DmlplCrawler
21from crawler.by_source.edpsci_crawler import EdpsciCrawler
22from crawler.by_source.ejc_crawler import EjcCrawler
23from crawler.by_source.elibm_crawler import ElibmCrawler
24from crawler.by_source.emis_aas_crawler import Emis_aasCrawler
25from crawler.by_source.emis_am_crawler import Emis_amCrawler
26from crawler.by_source.emis_hoa_crawler import Emis_hoaCrawler
27from crawler.by_source.ems_crawler import EmsCrawler
28from crawler.by_source.episciences_crawlers.episciences_crawler import EpisciencesCrawler
29from crawler.by_source.episciences_crawlers.episciences_dmtcs_crawler import (
30 EpisciencesDmtcsCrawler,
31)
32from crawler.by_source.episciences_crawlers.episciences_epidemes_crawler import (
33 EpisciencesEpidemesCrawler,
34)
35from crawler.by_source.episciences_crawlers.episciences_fi_crawler import EpisciencesFiCrawler
36from crawler.by_source.episciences_crawlers.episciences_rdm_crawler import EpisciencesRdmCrawler
37from crawler.by_source.eudml_crawler import EudmlCrawler
38from crawler.by_source.fcm_crawler import FCMCrawler
39from crawler.by_source.geodesic_crawler import GeodesicCrawler
40from crawler.by_source.hdml_crawler import HdmlCrawler
41from crawler.by_source.heldermann_crawler import HeldermannCrawler
42from crawler.by_source.impan_crawler import ImpanCrawler
43from crawler.by_source.ipb_crawler import IpbCrawler
44from crawler.by_source.isrp_crawler import IsrpCrawler
45from crawler.by_source.j_stage_crawler import JStageCrawler
46from crawler.by_source.jgaa_crawler import JgaaCrawler
47from crawler.by_source.journalfi_crawler import JournalfiCrawler
48from crawler.by_source.jsig_crawler import JsigCrawler
49from crawler.by_source.kobe_archive_crawler import KobeArchiveCrawler
50from crawler.by_source.lofpl_crawler import LofplCrawler
51from crawler.by_source.mathbas_crawler import MathbasCrawler
52from crawler.by_source.mathnetru_crawler import MathnetruCrawler
53from crawler.by_source.msp_crawler import MspCrawler
54from crawler.by_source.mta_crawler import MtaCrawler
55from crawler.by_source.nsjom.nsjom_crawler import NsjomCrawler
56from crawler.by_source.numdam_crawler import NumdamCrawler
57from crawler.by_source.ptm_crawler import PtmCrawler
58from crawler.by_source.rcm_crawler import RcmCrawler
59from crawler.by_source.sasa_crawler import SasaCrawler
60from crawler.by_source.scholastica_crawler import ScholasticaCrawler
61from crawler.by_source.seio_crawler import SeioCrawler
62from crawler.by_source.slc_crawler import Slc_Crawler
63from crawler.by_source.tac_crawler import TacCrawler
65if TYPE_CHECKING:
66 from collections.abc import Callable
68crawler_classes = (
69 AmcCrawler,
70 AmiCrawler,
71 AmpCrawler,
72 AmsCrawler,
73 ArsiaCrawler,
74 AsuoCrawler,
75 AulfmCrawler,
76 BdimCrawler,
77 BmmsCrawler,
78 CsisCrawler,
79 Dml_eCrawler,
80 DmlbulCrawler,
81 DmlczCrawler,
82 DmlplCrawler,
83 EdpsciCrawler,
84 EjcCrawler,
85 ElibmCrawler,
86 Emis_aasCrawler,
87 Emis_amCrawler,
88 Emis_hoaCrawler,
89 EmsCrawler,
90 EpisciencesCrawler,
91 EpisciencesDmtcsCrawler,
92 EpisciencesEpidemesCrawler,
93 EpisciencesRdmCrawler,
94 EpisciencesFiCrawler,
95 EudmlCrawler,
96 GeodesicCrawler,
97 HdmlCrawler,
98 HeldermannCrawler,
99 ImpanCrawler,
100 IpbCrawler,
101 IsrpCrawler,
102 JgaaCrawler,
103 JournalfiCrawler,
104 JsigCrawler,
105 LofplCrawler,
106 MathbasCrawler,
107 MathnetruCrawler,
108 MspCrawler,
109 MtaCrawler,
110 NsjomCrawler,
111 NumdamCrawler,
112 PtmCrawler,
113 RcmCrawler,
114 SasaCrawler,
115 ScholasticaCrawler,
116 SeioCrawler,
117 Slc_Crawler,
118 TacCrawler,
119 CupCrawler,
120 AmucCrawler,
121 FCMCrawler,
122 KobeArchiveCrawler,
123 JStageCrawler,
124)
126crawler_classes_map = {c.source_domain: c for c in crawler_classes}
129def get_crawler_class(source: str):
130 "source is the source domain"
131 return crawler_classes_map.get(source, None)
134def crawler_factory(
135 source: str,
136 colid: str,
137 username: str,
138 dry: bool = False,
139 collection_url: str | None = None,
140 backend: str | None = None,
141 pause_function: "Callable[[float], None]" = staticmethod(time.sleep),
142) -> BaseCollectionCrawler:
143 """
144 Factory for the crawlers
146 :param source: the source domain
147 :param colid: collection pid
148 :param col_url: url of the collection web page
149 :param username:
150 :param progress_bar: alive_bar progress_bar if you already have one (default: None)
151 :return: a crawler derived from base_crawler
152 """
153 klass = get_crawler_class(source)
155 if klass is None: 155 ↛ 156line 155 didn't jump to line 156 because the condition on line 155 was never true
156 raise NotImplementedError
158 crawler = klass(
159 collection_id=colid,
160 username=username,
161 dry=dry,
162 collection_url=collection_url,
163 backend=backend,
164 pause_function=pause_function,
165 )
167 return crawler