Coverage for src/crawler/cmds/augment/augment_cmd.py: 21%

109 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-11-19 16:36 +0000

1import logging 

2from concurrent.futures import Future 

3from typing import TYPE_CHECKING, NotRequired, TypedDict 

4 

5from history.model_data import HistoryChildDict, MatchingEventResult 

6from history.models import HistoryEventStatus 

7from history.utils import insert_history_event 

8from ptf import model_helpers 

9from ptf.cmds.base_cmds import baseCmd 

10from ptf.display.resolver import ExtIdName 

11from ptf.models import Article, Collection 

12from ptf.utils import DjangoConnectionThreadPoolExecutor 

13from ptf_external.matching import get_matching_backend 

14 

15from crawler.cmds.augment import update_article_with_xarticle 

16from crawler.models.source import Source 

17 

18if TYPE_CHECKING: 

19 from collections.abc import Callable 

20 

21 from ptf.models import ArticleQuerySet 

22 from ptf_external.matching.base_matching import MatchingBackend 

23 

24 class AugmentArticlesParams(TypedDict): 

25 source_id: str 

26 collection_pid: NotRequired[str] 

27 issue_pids: NotRequired[list[str]] 

28 update_pdf_link: NotRequired[bool] 

29 

30 from typing import NotRequired, TypedDict 

31 

32 class MatchingOperationMessage(TypedDict): 

33 pid: str 

34 zbl_id: "NotRequired[str]" 

35 score: "NotRequired[int]" 

36 status: HistoryEventStatus 

37 message: "NotRequired[str]" 

38 status_message: "NotRequired[str]" 

39 

40 

41_logger = logging.getLogger(__name__) 

42 

43 

44def compile_history_events(event_messages: "list[MatchingOperationMessage]"): 

45 children = [] 

46 for msg in event_messages: 

47 resource = model_helpers.get_resource(msg["pid"]) 

48 if not resource: 

49 raise ValueError("Resource not found") 

50 data: HistoryChildDict = { 

51 "resource": resource, 

52 "type": "zbl-id", 

53 "status": msg.get("status"), 

54 } 

55 

56 if "zbl_id" in msg and msg["zbl_id"] != "": 

57 data["url"] = f"https://zbmath.org/{msg['zbl_id']}" 

58 

59 if "score" in msg: 

60 data["score"] = msg["score"] 

61 

62 if "message" in msg: 

63 data["message"] = msg["message"] 

64 

65 if "status_message" in msg: 

66 data["status_message"] = msg["status_message"] 

67 

68 children.append(data) 

69 return children 

70 

71 

72class AugmentArticlesCmd(baseCmd[None]): 

73 """ 

74 Metadata augment command. 

75 Calls the desired Matching Backend defined in ptf-back. 

76 """ 

77 

78 required_params = ["backend", "source_id"] 

79 

80 backend: "type[MatchingBackend]" 

81 source_id: str 

82 collection_pid: str | None = None 

83 issue_pids: list[str] = [] 

84 update_pdf_link: bool = False 

85 collection: "Collection | None" = None 

86 queryset: "ArticleQuerySet" 

87 event_messages: list[dict] 

88 callback: "Callable" = lambda _: _ 

89 

90 def __init__(self, params: "AugmentArticlesParams", backend: ExtIdName): 

91 super().__init__(params) 

92 

93 self.backend = get_matching_backend(backend) 

94 

95 self.event_messages = [] 

96 self.queryset = Article.objects.prefetch_related("extid_set").all() 

97 

98 if self.collection_pid: 

99 self.queryset = self.queryset.filter( 

100 my_container__my_collection__pid=self.collection_pid 

101 ) 

102 self.collection = Collection.objects.get(pid=self.collection_pid) 

103 

104 if self.source_id: 

105 source = Source.objects.get(domain=self.source_id) 

106 self.queryset = self.queryset.filter(my_container__origin__source=source) 

107 

108 if self.issue_pids: 

109 self.queryset = self.queryset.filter(my_container__pid__in=self.issue_pids) 

110 

111 def internal_do(self) -> None: 

112 _logger.info( 

113 f"Start augment [{self.backend.name}] " 

114 f"source={self.source_id} collection={self.collection_pid} " 

115 f"({self.queryset.count()} articles)" 

116 ) 

117 promises: set[Future] = set() 

118 with DjangoConnectionThreadPoolExecutor(max_workers=1) as executor: 

119 for article in self.queryset.iterator(chunk_size=2000): 

120 xarticle, event = self._find_article(article) 

121 

122 if event: 

123 self.event_messages.append(event) 

124 

125 if not xarticle: 

126 self.callback() 

127 continue 

128 

129 # Handle asyncronous results and exceptions as soon as possible 

130 completed: set[Future] = set() 

131 for f in promises: 

132 if f.done(): 

133 completed.add(f) 

134 exc = f.exception() 

135 if exc: 

136 executor.shutdown(wait=False, cancel_futures=True) 

137 raise exc 

138 promises -= completed 

139 # Runs the database insertion in parallel 

140 promise = executor.submit( 

141 update_article_with_xarticle, 

142 article, 

143 xarticle, 

144 merge_titles=False, 

145 update_pdf_link=self.update_pdf_link, 

146 ) 

147 promise.add_done_callback(self.callback) 

148 promises.add(promise) 

149 _logger.info(f"Augment [{self.backend.name}] over") 

150 

151 def _find_article(self, article: Article) -> tuple: 

152 """ 

153 Calls ArticleData from backend. 

154 """ 

155 # Init matching backend instance 

156 self.matcher = self.backend.init_from_db_article(resource=article) 

157 # With external ID 

158 xarticle = self.matcher.get_xarticle_by_extids() 

159 if xarticle: 

160 _logger.debug(f"[{self.matcher.name}] {article.pid}: found via External ID") 

161 return xarticle, self._make_event(article, MatchingEventResult.ALREADY_PRESENT) 

162 

163 # Matching fuzzy 

164 xarticle = self.matcher.get_xarticle_by_matching() 

165 if xarticle: 

166 _logger.debug(f"[{self.matcher.name}] {article.pid}: found via Matching") 

167 return xarticle, self._make_event(article, MatchingEventResult.ADDED) 

168 

169 _logger.info( 

170 f"[{self.matcher.name}] {article.pid}: not found with External ID nor Matching" 

171 ) 

172 return None, self._make_event( 

173 article, MatchingEventResult.NOT_FOUND, status=HistoryEventStatus.WARNING 

174 ) 

175 

176 def _make_event( 

177 self, 

178 article: Article, 

179 status_message: str, 

180 status: HistoryEventStatus = HistoryEventStatus.OK, 

181 ): 

182 return { 

183 "pid": article.pid, 

184 "status": status, 

185 "status_message": status_message, 

186 "backend": self.matcher.name, 

187 } 

188 

189 def insert_history_event(self): 

190 children = [] 

191 for msg in self.event_messages: 

192 resource = model_helpers.get_resource(msg["pid"]) 

193 if not resource: 

194 raise ValueError(f"Resource not found: {msg['pid']}") 

195 data: HistoryChildDict = { 

196 "resource": resource, 

197 "type": msg.get("backend", self.backend.name), 

198 "status": msg.get("status"), 

199 } 

200 if "score" in msg: 

201 data["score"] = msg["score"] 

202 if "message" in msg: 

203 data["message"] = msg["message"] 

204 if "status_message" in msg: 

205 data["status_message"] = msg["status_message"] 

206 children.append(data) 

207 

208 insert_history_event( 

209 { 

210 "pid": f"{self.source_id}_{self.collection_pid or ''}_{self.backend.name}_matching", 

211 "col": self.collection, 

212 "source": self.source_id, 

213 "status": HistoryEventStatus.OK, 

214 "type": "matching", 

215 "children": children, 

216 } 

217 )