Coverage for src/pullapprove/matches.py: 90%

180 statements  

« prev     ^ index     » next       coverage.py v7.14.1, created at 2026-10-31 23:22 -0500

1from __future__ import annotations 

2 

3import hashlib 

4import json 

5from collections.abc import Generator, Iterator 

6from pathlib import Path 

7from typing import Any 

8 

9from pydantic import BaseModel, ConfigDict, Field, model_validator 

10 

11from .config import ( 

12 CompiledConfigModels, 

13 ConfigModel, 

14 LargeScaleChangeModel, 

15 ScopeModel, 

16) 

17from .diff import DiffCode, DiffFile, iterate_diff_parts 

18from .exceptions import LargeScaleChangeException 

19 

20 

21def match_path( 

22 *, path: Path, config: ConfigModel 

23) -> tuple[ScopePathMatch, list[ScopeModel]]: 

24 path_match = ScopePathMatch(path=str(path), scopes=[]) 

25 

26 scopes_matching_paths = [ 

27 scope for scope in config.scopes if scope.matches_path(path) 

28 ] 

29 code_scopes = [scope for scope in scopes_matching_paths if scope.code] 

30 path_scopes = [scope for scope in scopes_matching_paths if not scope.code] 

31 

32 # Set the scopes on the path itself 

33 for scope in path_scopes: 

34 path_match.add_scope(scope) 

35 

36 return path_match, code_scopes 

37 

38 

39def match_code( 

40 *, path: str, code: str, scopes: list[ScopeModel], line_offset: int = 0 

41) -> Generator[ScopeCodeMatch]: 

42 code_matches: dict[str, ScopeCodeMatch] = {} 

43 

44 for scope in scopes: 

45 for match in scope.matches_code(code): 

46 code_match = ScopeCodeMatch( 

47 path=path, 

48 start_line=line_offset + match["start_line"], 

49 end_line=line_offset + match["end_line"], 

50 start_column=match["start_col"], 

51 end_column=match["end_col"], 

52 scopes=[scope.name], 

53 location_id="", 

54 ) 

55 code_match._scopes = [scope] 

56 

57 if code_match.location_id in code_matches: 

58 # Just add the scopes to it 

59 code_matches[code_match.location_id].add_scope(scope) 

60 else: 

61 code_matches[code_match.location_id] = code_match 

62 

63 yield from code_matches.values() 

64 

65 

66def match_files(configs: CompiledConfigModels, files: Iterator[str]) -> ChangeMatches: 

67 def _iterate() -> Generator[ScopePathMatch | ScopeCodeMatch]: 

68 for f in files: 

69 file_path = Path(f) 

70 

71 config = configs.closest_config(file_path) 

72 

73 path_match, code_scopes = match_path( 

74 path=file_path, 

75 config=config, 

76 ) 

77 

78 # Yield the paths first 

79 yield path_match 

80 

81 # Then go line by line to find scopes that match lines 

82 if code_scopes: 

83 try: 

84 code = file_path.read_text() 

85 yield from match_code( 

86 path=str(file_path), 

87 code=code, 

88 scopes=code_scopes, 

89 ) 

90 except UnicodeDecodeError: 

91 # Skip binary files that can't be decoded as text 

92 pass 

93 except FileNotFoundError: 

94 # Deleted in the working tree, so there's no code left to 

95 # match (a diff still has its removed lines) 

96 pass 

97 

98 return ChangeMatches.from_config_matches(configs, _iterate()) 

99 

100 

101def iterate_diff( 

102 configs: CompiledConfigModels, diff: Iterator[str] | str 

103) -> Generator[tuple[DiffFile | DiffCode, list[ScopePathMatch | ScopeCodeMatch]]]: 

104 # We can still iterate a diff without configs, just by yield the diff objs 

105 if not configs: 

106 for diff_obj in iterate_diff_parts(diff): 

107 yield diff_obj, [] 

108 

109 return 

110 

111 # Keep track of these as we go and jump between file header 

112 # and raw code during iteration 

113 check_code_scopes: list[ScopeModel] = [] 

114 current_code_path = None 

115 

116 current_code_diffs = [] 

117 

118 # TODO get root config here, check diff size as we go and raise exception? 

119 # or we need to keep track per LSC? should be a compiled value... 

120 

121 def yield_code_diffs() -> Generator[ 

122 tuple[DiffCode, list[ScopePathMatch | ScopeCodeMatch]] 

123 ]: 

124 # We're passing the entire diff chunk to see if there's a match inside, 

125 # but if there is, it probably won't match EVERY line in the chunk 

126 assert current_code_path is not None, "current_code_path must be set" 

127 current_code_chunk = "\n".join([code.raw() for code in current_code_diffs]) 

128 current_code_line_number = current_code_diffs[0].line_number - 1 

129 

130 code_matches = match_code( 

131 path=current_code_path, 

132 code=current_code_chunk, 

133 scopes=check_code_scopes, 

134 line_offset=current_code_line_number, 

135 ) 

136 code_matches = list(code_matches) 

137 

138 for diff_line_index, diff_code in enumerate(current_code_diffs): 

139 subcode_matches: list[ScopePathMatch | ScopeCodeMatch] = [ 

140 code_match 

141 for code_match in code_matches 

142 if code_match.start_line 

143 <= (current_code_line_number + diff_line_index + 1) 

144 <= code_match.end_line 

145 ] 

146 yield diff_code, subcode_matches 

147 

148 for diff_obj in iterate_diff_parts(diff): 

149 if isinstance(diff_obj, DiffFile): 

150 # Yield a code chunk if we finished one 

151 if current_code_diffs: 

152 yield from yield_code_diffs() 

153 

154 current_code_path = None 

155 current_code_diffs = [] 

156 

157 diff_file = diff_obj 

158 file_path = Path(diff_file.new_path) 

159 config = configs.closest_config(file_path) 

160 

161 path_match, code_scopes = match_path( 

162 path=file_path, 

163 config=config, 

164 ) 

165 

166 current_code_path = str(file_path) 

167 check_code_scopes = code_scopes 

168 

169 yield diff_obj, [path_match] 

170 elif isinstance(diff_obj, DiffCode): 

171 if check_code_scopes: 

172 # It will be yielded later 

173 current_code_diffs.append(diff_obj) 

174 else: 

175 # Skip all code lines if we don't care about code 

176 yield diff_obj, [] 

177 

178 # Yield the last code chunk we saw 

179 if current_code_diffs: 

180 yield from yield_code_diffs() 

181 

182 

183def match_diff(configs: CompiledConfigModels, diff: Iterator[str] | str) -> DiffResults: 

184 config_paths_modified: set[str] = set() 

185 changed_paths: set[str] = set() 

186 additions = 0 

187 deletions = 0 

188 

189 def iterate() -> Generator[ScopePathMatch | ScopeCodeMatch]: 

190 nonlocal additions, deletions 

191 for diff_obj, matches in iterate_diff(configs, diff): 

192 # Track additions/deletions during existing iteration 

193 if isinstance(diff_obj, DiffCode): 

194 if diff_obj.is_addition(): 

195 additions += 1 

196 elif diff_obj.is_deletion(): 

197 deletions += 1 

198 

199 if isinstance(diff_obj, DiffFile): 

200 changed_paths.add(diff_obj.new_path) 

201 if diff_obj.new_path in configs: 

202 config_paths_modified.add(diff_obj.new_path) 

203 if diff_obj.old_path in configs: 

204 config_paths_modified.add(diff_obj.old_path) 

205 

206 yield from matches 

207 

208 try: 

209 return DiffResults( 

210 matches=ChangeMatches.from_config_matches(configs, iterate()), 

211 config_paths_modified=list(config_paths_modified), 

212 changed_paths=sorted(changed_paths), 

213 additions=additions, 

214 deletions=deletions, 

215 ) 

216 except LargeScaleChangeException: 

217 # Get the large scale change config from CODEREVIEW.toml 

218 lsc = configs.get_default_large_scale_change() 

219 

220 return DiffResults( 

221 matches=ChangeMatches.from_large_scale_change( 

222 configs=configs, 

223 large_scale_change=lsc, 

224 ), 

225 config_paths_modified=list(config_paths_modified), 

226 additions=additions, 

227 deletions=deletions, 

228 ) 

229 

230 

231class DiffResults(BaseModel): 

232 """Results from analyzing a diff against configs.""" 

233 

234 model_config = ConfigDict(extra="forbid") 

235 

236 matches: ChangeMatches 

237 config_paths_modified: list[str] = Field(default_factory=list) 

238 # Every file the diff touched. What a result depends on, config-wise, is 

239 # the compiled config governing each of these; empty for a large-scale 

240 # change, whose verdict comes from the root config alone and whose 

241 # iteration stopped at the threshold. 

242 changed_paths: list[str] = Field(default_factory=list) 

243 additions: int = 0 

244 deletions: int = 0 

245 

246 

247class ChangeMatches(BaseModel): 

248 """ 

249 The matches for a given diff or set of files. 

250 

251 This knows nothing about a pull request (branches, commits, etc.) 

252 """ 

253 

254 model_config = ConfigDict(extra="forbid") 

255 

256 # Instead we could do 

257 # - scopes 

258 # - config 

259 # - paths 

260 # - code 

261 # could add points, reviewers, etc to this 

262 # but then we're mixing concerns... looking at raw files will just have empty values? 

263 

264 # Three modes are: 

265 # - raw files 

266 # - raw diff 

267 # - pull request (has reviews) 

268 

269 configs: dict[str, ConfigModel] = {} 

270 

271 # The matching LSC, if there is one. 

272 large_scale_change: LargeScaleChangeModel | None = None 

273 

274 # All scopes found in the results 

275 scopes: dict[str, ScopeModel] = {} 

276 

277 # All evaluated paths 

278 paths: dict[str, ScopePathMatch] = {} 

279 

280 # All code matches 

281 code: dict[str, ScopeCodeMatch] = {} 

282 

283 def as_dict(self) -> dict[str, Any]: 

284 return self.model_dump() 

285 

286 def __bool__(self) -> bool: 

287 return bool(self.scopes) 

288 

289 @classmethod 

290 def from_config_matches( 

291 cls, 

292 configs: CompiledConfigModels, 

293 matches: Iterator[ScopePathMatch | ScopeCodeMatch], 

294 ) -> ChangeMatches: 

295 scopes: dict[str, ScopeModel] = {} 

296 paths: dict[str, ScopePathMatch] = {} 

297 code: dict[str, ScopeCodeMatch] = {} 

298 

299 # Where each scope sits in the configs: which config it came from, 

300 # then its position in that config's scope list. Keyed by object 

301 # identity, not by name. Two configs may declare the same scope name, 

302 # and a name key would give one of them the other's position. The 

303 # matched scopes are the same ScopeModel instances these configs hold, 

304 # so identity is exact. 

305 # 

306 # The `scopes` dict below is keyed by name and so still collapses 

307 # duplicate names across configs. That is pre-existing; ordering by 

308 # identity keeps whichever scope survives that collapse in its own 

309 # declared position rather than making it worse. 

310 declaration_order = { 

311 id(scope): (config_index, scope_index) 

312 for config_index, config in enumerate(configs.root.values()) 

313 for scope_index, scope in enumerate(config.scopes) 

314 } 

315 

316 for match in matches: 

317 # Store seen scopes as we go from all matches 

318 for scope in match._scopes: 

319 scopes[scope.name] = scope 

320 

321 if isinstance(match, ScopePathMatch): 

322 if not match._scopes: 

323 # Right now we don't care about storing anything that doesn't have scopes. 

324 # This prevents an unnecessarily huge dump on big repos or PRs. 

325 continue 

326 

327 paths[match.path] = match 

328 

329 elif isinstance(match, ScopeCodeMatch): 

330 code_location_id = match.location_id 

331 

332 # Store it in the code results 

333 code[code_location_id] = match 

334 

335 else: 

336 raise TypeError(f"Unknown match type: {match}") 

337 

338 # Sort into declaration order. Without this, the order is whichever 

339 # scope the first changed file matched, which is a property of the 

340 # diff rather than the configs. Consumers that walk scopes in sequence 

341 # need declaration order: review requesting counts what the scopes 

342 # before this one already asked for, so the order decides who gets 

343 # asked. A scope missing from the map sorts last and keeps its 

344 # position; the match came from these configs, so this cannot 

345 # normally happen. 

346 ordered_scopes = { 

347 name: scopes[name] 

348 for name in sorted( 

349 scopes, 

350 key=lambda name: declaration_order.get( 

351 id(scopes[name]), (len(configs.root), 0) 

352 ), 

353 ) 

354 } 

355 

356 return cls( 

357 large_scale_change=None, 

358 scopes=ordered_scopes, 

359 paths=paths, 

360 code=code, 

361 # These are the compiled, effective configs (extends merged, aliases 

362 # expanded, paths anchored, PR-disabled scopes dropped) — i.e. what 

363 # actually applied to this diff. 

364 configs=configs.get_config_models(), 

365 ) 

366 

367 @classmethod 

368 def from_large_scale_change( 

369 cls, 

370 configs: CompiledConfigModels, 

371 large_scale_change: LargeScaleChangeModel, 

372 ) -> ChangeMatches: 

373 return cls( 

374 configs=configs.get_config_models(), 

375 large_scale_change=large_scale_change, 

376 scopes={}, 

377 paths={}, 

378 code={}, 

379 ) 

380 

381 

382class ScopePathMatch(BaseModel): 

383 model_config = ConfigDict(extra="forbid") 

384 

385 path: str = Field(min_length=1) 

386 scopes: list[str] # Field(min_length=1) 

387 # code: list[str] = [] 

388 

389 # Store this internally during processing (full reference of scope models) 

390 _scopes: list[ScopeModel] = [] 

391 

392 def add_scope(self, scope: ScopeModel) -> None: 

393 if not scope.ownership: 

394 # Remove any other scopes that don't have special ownership rules 

395 # (i.e. we only want one primary scope in the end) 

396 self._scopes = [s for s in self._scopes if s.ownership] 

397 

398 self._scopes.append(scope) 

399 

400 self.scopes = [s.name for s in self._scopes] 

401 

402 

403class ScopeCodeMatch(BaseModel): 

404 model_config = ConfigDict(extra="forbid") 

405 

406 # In a diff match, we could see both sides of the diff, i.e. repeated lines if the before and after both match... 

407 path: str = Field(min_length=1) 

408 start_line: int 

409 end_line: int 

410 start_column: int 

411 end_column: int 

412 scopes: list[str] # Field(min_length=1) 

413 location_id: str 

414 

415 # Store this internally during processing (full reference of scope models) 

416 _scopes: list[ScopeModel] = [] 

417 

418 def printed_location(self) -> str: 

419 if self.start_line == self.end_line: 

420 return f"Ln {self.start_line}, Col {self.start_column}-{self.end_column}" 

421 else: 

422 return f"Ln {self.start_line}-{self.end_line}" 

423 

424 def add_scope(self, scope: ScopeModel) -> None: 

425 if not scope.ownership: 

426 # Remove any other scopes that don't have special ownership rules 

427 # (i.e. we only want one primary scope in the end) 

428 self._scopes = [s for s in self._scopes if s.ownership] 

429 

430 self._scopes.append(scope) 

431 

432 self.scopes = [s.name for s in self._scopes] 

433 

434 @model_validator(mode="after") 

435 def compute_location_id(self) -> ScopeCodeMatch: 

436 # only compute if the caller didn't provide one 

437 if not self.location_id: 

438 loc = { 

439 "path": self.path, 

440 "start_line": self.start_line, 

441 "end_line": self.end_line, 

442 "start_column": self.start_column, 

443 "end_column": self.end_column, 

444 } 

445 raw = json.dumps(loc, sort_keys=True, separators=(",", ":")).encode() 

446 self.location_id = hashlib.md5(raw).hexdigest() 

447 return self 

448 

449 

450# how to store what was reviewed? ideally we could be fine-grained, at some point 

451# so we need to know who, which scopes, which paths, which codes (location hash) then we can cross reference everything?