Coverage for src/pullapprove/trust/tokens.py: 96%

1"""Minimal source tokenizer for the type-annotation trust rule. 

2 

3The rule has to tell a type-annotation ``:`` from a suite / dict / ternary / 

4lambda / object-value / case-label colon. That is a lexing question, so rather 

5than scan bytes with ever-more special cases (which kept springing leaks) we 

6tokenize the line first and reason about token *roles*. 

7 

8Deliberately conservative and NOT a full parser: a line it can't cleanly 

9tokenize — an unterminated string or block comment, or brackets left open, i.e. 

10part of a multi-line construct it can't judge in isolation — yields ``None``, and 

11the rule then declines to label rather than guess. Only the two languages the 

12annotation rule supports (Python and TypeScript/TSX) are modeled. 

13""" 

14 

15from __future__ import annotations 

16 

17from functools import lru_cache 

18 

19from .helpers import BLOCK_COMMENT_DELIMITERS, LINE_COMMENT_PREFIXES 

20from .linescan import scan_string 

21 

22WORD = "word" # identifier or keyword 

23OP = "op" # a punctuation/operator token (`:`, `->`, `(`, `,`, `=`, …) 

24STRING = "string" # a whole string literal, delimiters included 

25NUMBER = "number" 

26 

27 

28class Token: 

29 __slots__ = ("kind", "text", "start", "end") 

30 

31 def __init__(self, kind: str, text: str, start: int, end: int) -> None: 

32 self.kind = kind 

33 self.text = text 

34 self.start = start # char offset of the token in its source line 

35 self.end = end # char offset just past the token 

36 

37 def __repr__(self) -> str: # pragma: no cover - debugging aid 

38 return f"<{self.kind} {self.text!r}>" 

39 

40 

41# extension -> (line-comment prefixes, (block open, close) | None), derived from 

42# the shared syntax tables (helpers.py) so the tokenizer and the comments rule 

43# can never disagree about what starts a comment. 

44_COMMENTS: dict[str, tuple[tuple[str, ...], tuple[str, str] | None]] = { 

45 ext: (LINE_COMMENT_PREFIXES[ext], BLOCK_COMMENT_DELIMITERS.get(ext)) 

46 for ext in ("py", "ts", "tsx") 

47} 

48 

49_QUOTES = "\"'`" 

50# The bracket vocabulary, shared with the annotation span-walker (annotations.py) 

51# so depth tracking can't diverge between the two. 

52OPEN_BRACKETS, CLOSE_BRACKETS = "([{", ")]}" 

53 

54 

55# Cached: the three token-delta rules (`_style`, `_spacing`, 

56# `_type_annotations`) each tokenize the same changed lines as a hunk walks the 

57# rule chain, and tokenizing is the dominant per-hunk cost. The cache returns 

58# the SAME list to every caller — treat it as read-only. 

59@lru_cache(maxsize=4096) 

60def tokenize(line: str, ext: str) -> list[Token] | None: 

61 """Tokenize a single source line, or None if it can't be judged in isolation. 

62 

63 Returns None on an unterminated string/block comment or unbalanced brackets — 

64 all signs the line is part of a multi-line construct — so callers stay 

65 conservative (no label) instead of guessing at a fragment. 

66 """ 

67 comment_prefixes, block = _COMMENTS.get(ext, ((), None)) 

68 tokens: list[Token] = [] 

69 i, n = 0, len(line) 

70 depth = 0 

71 while i < n: 

72 ch = line[i] 

73 if ch.isspace(): 

74 i += 1 

75 elif comment_prefixes and line.startswith(comment_prefixes, i): 

76 break # rest of the line is a comment 

77 elif block and line.startswith(block[0], i): 

78 end = line.find(block[1], i + len(block[0])) 

79 if end == -1: 

80 return None # unterminated block comment -> multi-line 

81 i = end + len(block[1]) 

82 elif ch in _QUOTES: 

83 close = scan_string(line, i) 

84 if close is None: 

85 return None # unterminated string -> multi-line 

86 tokens.append(Token(STRING, line[i:close], i, close)) 

87 i = close 

88 elif ch.isalpha() or ch == "_": 

89 j = i + 1 

90 while j < n and (line[j].isalnum() or line[j] == "_"): 

91 j += 1 

92 tokens.append(Token(WORD, line[i:j], i, j)) 

93 i = j 

94 elif ch.isdigit(): 

95 j = i + 1 

96 while j < n and (line[j].isalnum() or line[j] in "._"): 

97 j += 1 

98 tokens.append(Token(NUMBER, line[i:j], i, j)) 

99 i = j 

100 elif line.startswith("->", i): 

101 tokens.append(Token(OP, "->", i, i + 2)) 

102 i += 2 

103 else: 

104 if ch in OPEN_BRACKETS: 

105 depth += 1 

106 elif ch in CLOSE_BRACKETS: 

107 depth -= 1 

108 if depth < 0: 

109 return None # more closers than openers -> can't be a line 

110 tokens.append(Token(OP, ch, i, i + 1)) 

111 i += 1 

112 if depth != 0: 

113 return None # brackets left open -> part of a multi-line construct 

114 return tokens