Coverage for pgsql_upserter / column_matcher.py: 100.00%

1"""Column matching utilities for PostgreSQL upsert operations.""" 

2 

3import logging 

4from typing import Any 

5 

6from .schema_inspector import TableSchema 

7 

8logger = logging.getLogger(__name__) 

9 

10 

11def match_columns( 

12 data_list: list[dict[str, Any]], 

13 table_schema: TableSchema, 

14 ignore_columns: list[str] | None = None 

15) -> dict[str, list[str]]: 

16 """Match data columns against table schema with union approach. 

17 

18 Args: 

19 data_list: List of dictionaries containing data to be inserted 

20 table_schema: Table schema information from schema inspector 

21 ignore_columns: Optional list of column names to ignore (case insensitive) 

22 

23 Returns: 

24 Dict with keys: matched_columns, ignored_columns, missing_columns 

25 Each containing respective lists of column names 

26 """ 

27 # Handle edge cases 

28 if not data_list: 

29 return { 

30 'matched_columns': [], 

31 'ignored_columns': [], 

32 'missing_columns': [] 

33 } 

34 

35 # Collect all unique keys from entire data_list 

36 all_data_columns = set() 

37 for row in data_list: 

38 all_data_columns.update(row.keys()) 

39 

40 # Convert to list for consistent ordering 

41 all_data_columns = list(all_data_columns) 

42 

43 # Get valid table columns (excludes auto-generated) 

44 valid_table_columns = set(table_schema.valid_columns) 

45 

46 # Normalize ignore_columns to lowercase (PostgreSQL standard) 

47 ignore_set = set() 

48 if ignore_columns: 

49 ignore_set = {col.lower() for col in ignore_columns} 

50 

51 # Add auto-generated columns to ignore_set 

52 auto_gen_columns = [col.name for col in table_schema.columns if col.is_auto_generated] 

53 ignore_set.update(col for col in auto_gen_columns) 

54 

55 # Categorize columns 

56 matched_columns = [] 

57 ignored_columns = [] 

58 missing_columns = [] 

59 

60 for col in all_data_columns: 

61 col_lower = col.lower() 

62 

63 # Check if column should be ignored 

64 if col_lower in ignore_set: 

65 ignored_columns.append(col) 

66 continue 

67 

68 # Check if column exists in table schema 

69 if col in valid_table_columns: 

70 matched_columns.append(col) 

71 else: 

72 missing_columns.append(col) 

73 logger.debug( 

74 f"Column '{col}' does not exist in table '{table_schema.schema_name}.{table_schema.table_name}'") 

75 

76 logger.debug(f"Column matching results: {len(matched_columns)} matched, " 

77 f"{len(ignored_columns)} ignored, {len(missing_columns)} missing") 

78 

79 return { 

80 'matched_columns': matched_columns, 

81 'ignored_columns': ignored_columns, 

82 'missing_columns': missing_columns 

83 }