Coverage for /var/devmt/py/utils4_1.8.0/utils4/dfdiff.py: 100%

1#!/usr/bin/env python3 

2# -*- coding: utf-8 -*- 

3""" 

4:Purpose: This module provides DataFrame differencing logic. 

5 

6 The caller creates an instance which accepts the two DataFrames 

7 to be compared as the arguments. When the 

8 :meth:`~DataFrameDiff.diff` method is called, a list of columns 

9 containing value mismatches is compiled. Then, the list of column 

10 mismatches is iterated with each value in the column being 

11 compared. All value mismatches are reported to the terminal. 

12 

13:Developer: J Berendt 

14:Email: support@s3dev.uk 

15 

16:Note: It's worth noting that current functionality **does not 

17 check data types**, unlike the pandas ``pd.DataFrame.equals()`` 

18 method. This functionality may be added in a future release. 

19 

20:Example: 

21 

22 Short example for differencing two DataFrames:: 

23 

24 >>> from utils4 import dfdiff 

25 

26 >>> d = dfdiff.DataFrameDiff(df_source, df_test) 

27 >>> d.diff() 

28 

29""" 

30# pylint: disable=wrong-import-order 

31 

32import pandas as pd 

33from itertools import zip_longest 

34try: 

35 from .user_interface import ui 

36except ImportError: # pragma: nocover 

37 from user_interface import ui 

38 

39 

40class _Messages: 

41 """This private class handles the messaging for DataFrame differencing.""" 

42 

43 _FMT = '{:<10}\t{:<10}\t{:<25}\t{:<25}' 

44 

45 @staticmethod 

46 def column_mismatches(columns: list): 

47 """List columns with mismatches. 

48 

49 Args: 

50 columns (list): A list of columns containing mismatches. 

51 

52 """ 

53 # pylint: disable=consider-using-f-string 

54 ui.print_('\nColumn mismatches:', fore='cyan', style='normal') 

55 print(*map('- {}'.format, columns), sep='\n') 

56 

57 @staticmethod 

58 def column_mismatches_none(): 

59 """Print message for no column mismatches.""" 

60 ui.print_('\nNo mismatches for this set.', fore='green') 

61 

62 def data_mismatches(self, column: str, mismatches: list): 

63 """Print the data mismatches. 

64 

65 Args: 

66 column (str): Name of the column being analysed. 

67 mismatches (list): A list of tuples containing data mismatches, 

68 as:: 

69 

70 [(0, 0, 1, 2), (1, 1, 3, 4)] 

71 

72 """ 

73 ui.print_(f'Data mismatches for column: {column}', fore='yellow') 

74 print(self._FMT.format('SrcRow', 'TstRow', 'SrcValue', 'TstValue')) 

75 print('-'*92) 

76 print(*(self._FMT.format(*m) for m in mismatches), sep='\n') 

77 print() 

78 

79 @staticmethod 

80 def data_mismatches_none(column: str): 

81 """Print message for no data mismatches. 

82 

83 Args: 

84 column (str): Name of the column being analysed. 

85 

86 """ 

87 ui.print_(f'\nNo data mismatches for {column}', fore='green') 

88 

89 

90class DataFrameDiff: 

91 """Test and report differences in two pandas DataFrames. 

92 

93 Args: 

94 df_source (pd.DataFrame): DataFrame containing **source** data. 

95 This dataset holds the **expected** results. 

96 df_test (pd.DataFrame): DataFrame containing the **test** data. 

97 This dataset is compared against the 'expected' dataset. 

98 

99 """ 

100 

101 def __init__(self, df_source: pd.DataFrame, df_test: pd.DataFrame): 

102 """DataFrame difference class initialiser.""" 

103 self._df_s = df_source 

104 self._df_t = df_test 

105 self._col_mismatches = [] 

106 self._msg = _Messages() 

107 

108 def diff(self): 

109 """Compare DataFrames and report the differences.""" 

110 self._get_mismatches() 

111 self._report() 

112 

113 def _get_mismatches(self): 

114 """Build a list of columns with mismatches.""" 

115 # Add column to list if it contains a mismatch. 

116 mis = [col for col in self._df_s.columns 

117 if not self._df_t[col].equals(self._df_s[col])] 

118 if mis: 

119 self._msg.column_mismatches(columns=self._col_mismatches) 

120 else: 

121 self._msg.column_mismatches_none() 

122 self._col_mismatches = mis 

123 

124 def _report(self) -> None: 

125 """Compare values in mismatched columns and report.""" 

126 for col in self._col_mismatches: 

127 mismatches = [] 

128 # Zip source and test datasets. 

129 for (idx1, row1), (idx2, row2) in zip_longest(self._df_s.iterrows(), 

130 self._df_t.iterrows(), 

131 fillvalue=(None, None)): 

132 # Catch if a row exists in one dataset and not the other. 

133 if any([row1 is None, row2 is None]): 

134 idx1 = idx1 if idx1 is not None else idx2 

135 idx2 = idx2 if idx2 is not None else idx1 

136 val1 = str(row1[col]) if row1 is not None else 'no value (source)' 

137 val2 = str(row2[col]) if row2 is not None else 'no value (test)' 

138 # Convert datetimes to string for compare. 

139 elif isinstance(row2[col], pd.Timestamp): 

140 val1 = str(row1[col]) 

141 val2 = str(row2[col]) 

142 # Enable compare of nan types. 

143 elif any([pd.isna(row1[col]), pd.isna(row2[col])]): 

144 # Convert mismatched nan/NaT types to 'NaT' string. 

145 if all([pd.isna(row1[col]), row2[col] is pd.NaT]): 

146 val1 = 'NaT' 

147 val2 = 'NaT' 

148 else: 

149 val1 = str(row1[col]) 

150 val2 = str(row2[col]) 

151 # Reformat floats to align. 

152 elif any([isinstance(row1[col], float), isinstance(row2[col], float)]): 

153 val1 = round(float(row1[col]), 5) 

154 val2 = round(float(row2[col]), 5) 

155 else: 

156 # Convert to string for each compare. 

157 val1 = str(row1[col]) 

158 val2 = str(row2[col]) 

159 # Do the compare. 

160 if val1 != val2: 

161 # Add any mismatches to a list for reporting. 

162 mismatches.append((idx1, idx2, val1, val2)) 

163 if mismatches: 

164 self._msg.data_mismatches(column=col, mismatches=mismatches) 

165 else: 

166 self._msg.data_mismatches_none(column=col)