1. Get null percentage of columns
def get_null_percentage(df):
count = df.isnull().sum()
percent = (df.isnull().mean() * 100).round(2)
summary = pd.DataFrame({'count': count, 'percentage': percent})
return summary[summary["count"] > 0].sort_values(ascending=False, by="percentage")2. Get zero percentage of columns
def get_zero_percentage(df):
count = df.eq(0).sum()
percent = (df.eq(0).mean() * 100).round(2)
summary = pd.DataFrame({'count': count, 'percentage': percent})
return summary[summary["count"] > 0].sort_values(ascending=False, by="percentage")3. Remove Outliers for one column
3.1 using IQR
def remove_outliers_by_iqr(df, column):
q1 = df[column].quantile(0.25)
q3 = df[column].quantile(0.75)
value = q3 - q1
lower_value = q1 - 1.5 * value
higer_value = q3 + 1.5 * value
return df[(df[column] < higer_value) & (df[column] > lower_value)]This might end up with an empty DataFrame if all or a significant portion of your data points are considered outliers and removed. e.g.,
We can use z-score to solve it.
3.2 using Z-Score
def remove_outliers_by_zscore(df, column_name, z_threshold=3):
# Calculate the Z-scores for the specified column
z_scores = (df[column_name] - df[column_name].mean()) / df[column_name].std()
# Filter out rows where the Z-score is greater than the threshold
return df[abs(z_scores) <= z_threshold]4 Get extreme distribution columns
def check_extreme_distribution(df, threshold=0.95):
extreme_columns = []
for column in df.columns:
mode_value = df[column].mode().iloc[0] # first value, it is also the most contributed value
mode_percentage = (df[column] == mode_value).sum() / len(df)
if mode_percentage >= threshold:
extreme_columns.append(column)
return extreme_columns5 Get same percentile columns
def get_same_percentile_columns(df, value):
column_stats = df.describe()
return column_stats.columns[(column_stats.loc['min'] == value) &
(column_stats.loc['25%'] == value) &
(column_stats.loc['50%'] == value) &
(column_stats.loc['75%'] == value)]We can use it to check the columns that all percentiles are 0:
get_percentile_same_columns(df, 0)