EDA - Exploratory Data Analysis

EDA is the process of discovering data - understanding its structure, patterns, and anomalies!

EDA Workflow

1import pandas as pd
2import numpy as np
3import matplotlib.pyplot as plt
4import seaborn as sns
5
6# 1. Load data
7df = pd.read_csv('safari_observations.csv')
8
9# 2. First look
10print("Shape:", df.shape)
11print("\nFirst rows:")
12print(df.head())
13print("\nInfo:")
14print(df.info())
15print("\nStatistics:")
16print(df.describe())

Data Type Analysis

1# Data types
2print(df.dtypes)
3
4# Categorical vs numerical columns
5categorical = df.select_dtypes(include=['object', 'category']).columns
6numerical = df.select_dtypes(include=[np.number]).columns
7
8print(f"Categorical: {list(categorical)}")
9print(f"Numerical: {list(numerical)}")
10
11# Unique categorical values
12for col in categorical:
13    print(f"\n{col}: {df[col].nunique()} unique")
14    print(df[col].value_counts().head())

Missing Data Analysis

1# Missing values
2missing = df.isnull().sum()
3missing_pct = (missing / len(df)) * 100
4
5missing_df = pd.DataFrame({
6    'missing': missing,
7    'percent': missing_pct
8}).sort_values('missing', ascending=False)
9
10print(missing_df[missing_df['missing'] > 0])
11
12# Visualizing missing data
13plt.figure(figsize=(10, 6))
14sns.heatmap(df.isnull(), cbar=True, yticklabels=False)
15plt.title('Missing Values Map')
16plt.show()

Distribution Analysis

1# Histograms for all numerical columns
2df[numerical].hist(figsize=(15, 10), bins=30, edgecolor='black')
3plt.tight_layout()
4plt.show()
5
6# Boxplots - outliers
7fig, axes = plt.subplots(1, len(numerical), figsize=(15, 5))
8for i, col in enumerate(numerical):
9    df.boxplot(column=col, ax=axes[i])
10    axes[i].set_title(col)
11plt.tight_layout()
12plt.show()
13
14# Detecting outliers (IQR method)
15def find_outliers_iqr(df, column):
16    Q1 = df[column].quantile(0.25)
17    Q3 = df[column].quantile(0.75)
18    IQR = Q3 - Q1
19    lower = Q1 - 1.5 * IQR
20    upper = Q3 + 1.5 * IQR
21    return df[(df[column] < lower) | (df[column] > upper)]
22
23outliers = find_outliers_iqr(df, 'population')
24print(f"Outliers in population: {len(outliers)}")

Correlation Analysis

1# Correlation matrix
2correlation = df[numerical].corr()
3
4plt.figure(figsize=(12, 8))
5sns.heatmap(correlation, annot=True, cmap='coolwarm',
6            center=0, fmt='.2f', linewidths=0.5)
7plt.title('Correlation Matrix')
8plt.show()
9
10# Strongest correlations
11corr_pairs = correlation.unstack().sort_values(ascending=False)
12print("Strongest correlations:")
13print(corr_pairs[corr_pairs < 1].head(10))

Category Analysis

1# Category distribution
2for col in categorical:
3    plt.figure(figsize=(10, 5))
4    df[col].value_counts().plot(kind='bar')
5    plt.title(f'Distribution: {col}')
6    plt.xticks(rotation=45)
7    plt.show()
8
9# Categories vs numerical variables
10for cat in categorical:
11    for num in numerical:
12        plt.figure(figsize=(10, 5))
13        sns.boxplot(data=df, x=cat, y=num)
14        plt.title(f'{num} by {cat}')
15        plt.xticks(rotation=45)
16        plt.show()

EDA Report - Summary

1def eda_summary(df):
2    """Generates an EDA summary."""
3    print("=" * 50)
4    print("EDA REPORT")
5    print("=" * 50)
6    print(f"\nRows: {df.shape[0]}")
7    print(f"Columns: {df.shape[1]}")
8    print(f"\nMissing data: {df.isnull().sum().sum()}")
9    print(f"Duplicates: {df.duplicated().sum()}")
10    print("\nData types:")
11    print(df.dtypes.value_counts())
12    print("\nNumerical statistics:")
13    print(df.describe())
14    print("=" * 50)
15
16eda_summary(df)