Statistics - The Mathematics of Data

Statistics is the foundation of Data Science - it allows you to draw conclusions from data!

Descriptive Statistics

1import numpy as np
2import scipy.stats as stats
3
4data = np.array([120, 450, 85, 200, 330, 150, 280, 95, 400, 175])
5
6# Measures of central tendency
7print(f"Mean: {np.mean(data):.2f}")
8print(f"Median: {np.median(data):.2f}")
9print(f"Mode: {stats.mode(data, keepdims=True)[0][0]}")
10
11# Measures of dispersion
12print(f"Standard deviation: {np.std(data):.2f}")
13print(f"Variance: {np.var(data):.2f}")
14print(f"Range: {np.ptp(data)}")
15print(f"IQR: {stats.iqr(data):.2f}")
16
17# Quartiles and percentiles
18print(f"Q1 (25%): {np.percentile(data, 25):.2f}")
19print(f"Q2 (50%): {np.percentile(data, 50):.2f}")
20print(f"Q3 (75%): {np.percentile(data, 75):.2f}")
21print(f"95th percentile: {np.percentile(data, 95):.2f}")
22
23# Skewness and kurtosis
24print(f"Skewness: {stats.skew(data):.2f}")
25print(f"Kurtosis: {stats.kurtosis(data):.2f}")

Probability Distributions

1import matplotlib.pyplot as plt
2
3# Normal distribution
4x = np.linspace(-4, 4, 1000)
5y_normal = stats.norm.pdf(x, loc=0, scale=1)
6
7plt.figure(figsize=(10, 6))
8plt.plot(x, y_normal, label='Normal(0,1)')
9plt.fill_between(x, y_normal, alpha=0.3)
10plt.title('Normal Distribution')
11plt.legend()
12plt.show()
13
14# Generating data from distributions
15normal_data = np.random.normal(loc=100, scale=15, size=1000)
16poisson_data = np.random.poisson(lam=5, size=1000)
17exponential_data = np.random.exponential(scale=2, size=1000)
18
19# Normality test (Shapiro-Wilk)
20stat, p_value = stats.shapiro(normal_data)
21print(f"Shapiro-Wilk: statistic={stat:.4f}, p-value={p_value:.4f}")
22if p_value > 0.05:
23    print("The data likely comes from a normal distribution")

Statistical Tests

1# T-test - comparing means of two groups
2group_a = np.random.normal(100, 15, 50)
3group_b = np.random.normal(105, 15, 50)
4
5t_stat, p_value = stats.ttest_ind(group_a, group_b)
6print(f"T-test: t={t_stat:.4f}, p={p_value:.4f}")
7
8# Chi-squared test - categorical variable dependency
9observed = np.array([[50, 30], [20, 40]])
10chi2, p_value, dof, expected = stats.chi2_contingency(observed)
11print(f"Chi-squared: chi2={chi2:.4f}, p={p_value:.4f}")
12
13# Pearson correlation
14x = np.random.rand(100)
15y = x + np.random.rand(100) * 0.5
16corr, p_value = stats.pearsonr(x, y)
17print(f"Pearson correlation: r={corr:.4f}, p={p_value:.4f}")
18
19# Spearman correlation (rank-based)
20rho, p_value = stats.spearmanr(x, y)
21print(f"Spearman correlation: rho={rho:.4f}, p={p_value:.4f}")

Confidence Intervals

1# Confidence interval for the mean
2data = np.random.normal(100, 15, 100)
3mean = np.mean(data)
4sem = stats.sem(data)  # Standard error of mean
5ci_95 = stats.t.interval(0.95, len(data)-1, loc=mean, scale=sem)
6
7print(f"Mean: {mean:.2f}")
8print(f"95% CI: ({ci_95[0]:.2f}, {ci_95[1]:.2f})")

Linear Regression

1from scipy import stats
2
3# Data
4x = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
5y = 2 * x + 1 + np.random.randn(10)
6
7# Regression
8slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)
9
10print(f"Slope: {slope:.4f}")
11print(f"Intercept: {intercept:.4f}")
12print(f"R-squared: {r_value**2:.4f}")
13print(f"P-value: {p_value:.4f}")
14
15# Plot
16plt.figure(figsize=(10, 6))
17plt.scatter(x, y, label='Data')
18plt.plot(x, slope * x + intercept, 'r-', label=f'y = {slope:.2f}x + {intercept:.2f}')
19plt.legend()
20plt.title('Linear Regression')
21plt.show()