Pandas - Data Manipulation

Pandas is the most important library for data analysis! It provides DataFrame and Series structures ideal for working with tabular data.

Installation

1pip install pandas

Series - One-Dimensional Structure

1import pandas as pd
2
3# From a list
4populations = pd.Series([120, 450, 85, 200], name='population')
5
6# With an index
7species = pd.Series(
8    [120, 450, 85, 200],
9    index=['Lion', 'Elephant', 'Cheetah', 'Giraffe'],
10    name='population'
11)
12
13print(species['Lion'])      # 120
14print(species[species > 100])  # Filtering

DataFrame - Data Table

1# Creating a DataFrame
2safari_data = pd.DataFrame({
3    'species': ['Lion', 'Elephant', 'Cheetah', 'Giraffe'],
4    'population': [120, 450, 85, 200],
5    'habitat': ['Savanna', 'Forest', 'Savanna', 'Savanna'],
6    'endangered': [True, False, True, False]
7})
8
9# From a CSV file
10df = pd.read_csv('safari_species.csv')
11
12# From an Excel file
13df = pd.read_excel('safari_data.xlsx')
14
15# From JSON
16df = pd.read_json('species.json')

Exploring Data

1# Basic information
2print(df.head())       # First 5 rows
3print(df.tail())       # Last 5 rows
4print(df.info())       # Data types, nulls
5print(df.describe())   # Descriptive statistics
6print(df.shape)        # (rows, columns)
7print(df.columns)      # Column names
8print(df.dtypes)       # Column data types

Data Selection

1# Single column (Series)
2populations = df['population']
3
4# Multiple columns
5subset = df[['species', 'population']]
6
7# Rows by index
8df.iloc[0]       # First row
9df.iloc[0:3]     # Rows 0-2
10df.iloc[0, 1]    # Row 0, column 1
11
12# Rows by label
13df.loc[0]              # Row with index 0
14df.loc[0:2, 'species'] # Rows 0-2, column 'species'
15
16# Conditional filtering
17endangered = df[df['endangered'] == True]
18large_pop = df[df['population'] > 100]
19savanna = df[df['habitat'] == 'Savanna']
20
21# Combined conditions
22df[(df['population'] > 100) & (df['endangered'] == False)]

Modifying Data

1# New column
2df['population_thousands'] = df['population'] / 1000
3
4# Modifying existing column
5df['population'] = df['population'] * 1.1  # 10% growth
6
7# Apply - apply a function
8df['status'] = df['population'].apply(
9    lambda x: 'endangered' if x < 100 else 'stable'
10)
11
12# Dropping
13df_clean = df.drop('temp_column', axis=1)  # Drop column
14df_clean = df.drop(0, axis=0)               # Drop row
15
16# Renaming columns
17df = df.rename(columns={'old_name': 'new_name'})

Handling Missing Data

1# Check for nulls
2print(df.isnull().sum())
3
4# Drop rows with nulls
5df_clean = df.dropna()
6
7# Fill nulls
8df['population'] = df['population'].fillna(0)
9df['population'] = df['population'].fillna(df['population'].mean())

Grouping and Aggregation

1# Group by habitat
2by_habitat = df.groupby('habitat')
3
4# Aggregations
5print(by_habitat['population'].mean())   # Average population
6print(by_habitat['population'].sum())    # Sum
7print(by_habitat.size())                 # Number of species
8
9# Multiple aggregations
10stats = df.groupby('habitat').agg({
11    'population': ['mean', 'sum', 'count'],
12    'endangered': 'sum'
13})

Sorting

1# Sort by values
2df_sorted = df.sort_values('population', ascending=False)
3
4# Sort by multiple columns
5df_sorted = df.sort_values(['habitat', 'population'])
6
7# Sort by index
8df_sorted = df.sort_index()

Merging DataFrames

1# Merge (like SQL JOIN)
2df_merged = pd.merge(
3    df_species,
4    df_habitats,
5    on='habitat_id',
6    how='left'  # 'inner', 'outer', 'right'
7)
8
9# Concat - combining rows/columns
10df_combined = pd.concat([df1, df2], axis=0)  # Rows
11df_combined = pd.concat([df1, df2], axis=1)  # Columns

Saving Data

1# To CSV
2df.to_csv('output.csv', index=False)
3
4# To Excel
5df.to_excel('output.xlsx', index=False)
6
7# To JSON
8df.to_json('output.json', orient='records')