Key Code Examples
1. Pandas — Data Cleaning Pipeline
import pandas as pd
import numpy as np
# Load data
df = pd.read_csv("sales_data.csv")
# Explore
print(df.shape) # Dimensions
print(df.dtypes) # Column types
print(df.describe()) # Statistics
print(df.isnull().sum()) # Missing values per column
# Handle missing values
df["price"].fillna(df["price"].median(), inplace=True)
df["category"].fillna("Unknown", inplace=True)
# Remove duplicates
df.drop_duplicates(inplace=True)
# Type conversion
df["date"] = pd.to_datetime(df["date"])
df["price"] = pd.to_numeric(df["price"], errors="coerce")
# Feature engineering
df["revenue"] = df["price"] * df["quantity"]
df["month"] = df["date"].dt.month
df["day_of_week"] = df["date"].dt.day_name()
# Filter and group
high_value = df[df["revenue"] > 10000]
monthly_revenue = df.groupby("month")["revenue"].sum()
2. NumPy — Statistical Analysis
import numpy as np
# Generate sample data
np.random.seed(42)
data = np.random.normal(loc=100, scale=15, size=1000)
# Descriptive statistics
print(f"Mean: {np.mean(data):.2f}") # 100.05
print(f"Median: {np.median(data):.2f}") # 100.12
print(f"Std Dev:{np.std(data):.2f}") # 14.89
print(f"Variance:{np.var(data):.2f}") # 221.67
print(f"Min: {np.min(data):.2f}")
print(f"Max: {np.max(data):.2f}")
print(f"Range: {np.ptp(data):.2f}")
# Percentiles
percentiles = [5, 25, 50, 75, 95]
values = np.percentile(data, percentiles)
for p, v in zip(percentiles, values):
print(f"P{p}: {v:.2f}")
# Z-score normalization
z_scores = (data - np.mean(data)) / np.std(data)
print(f"Z-scores mean: {np.mean(z_scores):.4f}") # ~0
print(f"Z-scores std: {np.std(z_scores):.4f}") # ~1
# Correlation
x = np.array([1, 2, 3, 4, 5, 6, 7, 8])
y = np.array([2, 4, 5, 4, 5, 8, 7, 9])
correlation = np.corrcoef(x, y)[0, 1]
print(f"Correlation: {correlation:.4f}") # 0.87
3. Linear Regression from Scratch
import numpy as np
class LinearRegression:
"""Simple linear regression using gradient descent."""
def __init__(self, learning_rate=0.01, n_iterations=1000):
self.lr = learning_rate
self.n_iter = n_iterations
self.weights = None
self.bias = None
self.cost_history = []
def fit(self, X, y):
n_samples, n_features = X.shape
self.weights = np.zeros(n_features)
self.bias = 0
for i in range(self.n_iter):
# Forward pass
y_pred = np.dot(X, self.weights) + self.bias
# Compute gradients
dw = (1 / n_samples) * np.dot(X.T, (y_pred - y))
db = (1 / n_samples) * np.sum(y_pred - y)
# Update parameters
self.weights -= self.lr * dw
self.bias -= self.lr * db
# Track cost (MSE)
cost = np.mean((y_pred - y) ** 2)
self.cost_history.append(cost)
def predict(self, X):
return np.dot(X, self.weights) + self.bias
# Example usage
X = np.array([[1], [2], [3], [4], [5]], dtype=float)
y = np.array([2, 4, 5, 4, 5], dtype=float)
model = LinearRegression(learning_rate=0.1, n_iterations=1000)
model.fit(X, y)
predictions = model.predict(np.array([[6], [7]]))
print(f"Predictions for [6, 7]: {predictions}")
print(f"Final weights: {model.weights}, bias: {model.bias}")
4. K-Means Clustering from Scratch
import numpy as np
class KMeans:
"""K-Means clustering algorithm."""
def __init__(self, k=3, max_iters=100):
self.k = k
self.max_iters = max_iters
def fit(self, X):
n_samples, n_features = X.shape
# Initialize centroids randomly from data points
random_idx = np.random.choice(n_samples, self.k, replace=False)
self.centroids = X[random_idx].copy()
for _ in range(self.max_iters):
# Assign each point to nearest centroid
distances = self._compute_distances(X)
self.labels = np.argmin(distances, axis=1)
# Update centroids
new_centroids = np.array([
X[self.labels == j].mean(axis=0)
if np.sum(self.labels == j) > 0
else self.centroids[j]
for j in range(self.k)
])
# Check convergence
if np.allclose(self.centroids, new_centroids):
break
self.centroids = new_centroids
def _compute_distances(self, X):
"""Euclidean distance from each point to each centroid."""
distances = np.zeros((X.shape[0], self.k))
for j in range(self.k):
distances[:, j] = np.linalg.norm(X - self.centroids[j], axis=1)
return distances
def predict(self, X):
distances = self._compute_distances(X)
return np.argmin(distances, axis=1)
# Example
X = np.array([[1, 2], [1, 4], [1, 0],
[10, 2], [10, 4], [10, 0]])
kmeans = KMeans(k=2)
kmeans.fit(X)
print(f"Labels: {kmeans.labels}") # [0 0 0 1 1 1]
print(f"Centroids:\n{kmeans.centroids}") # [[1,2], [10,2]]
5. Decision Tree Classifier
import numpy as np
from collections import Counter
class DecisionTree:
"""Simple decision tree using Gini impurity."""
def __init__(self, max_depth=5):
self.max_depth = max_depth
def fit(self, X, y, depth=0):
n_samples, n_features = X.shape
n_classes = len(np.unique(y))
# Stopping criteria
if depth >= self.max_depth or n_classes == 1:
self.prediction = Counter(y).most_common(1)[0][0]
return
# Find best split
best_gini = 1.0
best_feature = None
best_threshold = None
for feature in range(n_features):
thresholds = np.unique(X[:, feature])
for threshold in thresholds:
left_mask = X[:, feature] <= threshold
right_mask = ~left_mask
if np.sum(left_mask) == 0 or np.sum(right_mask) == 0:
continue
gini = self._gini(y[left_mask], y[right_mask],
n_samples)
if gini < best_gini:
best_gini = gini
best_feature = feature
best_threshold = threshold
if best_feature is None:
self.prediction = Counter(y).most_common(1)[0][0]
return
# Split data
left_mask = X[:, best_feature] <= best_threshold
self.feature = best_feature
self.threshold = best_threshold
self.left = DecisionTree(self.max_depth)
self.right = DecisionTree(self.max_depth)
self.left.fit(X[left_mask], y[left_mask], depth + 1)
self.right.fit(X[~left_mask], y[~left_mask], depth + 1)
def _gini(self, left, right, total):
def gini_impurity(y):
counts = Counter(y)
return 1 - sum((c / len(y))**2 for c in counts.values())
w_left = len(left) / total
w_right = len(right) / total
return w_left * gini_impurity(left) + w_right * gini_impurity(right)
def predict_one(self, x):
if hasattr(self, 'prediction'):
return self.prediction
if x[self.feature] <= self.threshold:
return self.left.predict_one(x)
return self.right.predict_one(x)
def predict(self, X):
return np.array([self.predict_one(x) for x in X])
6. Matplotlib — Data Visualization
import matplotlib.pyplot as plt
import numpy as np
# Sample data
months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun']
sales_2025 = [15000, 18000, 22000, 20000, 25000, 28000]
sales_2024 = [12000, 14000, 19000, 17000, 21000, 24000]
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
# 1. Line plot — Sales trend
axes[0, 0].plot(months, sales_2025, 'o-', label='2025', color='blue')
axes[0, 0].plot(months, sales_2024, 's--', label='2024', color='orange')
axes[0, 0].set_title('Monthly Sales Comparison')
axes[0, 0].legend()
axes[0, 0].set_ylabel('Sales ($)')
# 2. Histogram — Distribution
data = np.random.normal(100, 15, 1000)
axes[0, 1].hist(data, bins=30, edgecolor='black', alpha=0.7)
axes[0, 1].set_title('Distribution of Values')
axes[0, 1].axvline(np.mean(data), color='red', linestyle='--', label='Mean')
# 3. Scatter plot — Correlation
x = np.random.randn(100)
y = 2 * x + np.random.randn(100) * 0.5
axes[1, 0].scatter(x, y, alpha=0.6)
axes[1, 0].set_title('Correlation Plot')
# 4. Bar chart — Category comparison
categories = ['A', 'B', 'C', 'D']
values = [23, 45, 56, 78]
axes[1, 1].bar(categories, values, color=['red', 'blue', 'green', 'purple'])
axes[1, 1].set_title('Category Comparison')
plt.tight_layout()
plt.savefig('analysis.png', dpi=150)
plt.show()
7. SQL for Data Analysis
-- Revenue analysis with window functions
WITH monthly_revenue AS (
SELECT
DATE_TRUNC('month', order_date) AS month,
SUM(amount) AS revenue,
COUNT(*) AS orders
FROM orders
WHERE order_date >= '2025-01-01'
GROUP BY 1
)
SELECT
month,
revenue,
orders,
ROUND(revenue::NUMERIC / orders, 2) AS avg_order_value,
SUM(revenue) OVER (ORDER BY month) AS running_total,
revenue - LAG(revenue) OVER (ORDER BY month) AS revenue_change,
ROUND(
(revenue - LAG(revenue) OVER (ORDER BY month))::NUMERIC /
LAG(revenue) OVER (ORDER BY month) * 100, 2
) AS pct_growth
FROM monthly_revenue
ORDER BY month;
-- Customer segmentation (RFM Analysis)
SELECT
customer_id,
NTILE(5) OVER (ORDER BY recency_days) AS recency_score,
NTILE(5) OVER (ORDER BY frequency) AS frequency_score,
NTILE(5) OVER (ORDER BY monetary) AS monetary_score
FROM (
SELECT
customer_id,
CURRENT_DATE - MAX(order_date)::DATE AS recency_days,
COUNT(*) AS frequency,
SUM(amount) AS monetary
FROM orders
GROUP BY customer_id
) rfm_base;
8. Scikit-learn — Complete ML Pipeline
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.pipeline import Pipeline
import pandas as pd
# Load and prepare data
df = pd.read_csv("churn_data.csv")
X = df.drop("churn", axis=1)
y = df["churn"]
# Encode categorical variables
X = pd.get_dummies(X, columns=["plan_type", "payment_method"])
# Split data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42, stratify=y
)
# Create pipeline
pipeline = Pipeline([
("scaler", StandardScaler()),
("classifier", RandomForestClassifier(
n_estimators=100,
max_depth=10,
random_state=42
))
])
# Train
pipeline.fit(X_train, y_train)
# Evaluate
train_score = pipeline.score(X_train, y_train)
test_score = pipeline.score(X_test, y_test)
print(f"Train accuracy: {train_score:.4f}")
print(f"Test accuracy: {test_score:.4f}")
y_pred = pipeline.predict(X_test)
print(classification_report(y_test, y_pred))
# Feature importance
rf = pipeline.named_steps["classifier"]
importances = pd.Series(
rf.feature_importances_,
index=X.columns
).sort_values(ascending=False)
print("\nTop 5 Features:")
print(importances.head())
9. Time Series Analysis — ARIMA
import pandas as pd
import numpy as np
from statsmodels.tsa.arima.model import ARIMA
from statsmodels.tsa.stattools import adfuller
# Load time series data
df = pd.read_csv("monthly_sales.csv", parse_dates=["date"])
df.set_index("date", inplace=True)
# Check stationarity
result = adfuller(df["sales"])
print(f"ADF Statistic: {result[0]:.4f}")
print(f"p-value: {result[1]:.4f}")
# p < 0.05 → stationary
# Differencing if not stationary
df["sales_diff"] = df["sales"].diff()
# Fit ARIMA model (p=1, d=1, q=1)
model = ARIMA(df["sales"], order=(1, 1, 1))
fitted = model.fit()
print(fitted.summary())
# Forecast next 12 months
forecast = fitted.forecast(steps=12)
conf_int = fitted.get_forecast(steps=12).conf_int()
print("\n12-Month Forecast:")
for date, value in forecast.items():
print(f" {date.strftime('%Y-%m')}: ${value:,.0f}")
# Model diagnostics
residuals = fitted.resid
print(f"\nResidual mean: {residuals.mean():.4f}")
print(f"Residual std: {residuals.std():.4f}")
10. Natural Language Processing — Text Classification
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
# Sample data
texts = [
"Great product, highly recommend",
"Terrible quality, do not buy",
"Amazing customer service",
"Waste of money",
"Best purchase ever",
"Disappointed with the results",
"Excellent value for money",
"Poor packaging and slow delivery",
]
labels = ["positive", "negative", "positive", "negative",
"positive", "negative", "positive", "negative"]
# Build pipeline
pipeline = Pipeline([
("tfidf", TfidfVectorizer(
max_features=1000,
ngram_range=(1, 2),
stop_words="english"
)),
("classifier", MultinomialNB())
])
# Train
pipeline.fit(texts, labels)
# Predict
test_texts = ["This is wonderful", "Awful experience"]
predictions = pipeline.predict(test_texts)
print(dict(zip(test_texts, predictions)))
# {'This is wonderful': 'positive', 'Awful experience': 'negative'}
Core Concepts
The Data Science Lifecycle
Definition: The end-to-end process of extracting value from data consists of:
- Problem Definition: Understand the business question
- Data Collection: Gather data from databases, APIs, files, sensors
- Data Cleaning: Handle missing values, outliers, duplicates, type errors
- Exploratory Data Analysis (EDA): Visualize distributions, correlations, patterns
- Feature Engineering: Create/transform features for modeling
- Model Selection & Training: Choose algorithm, split data, train
- Evaluation: Measure performance with appropriate metrics
- Deployment: Put model into production
- Monitoring: Track model drift, retrain as needed
Supervised vs. Unsupervised Learning
Supervised Learning: Model learns from labeled data (input → known output).
- Classification: Predict a category (spam/not spam, disease/no disease)
- Regression: Predict a continuous value (price, temperature)
- Algorithms: Linear Regression, Logistic Regression, Decision Trees, SVM, Neural Networks
Unsupervised Learning: Model finds patterns in unlabeled data.
- Clustering: Group similar items (customer segmentation, document grouping)
- Dimensionality Reduction: Reduce features (PCA, t-SNE)
- Algorithms: K-Means, DBSCAN, Hierarchical Clustering, Autoencoders
Statistical Foundations
- Mean (μ): Average = Σx / n
- Median: Middle value when sorted
- Mode: Most frequent value
- Variance (σ²): Average squared deviation from mean = Σ(x-μ)² / n
- Standard Deviation (σ): √Variance — spread of data
- Correlation (r): Measures linear relationship, -1 ≤ r ≤ 1
- p-value: Probability of observing results by chance; p < 0.05 typically significant
- Central Limit Theorem: Sample means approach normal distribution as n increases
Bias-Variance Tradeoff
Definition: The balance between a model's ability to fit training data (low bias) and generalize to unseen data (low variance).
- High Bias (Underfitting): Model too simple, misses patterns. Fix: more features, more complex model
- High Variance (Overfitting): Model too complex, memorizes noise. Fix: more data, regularization, simpler model
- Bias-Variance Decomposition: Total Error = Bias² + Variance + Irreducible Error
- Cross-Validation: k-fold CV helps estimate generalization error
Feature Engineering
Definition: The process of transforming raw data into meaningful features that improve model performance.
- Scaling: StandardScaler (z-score: (x-μ)/σ), MinMaxScaler (range [0,1])
- Encoding: One-hot encoding, label encoding, target encoding
- Binning: Convert continuous to categorical (age groups)
- Polynomial Features: Create interaction terms (x₁, x₂, x₁x₂, x₁²)
- Domain Features: Day-of-week, is-weekend, time-since-last-purchase
Model Evaluation Metrics
Classification:
- Accuracy: (TP+TN) / (TP+TN+FP+FN)
- Precision: TP / (TP+FP) — "Of predicted positives, how many are correct?"
- Recall (Sensitivity): TP / (TP+FN) — "Of actual positives, how many did we find?"
- F1-Score: 2 × (Precision × Recall) / (Precision + Recall)
- AUC-ROC: Area under the ROC curve (true positive rate vs. false positive rate)
Regression:
- MAE: Mean Absolute Error = Σ|ŷ - y| / n
- MSE: Mean Squared Error = Σ(ŷ - y)² / n
- RMSE: √MSE — penalizes large errors more
- R² (R-squared): Proportion of variance explained = 1 - (SS_res / SS_tot)
Dimensionality Reduction
Definition: Reducing the number of input features while preserving important information.
- PCA (Principal Component Analysis): Find directions of maximum variance; project data onto them. Linear method.
- t-SNE: Non-linear; good for visualization of high-dimensional data in 2D/3D
- UMAP: Faster than t-SNE, preserves more global structure
- Feature Selection: SelectKBest, Recursive Feature Elimination (RFE), LASSO
- When to use: High-dimensional data, multicollinearity, visualization, faster training
Bayesian Thinking in Data Science
Bayes' Theorem: P(A|B) = P(B|A) × P(A) / P(B)
- Prior P(A): Initial belief before seeing data
- Likelihood P(B|A): Probability of data given hypothesis
- Posterior P(A|B): Updated belief after seeing data
- Naive Bayes: Assumes features are conditionally independent given the class
- Applications: Spam filtering, A/B testing, recommendation systems
Worked Examples
Example 1: Calculate Statistical Measures by Hand
Problem: Given dataset [12, 15, 18, 22, 25, 28, 30], find mean, median, variance, and standard deviation.
Solution:
- Mean: (12+15+18+22+25+28+30) / 7 = 150 / 7 = 21.43
- Median: Sorted already, middle (4th) value = 22
- Variance:
- Deviations: -9.43, -6.43, -3.43, 0.57, 3.57, 6.57, 8.57
- Squared: 88.90, 41.34, 11.76, 0.32, 12.74, 43.16, 73.44
- Sum = 271.67
- Variance = 271.67 / 7 = 38.81 (population) or 271.67/6 = 45.28 (sample)
- Standard Deviation: √38.81 = 6.23
Example 2: K-Means Clustering Walkthrough
Problem: Cluster points A(1,1), B(1,2), C(5,5), D(6,6) into 2 clusters.
Solution:
- Step 1: Initialize centroids: C1=(1,1), C5=(5,5)
- Step 2: Assign points to nearest centroid:
- A to C1: d=0, A to C5: d=5.66 → A→C1
- B to C1: d=1, B to C5: d=4.47 → B→C1
- C to C1: d=5.66, C to C5: d=0 → C→C5
- D to C1: d=7.07, D to C5: d=1 → D→C5
- Step 3: Update centroids:
- C1 = midpoint(A,B) = (1, 1.5)
- C5 = midpoint(C,D) = (5.5, 5.5)
- Step 4: Re-assign (same result → converged)
- Final Clusters: {A, B} and {C, D}
Example 3: Linear Regression Calculation
Problem: Fit y = wx + b to points (1,2), (2,3), (3,5), (4,4).
Solution using Least Squares:
- n = 4, Σx = 10, Σy = 14, Σxy = 37, Σx² = 30
- w = (nΣxy - ΣxΣy) / (nΣx² - (Σx)²) = (4×37 - 10×14) / (4×30 - 100) = 8/20 = 0.4
- b = (Σy - wΣx) / n = (14 - 0.4×10) / 4 = 10/4 = 2.5
- Model: y = 0.4x + 2.5
- Predictions: x=1→2.9, x=2→3.3, x=3→3.7, x=4→4.1
Example 4: SQL Window Function Analysis
Problem: Given a sales table, calculate running total and moving average per month.
-- Given: sales(month, amount)
-- month | amount
-- 2025-01 | 100
-- 2025-02 | 150
-- 2025-03 | 120
-- 2025-04 | 200
SELECT
month,
amount,
SUM(amount) OVER (ORDER BY month) AS running_total,
ROUND(AVG(amount) OVER (
ORDER BY month ROWS BETWEEN 2 PRECEDING AND CURRENT ROW
), 2) AS moving_avg_3
FROM sales ORDER BY month;
-- Result:
-- 2025-01 | 100 | 100 | 100.00
-- 2025-02 | 150 | 250 | 125.00
-- 2025-03 | 120 | 370 | 123.33
-- 2025-04 | 200 | 570 | 156.67
Example 5: Hypothesis Testing — A/B Test
Problem: A website tests two button colors. Version A: 200 clicks / 1000 views (20%). Version B: 240 clicks / 1000 views (24%). Is B significantly better?
Solution (Two-proportion z-test):
- H₀: p_A = p_B (no difference)
- H₁: p_A ≠ p_B (significant difference)
- Pooled proportion: p̂ = (200+240) / (1000+1000) = 0.22
- Standard error: SE = √(0.22×0.78 × (1/1000 + 1/1000)) = 0.0186
- z = (0.24 - 0.20) / 0.0186 = 2.15
- p-value for z=2.15 (two-tailed) ≈ 0.031
- Since p < 0.05: Reject H₀ — Version B is significantly better
Practice Problems
Problem 1: Pandas Data Manipulation
Given a DataFrame with columns [name, department, salary], write code to find the average salary per department.
import pandas as pd
df = pd.DataFrame({
"name": ["Alice", "Bob", "Charlie", "Diana", "Eve"],
"dept": ["Eng", "Eng", "HR", "HR", "Eng"],
"salary": [90000, 85000, 70000, 75000, 95000]
})
# Average salary per department
avg_by_dept = df.groupby("dept")["salary"].mean()
print(avg_by_dept)
# Eng: 90000, HR: 72500
# Additional: departments with avg salary > 80000
high_paying = df.groupby("dept")["salary"].mean()
print(high_paying[high_paying > 80000]) # Eng only
Problem 2: NumPy Array Operations
Create a 5×5 identity matrix, a random 5×5 matrix, and compute their dot product.
import numpy as np
# Identity matrix
I = np.eye(5)
print("Identity:\n", I)
# Random matrix
A = np.random.rand(5, 5)
print("Random:\n", A.round(3))
# Dot product
result = np.dot(I, A)
print("I × A =\n", result.round(3))
# Same as A — identity matrix property!
# Matrix multiplication vs element-wise
print("A × A (element-wise):\n", (A * A).round(3))
print("A @ A (matrix mult):\n", (A @ A).round(3))
Problem 3: Implement Gradient Descent
Write a function that performs gradient descent to minimize f(x) = x² + 5x + 6.
def gradient_descent(learning_rate=0.1, n_iterations=100, x_start=0):
"""Minimize f(x) = x² + 5x + 6 using gradient descent.
f'(x) = 2x + 5"""
x = x_start
history = [x]
for i in range(n_iterations):
gradient = 2 * x + 5 # derivative
x = x - learning_rate * gradient
history.append(x)
if i % 20 == 0:
f_x = x**2 + 5*x + 6
print(f"Iter {i}: x={x:.4f}, f(x)={f_x:.4f}")
return x, history
# Minimum of f(x) is at x = -b/2a = -5/2 = -2.5
x_min, history = gradient_descent()
print(f"\nConverged to x = {x_min:.4f} (expected -2.5)")
print(f"f(-2.5) = {(-2.5)**2 + 5*(-2.5) + 6}") # -0.25
Problem 4: Feature Scaling
Given a dataset with features on different scales, implement standard scaling and min-max scaling.
import numpy as np
data = np.array([
[1000, 2, 50],
[2000, 5, 30],
[1500, 3, 40],
[3000, 8, 20],
])
# Standard Scaling (Z-score): (x - mean) / std
def standard_scale(X):
mean = X.mean(axis=0)
std = X.std(axis=0)
return (X - mean) / std
# Min-Max Scaling: (x - min) / (max - min)
def min_max_scale(X):
min_val = X.min(axis=0)
max_val = X.max(axis=0)
return (X - min_val) / (max_val - min_val)
print("Original:\n", data)
print("Standard Scaled:\n", standard_scale(data).round(3))
print("Min-Max Scaled:\n", min_max_scale(data).round(3))
Problem 5: SQL — Find Duplicates
Write a SQL query to find duplicate email addresses in a users table.
SELECT email, COUNT(*) as count
FROM users
GROUP BY email
HAVING COUNT(*) > 1
ORDER BY count DESC;
-- Find all rows with duplicate emails
SELECT u.*
FROM users u
INNER JOIN (
SELECT email
FROM users
GROUP BY email
HAVING COUNT(*) > 1
) dup ON u.email = dup.email
ORDER BY u.email;
Problem 6: Implement PCA from Scratch
Write a function that performs PCA on a dataset and returns the principal components.
import numpy as np
def pca(X, n_components=2):
"""Principal Component Analysis from scratch."""
# Step 1: Center the data
X_centered = X - X.mean(axis=0)
# Step 2: Compute covariance matrix
cov_matrix = np.cov(X_centered, rowvar=False)
# Step 3: Eigendecomposition
eigenvalues, eigenvectors = np.linalg.eigh(cov_matrix)
# Step 4: Sort by eigenvalue (descending)
idx = np.argsort(eigenvalues)[::-1]
eigenvalues = eigenvalues[idx]
eigenvectors = eigenvectors[:, idx]
# Step 5: Select top n_components
components = eigenvectors[:, :n_components]
# Step 6: Project data
X_transformed = X_centered @ components
# Explained variance ratio
explained_variance = eigenvalues[:n_components] / eigenvalues.sum()
return X_transformed, explained_variance, components
# Example
np.random.seed(42)
X = np.random.randn(100, 5)
X_proj, var_ratio, _ = pca(X, n_components=2)
print(f"Explained variance: {var_ratio}")
print(f"Original shape: {X.shape}, Reduced shape: {X_proj.shape}")
Problem 7: Cross-Validation Implementation
Implement k-fold cross-validation from scratch.
import numpy as np
def k_fold_split(X, y, k=5):
"""Generate k-fold train/test indices."""
n = len(X)
indices = np.arange(n)
np.random.shuffle(indices)
fold_size = n // k
folds = []
for i in range(k):
test_start = i * fold_size
test_end = test_start + fold_size if i < k-1 else n
test_idx = indices[test_start:test_end]
train_idx = np.concatenate([indices[:test_start], indices[test_end:]])
folds.append((train_idx, test_idx))
return folds
def cross_validate(X, y, model_class, k=5):
"""Perform k-fold cross-validation."""
scores = []
for fold, (train_idx, test_idx) in enumerate(k_fold_split(X, y, k)):
model = model_class()
model.fit(X[train_idx], y[train_idx])
score = model.score(X[test_idx], y[test_idx])
scores.append(score)
print(f"Fold {fold+1}: accuracy = {score:.4f}")
print(f"Mean: {np.mean(scores):.4f} ± {np.std(scores):.4f}")
return scores
Problem 8: Feature Importance
Given a trained RandomForestClassifier, extract and plot the top 10 feature importances.
import matplotlib.pyplot as plt
def plot_feature_importance(model, feature_names, top_n=10):
importances = model.feature_importances_
indices = np.argsort(importances)[::-1][:top_n]
plt.figure(figsize=(10, 6))
plt.barh(range(top_n), importances[indices], align="center")
plt.yticks(range(top_n), [feature_names[i] for i in indices])
plt.xlabel("Feature Importance")
plt.title("Top 10 Most Important Features")
plt.gca().invert_yaxis()
plt.tight_layout()
plt.show()
# Also return as dict
return {feature_names[i]: importances[i] for i in indices}
Problem 9: Time Series Decomposition
Decompose a time series into trend, seasonal, and residual components.
from statsmodels.tsa.seasonal import seasonal_decompose
import pandas as pd
# Load data
df = pd.read_csv("monthly_sales.csv", parse_dates=["date"])
df.set_index("date", inplace=True)
# Decomposition (additive model)
decomposition = seasonal_decompose(
df["sales"], model="additive", period=12
)
# Plot components
fig, axes = plt.subplots(4, 1, figsize=(12, 8))
decomposition.observed.plot(ax=axes[0], title="Observed")
decomposition.trend.plot(ax=axes[1], title="Trend")
decomposition.seasonal.plot(ax=axes[2], title="Seasonal")
decomposition.resid.plot(ax=axes[3], title="Residual")
plt.tight_layout()
plt.show()
# Check if residual is random (good decomposition)
residual_std = decomposition.resid.dropna().std()
print(f"Residual std: {residual_std:.2f}")
Problem 10: A/B Test Sample Size
Calculate the required sample size for an A/B test with 5% baseline conversion, minimum detectable effect of 10%, 80% power, 5% significance.
import math
from scipy import stats
def sample_size_per_variant(baseline, mde, alpha=0.05, power=0.8):
"""Calculate sample size per variant for two-proportion z-test."""
p1 = baseline
p2 = baseline * (1 + mde)
z_alpha = stats.norm.ppf(1 - alpha/2) # 1.96
z_beta = stats.norm.ppf(power) # 0.84
p_bar = (p1 + p2) / 2
numerator = (z_alpha * math.sqrt(2 * p_bar * (1 - p_bar)) +
z_beta * math.sqrt(p1*(1-p1) + p2*(1-p2)))**2
denominator = (p2 - p1)**2
return math.ceil(numerator / denominator)
# Example: 5% baseline, 10% relative improvement
n = sample_size_per_variant(baseline=0.05, mde=0.10)
print(f"Sample size per variant: {n}") # ~15,700
print(f"Total sample size: {2 * n}") # ~31,400