Complete Exploratory Data Analysis

# Data type conversion function
def convert_string_to_numeric(value):
    """Convert string values like '4.1M' to numeric"""
    if pd.isna(value):
        return value
    
    value = str(value).strip()
    
    if value.endswith('M'):
        return float(value[:-1]) * 1_000_000
    elif value.endswith('K'):
        return float(value[:-1]) * 1_000
    else:
        try:
            return float(value)
        except ValueError:
            return np.nan

def fix_data_types(df):
    """Fix data type issues identified in the analysis"""
    df_fixed = df.copy()
    
    # Convert accepted and submissions columns
    df_fixed['accepted_numeric'] = df_fixed['accepted'].apply(convert_string_to_numeric)
    df_fixed['submissions_numeric'] = df_fixed['submissions'].apply(convert_string_to_numeric)
    
    # Verify conversions
    print("Data type conversions completed:")
    print(f"accepted_numeric dtype: {df_fixed['accepted_numeric'].dtype}")
    print(f"submissions_numeric dtype: {df_fixed['submissions_numeric'].dtype}")
    
    return df_fixed

# Execute the solution
df_final = fix_data_types(df_cleaned)

# Outlier detection using IQR method
def detect_outliers_iqr(df, column):
    """Detect outliers using IQR method"""
    Q1 = df[column].quantile(0.25)
    Q3 = df[column].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5 * IQR
    upper_bound = Q3 + 1.5 * IQR
    
    outliers = df[(df[column] < lower_bound) | (df[column] > upper_bound)]
    return outliers, lower_bound, upper_bound

# Detect outliers in key numeric columns
numeric_columns = ['likes', 'dislikes', 'acceptance_rate', 'frequency']
outlier_summary = {}

for col in numeric_columns:
    outliers, lower, upper = detect_outliers_iqr(df_final, col)
    outlier_summary[col] = {
        'count': len(outliers),
        'percentage': (len(outliers) / len(df_final)) * 100,
        'bounds': (lower, upper)
    }
    
print("Outlier Summary:")
for col, info in outlier_summary.items():
    print(f"{col}: {info['count']} outliers ({info['percentage']:.2f}%)")

# Handle outliers using capping method
def handle_outliers(df, method='cap'):
    """Handle outliers using specified method"""
    df_handled = df.copy()
    
    if method == 'cap':
        # Cap outliers at 95th percentile
        for col in numeric_columns:
            upper_cap = df[col].quantile(0.95)
            df_handled[col] = df_handled[col].clip(upper=upper_cap)
    
    return df_handled

# Apply capping method (preserves all data)
df_no_outliers = handle_outliers(df_final, method='cap')
print(f"Shape after handling outliers: {df_no_outliers.shape}")

# Question 12: Top 20 companies
companies_expanded = df_final['companies'].str.split(',').explode()
top_companies = companies_expanded.value_counts().head(20)
plt.figure(figsize=(12, 8))
top_companies.plot(kind='barh')
plt.title('Top 20 Companies by Number of Problems')
plt.xlabel('Number of Problems')
plt.tight_layout()
plt.show()

# Question 13: Difficulty distribution
plt.figure(figsize=(8, 6))
difficulty_counts = df_final['difficulty'].value_counts()
plt.bar(difficulty_counts.index, difficulty_counts.values)
plt.title('Distribution of Problem Difficulty')
plt.xlabel('Difficulty Level')
plt.ylabel('Number of Problems')
plt.show()

# Question 14: Acceptance rate distribution
plt.figure(figsize=(8, 6))
plt.hist(df_final['acceptance_rate'], bins=30, alpha=0.7)
plt.title('Distribution of Acceptance Rate')
plt.xlabel('Acceptance Rate (%)')
plt.ylabel('Frequency')
plt.show()

# Question 15: Frequency distribution
plt.figure(figsize=(8, 6))
plt.hist(df_final['frequency'], bins=30, alpha=0.7)
plt.title('Distribution of Problem Frequency')
plt.xlabel('Frequency')
plt.ylabel('Count')
plt.show()

# Question 16: Likes vs dislikes scatter plot
plt.figure(figsize=(8, 6))
plt.scatter(df_final['likes'], df_final['dislikes'], alpha=0.6)
plt.xlabel('Likes')
plt.xlabel('Likes')
plt.ylabel('Dislikes')
plt.title('Likes vs Dislikes Scatter Plot')
plt.show()

# Question 17: Accepted vs submissions scatter plot
plt.figure(figsize=(8, 6))
plt.scatter(df_final['accepted_numeric'], df_final['submissions_numeric'], alpha=0.6)
plt.xlabel('Accepted')
plt.ylabel('Submissions')
plt.title('Accepted vs Submissions Scatter Plot')
plt.show()

# Question 18: Average acceptance rate by difficulty
avg_acceptance = df_final.groupby('difficulty')['acceptance_rate'].mean().sort_values(ascending=False)
plt.figure(figsize=(8, 6))
avg_acceptance.plot(kind='bar')
plt.title('Average Acceptance Rate by Difficulty')
plt.xlabel('Difficulty')
plt.ylabel('Average Acceptance Rate (%)')
plt.xticks(rotation=45)
plt.show()

# Question 19: Acceptance rate vs frequency scatter plot
plt.figure(figsize=(8, 6))
plt.scatter(df_final['acceptance_rate'], df_final['frequency'], alpha=0.6)
plt.xlabel('Acceptance Rate (%)')
plt.ylabel('Frequency')
plt.title('Acceptance Rate vs Frequency')
plt.show()

# Complete EDA Solution for LeetCode Dataset
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# Load your dataset
# leetcode_df = pd.read_csv("your_leetcode_dataset.csv")

# Step 1: Handle missing data
df_cleaned = handle_missing_data(leetcode_df)

# Step 2: Fix data types
df_final = fix_data_types(df_cleaned)

# Step 3: Handle outliers
df_no_outliers = handle_outliers(df_final, method='cap')

# Step 4: Generate all visualizations
# (Use the visualization code from the previous tab)

# Step 5: Final analysis summary
print("🎯 EDA Assignment Complete!")
print(f"Final dataset shape: {df_no_outliers.shape}")
print(f"Missing data: {df_no_outliers.isnull().sum().sum()}")
print(f"Ready for advanced analysis and modeling!")