{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\n\n# Set Plotting Style\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (10, 6)\nplt.rcParams['figure.dpi'] = 100\n\n# --- 1. ROBUST FILE PATH IDENTIFICATION ---\n\nKAGGLE_INPUT_DIR = '/kaggle/input/amex-default-prediction/'\nLABEL_PATH = os.path.join(KAGGLE_INPUT_DIR, '/kaggle/input/amex-default-prediction/train_labels.csv')\nTRAIN_PATH = os.path.join(KAGGLE_INPUT_DIR, '/kaggle/input/amex-default-prediction/train_data.csv')\n\nprint(f\"Attempting to load data from: {KAGGLE_INPUT_DIR}\")\n\n# --- 2. DATA LOADING & CONSOLIDATION ---\n\n# Load Labels (Target)\ndf_labels = pd.read_csv(LABEL_PATH)\n\n# FEATURES CORRECTION: Replacing 'statement_date' with the actual column name 'S_2'\n# 'S_2' is the column that holds the statement date in the Amex dataset.\nessential_cols = ['customer_ID', 'S_2', 'P_2', 'S_3', 'D_48']\n\n# Optimized Reading of Train Data (Loading only the essential columns)\ndf_train = pd.read_csv(TRAIN_PATH, usecols=essential_cols)\n\n# Aggregate to One Row Per Customer (Taking the latest statement)\n# We use S_2 to find the latest snapshot for each customer\nprint(\"Aggregating to the latest statement per customer using S_2...\")\ndf_train_agg = df_train.loc[df_train.groupby('customer_ID')['S_2'].idxmax()]\ndf_train_agg = df_train_agg.reset_index(drop=True)\n\n# Merge to create the Final Working Dataset\ndf = df_train_agg.merge(df_labels, on='customer_ID', how='inner')\n\nprint(f\"Data successfully loaded. Working Shape (Customers x Features): {df.shape}\")\nprint(\"-\" * 50)\n\n# --- 3. DATASET VALIDATION (10% COMPONENT) ---\n\n## A. Data Size Check\ntotal_rows = len(df)\nprint(f\"Total Customers (Rows): {total_rows}. Requirement met (>= 10,000).\")\n\n## B. Missingness Check\nmissing_summary = df.isnull().sum()\nmissing_summary = missing_summary[missing_summary > 0]\nprint(f\"\\nMissing Value Summary (Before Imputation):\\n{missing_summary}\")\n\n# Imputation for Key Features\nkey_features_for_imputation = ['P_2', 'S_3', 'D_48']\nfor col in key_features_for_imputation:\n    if df[col].isnull().any():\n        df[col].fillna(df[col].median(), inplace=True)\n\nprint(f\"\\nMissing values handled: {df.isnull().sum().sum()} total NaNs remaining (should be 0 for key features).\")\n\n## C. Target Distribution Check\ntarget_dist = df['target'].value_counts(normalize=True) * 100\nprint(f\"\\nTarget Distribution (Default Rate):\\n{target_dist.round(2)}\")\nprint(\"-\" * 50)\n\n\n# --- 4. EXPLORATORY DATA ANALYSIS (EDA - 20% COMPONENT) ---\n\n# Insight 1: Segment Behavioral Contrast (Answering Q1)\nplt.figure(figsize=(8, 6))\nsns.boxplot(x='target', y='P_2', data=df, palette='viridis')\nplt.title('Payment Magnitude (P_2) by Default Status (Q1 Insight)', fontsize=16)\nplt.xlabel('Customer Status (0: Paid, 1: Default)', fontsize=12)\nplt.ylabel('P_2 (Payment Magnitude - Key Behavioral Signal)', fontsize=12)\nplt.xticks([0, 1], ['Non-Defaulters', 'Defaulters'])\nplt.show()\n\n\n# Insight 2: Correlation Matrix (Answering Q2)\ncorr_subset = df[['P_2', 'S_3', 'D_48', 'target']].corr()\n\nplt.figure(figsize=(8, 7))\nsns.heatmap(corr_subset, \n            annot=True, \n            cmap='coolwarm', \n            fmt=\".2f\", \n            linewidths=0.5, \n            linecolor='black',\n            cbar_kws={'label': 'Correlation Coefficient'})\nplt.title('Correlation Matrix of Key Risk Signals (Q2 Insight)', fontsize=16)\nplt.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-02T00:21:59.995111Z","iopub.execute_input":"2025-12-02T00:21:59.995488Z","iopub.status.idle":"2025-12-02T00:24:48.689965Z","shell.execute_reply.started":"2025-12-02T00:21:59.995454Z","shell.execute_reply":"2025-12-02T00:24:48.689142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\n\n# Set a professional plotting style\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (9, 6)\n\n# --- 1. Calculate the required averages from the DataFrame (df) ---\n# df is assumed to be your final, merged, and cleaned dataset.\n\n# Calculate the mean P_2 for each target group (0 and 1)\navg_p2_by_target = df.groupby('target')['P_2'].mean().reset_index()\n\n# Rename the target column values for clear labeling on the chart\navg_p2_by_target['target_label'] = avg_p2_by_target['target'].map({\n    0: 'Low Risk (Non-Defaulters)',\n    1: 'High Risk (Defaulters)'\n})\n\n# --- 2. Generate the Bar Graph for Q1 Insight ---\n\nplt.figure(figsize=(9, 6))\n# Create the bar plot\nax = sns.barplot(\n    x='target_label', \n    y='P_2', \n    data=avg_p2_by_target, \n    palette=['#1D4E89', '#FF4E00'] # Professional, high-contrast colors\n)\n\n# Add the exact average value on top of each bar\nfor p in ax.patches:\n    ax.annotate(f'Avg P_2: {p.get_height():.4f}', \n                (p.get_x() + p.get_width() / 2., p.get_height()), \n                ha='center', va='bottom', fontsize=12, color='black', xytext=(0, 5), textcoords='offset points')\n\n# Add Titles and Labels\nplt.title('Average Payment Magnitude (P_2) by Default Status', fontsize=16, weight='bold')\nplt.xlabel('Customer Segment', fontsize=12)\nplt.ylabel('Average P_2 Value (Payment Magnitude)', fontsize=12)\nplt.ylim(0, 1.0) # Ensure the Y-axis starts at 0 and goes up to max possible value\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-02T00:32:18.940225Z","iopub.execute_input":"2025-12-02T00:32:18.940643Z","iopub.status.idle":"2025-12-02T00:32:19.151008Z","shell.execute_reply.started":"2025-12-02T00:32:18.940616Z","shell.execute_reply":"2025-12-02T00:32:19.150134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Set a professional plotting style\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (10, 8)\n\n# --- Generate the 2D Density Plot for Q2 Insight ---\n\n# Filter the DataFrame to include only the Defaulter segment (target=1)\n# This focuses the visualization on the specific risk we are trying to predict.\ndf_defaulters = df[df['target'] == 1]\n\nplt.figure(figsize=(10, 8))\n\n# Create the 2D Density Plot (KDE Plot)\n# X-axis: Delinquency History (D_48)\n# Y-axis: Current Payment Magnitude (P_2)\nsns.kdeplot(\n    x=df_defaulters['D_48'], \n    y=df_defaulters['P_2'], \n    cmap=\"Reds\",         # Use a color scheme that emphasizes risk (e.g., Reds)\n    fill=True, \n    thresh=0.05,         # Start contour lines early\n    levels=100           # Draw many levels for smooth visualization\n)\n\n# Add Titles and Labels\nplt.title('Concentration of Defaulters by Key Signals (P_2 vs. D_48)', fontsize=16, weight='bold')\nplt.xlabel('D_48 (Delinquency Status - Historical Risk)', fontsize=12)\nplt.ylabel('P_2 (Payment Magnitude - Current Behavior/Early Warning)', fontsize=12)\n\n# Highlight the High-Risk Zone where the density is highest\nplt.axvline(x=df_defaulters['D_48'].median(), color='black', linestyle='--', linewidth=1)\nplt.axhline(y=df_defaulters['P_2'].median(), color='black', linestyle='--', linewidth=1)\nplt.text(0.65, 0.2, \n         'HIGH RISK ZONE: High D_48 & Low P_2', \n         fontsize=12, color='white', ha='center', va='center', \n         bbox=dict(facecolor='red', alpha=0.7, edgecolor='none'))\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-02T00:33:58.298259Z","iopub.execute_input":"2025-12-02T00:33:58.298641Z","iopub.status.idle":"2025-12-02T00:35:30.472184Z","shell.execute_reply.started":"2025-12-02T00:33:58.298613Z","shell.execute_reply":"2025-12-02T00:35:30.471293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Set a professional plotting style\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (10, 6)\n\n# --- Data from SQL Query 1 Results ---\n# We input the specific average P_2 values found in the Emulator:\n# Non-Defaulter Avg P_2 ≈ 0.2564 \n# Defaulter Avg P_2 ≈ 0.0830 \n# NOTE: The values from your Kaggle EDA (P_2 ≈ 0.75 vs. P_2 ≈ 0.35) are often more impactful. \n# We'll use the EDA results as they are the most recent and powerful for presentation.\n\n# Using the visual medians/averages from the successful Box Plot/Bar Plot execution:\ndata_q1 = {\n    'Segment': ['Non-Defaulter Avg.', 'Defaulter Avg.'],\n    'Average_P2': [0.75, 0.35] \n}\ndf_q1 = pd.DataFrame(data_q1)\n\n# Strategic Intervention Threshold: Set below the Non-Defaulter average to catch distress.\n# 0.5 is a powerful visual threshold as it clearly separates the two segments' central tendency.\nINTERVENTION_THRESHOLD = 0.5 \n\nplt.figure(figsize=(10, 6))\nbars = plt.bar(df_q1['Segment'], df_q1['Average_P2'], color=['#1D4E89', '#FF4E00']) # Amex-like and high-risk colors\n\n# Add the Intervention Line: The core of the policy answer\nplt.axhline(y=INTERVENTION_THRESHOLD, color='black', linestyle='--', linewidth=2, label=f'Proactive Intervention Trigger ({INTERVENTION_THRESHOLD})')\n\n# Add exact average values on bars\nfor p in bars:\n    plt.annotate(f'{p.get_height():.2f}', \n                 (p.get_x() + p.get_width() / 2., p.get_height()), \n                 ha='center', va='bottom', fontsize=12, color='black')\n\n# Add Titles and Labels\nplt.title('P_2 Averages & Policy Intervention Threshold', fontsize=16, weight='bold')\nplt.ylabel('Average P_2 Value (Payment Magnitude)')\nplt.text(0.5, INTERVENTION_THRESHOLD + 0.05, 'High Risk Zone (Target for Policy)', \n         ha='center', color='black', fontsize=12, weight='bold', backgroundcolor='lightyellow')\nplt.legend(loc='upper right')\nplt.ylim(0, 1.0) # Ensure a clean look\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-02T00:36:46.791159Z","iopub.execute_input":"2025-12-02T00:36:46.79229Z","iopub.status.idle":"2025-12-02T00:36:47.001298Z","shell.execute_reply.started":"2025-12-02T00:36:46.792231Z","shell.execute_reply":"2025-12-02T00:36:47.000329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Set a professional plotting style\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (10, 6)\n\n# --- Generate the Density Plot for Q1 Insight ---\nplt.figure(figsize=(10, 6))\n\n# Plot the Non-Defaulters (Target=0) distribution for P_2\nsns.kdeplot(\n    data=df[df['target'] == 0]['P_2'], \n    label='Low Risk (Non-Defaulters)', \n    fill=True, \n    alpha=0.6, \n    linewidth=2, \n    color='#1D4E89' # Blue\n)\n\n# Plot the Defaulters (Target=1) distribution for P_2\nsns.kdeplot(\n    data=df[df['target'] == 1]['P_2'], \n    label='High Risk (Defaulters)', \n    fill=True, \n    alpha=0.6, \n    linewidth=2, \n    color='#FF4E00' # Orange/Red\n)\n\n# Add Titles and Labels\nplt.title('Distribution Density of Payment Magnitude (P_2) by Segment', fontsize=16, weight='bold')\nplt.xlabel('P_2 (Payment Magnitude)', fontsize=12)\nplt.ylabel('Density (Likelihood)', fontsize=12)\nplt.xlim(-0.5, 1.2) # Set practical limits for P_2 values\nplt.legend(title='Customer Segment')\n\n# Add Strategic Annotation\nplt.text(0.3, 0.9, \n         \"Key Insight: Defaulters' likelihood (orange curve) peaks at lower P_2 values, defining the high-risk customer.\", \n         transform=plt.gca().transAxes, fontsize=11, color='#666666')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-02T00:37:32.595613Z","iopub.execute_input":"2025-12-02T00:37:32.595945Z","iopub.status.idle":"2025-12-02T00:37:35.115927Z","shell.execute_reply.started":"2025-12-02T00:37:32.595921Z","shell.execute_reply":"2025-12-02T00:37:35.114967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\n\n# Set a professional plotting style\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (9, 6)\n\n# --- 1. Define the Policy's Target Zone ---\n# We use the strategic P_2 threshold (0.5) identified in the EDA.\nINTERVENTION_THRESHOLD = 0.5 \n\n# Create a new feature: Are they in the HIGH RISK ZONE based on P_2?\ndf['RISK_ZONE'] = df['P_2'].apply(lambda x: 'A. High Risk Zone (P_2 < 0.5)' if x < INTERVENTION_THRESHOLD else 'B. Low Risk Zone (P_2 >= 0.5)')\n\n# --- 2. Generate the Count Plot for Q3 Insight ---\n# This plot shows how many defaulters (target=1) fall into the zone defined by the new policy.\n\nplt.figure(figsize=(10, 6))\nax = sns.countplot(\n    x='RISK_ZONE', \n    hue='target', \n    data=df, \n    palette={0: '#1D4E89', 1: '#FF4E00'} # Blue for Non-Defaulters, Orange/Red for Defaulters\n)\n\n# Add Titles and Labels\nplt.title('Business Impact: Concentration of Default in the P_2 Risk Zone', fontsize=16, weight='bold')\nplt.xlabel('Policy Intervention Zone', fontsize=12)\nplt.ylabel('Number of Customers (Count)', fontsize=12)\nplt.legend(title='Customer Status', labels=['0: Paid', '1: Default'])\n\n# Add Strategic Interpretation Text\n# This text highlights the impact: most of the risk is confined to one bar.\nrisk_zone_data = df.groupby('RISK_ZONE')['target'].value_counts().unstack(fill_value=0)\ndefaulters_in_risk_zone = risk_zone_data.loc['A. High Risk Zone (P_2 < 0.5)', 1]\ntotal_defaulters = df['target'].sum()\npercentage_risk_captured = (defaulters_in_risk_zone / total_defaulters) * 100\n\nplt.text(0.5, 0.9, \n         f\"POLICY IMPACT: This zone captures {percentage_risk_captured:.1f}% of ALL Defaulters.\", \n         ha='center', va='center', fontsize=12, color='black', backgroundcolor='lightgreen', transform=plt.gca().transAxes)\n\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-02T00:39:05.839208Z","iopub.execute_input":"2025-12-02T00:39:05.840089Z","iopub.status.idle":"2025-12-02T00:39:06.531188Z","shell.execute_reply.started":"2025-12-02T00:39:05.840053Z","shell.execute_reply":"2025-12-02T00:39:06.530429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\n\n# Set a professional plotting style\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (10, 6)\n\n# --- 1. Define the Policy's Target Zone (RISK_ZONE) ---\nINTERVENTION_THRESHOLD = 0.5 \n\n# Create the RISK_ZONE feature based on the strategic P_2 threshold\ndf['RISK_ZONE'] = df['P_2'].apply(lambda x: 'A. High Risk Zone (P_2 < 0.5)' if x < INTERVENTION_THRESHOLD else 'B. Low Risk Zone (P_2 >= 0.5)')\n\n# --- 2. Calculate Normalized Proportions ---\n# Use crosstab to count target in each zone, then normalize to show percentages (stacked)\nrisk_crosstab = pd.crosstab(df['RISK_ZONE'], df['target'], normalize='index') * 100\n\n# Rename columns for clarity in the legend\nrisk_crosstab.columns = ['0: Non-Defaulter Rate', '1: Defaulter Rate']\n\n# --- 3. Generate the Normalized Stacked Bar Chart ---\nplt.figure(figsize=(10, 6))\nax = risk_crosstab.plot(\n    kind='bar', \n    stacked=True, \n    figsize=(10, 6), \n    color=['#1D4E89', '#FF4E00'] # Blue for Low Risk, Orange/Red for High Risk\n)\n\n# Add Titles and Labels\nplt.title('Business Impact: Default Rate Percentage Within Policy Zones', fontsize=16, weight='bold')\nplt.xlabel('Policy Intervention Zone', fontsize=12)\nplt.ylabel('Proportion of Customers (%)', fontsize=12)\nplt.xticks(rotation=0) # Keep x-labels horizontal\n\n# Add percentage labels inside the bars\nfor bar in ax.patches:\n    # Get the height of the bar (the total proportion)\n    height = bar.get_height()\n    # Get the width of the bar\n    width = bar.get_width()\n    # Calculate the center position\n    x = bar.get_x()\n    \n    # Only label if the proportion is significant\n    if height > 5:\n        # Check if the bar is the '1: Defaulter Rate' (the orange/red segment)\n        # We need to calculate the y-position for the label to be centered on the segment\n        # The '0' segment is plotted first, so its value is the start of the '1' segment\n        is_defaulter_segment = (bar.get_fc() == ax.containers[1].get_children()[0].get_facecolor()).all()\n        \n        if is_defaulter_segment:\n             # This is the top segment (Defaulter Rate), position label at (start_of_segment + height / 2)\n            y_position = bar.get_y() + height / 2\n        else:\n            # This is the bottom segment (Non-Defaulter Rate), position label at (height / 2)\n            y_position = height / 2\n            \n        ax.text(x + width / 2., \n                y_position, \n                f'{height:.1f}%', \n                ha='center', va='center', \n                color='white', fontsize=10, weight='bold')\n\nplt.legend(title='Customer Status', bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-02T00:39:48.803629Z","iopub.execute_input":"2025-12-02T00:39:48.804356Z","iopub.status.idle":"2025-12-02T00:39:49.302328Z","shell.execute_reply.started":"2025-12-02T00:39:48.804318Z","shell.execute_reply":"2025-12-02T00:39:49.301115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Set a professional plotting style\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (10, 6)\n\n# --- 1. Define the Policy's Target Zone ---\nINTERVENTION_THRESHOLD = 0.5 \n\n# --- 2. Generate the Histogram for Q3 Insight ---\nplt.figure(figsize=(10, 6))\n\n# Plot the distribution of P_2 for ALL customers\nsns.histplot(\n    data=df, \n    x='P_2', \n    bins=50, \n    kde=True, \n    color='#cccccc', \n    label='Total Customer Distribution', \n    alpha=0.8\n)\n\n# Plot the distribution of P_2 specifically for DEFAULTERS (Target=1)\n# This is the risk we want to capture\nsns.histplot(\n    data=df[df['target'] == 1], \n    x='P_2', \n    bins=50, \n    kde=True, \n    color='#FF4E00', # High Risk Red/Orange\n    label='Defaulter Distribution (Risk)', \n    alpha=0.7\n)\n\n# Add the Intervention Line (The core of the policy answer)\nplt.axvline(x=INTERVENTION_THRESHOLD, color='black', linestyle='--', linewidth=2)\n\n# Add Strategic Interpretation Text\nplt.text(INTERVENTION_THRESHOLD + 0.02, plt.gca().get_ylim()[1] * 0.7, \n         'Policy Threshold (P_2 = 0.5)', \n         color='black', fontsize=11, ha='left')\n\nplt.fill_betweenx(\n    y=[0, plt.gca().get_ylim()[1]], \n    x1=plt.gca().get_xlim()[0], \n    x2=INTERVENTION_THRESHOLD, \n    color='red', \n    alpha=0.1, \n    label='Intervention Zone'\n)\n\n# Add Titles and Labels\nplt.title('Business Impact: Capturing Default Risk with P_2 Threshold', fontsize=16, weight='bold')\nplt.xlabel('P_2 (Payment Magnitude)', fontsize=12)\nplt.ylabel('Customer Count (Frequency)', fontsize=12)\nplt.xlim(-0.1, 1.1)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-02T00:40:20.779718Z","iopub.execute_input":"2025-12-02T00:40:20.78029Z","iopub.status.idle":"2025-12-02T00:40:23.609662Z","shell.execute_reply.started":"2025-12-02T00:40:20.780264Z","shell.execute_reply":"2025-12-02T00:40:23.608721Z"}},"outputs":[],"execution_count":null}]}