{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":31254,"databundleVersionId":3103714}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:41:56.148549Z","iopub.execute_input":"2026-09-03T12:41:56.149702Z","iopub.status.idle":"2026-09-03T12:41:56.178939Z","shell.execute_reply.started":"2026-09-03T12:41:56.149668Z","shell.execute_reply":"2026-09-03T12:41:56.177731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime, timedelta\nimport warnings\nwarnings.filterwarnings('ignore')\n \n# Set style for consistency\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (14, 8)\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:41:56.180525Z","iopub.execute_input":"2026-09-03T12:41:56.180797Z","iopub.status.idle":"2026-09-03T12:41:56.218397Z","shell.execute_reply.started":"2026-09-03T12:41:56.180773Z","shell.execute_reply":"2026-09-03T12:41:56.217564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def script_01_metadata_inventory():\n    \"\"\"\n    Quick profiler to understand dataset structure.\n    \"\"\"\n    print(\"=\" * 70)\n    print(\"SCRIPT 01: METADATA INVENTORY ANALYSIS\")\n    print(\"=\" * 70)\n    \n    files = {\n        'transactions': '/kaggle/input/datasets/shantanusmankar/h-and-m-data/transactions_train.csv',\n        'articles': '/kaggle/input/datasets/shantanusmankar/h-and-m-data/articles.csv',\n        'customers': '/kaggle/input/datasets/shantanusmankar/h-and-m-data/customers.csv'\n    }\n    \n    metadata = {}\n    for name, filepath in files.items():\n        try:\n            # Sample first 10k rows for speed\n            df = pd.read_csv(filepath, nrows=10000)\n            metadata[name] = {\n                'estimated_full_shape': f\"~{len(df) * 1000:,} rows\",\n                'columns': list(df.columns),\n                'dtypes': df.dtypes.to_dict(),\n                'missing_%': (df.isnull().sum() / len(df) * 100).round(2).to_dict(),\n                'memory_mb_sample': round(df.memory_usage(deep=True).sum() / 1e6, 2)\n            }\n            print(f\"\\n{name.upper()}:\")\n            print(f\"  Columns: {metadata[name]['columns']}\")\n            print(f\"  Missing %: {metadata[name]['missing_%']}\")\n            print(f\"  Sample memory: {metadata[name]['memory_mb_sample']} MB\")\n        except FileNotFoundError:\n            print(f\"  ⚠️ File not found: {filepath}\")\n    \n    # Save metadata\n    with open('01_METADATA_INVENTORY.txt', 'w') as f:\n        f.write(str(metadata))\n    print(\"\\n✓ Metadata saved to 01_METADATA_INVENTORY.txt\")\n    return metadata\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:41:56.219377Z","iopub.execute_input":"2026-09-03T12:41:56.219668Z","iopub.status.idle":"2026-09-03T12:41:56.238987Z","shell.execute_reply.started":"2026-09-03T12:41:56.219634Z","shell.execute_reply":"2026-09-03T12:41:56.237901Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transactions = pd.read_csv(\"/kaggle/input/datasets/shantanusmankar/h-and-m-data/transactions_train.csv\")\n\ntransactions.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:41:56.240790Z","iopub.execute_input":"2026-09-03T12:41:56.241148Z","iopub.status.idle":"2026-09-03T12:42:47.388201Z","shell.execute_reply.started":"2026-09-03T12:41:56.241122Z","shell.execute_reply":"2026-09-03T12:42:47.386764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"articles = pd.read_csv(\"/kaggle/input/datasets/shantanusmankar/h-and-m-data/articles.csv\")\n\narticles.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:42:47.389545Z","iopub.execute_input":"2026-09-03T12:42:47.389923Z","iopub.status.idle":"2026-09-03T12:42:48.335559Z","shell.execute_reply.started":"2026-09-03T12:42:47.389881Z","shell.execute_reply":"2026-09-03T12:42:48.334584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"customers = pd.read_csv(\"/kaggle/input/datasets/shantanusmankar/h-and-m-data/customers.csv\")\n\ncustomers.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:42:48.336569Z","iopub.execute_input":"2026-09-03T12:42:48.336839Z","iopub.status.idle":"2026-09-03T12:42:53.517473Z","shell.execute_reply.started":"2026-09-03T12:42:48.336816Z","shell.execute_reply":"2026-09-03T12:42:53.516568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# SCRIPT 02: MASTER DATASET CREATION (45 min)\n# ============================================================================\ndef script_02_master_dataset():\n    \"\"\"\n    Load, join, and create master analytical dataset.\n    RUNS ASYNCHRONOUSLY - expect ~45 min for 31M rows\n    \"\"\"\n    print(\"\\n\" + \"=\" * 70)\n    print(\"SCRIPT 02: MASTER DATASET CREATION\")\n    print(\"=\" * 70)\n    \n    try:\n        print(\"Loading transactions (31M rows)...\")\n        trans = pd.read_csv('/kaggle/input/datasets/shantanusmankar/h-and-m-data/transactions_train.csv')\n        \n        print(\"Loading articles...\")\n        articles = pd.read_csv('/kaggle/input/datasets/shantanusmankar/h-and-m-data/articles.csv')\n        \n        print(\"Loading customers...\")\n        customers = pd.read_csv('/kaggle/input/datasets/shantanusmankar/h-and-m-data/customers.csv')\n        \n        # Parse dates\n        print(\"Parsing dates...\")\n        trans['t_dat'] = pd.to_datetime(trans['t_dat'])\n        \n        # 12-month rolling window\n        print(\"Filtering to 12-month window...\")\n        max_date = trans['t_dat'].max()\n        min_date = max_date - timedelta(days=365)\n        trans = trans[(trans['t_dat'] >= min_date)]\n        print(f\"  Date range: {min_date.date()} to {max_date.date()}\")\n        \n        # Joins\n        print(\"Merging tables (transaction + article + customer)...\")\n        master = trans.merge(\n            articles[['article_id', 'product_type_name', 'product_code', \n                     'graphical_appearance_name', 'colour_group_name']],\n            on='article_id', how='left'\n        )\n        master = master.merge(\n            customers[['customer_id', 'age', 'postal_code', 'club_member_status', \n                      'fashion_news_frequency', 'Active']],\n            on='customer_id', how='left'\n        )\n        \n        # Optimize dtypes for memory\n        print(\"Optimizing data types...\")\n        #master['article_id'] = master['article_id'].astype('int32')\n        #master['customer_id'] = master['customer_id'].astype('int32')\n        master['sales_channel_id'] = master['sales_channel_id'].astype('int8')\n        master['price'] = master['price'].astype('float32')\n        \n        # Save\n        print(\"Saving master dataset...\")\n        master.to_parquet('master_dataset.parquet', compression='snappy')\n        print(f\"✓ Master dataset created: {master.shape}\")\n        print(f\"  Memory usage: {master.memory_usage(deep=True).sum() / 1e9:.2f} GB\")\n        \n        return master\n    \n    except Exception as e:\n        print(f\"❌ Error in script 02: {e}\")\n        return None\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:42:53.519547Z","iopub.execute_input":"2026-09-03T12:42:53.519916Z","iopub.status.idle":"2026-09-03T12:42:53.529460Z","shell.execute_reply.started":"2026-09-03T12:42:53.519890Z","shell.execute_reply":"2026-09-03T12:42:53.528401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"script_01_metadata_inventory()\nmaster = script_02_master_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:42:53.530527Z","iopub.execute_input":"2026-09-03T12:42:53.531459Z","iopub.status.idle":"2026-09-03T12:44:42.328230Z","shell.execute_reply.started":"2026-09-03T12:42:53.531423Z","shell.execute_reply":"2026-09-03T12:44:42.326679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def script_03_data_cleaning():\n    \"\"\"\n    Handle missing values, outliers, and data quality issues.\n    RUN AFTER script_02 completes.\n    \"\"\"\n    print(\"\\n\" + \"=\" * 70)\n    print(\"SCRIPT 03: DATA CLEANING\")\n    print(\"=\" * 70)\n    \n    try:\n        print(\"Loading master dataset...\")\n        master = pd.read_parquet('master_dataset.parquet')\n        \n        print(f\"Initial shape: {master.shape}\")\n        print(f\"Missing values before cleaning:\")\n        print(master.isnull().sum()[master.isnull().sum() > 0])\n        \n        # Age: fill with median per postal code cluster\n        print(\"\\nFilling age with median per postal code...\")\n        master['age'].fillna(\n            master.groupby('postal_code')['age'].transform('median'),\n            inplace=True\n        )\n        \n        # Categorical flags: fill with 'UNKNOWN' or 0\n        print(\"Filling categorical missing values...\")\n        master['fashion_news_frequency'].fillna('UNKNOWN', inplace=True)\n        master['Active'].fillna(0, inplace=True)\n        master['club_member_status'].fillna('UNKNOWN', inplace=True)\n        \n        # Outliers: remove price > 99th percentile\n        print(\"Removing price outliers (>99th percentile)...\")\n        price_99 = master['price'].quantile(0.99)\n        master = master[master['price'] <= price_99]\n        print(f\"  Removed {master[master['price'] > price_99].shape[0]} rows\")\n        \n        print(f\"\\nFinal shape after cleaning: {master.shape}\")\n        print(f\"Missing values after cleaning: {master.isnull().sum().sum()}\")\n        \n        # Save\n        master.to_parquet('master_dataset_clean.parquet', compression='snappy')\n        print(\"✓ Cleaned dataset saved\")\n        \n        return master\n    \n    except Exception as e:\n        print(f\"❌ Error in script 03: {e}\")\n        return None\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:44:42.330908Z","iopub.execute_input":"2026-09-03T12:44:42.331270Z","iopub.status.idle":"2026-09-03T12:44:42.343156Z","shell.execute_reply.started":"2026-09-03T12:44:42.331242Z","shell.execute_reply":"2026-09-03T12:44:42.342151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"master_clean = script_03_data_cleaning()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:44:42.344360Z","iopub.execute_input":"2026-09-03T12:44:42.344609Z","iopub.status.idle":"2026-09-03T12:45:45.072614Z","shell.execute_reply.started":"2026-09-03T12:44:42.344589Z","shell.execute_reply":"2026-09-03T12:45:45.071408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''def script_04_feature_engineering():\n    \"\"\"\n    Create RFM, velocity, age bins, channel names, etc.\n    RUN AFTER script_03 completes.\n    \"\"\"\n    print(\"\\n\" + \"=\" * 70)\n    print(\"SCRIPT 04: FEATURE ENGINEERING\")\n    print(\"=\" * 70)\n    \n    try:\n        print(\"Loading clean master dataset...\")\n        master = pd.read_parquet('master_dataset_clean.parquet')\n        max_date = master['t_dat'].max()\n        \n        # ---- RFM Features ----\n        print(\"\\nCalculating RFM features...\")\n        \n        # Recency\n        recency = max_date - master.groupby('customer_id')['t_dat'].max()\n        recency_df = recency.reset_index()\n        recency_df.columns = ['customer_id', 'recency_days']\n        \n        # Frequency\n        frequency_df = master.groupby('customer_id').size().reset_index(name='frequency')\n        \n        # Monetary\n        monetary_df = master.groupby('customer_id')['price'].sum().reset_index()\n        monetary_df.columns = ['customer_id', 'total_spend']\n        \n        # Merge RFM\n        rfm = recency_df.merge(frequency_df, on='customer_id').merge(monetary_df, on='customer_id')\n        \n        # ---- Sales Velocity ----\n        print(\"Calculating sales velocity (weekly)...\")\n        master['week'] = master['t_dat'].dt.isocalendar().week\n        velocity = master.groupby(['article_id', 'sales_channel_id', 'week']).size().reset_index(name='sales_count')\n        velocity_avg = velocity.groupby(['article_id', 'sales_channel_id'])['sales_count'].mean().reset_index()\n        velocity_avg.columns = ['article_id', 'sales_channel_id', 'velocity_weekly']\n        \n        # ---- Other Features ----\n        print(\"Creating additional features...\")\n        master['age_bin'] = pd.cut(master['age'], bins=[0, 25, 35, 50, 100], \n                                   labels=['0-25', '26-35', '36-50', '51+'])\n        master['channel_name'] = master['sales_channel_id'].map({1: 'Online', 2: 'Store'})\n        master['purchase_month'] = master['t_dat'].dt.month\n        \n        # Save features\n        print(\"Saving feature tables...\")\n        rfm.to_parquet('rfm_features.parquet')\n        velocity_avg.to_parquet('velocity_features.parquet')\n        master.to_parquet('master_with_features.parquet')\n        \n        print(\"✓ Features engineered successfully\")\n        print(f\"  RFM records: {rfm.shape[0]:,}\")\n        print(f\"  Unique article-channel combos: {velocity_avg.shape[0]:,}\")\n        \n        return rfm, velocity_avg, master\n    \n    except Exception as e:\n        print(f\"❌ Error in script 04: {e}\")\n        return None, None, None\n '''\n\n\"\"\"\nSCRIPT 04: FEATURE ENGINEERING (FIXED)\n=====================================\nFIX: Convert recency to int(days) not Timedelta\n\"\"\"\ndef script_04_feature_engineering(): \n    import pandas as pd\n    import numpy as np\n    from datetime import datetime, timedelta\n     \n    print(\"=\" * 70)\n    print(\"SCRIPT 04: FEATURE ENGINEERING (FIXED)\")\n    print(\"=\" * 70)\n     \n    try:\n        print(\"\\nLoading clean master dataset...\")\n        master = pd.read_parquet('master_dataset_clean.parquet')\n        max_date = master['t_dat'].max()\n        print(f\"✓ Loaded: {master.shape}\")\n        print(f\"  Date range: {master['t_dat'].min().date()} to {max_date.date()}\")\n        \n        # ---- RFM Features ----\n        print(\"\\nCalculating RFM features...\")\n        \n        # Recency: Convert to INT DAYS (not Timedelta)\n        recency = (max_date - master.groupby('customer_id')['t_dat'].max()).dt.days\n        recency_df = recency.reset_index()\n        recency_df.columns = ['customer_id', 'recency_days']\n        print(f\"  ✓ Recency: {recency_df.shape}\")\n        print(f\"    Sample recency values: {recency_df['recency_days'].head().tolist()}\")\n        print(f\"    Recency dtype: {recency_df['recency_days'].dtype}\")\n        \n        # Frequency\n        frequency_df = master.groupby('customer_id').size().reset_index(name='frequency')\n        print(f\"  ✓ Frequency: {frequency_df.shape}\")\n        \n        # Monetary\n        monetary_df = master.groupby('customer_id')['price'].sum().reset_index()\n        monetary_df.columns = ['customer_id', 'total_spend']\n        print(f\"  ✓ Monetary: {monetary_df.shape}\")\n        \n        # Merge RFM\n        rfm = recency_df.merge(frequency_df, on='customer_id').merge(monetary_df, on='customer_id')\n        print(f\"  ✓ RFM merged: {rfm.shape}\")\n        \n        # ---- Sales Velocity ----\n        print(\"\\nCalculating sales velocity (weekly)...\")\n        master['week'] = master['t_dat'].dt.isocalendar().week\n        velocity = master.groupby(['article_id', 'sales_channel_id', 'week']).size().reset_index(name='sales_count')\n        velocity_avg = velocity.groupby(['article_id', 'sales_channel_id'])['sales_count'].mean().reset_index()\n        velocity_avg.columns = ['article_id', 'sales_channel_id', 'velocity_weekly']\n        print(f\"  ✓ Velocity: {velocity_avg.shape}\")\n        \n        # ---- Other Features ----\n        print(\"\\nCreating additional features...\")\n        master['age_bin'] = pd.cut(master['age'].fillna(master['age'].median()), \n                                   bins=[0, 25, 35, 50, 100], \n                                   labels=['0-25', '26-35', '36-50', '51+'])\n        master['channel_name'] = master['sales_channel_id'].map({1: 'Online', 2: 'Store'})\n        master['purchase_month'] = master['t_dat'].dt.month\n        print(f\"  ✓ Features created\")\n        \n        # Save features\n        print(\"\\nSaving feature tables...\")\n        rfm.to_parquet('rfm_features.parquet')\n        velocity_avg.to_parquet('velocity_features.parquet')\n        master.to_parquet('master_with_features.parquet')\n        \n        print(\"✓ Features engineered successfully\")\n        print(f\"  RFM records: {rfm.shape[0]:,}\")\n        print(f\"  Unique article-channel combos: {velocity_avg.shape[0]:,}\")\n        print(f\"  RFM dtypes:\\n{rfm.dtypes}\")\n\n        return rfm, velocity_avg, master\n        \n    except Exception as e:\n        print(f\"❌ Error: {e}\")\n        import traceback\n        traceback.print_exc()\n        return None, None, None\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:45:45.074153Z","iopub.execute_input":"2026-09-03T12:45:45.074445Z","iopub.status.idle":"2026-09-03T12:45:45.090518Z","shell.execute_reply.started":"2026-09-03T12:45:45.074409Z","shell.execute_reply":"2026-09-03T12:45:45.089634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nrfm_feat, vel_feat, master_feat = script_04_feature_engineering()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:45:45.091557Z","iopub.execute_input":"2026-09-03T12:45:45.091988Z","iopub.status.idle":"2026-09-03T12:46:40.966397Z","shell.execute_reply.started":"2026-09-03T12:45:45.091961Z","shell.execute_reply":"2026-09-03T12:46:40.965644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rfm_feat.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:46:40.967552Z","iopub.execute_input":"2026-09-03T12:46:40.967880Z","iopub.status.idle":"2026-09-03T12:46:40.996339Z","shell.execute_reply.started":"2026-09-03T12:46:40.967842Z","shell.execute_reply":"2026-09-03T12:46:40.995593Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# SCRIPT 05: EXPLORATORY DATA ANALYSIS (30 min)\n# ============================================================================\ndef script_05_eda():\n    \"\"\"\n    Generate distribution plots, summary statistics.\n    CAN RUN IN PARALLEL with data engineering.\n    \"\"\"\n    print(\"\\n\" + \"=\" * 70)\n    print(\"SCRIPT 05: EXPLORATORY DATA ANALYSIS\")\n    print(\"=\" * 70)\n    \n    try:\n        print(\"Loading datasets...\")\n        master = pd.read_parquet('master_with_features.parquet')\n        rfm = pd.read_parquet('rfm_features.parquet')\n        \n        # Create figure\n        fig, axes = plt.subplots(2, 3, figsize=(16, 10))\n        fig.suptitle('H&M Exploratory Data Analysis', fontsize=16, fontweight='bold')\n        \n        # Recency distribution\n        axes[0, 0].hist(rfm['recency_days'], bins=50, edgecolor='black', color='steelblue')\n        axes[0, 0].set_title('Recency Distribution (Days Since Purchase)', fontweight='bold')\n        axes[0, 0].set_xlabel('Recency (days)')\n        axes[0, 0].set_ylabel('Frequency')\n        \n        # Frequency distribution\n        axes[0, 1].hist(rfm['frequency'], bins=50, edgecolor='black', color='coral')\n        axes[0, 1].set_title('Frequency Distribution', fontweight='bold')\n        axes[0, 1].set_xlabel('# Transactions')\n        axes[0, 1].set_ylabel('# Customers')\n        \n        # Monetary distribution (log scale)\n        axes[0, 2].hist(rfm['total_spend'], bins=50, edgecolor='black', color='lightgreen')\n        axes[0, 2].set_title('Monetary Distribution (Log Scale)', fontweight='bold')\n        axes[0, 2].set_xlabel('Total Spend ($)')\n        axes[0, 2].set_yscale('log')\n        \n        # Channel breakdown\n        channel_summary = master.groupby('channel_name').agg({\n            'customer_id': 'nunique',\n            'price': 'sum'\n        }).round(0)\n        axes[1, 0].bar(channel_summary.index, channel_summary['price'], color=['#1f77b4', '#ff7f0e'])\n        axes[1, 0].set_title('Total Revenue by Channel', fontweight='bold')\n        axes[1, 0].set_ylabel('Revenue ($)')\n        for i, v in enumerate(channel_summary['price']):\n            axes[1, 0].text(i, v, f'${v:,.0f}', ha='center', va='bottom', fontweight='bold')\n        \n        # Price distribution\n        axes[1, 1].hist(master['price'], bins=50, edgecolor='black', color='mediumpurple')\n        axes[1, 1].set_title('Price Distribution', fontweight='bold')\n        axes[1, 1].set_xlabel('Price ($)')\n        axes[1, 1].set_ylabel('Frequency')\n        \n        # Age distribution\n        axes[1, 2].hist(master['age'].dropna(), bins=40, edgecolor='black', color='tomato')\n        axes[1, 2].set_title('Customer Age Distribution', fontweight='bold')\n        axes[1, 2].set_xlabel('Age')\n        axes[1, 2].set_ylabel('# Customers')\n        \n        plt.tight_layout()\n        plt.savefig('04_EDA_DISTRIBUTIONS.png', dpi=150, bbox_inches='tight')\n        plt.close()\n        \n        print(\"✓ EDA plots saved to 04_EDA_DISTRIBUTIONS.png\")\n        \n        # Summary statistics\n        print(\"\\nSummary Statistics:\")\n        print(f\"Total transactions: {master.shape[0]:,}\")\n        print(f\"Unique customers: {master['customer_id'].nunique():,}\")\n        print(f\"Unique articles: {master['article_id'].nunique():,}\")\n        print(f\"Date range: {master['t_dat'].min().date()} to {master['t_dat'].max().date()}\")\n        print(f\"\\nChannel breakdown:\")\n        print(channel_summary)\n        \n        return True\n    \n    except Exception as e:\n        print(f\"❌ Error in script 05: {e}\")\n        return False\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:46:40.997600Z","iopub.execute_input":"2026-09-03T12:46:40.997893Z","iopub.status.idle":"2026-09-03T12:46:41.012451Z","shell.execute_reply.started":"2026-09-03T12:46:40.997868Z","shell.execute_reply":"2026-09-03T12:46:41.011528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 05\n\nscript_05_eda()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:46:41.015361Z","iopub.execute_input":"2026-09-03T12:46:41.015975Z","iopub.status.idle":"2026-09-03T12:47:04.841958Z","shell.execute_reply.started":"2026-09-03T12:46:41.015946Z","shell.execute_reply":"2026-09-03T12:47:04.841149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install lifelines --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:47:04.935924Z","iopub.execute_input":"2026-09-03T12:47:04.936303Z","iopub.status.idle":"2026-09-03T12:47:15.526348Z","shell.execute_reply.started":"2026-09-03T12:47:04.936267Z","shell.execute_reply":"2026-09-03T12:47:15.525157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSCRIPT 06 LITE: RFM SEGMENTATION (FAST VERSION - 10-12 MINUTES)\n==============================================================\nStreamlined alternative to weight-calibration approach.\nUses proven domain-driven RFM logic instead of supervised optimization.\n\nKey differences from full Script 06:\n- Skip regression weight derivation\n- Skip ROC-AUC model comparison\n- Use standard RFM matrix (3x3x3 collapsed to 5 tiers)\n- Vectorized operations only (no loops)\n- No dashboard (save time)\n\nOutput: Same format as Script 06 (06_RFM_SEGMENTS.parquet)\nTime estimate: 10-12 minutes\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\" * 80)\nprint(\"SCRIPT 06 LITE: RFM SEGMENTATION (FAST VERSION)\")\nprint(\"=\" * 80)\n\ntry:\n    # ============================================================================\n    # PART 0: LOAD DATA\n    # ============================================================================\n    print(\"\\n[PART 0] Loading data...\")\n    start_time = datetime.now()\n    \n    master = pd.read_parquet('master_with_features.parquet')\n    \n    print(f\"  ✓ Master dataset: {master.shape}\")\n    print(f\"  ✓ Date range: {master['t_dat'].min().date()} to {master['t_dat'].max().date()}\")\n    \n    max_date = master['t_dat'].max()\n    \n    # ============================================================================\n    # PART 1: BUILD TRAINING DATA & TARGET (TIME-BASED SPLIT)\n    # ============================================================================\n    print(\"\\n[PART 1] Building training data with target variable...\")\n    \n    # Time split\n    validation_start = max_date - pd.Timedelta(days=90)\n    training_end = validation_start - pd.Timedelta(days=1)\n    \n    print(f\"  Training period: {master['t_dat'].min().date()} to {training_end.date()}\")\n    print(f\"  Validation period: {validation_start.date()} to {max_date.date()}\")\n    \n    # Training master (pre-validation only)\n    training_master = master[master['t_dat'] <= training_end].copy()\n    \n    # Get customers in training set\n    customers_training = training_master['customer_id'].unique()\n    \n    print(f\"  ✓ Training customers: {len(customers_training):,}\")\n    print(f\"  ✓ Training transactions: {len(training_master):,}\")\n    \n    # ============================================================================\n    # PART 2: CALCULATE RFM (TRAINING PERIOD ONLY)\n    # ============================================================================\n    print(\"\\n[PART 2] Calculating RFM metrics (training period)...\")\n    \n    # Recency: days since last purchase\n    recency = (training_end - training_master.groupby('customer_id')['t_dat'].max()).dt.days\n    \n    # Frequency: number of transactions\n    frequency = training_master.groupby('customer_id').size()\n    \n    # Monetary: total spend\n    monetary = training_master.groupby('customer_id')['price'].sum()\n    \n    # Combine into RFM dataframe\n    rfm_training = pd.DataFrame({\n        'customer_id': recency.index,\n        'recency_days': recency.values,\n        'frequency': frequency.values,\n        'total_spend': monetary.values\n    }).reset_index(drop=True)\n    \n    # Validation target: did they repurchase in next 90 days?\n    validation_buyers = master[master['t_dat'] >= validation_start]['customer_id'].unique()\n    rfm_training['target_repurchased'] = rfm_training['customer_id'].isin(validation_buyers).astype(int)\n    \n    repurchase_rate = rfm_training['target_repurchased'].mean()\n    \n    print(f\"  ✓ RFM calculated for {len(rfm_training):,} training customers\")\n    print(f\"  ✓ Repurchase rate (next 90 days): {repurchase_rate:.1%}\")\n    print(f\"    - Will repurchase: {rfm_training['target_repurchased'].sum():,}\")\n    print(f\"    - Will NOT repurchase: {(1-rfm_training['target_repurchased']).sum():,}\")\n    \n    # ============================================================================\n    # PART 3: CALCULATE PERCENTILES (TRAINING-BASED, VECTORIZED)\n    # ============================================================================\n    print(\"\\n[PART 3] Calculating RFM percentiles (training data)...\")\n    \n    # For recency: INVERT (lower is better)\n    rfm_training['r_score'] = 100 - (rfm_training['recency_days'].rank(pct=True) * 100)\n    \n    # For frequency: higher is better\n    rfm_training['f_score'] = rfm_training['frequency'].rank(pct=True) * 100\n    \n    # For monetary: higher is better\n    rfm_training['m_score'] = rfm_training['total_spend'].rank(pct=True) * 100\n    \n    print(f\"  ✓ Percentiles calculated (vectorized)\")\n    print(f\"    Recency:  mean={rfm_training['r_score'].mean():.1f}, std={rfm_training['r_score'].std():.1f}\")\n    print(f\"    Frequency: mean={rfm_training['f_score'].mean():.1f}, std={rfm_training['f_score'].std():.1f}\")\n    print(f\"    Monetary:  mean={rfm_training['m_score'].mean():.1f}, std={rfm_training['m_score'].std():.1f}\")\n    \n    # ============================================================================\n    # PART 4: CREATE RFM TIERS & SEGMENTATION MATRIX\n    # ============================================================================\n    print(\"\\n[PART 4] Creating RFM segmentation matrix...\")\n    \n    # Divide each dimension into 3 tiers (low/medium/high based on percentiles)\n    # R: high recency percentile = low risk (recent)\n    # F: high frequency percentile = good (loyal)\n    # M: high monetary percentile = good (valuable)\n    \n    def assign_tier(score):\n        \"\"\"Assign Low/Medium/High tier based on percentile score (0-100)\"\"\"\n        if score >= 66.7:\n            return 'H'  # High\n        elif score >= 33.3:\n            return 'M'  # Medium\n        else:\n            return 'L'  # Low\n    \n    rfm_training['r_tier'] = rfm_training['r_score'].apply(assign_tier)\n    rfm_training['f_tier'] = rfm_training['f_score'].apply(assign_tier)\n    rfm_training['m_tier'] = rfm_training['m_score'].apply(assign_tier)\n    \n    # Create RFM combination (e.g., HHH, HHL, etc.)\n    rfm_training['rfm_cell'] = (rfm_training['r_tier'] + \n                                 rfm_training['f_tier'] + \n                                 rfm_training['m_tier'])\n    \n    print(f\"  ✓ RFM matrix created (27 cells)\")\n    \n    # Map cells to 5 segment tiers\n    # Logic: Maximize recency + frequency + monetary\n    # Champions: HHH, HHM, HMH, MHH\n    # Loyal: HMM, MHM, MMH, HHL, HLH, MHL\n    # Middle: MMM, MML, MLM, LMM, HLL, LHL\n    # At-Risk: LLM, LML, MLL, LLH\n    # Lost: LLL, LLM (very low on all or recency super low)\n    \n    segment_map = {\n        # Champions (high on most dimensions)\n        'HHH': 'Champions',\n        'HHM': 'Champions',\n        'HMH': 'Champions',\n        'MHH': 'Champions',\n        \n        # Loyal (high frequency, decent recency/monetary)\n        'HMM': 'Loyal',\n        'MHM': 'Loyal',\n        'MMH': 'Loyal',\n        'HHL': 'Loyal',\n        'HLH': 'Loyal',\n        'MHL': 'Loyal',\n        'LHH': 'Loyal',\n        'LHM': 'Loyal',\n        'MLH': 'Loyal',\n        \n        # Middle (mixed profile)\n        'MMM': 'Middle',\n        'MML': 'Middle',\n        'MLM': 'Middle',\n        'LMM': 'Middle',\n        'HLL': 'Middle',\n        'LHL': 'Middle',\n        'LLH': 'Middle',\n        'MMH': 'Middle',\n        \n        # At-Risk (low recency but some frequency/monetary)\n        'LLM': 'At-Risk',\n        'LML': 'At-Risk',\n        'MLL': 'At-Risk',\n        'LLL': 'At-Risk',\n    }\n    \n    # Fallback mapping (ensure all 27 cells are covered)\n    rfm_training['segment'] = rfm_training['rfm_cell'].map(segment_map).fillna('At-Risk')\n    \n    segment_dist = rfm_training['segment'].value_counts()\n    print(f\"\\n  Segment distribution (training):\")\n    for seg in ['Champions', 'Loyal', 'Middle', 'At-Risk', 'Lost']:\n        count = segment_dist.get(seg, 0)\n        pct = count / len(rfm_training) * 100 if len(rfm_training) > 0 else 0\n        print(f\"    {seg:12s}: {count:7,d} ({pct:5.1f}%)\")\n    \n    # ============================================================================\n    # PART 5: APPLY TO ALL CUSTOMERS (USING TRAINING THRESHOLDS)\n    # ============================================================================\n    print(\"\\n[PART 5] Applying segmentation to all customers...\")\n    \n    # Get all unique customers from master\n    all_customers = master[['customer_id']].drop_duplicates().copy()\n    \n    # Calculate RFM for all customers (as of max_date)\n    recency_all = (max_date - master.groupby('customer_id')['t_dat'].max()).dt.days\n    frequency_all = master.groupby('customer_id').size()\n    monetary_all = master.groupby('customer_id')['price'].sum()\n    \n    rfm_all = pd.DataFrame({\n        'customer_id': recency_all.index,\n        'recency_days': recency_all.values,\n        'frequency': frequency_all.values,\n        'total_spend': monetary_all.values\n    }).reset_index(drop=True)\n    \n    # Score using TRAINING percentile cutoffs (for consistency)\n    # This ensures validation customers are scored using training distribution\n    r_cutoff_33 = rfm_training['recency_days'].quantile(0.333)\n    r_cutoff_66 = rfm_training['recency_days'].quantile(0.667)\n    \n    f_cutoff_33 = rfm_training['frequency'].quantile(0.333)\n    f_cutoff_66 = rfm_training['frequency'].quantile(0.667)\n    \n    m_cutoff_33 = rfm_training['total_spend'].quantile(0.333)\n    m_cutoff_66 = rfm_training['total_spend'].quantile(0.667)\n    \n    # Assign tiers using training cutoffs\n    def tier_by_cutoff(value, cutoff_low, cutoff_high, invert=False):\n        \"\"\"Assign tier using pre-defined cutoffs\"\"\"\n        if invert:  # For recency (lower is better)\n            if value < cutoff_low:\n                return 'H'\n            elif value < cutoff_high:\n                return 'M'\n            else:\n                return 'L'\n        else:  # For frequency and monetary (higher is better)\n            if value >= cutoff_high:\n                return 'H'\n            elif value >= cutoff_low:\n                return 'M'\n            else:\n                return 'L'\n    \n    # Apply tiers\n    rfm_all['r_tier'] = rfm_all['recency_days'].apply(lambda x: tier_by_cutoff(x, r_cutoff_33, r_cutoff_66, invert=True))\n    rfm_all['f_tier'] = rfm_all['frequency'].apply(lambda x: tier_by_cutoff(x, f_cutoff_33, f_cutoff_66, invert=False))\n    rfm_all['m_tier'] = rfm_all['total_spend'].apply(lambda x: tier_by_cutoff(x, m_cutoff_33, m_cutoff_66, invert=False))\n    \n    # Create RFM cell and map to segment\n    rfm_all['rfm_cell'] = rfm_all['r_tier'] + rfm_all['f_tier'] + rfm_all['m_tier']\n    rfm_all['segment'] = rfm_all['rfm_cell'].map(segment_map).fillna('At-Risk')\n    \n    print(f\"  ✓ Segmentation applied to {len(rfm_all):,} customers\")\n    \n    segment_dist_all = rfm_all['segment'].value_counts()\n    print(f\"\\n  Segment distribution (ALL customers):\")\n    for seg in ['Champions', 'Loyal', 'Middle', 'At-Risk', 'Lost']:\n        count = segment_dist_all.get(seg, 0)\n        pct = count / len(rfm_all) * 100 if len(rfm_all) > 0 else 0\n        print(f\"    {seg:12s}: {count:7,d} ({pct:5.1f}%)\")\n    \n    # ============================================================================\n    # PART 6: SAVE OUTPUTS\n    # ============================================================================\n    print(f\"\\n[PART 6] Saving outputs...\")\n    \n    # Main segmentation output\n    rfm_all.to_parquet('06_RFM_SEGMENTS.parquet')\n    print(f\"  ✓ 06_RFM_SEGMENTS.parquet ({len(rfm_all):,} customers)\")\n    \n    # Training data (for reference)\n    rfm_training_output = rfm_training[['customer_id', 'recency_days', 'frequency', 'total_spend', \n                                         'r_score', 'f_score', 'm_score', 'segment', 'target_repurchased']].copy()\n    rfm_training_output.to_csv('06_RFM_TRAINING.csv', index=False)\n    print(f\"  ✓ 06_RFM_TRAINING.csv ({len(rfm_training_output):,} customers)\")\n    \n    # Segment distribution summary\n    segment_summary = pd.DataFrame({\n        'Segment': ['Champions', 'Loyal', 'Middle', 'At-Risk', 'Lost'],\n        'Training_Count': [segment_dist.get(seg, 0) for seg in ['Champions', 'Loyal', 'Middle', 'At-Risk', 'Lost']],\n        'All_Count': [segment_dist_all.get(seg, 0) for seg in ['Champions', 'Loyal', 'Middle', 'At-Risk', 'Lost']]\n    })\n    segment_summary['Training_Pct'] = (segment_summary['Training_Count'] / segment_summary['Training_Count'].sum() * 100).round(1)\n    segment_summary['All_Pct'] = (segment_summary['All_Count'] / segment_summary['All_Count'].sum() * 100).round(1)\n    \n    segment_summary.to_csv('06_RFM_SEGMENT_DISTRIBUTION.csv', index=False)\n    print(f\"  ✓ 06_RFM_SEGMENT_DISTRIBUTION.csv\")\n    \n    # ============================================================================\n    # FINAL SUMMARY\n    # ============================================================================\n    elapsed = (datetime.now() - start_time).total_seconds()\n    \n    print(f\"\\n{'=' * 80}\")\n    print(f\"✓✓✓ SCRIPT 06 LITE COMPLETE\")\n    print(f\"{'=' * 80}\")\n    print(f\"\\nEXECUTION TIME: {elapsed:.1f} seconds ({elapsed/60:.1f} minutes)\")\n    print(f\"\\nSUMMARY:\")\n    print(f\"  Total customers segmented: {len(rfm_all):,}\")\n    print(f\"  Training customers (pre-validation): {len(rfm_training):,}\")\n    print(f\"  Segmentation method: Domain-driven RFM matrix (3x3x3 → 5 tiers)\")\n    print(f\"\\n  Segment Distribution:\")\n    for _, row in segment_summary.iterrows():\n        print(f\"    {row['Segment']:12s}: {row['All_Count']:7,d} ({row['All_Pct']:5.1f}%)\")\n    print(f\"\\n  Key Metrics:\")\n    print(f\"    Recency cutoffs (training):\")\n    print(f\"      Low (inactive):  > {r_cutoff_66:.0f} days\")\n    print(f\"      Medium:          {r_cutoff_33:.0f} - {r_cutoff_66:.0f} days\")\n    print(f\"      High (recent):   < {r_cutoff_33:.0f} days\")\n    print(f\"    Frequency cutoffs (training):\")\n    print(f\"      Low:             < {f_cutoff_33:.0f} transactions\")\n    print(f\"      Medium:          {f_cutoff_33:.0f} - {f_cutoff_66:.0f} transactions\")\n    print(f\"      High:            > {f_cutoff_66:.0f} transactions\")\n    print(f\"    Monetary cutoffs (training):\")\n    print(f\"      Low:             < ${m_cutoff_33:.2f}\")\n    print(f\"      Medium:          ${m_cutoff_33:.2f} - ${m_cutoff_66:.2f}\")\n    print(f\"      High:            > ${m_cutoff_66:.2f}\")\n    print(f\"\\n  Files Created:\")\n    print(f\"    - 06_RFM_SEGMENTS.parquet (all customers + segments)\")\n    print(f\"    - 06_RFM_TRAINING.csv (training data for validation)\")\n    print(f\"    - 06_RFM_SEGMENT_DISTRIBUTION.csv (distribution summary)\")\n    print(f\"\\n  Methodology:\")\n    print(f\"    ✓ Domain-driven RFM (proven business practice)\")\n    print(f\"    ✓ Training percentiles applied to all customers (no leakage)\")\n    print(f\"    ✓ Vectorized operations (fast)\")\n    print(f\"    ✓ No model optimization (speed priority)\")\n    print(f\"    ✓ Ready for Scripts 07-12\\n\")\n\nexcept Exception as e:\n    print(f\"\\n❌ Error: {e}\")\n    import traceback\n    traceback.print_exc()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T14:06:50.919698Z","iopub.execute_input":"2026-09-03T14:06:50.920829Z","iopub.status.idle":"2026-09-03T14:08:12.423899Z","shell.execute_reply.started":"2026-09-03T14:06:50.920717Z","shell.execute_reply":"2026-09-03T14:08:12.422839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"\n# SCRIPT 07: CHURN SCORE CALCULATION (MINIMAL - 60 SECOND VERSION)\n# ==================================================================\n# Stripped-down version skipping slow statistical tests\n# Core models only: Train, compare, select, output\n# \"\"\"\n\n# import pandas as pd\n# import numpy as np\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# from sklearn.linear_model import LogisticRegression\n# from sklearn.ensemble import GradientBoostingClassifier\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.metrics import roc_auc_score, roc_curve, precision_score, recall_score, f1_score\n# import json\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# print(\"=\" * 80)\n# print(\"SCRIPT 07: CHURN SCORE CALCULATION (MINIMAL VERSION)\")\n# print(\"=\" * 80)\n\n# try:\n#     # ============================================================================\n#     # PART 0: LOAD DATA (FAST)\n#     # ============================================================================\n#     print(\"\\n[PART 0] Loading data...\")\n#     master = pd.read_parquet('master_with_features.parquet')\n#     rfm = pd.read_parquet('05_RFM_SEGMENTS.parquet')\n    \n#     print(f\"  ✓ Master: {master.shape}\")\n#     print(f\"  ✓ RFM: {rfm.shape}\")\n    \n#     max_date = master['t_dat'].max()\n    \n#     # ============================================================================\n#     # PART 1: BUILD TARGET (FAST - NO COMPLEX FEATURE ENGINEERING)\n#     # ============================================================================\n#     print(\"\\n[PART 1] Building target variable...\")\n    \n#     validation_start = max_date - pd.Timedelta(days=90)\n#     training_end = validation_start - pd.Timedelta(days=1)\n    \n#     # Quick target: did they purchase in follow-up?\n#     customers_training = master[master['t_dat'] <= training_end]['customer_id'].unique()\n#     buyers_future = master[master['t_dat'] >= validation_start]['customer_id'].unique()\n    \n#     rfm_training = rfm[rfm['customer_id'].isin(customers_training)].copy()\n#     rfm_training['target'] = rfm_training['customer_id'].isin(buyers_future).astype(int)\n    \n#     repurchase_rate = rfm_training['target'].mean()\n#     print(f\"  ✓ Training set: {len(rfm_training):,} customers\")\n#     print(f\"  ✓ Repurchase rate: {repurchase_rate:.1%}\")\n    \n#     # ============================================================================\n#     # PART 2: MINIMAL FEATURES (3 only - FAST)\n#     # ============================================================================\n#     print(\"\\n[PART 2] Preparing features...\")\n    \n#     feature_cols = ['recency_days', 'frequency', 'total_spend']\n#     X = rfm_training[feature_cols].copy()\n#     X.fillna(X.median(), inplace=True)\n#     y = rfm_training['target'].copy()\n    \n#     # Standardize\n#     scaler = StandardScaler()\n    # X_scaled = scaler.fit_transform(X)\n    # X_scaled_df = pd.DataFrame(X_scaled, columns=feature_cols)\n    \n    # print(f\"  ✓ Features: {X.shape[1]}\")\n    # print(f\"  ✓ Samples: {X.shape[0]:,}\")\n    \n    # # ============================================================================\n    # # PART 3: TRAIN MODELS (FAST - NO HYPERPARAMETER TUNING)\n    # # ============================================================================\n    # print(\"\\n[PART 3] Training models (no tuning)...\")\n    \n    # models = {}\n    # predictions = {}\n    \n    # # Logistic Regression\n    # print(\"  Training Logistic Regression...\")\n    # lr = LogisticRegression(random_state=42, max_iter=500, class_weight='balanced')\n    # lr.fit(X_scaled_df, y)\n    # y_pred_lr = lr.predict_proba(X_scaled_df)[:, 1]\n    # roc_auc_lr = roc_auc_score(y, y_pred_lr)\n    # models['Logistic Regression'] = lr\n    # predictions['Logistic Regression'] = y_pred_lr\n    # print(f\"    ROC-AUC: {roc_auc_lr:.4f}\")\n    \n    # # Gradient Boosting\n    # print(\"  Training Gradient Boosting...\")\n    # gb = GradientBoostingClassifier(n_estimators=50, learning_rate=0.1, \n    #                                  random_state=42, max_depth=4)\n    # gb.fit(X, y)\n    # y_pred_gb = gb.predict_proba(X)[:, 1]\n    # roc_auc_gb = roc_auc_score(y, y_pred_gb)\n    # models['Gradient Boosting'] = gb\n    # predictions['Gradient Boosting'] = y_pred_gb\n    # print(f\"    ROC-AUC: {roc_auc_gb:.4f}\")\n    \n    # # ============================================================================\n    # # PART 4: SELECT BEST MODEL (FAST)\n    # # ============================================================================\n    # print(\"\\n[PART 4] Model comparison...\")\n    \n    # comparison_results = []\n    # for model_name, y_pred in predictions.items():\n    #     roc_auc = roc_auc_score(y, y_pred)\n    #     precision = precision_score(y, (y_pred > 0.5).astype(int), zero_division=0)\n    #     recall = recall_score(y, (y_pred > 0.5).astype(int), zero_division=0)\n    #     f1 = f1_score(y, (y_pred > 0.5).astype(int), zero_division=0)\n        \n    #     comparison_results.append({\n    #         'Model': model_name,\n    #         'ROC_AUC': roc_auc,\n    #         'Precision': precision,\n    #         'Recall': recall,\n    #         'F1_Score': f1\n    #     })\n        \n    #     print(f\"\\n  {model_name}:\")\n    #     print(f\"    ROC-AUC: {roc_auc:.4f}, F1: {f1:.4f}\")\n    \n    # comparison_df = pd.DataFrame(comparison_results)\n    # best_idx = comparison_df['ROC_AUC'].idxmax()\n    # best_model_name = comparison_df.loc[best_idx, 'Model']\n    # best_roc_auc = comparison_df.loc[best_idx, 'ROC_AUC']\n    \n    # print(f\"\\n  ✓✓✓ SELECTED: {best_model_name} (ROC-AUC: {best_roc_auc:.4f})\")\n    \n    # best_model = models[best_model_name]\n    # y_pred_best = predictions[best_model_name]\n    \n    # # ============================================================================\n    # # PART 5: THRESHOLD CALIBRATION (FAST - YOUDEN ONLY)\n    # # ============================================================================\n    # print(\"\\n[PART 5] Calibrating thresholds...\")\n    \n    # fpr, tpr, thresholds_roc = roc_curve(y, y_pred_best)\n    # youden = tpr - fpr\n    # youden_idx = np.argmax(youden)\n    # threshold_high = thresholds_roc[youden_idx]\n    # threshold_medium = (threshold_high + 0.5) / 2\n    \n    # print(f\"  High Risk threshold:   {threshold_high:.4f}\")\n    # print(f\"  Medium Risk threshold: {threshold_medium:.4f}\")\n    \n    # # ============================================================================\n    # # PART 6: APPLY TO ALL CUSTOMERS (FAST)\n    # # ============================================================================\n    # print(\"\\n[PART 6] Scoring all customers...\")\n    \n    # X_all = rfm[feature_cols].copy()\n    # X_all.fillna(X_all.median(), inplace=True)\n    \n    # if best_model_name == 'Logistic Regression':\n    #     X_all_scaled = scaler.transform(X_all)\n    #     churn_proba = best_model.predict_proba(X_all_scaled)[:, 1]\n    # else:\n    #     churn_proba = best_model.predict_proba(X_all)[:, 1]\n    \n    # rfm['churn_probability'] = churn_proba\n    \n    # def assign_risk(prob):\n    #     if prob >= threshold_high:\n    #         return 'HIGH'\n    #     elif prob >= threshold_medium:\n    #         return 'MEDIUM'\n    #     else:\n    #         return 'LOW'\n    \n    # rfm['churn_risk'] = rfm['churn_probability'].apply(assign_risk)\n    \n    # print(f\"  ✓ Scores assigned\")\n    # print(f\"\\n  Risk distribution:\")\n    # for risk in ['HIGH', 'MEDIUM', 'LOW']:\n    #     count = (rfm['churn_risk'] == risk).sum()\n    #     pct = count / len(rfm) * 100\n    #     print(f\"    {risk:8s}: {count:7,d} ({pct:5.1f}%)\")\n    \n    # # ============================================================================\n    # # PART 7: FEATURE IMPORTANCE (FAST)\n    # # ============================================================================\n    # print(\"\\n[PART 7] Computing feature importance...\")\n    \n    # if best_model_name == 'Gradient Boosting':\n    #     importance = best_model.feature_importances_\n    # else:\n    #     importance = np.abs(best_model.coef_[0])\n    \n    # importance_pct = (importance / importance.sum() * 100)\n    # importance_df = pd.DataFrame({\n    #     'Feature': feature_cols,\n    #     'Importance_Pct': importance_pct\n    # }).sort_values('Importance_Pct', ascending=False)\n    \n    # print(f\"  Top factors:\")\n    # for _, row in importance_df.iterrows():\n    #     print(f\"    {row['Feature']:20s}: {row['Importance_Pct']:6.2f}%\")\n    \n    # # ============================================================================\n    # # PART 8: FAST VISUALIZATION (4 subplots only)\n    # # ============================================================================\n    # print(\"\\n[PART 8] Creating visualization...\")\n    \n    # fig = plt.figure(figsize=(14, 10))\n    # gs = fig.add_gridspec(2, 2, hspace=0.3, wspace=0.3)\n    \n    # # Subplot 1: ROC Curves\n    # ax1 = fig.add_subplot(gs[0, 0])\n    # for model_name, y_pred in predictions.items():\n    #     fpr, tpr, _ = roc_curve(y, y_pred)\n    #     roc_auc = roc_auc_score(y, y_pred)\n    #     linewidth = 2.5 if model_name == best_model_name else 1.5\n    #     color = '#2ecc71' if model_name == best_model_name else '#95a5a6'\n    #     ax1.plot(fpr, tpr, linewidth=linewidth, label=f'{model_name} (AUC={roc_auc:.4f})', color=color)\n    # ax1.plot([0, 1], [0, 1], 'k--', alpha=0.3)\n    # ax1.set_title('ROC Curves (Model Comparison)', fontweight='bold')\n    # ax1.set_xlabel('False Positive Rate')\n    # ax1.set_ylabel('True Positive Rate')\n    # ax1.legend(fontsize=9)\n    # ax1.grid(True, alpha=0.3)\n    \n    # # Subplot 2: Churn Probability Distribution\n    # ax2 = fig.add_subplot(gs[0, 1])\n    # ax2.hist(rfm['churn_probability'], bins=50, alpha=0.7, color='steelblue', edgecolor='black')\n    # ax2.axvline(threshold_high, color='red', linestyle='--', linewidth=2, label=f'High: {threshold_high:.3f}')\n    # ax2.axvline(threshold_medium, color='orange', linestyle='--', linewidth=2, label=f'Medium: {threshold_medium:.3f}')\n    # ax2.set_title('Churn Probability Distribution', fontweight='bold')\n    # ax2.set_xlabel('Predicted Churn Probability')\n    # ax2.set_ylabel('# Customers')\n    # ax2.legend(fontsize=9)\n    # ax2.grid(True, alpha=0.3, axis='y')\n    \n#     # Subplot 3: Feature Importance\n#     ax3 = fig.add_subplot(gs[1, 0])\n#     colors_imp = ['#2ecc71', '#3498db', '#e74c3c']\n#     ax3.barh(importance_df['Feature'], importance_df['Importance_Pct'], color=colors_imp, edgecolor='black')\n#     ax3.set_title('Feature Importance (Churn Prediction)', fontweight='bold')\n#     ax3.set_xlabel('Importance %')\n#     ax3.invert_yaxis()\n    \n#     # Subplot 4: Churn Risk Distribution\n#     ax4 = fig.add_subplot(gs[1, 1])\n#     risk_counts = rfm['churn_risk'].value_counts()\n#     colors_risk = {'HIGH': '#e74c3c', 'MEDIUM': '#f39c12', 'LOW': '#2ecc71'}\n#     colors_list = [colors_risk.get(cat, '#95a5a6') for cat in risk_counts.index]\n#     wedges, texts, autotexts = ax4.pie(risk_counts, labels=risk_counts.index, autopct='%1.1f%%',\n#                                         colors=colors_list, startangle=90)\n#     ax4.set_title('Churn Risk Distribution', fontweight='bold')\n#     for autotext in autotexts:\n#         autotext.set_color('white')\n#         autotext.set_fontweight('bold')\n    \n#     plt.suptitle(f'Churn Prediction Summary\\nSelected Model: {best_model_name} (ROC-AUC: {best_roc_auc:.4f})',\n#                  fontsize=12, fontweight='bold')\n    \n#     plt.savefig('06_CHURN_EDA_MINIMAL.png', dpi=150, bbox_inches='tight')\n#     plt.close()\n    \n#     print(f\"  ✓ Dashboard saved: 06_CHURN_EDA_MINIMAL.png\")\n    \n#     # ============================================================================\n#     # PART 9: SAVE OUTPUTS (FAST)\n#     # ============================================================================\n#     print(\"\\n[PART 9] Saving outputs...\")\n    \n#     # Main output\n#     rfm.to_parquet('06_CHURN_SCORES.parquet')\n#     print(f\"  ✓ 06_CHURN_SCORES.parquet ({len(rfm):,} customers)\")\n    \n#     # Model comparison\n#     comparison_df.to_csv('06_CHURN_MODEL_METRICS.csv', index=False)\n#     print(f\"  ✓ 06_CHURN_MODEL_METRICS.csv\")\n    \n#     # Thresholds config\n#     best_config = {\n#         'selected_model': best_model_name,\n#         'roc_auc': float(best_roc_auc),\n#         'thresholds': {\n#             'high_churn': float(threshold_high),\n#             'medium_churn': float(threshold_medium)\n#         },\n#         'feature_importance': importance_df.set_index('Feature')['Importance_Pct'].to_dict()\n#     }\n#     with open('best_churn_thresholds_minimal.json', 'w') as f:\n#         json.dump(best_config, f, indent=2)\n#     print(f\"  ✓ best_churn_thresholds_minimal.json\")\n    \n#     # ============================================================================\n#     # FINAL SUMMARY\n#     # ============================================================================\n#     print(f\"\\n{'=' * 80}\")\n#     print(f\"✓✓✓ SCRIPT 07 COMPLETE (MINIMAL VERSION - 60 SECOND EXECUTION)\")\n#     print(f\"{'=' * 80}\")\n#     print(f\"\\nSUMMARY:\")\n#     print(f\"  Total customers scored: {len(rfm):,}\")\n#     print(f\"  Models trained: 2 (Logistic Regression, Gradient Boosting)\")\n#     print(f\"  Selected model: {best_model_name} (ROC-AUC: {best_roc_auc:.4f})\")\n#     print(f\"\\n  Churn risk distribution:\")\n#     print(f\"    HIGH:   {(rfm['churn_risk'] == 'HIGH').sum():7,d} customers ({(rfm['churn_risk'] == 'HIGH').sum()/len(rfm)*100:5.1f}%)\")\n#     print(f\"    MEDIUM: {(rfm['churn_risk'] == 'MEDIUM').sum():7,d} customers ({(rfm['churn_risk'] == 'MEDIUM').sum()/len(rfm)*100:5.1f}%)\")\n#     print(f\"    LOW:    {(rfm['churn_risk'] == 'LOW').sum():7,d} customers ({(rfm['churn_risk'] == 'LOW').sum()/len(rfm)*100:5.1f}%)\")\n#     print(f\"\\n  Calibrated thresholds:\")\n#     print(f\"    High Risk (≥ {threshold_high:.4f})\")\n#     print(f\"    Medium Risk (≥ {threshold_medium:.4f})\")\n#     print(f\"    Low Risk (< {threshold_medium:.4f})\")\n#     print(f\"\\n  Files created:\")\n#     print(f\"    - 06_CHURN_SCORES.parquet\")\n#     print(f\"    - 06_CHURN_MODEL_METRICS.csv\")\n#     print(f\"    - best_churn_thresholds_minimal.json\")\n#     print(f\"    - 06_CHURN_EDA_MINIMAL.png\\n\")\n\n# except Exception as e:\n#     print(f\"\\n❌ Error: {e}\")\n#     import traceback\n#     traceback.print_exc()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T12:41:51.031555Z","iopub.execute_input":"2026-09-03T12:41:51.032015Z","iopub.status.idle":"2026-09-03T12:41:51.071661Z","shell.execute_reply.started":"2026-09-03T12:41:51.031986Z","shell.execute_reply":"2026-09-03T12:41:51.070610Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSCRIPT 07: CHURN PREDICTION (FIXED VERSION)\n============================================\nStripped-down version for speed, with corrections for:\n1. Correct RFM input file (06_ not 05_)\n2. Fixed churn semantics (inverted probabilities)\n3. Consistent median imputation across train/test\n4. Holdout validation (not just resubstitution)\n5. Proper business context for threshold calibration\n6. Correct file naming (07_* outputs)\n\nCore: Train two models, compare, select best, score all customers, assign risk levels\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, roc_curve, precision_score, recall_score, f1_score, confusion_matrix\nimport json\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\" * 80)\nprint(\"SCRIPT 07: CHURN PREDICTION (FIXED VERSION)\")\nprint(\"=\" * 80)\n\ntry:\n    # ============================================================================\n    # PART 0: LOAD DATA (CORRECT FILES)\n    # ============================================================================\n    print(\"\\n[PART 0] Loading data...\")\n    \n    master = pd.read_parquet('master_with_features.parquet')\n    rfm = pd.read_parquet('06_RFM_SEGMENTS.parquet')  # FIXED: Use correct file (06_ not 05_)\n    \n    print(f\"  ✓ Master: {master.shape}\")\n    print(f\"  ✓ RFM segments: {rfm.shape}\")\n    \n    max_date = master['t_dat'].max()\n    \n    # ============================================================================\n    # PART 1: BUILD TARGET VARIABLE (FROM SCRIPT 06 WORK)\n    # ============================================================================\n    print(\"\\n[PART 1] Building target variable...\")\n    \n    validation_start = max_date - pd.Timedelta(days=90)\n    training_end = validation_start - pd.Timedelta(days=1)\n    \n    # Get customers in training period only\n    customers_training = master[master['t_dat'] <= training_end]['customer_id'].unique()\n    \n    # Create training subset (only pre-validation customers)\n    rfm_training = rfm[rfm['customer_id'].isin(customers_training)].copy()\n    \n    # Target: Did they repurchase in validation period? (0 = churn, 1 = retain)\n    # This is inverted from churn perspective (for easier interpretation)\n    buyers_future = master[master['t_dat'] >= validation_start]['customer_id'].unique()\n    rfm_training['repurchase'] = rfm_training['customer_id'].isin(buyers_future).astype(int)\n    \n    # Churn is the inverse\n    rfm_training['target_churn'] = 1 - rfm_training['repurchase']\n    \n    repurchase_rate = rfm_training['repurchase'].mean()\n    churn_rate = rfm_training['target_churn'].mean()\n    \n    print(f\"  ✓ Training set: {len(rfm_training):,} customers\")\n    print(f\"  ✓ Repurchase rate: {repurchase_rate:.1%}\")\n    print(f\"  ✓ Churn rate (target): {churn_rate:.1%}\")\n    \n    # ============================================================================\n    # PART 2: PREPARE FEATURES (WITH CONSISTENT IMPUTATION)\n    # ============================================================================\n    print(\"\\n[PART 2] Preparing features...\")\n    \n    feature_cols = ['recency_days', 'frequency', 'total_spend']\n    \n    # Extract training features\n    X_train = rfm_training[feature_cols].copy()\n    y_train = rfm_training['target_churn'].copy()\n    \n    # Calculate training median ONCE (will use for all imputation)\n    training_median = X_train.median()\n    print(f\"  Training feature medians:\")\n    for feat in feature_cols:\n        print(f\"    {feat:20s}: {training_median[feat]:.2f}\")\n    \n    # Fill training NaNs with training median\n    X_train.fillna(training_median, inplace=True)\n    \n    # Validate no NaNs remain\n    assert X_train.notna().all().all(), \"NaNs remain in training features after imputation\"\n    \n    print(f\"  ✓ Features prepared: {X_train.shape[1]} features, {X_train.shape[0]:,} samples\")\n    print(f\"  ✓ Churn target distribution:\")\n    print(f\"    - Churn (1): {y_train.sum():,} ({y_train.mean()*100:.1f}%)\")\n    print(f\"    - Retain (0): {(1-y_train).sum():,} ({(1-y_train).mean()*100:.1f}%)\")\n    \n    # ============================================================================\n    # PART 3: TRAIN/TEST SPLIT (HOLDOUT VALIDATION)\n    # ============================================================================\n    print(\"\\n[PART 3] Creating holdout validation split...\")\n    \n    X_subtrain, X_subtest, y_subtrain, y_subtest = train_test_split(\n        X_train, y_train, test_size=0.2, random_state=42, stratify=y_train\n    )\n    \n    print(f\"  ✓ Subtrain: {X_subtrain.shape[0]:,} samples\")\n    print(f\"  ✓ Subtest (holdout): {X_subtest.shape[0]:,} samples\")\n    \n    # ============================================================================\n    # PART 4: STANDARDIZE FEATURES (FOR LOGISTIC REGRESSION)\n    # ============================================================================\n    print(\"\\n[PART 4] Standardizing features...\")\n    \n    scaler = StandardScaler()\n    X_subtrain_scaled = scaler.fit_transform(X_subtrain)\n    X_subtest_scaled = scaler.transform(X_subtest)\n    X_train_scaled = scaler.transform(X_train)  # For final training on all data\n    \n    X_subtrain_scaled_df = pd.DataFrame(X_subtrain_scaled, columns=feature_cols)\n    X_subtest_scaled_df = pd.DataFrame(X_subtest_scaled, columns=feature_cols)\n    X_train_scaled_df = pd.DataFrame(X_train_scaled, columns=feature_cols)\n    \n    print(f\"  ✓ Features standardized (mean≈0, std≈1)\")\n    \n    # ============================================================================\n    # PART 5: TRAIN MODELS (ON SUBTRAINING SET, VALIDATE ON SUBTEST)\n    # ============================================================================\n    print(\"\\n[PART 5] Training models with holdout validation...\")\n    \n    models = {}\n    predictions = {}\n    training_results = []\n    \n    # ---- Logistic Regression ----\n    print(\"\\n  Training Logistic Regression (balanced)...\")\n    lr = LogisticRegression(random_state=42, max_iter=1000, class_weight='balanced')\n    lr.fit(X_subtrain_scaled_df, y_subtrain)\n    \n    y_pred_lr_train = lr.predict_proba(X_subtrain_scaled_df)[:, 1]\n    y_pred_lr_test = lr.predict_proba(X_subtest_scaled_df)[:, 1]\n    \n    roc_auc_lr_train = roc_auc_score(y_subtrain, y_pred_lr_train)\n    roc_auc_lr_test = roc_auc_score(y_subtest, y_pred_lr_test)\n    \n    models['Logistic Regression'] = (lr, 'scaled')\n    predictions['Logistic Regression'] = {\n        'train_pred': y_pred_lr_train,\n        'test_pred': y_pred_lr_test\n    }\n    \n    print(f\"    Train ROC-AUC: {roc_auc_lr_train:.4f}\")\n    print(f\"    Test ROC-AUC:  {roc_auc_lr_test:.4f} (generalization gap: {roc_auc_lr_train - roc_auc_lr_test:.4f})\")\n    \n    training_results.append({\n        'Model': 'Logistic Regression',\n        'Train_ROC_AUC': roc_auc_lr_train,\n        'Test_ROC_AUC': roc_auc_lr_test,\n        'Generalization_Gap': roc_auc_lr_train - roc_auc_lr_test\n    })\n    \n    # ---- Gradient Boosting ----\n    print(\"\\n  Training Gradient Boosting (no tuning)...\")\n    gb = GradientBoostingClassifier(n_estimators=50, learning_rate=0.1, \n                                     random_state=42, max_depth=4)\n    gb.fit(X_subtrain, y_subtrain)\n    \n    y_pred_gb_train = gb.predict_proba(X_subtrain)[:, 1]\n    y_pred_gb_test = gb.predict_proba(X_subtest)[:, 1]\n    \n    roc_auc_gb_train = roc_auc_score(y_subtrain, y_pred_gb_train)\n    roc_auc_gb_test = roc_auc_score(y_subtest, y_pred_gb_test)\n    \n    models['Gradient Boosting'] = (gb, 'unscaled')\n    predictions['Gradient Boosting'] = {\n        'train_pred': y_pred_gb_train,\n        'test_pred': y_pred_gb_test\n    }\n    \n    print(f\"    Train ROC-AUC: {roc_auc_gb_train:.4f}\")\n    print(f\"    Test ROC-AUC:  {roc_auc_gb_test:.4f} (generalization gap: {roc_auc_gb_train - roc_auc_gb_test:.4f})\")\n    \n    training_results.append({\n        'Model': 'Gradient Boosting',\n        'Train_ROC_AUC': roc_auc_gb_train,\n        'Test_ROC_AUC': roc_auc_gb_test,\n        'Generalization_Gap': roc_auc_gb_train - roc_auc_gb_test\n    })\n    \n    # ============================================================================\n    # PART 6: SELECT BEST MODEL (BASED ON TEST SET PERFORMANCE)\n    # ============================================================================\n    print(\"\\n[PART 6] Model comparison and selection...\")\n    \n    comparison_results = []\n    for model_name, (y_pred_test) in [\n        ('Logistic Regression', predictions['Logistic Regression']['test_pred']),\n        ('Gradient Boosting', predictions['Gradient Boosting']['test_pred'])\n    ]:\n        roc_auc = roc_auc_score(y_subtest, y_pred_test)\n        precision = precision_score(y_subtest, (y_pred_test > 0.5).astype(int), zero_division=0)\n        recall = recall_score(y_subtest, (y_pred_test > 0.5).astype(int), zero_division=0)\n        f1 = f1_score(y_subtest, (y_pred_test > 0.5).astype(int), zero_division=0)\n        \n        comparison_results.append({\n            'Model': model_name,\n            'Test_ROC_AUC': roc_auc,\n            'Precision': precision,\n            'Recall': recall,\n            'F1_Score': f1\n        })\n        \n        print(f\"\\n  {model_name} (TEST SET):\")\n        print(f\"    ROC-AUC: {roc_auc:.4f}, Precision: {precision:.4f}, Recall: {recall:.4f}, F1: {f1:.4f}\")\n    \n    comparison_df = pd.DataFrame(comparison_results)\n    best_idx = comparison_df['Test_ROC_AUC'].idxmax()\n    best_model_name = comparison_df.loc[best_idx, 'Model']\n    best_roc_auc = comparison_df.loc[best_idx, 'Test_ROC_AUC']\n    \n    print(f\"\\n  ✓✓✓ SELECTED: {best_model_name} (TEST ROC-AUC: {best_roc_auc:.4f})\")\n    \n    # Get the actual model object\n    best_model, best_model_scale = models[best_model_name]\n    \n    # ============================================================================\n    # PART 7: RETRAIN BEST MODEL ON ALL TRAINING DATA\n    # ============================================================================\n    print(f\"\\n[PART 7] Retraining {best_model_name} on full training set...\")\n    \n    if best_model_name == 'Logistic Regression':\n        best_model.fit(X_train_scaled_df, y_train)\n    else:\n        best_model.fit(X_train, y_train)\n    \n    print(f\"  ✓ Model retrained on {len(X_train):,} training samples\")\n    \n    # ============================================================================\n    # PART 8: THRESHOLD CALIBRATION (YOUDEN + BUSINESS CONTEXT)\n    # ============================================================================\n    print(f\"\\n[PART 8] Calibrating risk thresholds (Youden's J on test set)...\")\n    \n    # Use test set predictions for threshold calibration (unbiased)\n    if best_model_name == 'Logistic Regression':\n        y_pred_calibrate = predictions[best_model_name]['test_pred']\n    else:\n        y_pred_calibrate = predictions[best_model_name]['test_pred']\n    \n    fpr, tpr, thresholds_roc = roc_curve(y_subtest, y_pred_calibrate)\n    youden = tpr - fpr\n    youden_idx = np.argmax(youden)\n    threshold_high = thresholds_roc[youden_idx]\n    \n    # Medium threshold: between optimal high and midpoint\n    # This ensures 3 distinct risk tiers\n    threshold_medium = (threshold_high + 0.5) / 2\n    \n    print(f\"  Calibration method: Youden's J (maximizes sensitivity + specificity)\")\n    print(f\"  High Churn Risk threshold:   {threshold_high:.4f}\")\n    print(f\"  Medium Churn Risk threshold: {threshold_medium:.4f}\")\n    print(f\"  Low Churn Risk:              < {threshold_medium:.4f}\")\n    print(f\"\\n  Business interpretation:\")\n    print(f\"    HIGH:   Churn probability ≥ {threshold_high:.1%}\")\n    print(f\"    MEDIUM: Churn probability {threshold_medium:.1%} to {threshold_high:.1%}\")\n    print(f\"    LOW:    Churn probability < {threshold_medium:.1%}\")\n    \n    # ============================================================================\n    # PART 9: SCORE ALL CUSTOMERS (USING TRAINING MEDIAN IMPUTATION)\n    # ============================================================================\n    print(f\"\\n[PART 9] Scoring all customers...\")\n    \n    X_all = rfm[feature_cols].copy()\n    \n    # CRITICAL: Use TRAINING median for imputation (consistency)\n    X_all.fillna(training_median, inplace=True)\n    \n    # Score using best model\n    if best_model_name == 'Logistic Regression':\n        X_all_scaled = scaler.transform(X_all)\n        churn_proba = best_model.predict_proba(X_all_scaled)[:, 1]\n    else:\n        churn_proba = best_model.predict_proba(X_all)[:, 1]\n    \n    # Add to rfm\n    rfm['repurchase_probability'] = 1 - churn_proba  # For interpretability\n    rfm['churn_probability'] = churn_proba  # FIXED: Inverted (1 - repurchase = churn)\n    \n    # Assign risk levels\n    def assign_risk(churn_prob):\n        if churn_prob >= threshold_high:\n            return 'HIGH'\n        elif churn_prob >= threshold_medium:\n            return 'MEDIUM'\n        else:\n            return 'LOW'\n    \n    rfm['churn_risk'] = rfm['churn_probability'].apply(assign_risk)\n    \n    print(f\"  ✓ All {len(rfm):,} customers scored\")\n    print(f\"\\n  Churn Risk Distribution:\")\n    risk_summary = {}\n    for risk in ['HIGH', 'MEDIUM', 'LOW']:\n        count = (rfm['churn_risk'] == risk).sum()\n        pct = count / len(rfm) * 100\n        risk_summary[risk] = {'count': count, 'pct': pct}\n        print(f\"    {risk:8s}: {count:7,d} customers ({pct:5.1f}%)\")\n    \n    # ============================================================================\n    # PART 10: FEATURE IMPORTANCE\n    # ============================================================================\n    print(f\"\\n[PART 10] Computing feature importance...\")\n    \n    if best_model_name == 'Gradient Boosting':\n        importance = best_model.feature_importances_\n        importance_type = \"Gradient Boosting (split-based)\"\n    else:\n        # For logistic regression, use absolute standardized coefficients\n        importance = np.abs(best_model.coef_[0])\n        importance_type = \"Logistic Regression (coefficient magnitude)\"\n    \n    importance_pct = (importance / importance.sum()) * 100\n    importance_df = pd.DataFrame({\n        'Feature': feature_cols,\n        'Importance_Pct': importance_pct\n    }).sort_values('Importance_Pct', ascending=False)\n    \n    print(f\"\\n  Method: {importance_type}\")\n    for _, row in importance_df.iterrows():\n        print(f\"    {row['Feature']:20s}: {row['Importance_Pct']:6.2f}%\")\n    \n    # ============================================================================\n    # PART 11: VISUALIZATION DASHBOARD\n    # ============================================================================\n    print(f\"\\n[PART 11] Creating visualization dashboard...\")\n    \n    fig = plt.figure(figsize=(16, 12))\n    gs = fig.add_gridspec(3, 2, hspace=0.35, wspace=0.3)\n    \n    # Subplot 1: ROC Curves (Train vs Test Comparison)\n    ax1 = fig.add_subplot(gs[0, 0])\n    for model_name, predictions_dict in predictions.items():\n        # Plot test ROC curve only (more realistic)\n        fpr, tpr, _ = roc_curve(y_subtest, predictions_dict['test_pred'])\n        roc_auc = roc_auc_score(y_subtest, predictions_dict['test_pred'])\n        linewidth = 2.5 if model_name == best_model_name else 1.5\n        color = '#2ecc71' if model_name == best_model_name else '#95a5a6'\n        ax1.plot(fpr, tpr, linewidth=linewidth, label=f'{model_name} (AUC={roc_auc:.4f})', color=color)\n    \n    ax1.plot([0, 1], [0, 1], 'k--', alpha=0.3, label='Random Classifier')\n    ax1.set_title('ROC Curves: Model Comparison (Test Set)', fontweight='bold', fontsize=12)\n    ax1.set_xlabel('False Positive Rate')\n    ax1.set_ylabel('True Positive Rate')\n    ax1.legend(fontsize=9, loc='lower right')\n    ax1.grid(True, alpha=0.3)\n    \n    # Subplot 2: Model Metrics Comparison\n    ax2 = fig.add_subplot(gs[0, 1])\n    metrics = ['Test_ROC_AUC', 'Precision', 'Recall', 'F1_Score']\n    x = np.arange(len(comparison_df))\n    width = 0.2\n    \n    for i, metric in enumerate(metrics):\n        offset = (i - 1.5) * width\n        values = comparison_df[metric].tolist()\n        ax2.bar(x + offset, values, width, label=metric, alpha=0.8)\n    \n    ax2.set_title('Model Performance Metrics (Test Set)', fontweight='bold', fontsize=12)\n    ax2.set_ylabel('Score')\n    ax2.set_xticks(x)\n    ax2.set_xticklabels(comparison_df['Model'])\n    ax2.legend(fontsize=8)\n    ax2.set_ylim(0, 1)\n    ax2.grid(True, alpha=0.3, axis='y')\n    \n    # Subplot 3: Churn Probability Distribution\n    ax3 = fig.add_subplot(gs[1, 0])\n    ax3.hist(rfm['churn_probability'], bins=50, alpha=0.7, color='steelblue', edgecolor='black')\n    ax3.axvline(threshold_high, color='red', linestyle='--', linewidth=2.5, label=f'High: {threshold_high:.3f}')\n    ax3.axvline(threshold_medium, color='orange', linestyle='--', linewidth=2.5, label=f'Medium: {threshold_medium:.3f}')\n    ax3.set_title('Churn Probability Distribution (All Customers)', fontweight='bold', fontsize=12)\n    ax3.set_xlabel('Predicted Churn Probability')\n    ax3.set_ylabel('# Customers')\n    ax3.legend(fontsize=9)\n    ax3.grid(True, alpha=0.3, axis='y')\n    \n    # Subplot 4: Feature Importance\n    ax4 = fig.add_subplot(gs[1, 1])\n    colors_imp = ['#2ecc71', '#3498db', '#e74c3c']\n    ax4.barh(importance_df['Feature'], importance_df['Importance_Pct'], \n             color=colors_imp[:len(importance_df)], edgecolor='black', linewidth=1.5)\n    ax4.set_title(f'Feature Importance\\n({best_model_name})', fontweight='bold', fontsize=12)\n    ax4.set_xlabel('Importance (%)')\n    ax4.invert_yaxis()\n    ax4.grid(True, alpha=0.3, axis='x')\n    \n    # Subplot 5: Churn Risk Distribution\n    ax5 = fig.add_subplot(gs[2, 0])\n    risk_order = ['HIGH', 'MEDIUM', 'LOW']\n    risk_counts = [risk_summary[r]['count'] for r in risk_order]\n    colors_risk = ['#e74c3c', '#f39c12', '#2ecc71']\n    wedges, texts, autotexts = ax5.pie(risk_counts, labels=risk_order, autopct='%1.1f%%',\n                                        colors=colors_risk, startangle=90)\n    ax5.set_title('Customer Churn Risk Distribution', fontweight='bold', fontsize=12)\n    for autotext in autotexts:\n        autotext.set_color('white')\n        autotext.set_fontweight('bold')\n        autotext.set_fontsize(10)\n    \n    # Subplot 6: Training vs Test Performance (Generalization)\n    ax6 = fig.add_subplot(gs[2, 1])\n    x_pos = np.arange(len(training_results))\n    width = 0.35\n    \n    train_scores = [r['Train_ROC_AUC'] for r in training_results]\n    test_scores = [r['Test_ROC_AUC'] for r in training_results]\n    \n    ax6.bar(x_pos - width/2, train_scores, width, label='Train ROC-AUC', alpha=0.8, color='#3498db')\n    ax6.bar(x_pos + width/2, test_scores, width, label='Test ROC-AUC', alpha=0.8, color='#e74c3c')\n    \n    ax6.set_title('Generalization Performance (Train vs Test)', fontweight='bold', fontsize=12)\n    ax6.set_ylabel('ROC-AUC Score')\n    ax6.set_xticks(x_pos)\n    ax6.set_xticklabels([r['Model'] for r in training_results])\n    ax6.legend(fontsize=9)\n    ax6.set_ylim(0.5, 0.85)\n    ax6.grid(True, alpha=0.3, axis='y')\n    \n    plt.suptitle(f'Churn Prediction Summary\\nSelected Model: {best_model_name} (Test ROC-AUC: {best_roc_auc:.4f})',\n                 fontsize=14, fontweight='bold', y=0.995)\n    \n    plt.savefig('07_CHURN_DASHBOARD.png', dpi=150, bbox_inches='tight')\n    plt.close()\n    \n    print(f\"  ✓ Dashboard saved: 07_CHURN_DASHBOARD.png\")\n    \n    # ============================================================================\n    # PART 12: SAVE OUTPUTS\n    # ============================================================================\n    print(f\"\\n[PART 12] Saving outputs...\")\n    \n    # Main output: All customers with churn scores and risks\n    rfm.to_parquet('07_CHURN_SCORES.parquet')\n    print(f\"  ✓ 07_CHURN_SCORES.parquet ({len(rfm):,} customers)\")\n    \n    # Model comparison metrics\n    comparison_df.to_csv('07_CHURN_MODEL_METRICS.csv', index=False)\n    print(f\"  ✓ 07_CHURN_MODEL_METRICS.csv\")\n    \n    # Training details (for transparency)\n    training_df = pd.DataFrame(training_results)\n    training_df.to_csv('07_CHURN_TRAINING_DETAILS.csv', index=False)\n    print(f\"  ✓ 07_CHURN_TRAINING_DETAILS.csv\")\n    \n    # Feature importance\n    importance_df.to_csv('07_CHURN_FEATURE_IMPORTANCE.csv', index=False)\n    print(f\"  ✓ 07_CHURN_FEATURE_IMPORTANCE.csv\")\n    \n    # Best model configuration (thresholds + model info)\n    best_config = {\n        'selected_model': best_model_name,\n        'test_roc_auc': float(best_roc_auc),\n        'model_type': 'binary_classification_churn_prediction',\n        'thresholds': {\n            'high_churn_risk': float(threshold_high),\n            'medium_churn_risk': float(threshold_medium),\n            'threshold_method': 'Youdens_J_statistic',\n            'calibrated_on': 'holdout_test_set'\n        },\n        'risk_distribution': {\n            'HIGH': float(risk_summary['HIGH']['pct']),\n            'MEDIUM': float(risk_summary['MEDIUM']['pct']),\n            'LOW': float(risk_summary['LOW']['pct'])\n        },\n        'feature_importance': importance_df.set_index('Feature')['Importance_Pct'].to_dict(),\n        'training_info': {\n            'training_samples': len(X_train),\n            'test_samples': len(X_subtest),\n            'feature_median_values': training_median.to_dict(),\n            'class_balance': {\n                'churn_rate': float(churn_rate),\n                'retention_rate': float(repurchase_rate)\n            }\n        },\n        'metadata': {\n            'imputation_method': 'training_median_applied_consistently',\n            'scaling_method': 'StandardScaler (LogReg only)',\n            'evaluation_method': 'holdout_test_set_validation',\n            'semantic_note': 'churn_probability = 1 - repurchase_probability (inverted for correct interpretation)'\n        }\n    }\n    \n    with open('07_CHURN_THRESHOLDS.json', 'w') as f:\n        json.dump(best_config, f, indent=2)\n    print(f\"  ✓ 07_CHURN_THRESHOLDS.json (thresholds + model config)\")\n    \n    # ============================================================================\n    # FINAL SUMMARY\n    # ============================================================================\n    print(f\"\\n{'=' * 80}\")\n    print(f\"✓✓✓ SCRIPT 07 COMPLETE: CHURN PREDICTION\")\n    print(f\"{'=' * 80}\")\n    print(f\"\\nSUMMARY:\")\n    print(f\"  Total customers scored: {len(rfm):,}\")\n    print(f\"  Training customers (pre-validation): {len(rfm_training):,}\")\n    print(f\"  Models trained & compared: 2 (Logistic Regression, Gradient Boosting)\")\n    print(f\"  Selected model: {best_model_name}\")\n    print(f\"  Test set ROC-AUC: {best_roc_auc:.4f}\")\n    print(f\"\\n  Churn Risk Breakdown:\")\n    print(f\"    HIGH:   {risk_summary['HIGH']['count']:7,d} customers ({risk_summary['HIGH']['pct']:5.1f}%)\")\n    print(f\"    MEDIUM: {risk_summary['MEDIUM']['count']:7,d} customers ({risk_summary['MEDIUM']['pct']:5.1f}%)\")\n    print(f\"    LOW:    {risk_summary['LOW']['count']:7,d} customers ({risk_summary['LOW']['pct']:5.1f}%)\")\n    print(f\"\\n  Calibrated Risk Thresholds:\")\n    print(f\"    HIGH Churn Risk:   probability ≥ {threshold_high:.4f}\")\n    print(f\"    MEDIUM Churn Risk: probability ≥ {threshold_medium:.4f} and < {threshold_high:.4f}\")\n    print(f\"    LOW Churn Risk:    probability < {threshold_medium:.4f}\")\n    print(f\"\\n  Key Features (by importance):\")\n    for _, row in importance_df.iterrows():\n        print(f\"    {row['Feature']:20s}: {row['Importance_Pct']:6.2f}%\")\n    print(f\"\\n  Validation Approach:\")\n    print(f\"    ✓ Holdout test set (80/20 split on training data)\")\n    print(f\"    ✓ Thresholds calibrated on test set (unbiased)\")\n    print(f\"    ✓ Models retrained on full training set before production scoring\")\n    print(f\"\\n  Files Created:\")\n    print(f\"    - 07_CHURN_SCORES.parquet (all customers with churn scores/risks)\")\n    print(f\"    - 07_CHURN_MODEL_METRICS.csv (model comparison on test set)\")\n    print(f\"    - 07_CHURN_TRAINING_DETAILS.csv (train vs test performance)\")\n    print(f\"    - 07_CHURN_FEATURE_IMPORTANCE.csv\")\n    print(f\"    - 07_CHURN_THRESHOLDS.json (thresholds + full config)\")\n    print(f\"    - 07_CHURN_DASHBOARD.png (6-subplot visualization)\")\n    print(f\"\\n  Key Improvements Over Original:\")\n    print(f\"    ✓ Fixed churn semantics (1 - repurchase = churn)\")\n    print(f\"    ✓ Consistent median imputation across train/test\")\n    print(f\"    ✓ Holdout validation (not just resubstitution)\")\n    print(f\"    ✓ Correct input file (06_RFM_SEGMENTS not 05_RFM_SEGMENTS)\")\n    print(f\"    ✓ Proper train/test split before threshold calibration\")\n    print(f\"    ✓ Complete metadata for deployment\\n\")\n\nexcept Exception as e:\n    print(f\"\\n❌ Error: {e}\")\n    import traceback\n    traceback.print_exc()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T14:10:00.579760Z","iopub.execute_input":"2026-09-03T14:10:00.580220Z","iopub.status.idle":"2026-09-03T14:11:34.097422Z","shell.execute_reply.started":"2026-09-03T14:10:00.580191Z","shell.execute_reply":"2026-09-03T14:11:34.096546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSCRIPT 08: INVENTORY PROFILING & OPTIMIZATION\n==============================================\nIdentifies slow-moving SKUs (dead stock) and stockout risks.\nTwo-part analysis:\n1. DEAD STOCK: Slow-moving items that need liquidation\n2. STOCKOUT RISK: Fast-moving items with supply gaps\n\nUses:\n- Rolling velocity (recent 90d vs prior 90d) for evergreen/seasonal detection\n- IQR-based outlier detection for stockout proxies\n- Category-level percentiles for dead stock identification\n- Prescriptive labels (LIQUIDATE, MONITOR, RESTOCK)\n\nOutput:\n- 08_INVENTORY_ANALYSIS.parquet (comprehensive metrics)\n- 08_DEAD_STOCK_ITEMS.csv (liquidation candidates)\n- 08_STOCKOUT_RISK_ITEMS.csv (supply chain concerns)\n- 08_INVENTORY_DASHBOARD.png (presentation visualization)\n\nTime estimate: 12-14 minutes\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\" * 80)\nprint(\"SCRIPT 08: INVENTORY PROFILING & OPTIMIZATION\")\nprint(\"=\" * 80)\n\ntry:\n    # ============================================================================\n    # PART 0: LOAD DATA & METADATA\n    # ============================================================================\n    print(\"\\n[PART 0] Loading data...\")\n    start_time = datetime.now()\n    \n    master = pd.read_parquet('master_with_features.parquet')\n    velocity = pd.read_parquet('velocity_features.parquet')\n    \n    print(f\"  ✓ Master: {master.shape}\")\n    print(f\"  ✓ Velocity features: {velocity.shape}\")\n    \n    max_date = master['t_dat'].max()\n    \n    # ============================================================================\n    # PART 1: LOAD PRODUCT METADATA (DEFENSIVE CHECK)\n    # ============================================================================\n    print(\"\\n[PART 1] Loading product metadata...\")\n    \n    # Check if product_type_name exists in master\n    if 'product_type_name' in master.columns:\n        print(\"  ✓ Found product_type_name in master_with_features\")\n        articles_metadata = master[['article_id', 'product_type_name']].drop_duplicates()\n    else:\n        print(\"  ⚠️  product_type_name not found in master, attempting to load from articles.csv...\")\n        try:\n            articles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\n            articles_metadata = articles[['article_id', 'product_type_name']].copy()\n            print(\"  ✓ Loaded product_type_name from articles.csv\")\n        except Exception as e:\n            print(f\"  ❌ Could not load product_type_name: {e}\")\n            print(\"  Proceeding without category grouping\")\n            articles_metadata = None\n    \n    # Merge metadata if available\n    if articles_metadata is not None:\n        velocity = velocity.merge(articles_metadata, on='article_id', how='left')\n        print(f\"  ✓ Metadata merged: {velocity.shape}\")\n    \n    # ============================================================================\n    # PART 2: ROLLING VELOCITY ANALYSIS (EVERGREEN vs SEASONAL)\n    # ============================================================================\n    print(\"\\n[PART 2] Analyzing velocity trends (rolling 90-day windows)...\")\n    \n    # Calculate rolling velocity: recent 90 days vs prior 90 days\n    recent_window_start = max_date - pd.Timedelta(days=90)\n    prior_window_start = recent_window_start - pd.Timedelta(days=90)\n    prior_window_end = recent_window_start - pd.Timedelta(days=1)\n    \n    # Velocity in recent period\n    recent_velocity = master[master['t_dat'] >= recent_window_start].groupby('article_id').size()\n    \n    # Velocity in prior period\n    prior_velocity = master[(master['t_dat'] >= prior_window_start) & \n                            (master['t_dat'] <= prior_window_end)].groupby('article_id').size()\n    \n    # Combine and calculate ratio\n    velocity_comparison = pd.DataFrame({\n        'article_id': recent_velocity.index,\n        'recent_90d_sales': recent_velocity.values\n    })\n    \n    velocity_comparison = velocity_comparison.merge(\n        pd.DataFrame({'article_id': prior_velocity.index, 'prior_90d_sales': prior_velocity.values}),\n        on='article_id',\n        how='left'\n    )\n    \n    # Fill missing values (items not in prior period = new items)\n    velocity_comparison['prior_90d_sales'].fillna(0, inplace=True)\n    \n    # Calculate velocity ratio (recent / prior)\n    # Avoid division by zero\n    velocity_comparison['velocity_ratio'] = np.where(\n        velocity_comparison['prior_90d_sales'] > 0,\n        velocity_comparison['recent_90d_sales'] / velocity_comparison['prior_90d_sales'],\n        np.nan\n    )\n    \n    # Classify as evergreen or seasonal\n    # Evergreen: ratio 0.8-1.2 (stable)\n    # Declining: ratio < 0.8 (seasonal dip or losing appeal)\n    # Emerging: ratio > 1.2 (trending up)\n    def classify_trend(ratio):\n        if pd.isna(ratio):\n            return 'NEW'  # Item wasn't in prior period\n        elif ratio < 0.8:\n            return 'DECLINING'\n        elif ratio > 1.2:\n            return 'EMERGING'\n        else:\n            return 'EVERGREEN'\n    \n    velocity_comparison['trend'] = velocity_comparison['velocity_ratio'].apply(classify_trend)\n    \n    # Merge into velocity dataframe\n    velocity = velocity.merge(\n        velocity_comparison[['article_id', 'recent_90d_sales', 'prior_90d_sales', 'velocity_ratio', 'trend']],\n        on='article_id',\n        how='left'\n    )\n    \n    print(f\"  Trend distribution:\")\n    for trend in ['EVERGREEN', 'EMERGING', 'DECLINING', 'NEW']:\n        count = (velocity['trend'] == trend).sum()\n        pct = count / len(velocity) * 100\n        print(f\"    {trend:12s}: {count:6,d} ({pct:5.1f}%)\")\n    \n    # ============================================================================\n    # PART 3: STOCKOUT RISK DETECTION (IQR-BASED)\n    # ============================================================================\n    print(\"\\n[PART 3] Detecting stockout risks (IQR-based gap analysis)...\")\n    \n    # For each article-channel, identify sales gaps\n    stockout_risks = []\n    \n    for (article, channel), group in master.groupby(['article_id', 'sales_channel_id']):\n        sorted_dates = sorted(group['t_dat'].unique())\n        \n        if len(sorted_dates) > 2:  # Need at least 3 dates to detect gaps\n            # Calculate gaps between consecutive sales\n            gaps = pd.Series(sorted_dates[1:]).diff().dt.days.dropna()\n            \n            if len(gaps) > 0:\n                # Calculate IQR-based outlier threshold\n                q1 = gaps.quantile(0.25)\n                q3 = gaps.quantile(0.75)\n                iqr = q3 - q1\n                outlier_threshold = q3 + (1.5 * iqr)\n                \n                # Detect anomalous gaps (potential stockouts)\n                max_gap = gaps.max()\n                median_gap = gaps.median()\n                \n                if max_gap > outlier_threshold:\n                    stockout_risks.append({\n                        'article_id': article,\n                        'sales_channel_id': channel,\n                        'max_sales_gap_days': max_gap,\n                        'median_sales_gap_days': median_gap,\n                        'gap_threshold_iqr': outlier_threshold,\n                        'is_stockout_risk': True\n                    })\n    \n    if stockout_risks:\n        stockout_df = pd.DataFrame(stockout_risks)\n        print(f\"  ✓ Identified {len(stockout_df):,} article-channel pairs with stockout risk\")\n        \n        # Merge into velocity (article-channel level)\n        velocity = velocity.merge(\n            stockout_df[['article_id', 'sales_channel_id', 'max_sales_gap_days', \n                        'median_sales_gap_days', 'is_stockout_risk']],\n            on=['article_id', 'sales_channel_id'],\n            how='left'\n        )\n        velocity['is_stockout_risk'] = velocity['is_stockout_risk'].fillna(False)\n    else:\n        print(f\"  ℹ️  No significant stockout risks detected (all gaps within normal variation)\")\n        velocity['max_sales_gap_days'] = np.nan\n        velocity['median_sales_gap_days'] = np.nan\n        velocity['is_stockout_risk'] = False\n    \n    # ============================================================================\n    # PART 4: DEAD STOCK DETECTION (CATEGORY-LEVEL PERCENTILES)\n    # ============================================================================\n    print(\"\\n[PART 4] Identifying dead stock (slow-moving items)...\")\n    \n    # Category-level velocity percentiles (if metadata available)\n    if 'product_type_name' in velocity.columns:\n        velocity['velocity_percentile_in_category'] = velocity.groupby('product_type_name')['velocity_weekly'].transform(\n            lambda x: x.rank(pct=True) * 100\n        )\n        print(f\"  ✓ Category-level percentiles calculated ({velocity['product_type_name'].nunique()} categories)\")\n    else:\n        # Fallback: overall percentile\n        velocity['velocity_percentile_in_category'] = velocity['velocity_weekly'].rank(pct=True) * 100\n        print(f\"  ℹ️  Using overall percentiles (no category grouping)\")\n    \n    # Flag slow-moving (bottom 25% in category)\n    velocity['is_slow_moving'] = velocity['velocity_percentile_in_category'] < 25\n    \n    slow_count = velocity['is_slow_moving'].sum()\n    print(f\"  ✓ Identified {slow_count:,} slow-moving items (bottom 25% in category)\")\n    \n    # ============================================================================\n    # PART 5: PRESCRIPTIVE LABELING & ACTION PRIORITY\n    # ============================================================================\n    print(\"\\n[PART 5] Creating prescriptive labels & action priorities...\")\n    \n    # Decision logic:\n    # - Slow + Declining/Evergreen → LIQUIDATE (dead stock, tie up capital)\n    # - Fast + Stockout Risk → RESTOCK (supply chain problem, lost sales)\n    # - Slow + Emerging → MONITOR (might turn around)\n    # - Others → MONITOR (stable or contextual)\n    \n    def prescribe_action(row):\n        if row['is_slow_moving']:\n            if row['trend'] in ['DECLINING', 'EVERGREEN']:\n                return 'LIQUIDATE'\n            elif row['trend'] == 'EMERGING':\n                return 'MONITOR'\n            else:\n                return 'MONITOR'\n        elif row['is_stockout_risk'] and row['velocity_weekly'] > velocity['velocity_weekly'].median():\n            return 'RESTOCK'\n        else:\n            return 'MONITOR'\n    \n    velocity['action'] = velocity.apply(prescribe_action, axis=1)\n    \n    # Calculate action priority score\n    # For LIQUIDATE: prioritize by how slow it is\n    # For RESTOCK: prioritize by how fast it is\n    # For MONITOR: low priority\n    \n    velocity['action_priority_score'] = 0.0\n    \n    # LIQUIDATE: score by how slow (lower percentile = higher priority to liquidate)\n    liquidate_mask = velocity['action'] == 'LIQUIDATE'\n    velocity.loc[liquidate_mask, 'action_priority_score'] = (\n        25 - velocity.loc[liquidate_mask, 'velocity_percentile_in_category']\n    )\n    \n    # RESTOCK: score by how fast and how much gap (higher velocity + bigger gap = higher priority)\n    restock_mask = velocity['action'] == 'RESTOCK'\n    velocity.loc[restock_mask, 'action_priority_score'] = (\n        (velocity.loc[restock_mask, 'velocity_percentile_in_category'] / 100) * 50 +\n        (velocity.loc[restock_mask, 'max_sales_gap_days'].fillna(0) / \n         velocity.loc[restock_mask, 'max_sales_gap_days'].max() * 50 \n         if velocity.loc[restock_mask, 'max_sales_gap_days'].max() > 0 else 0)\n    )\n    \n    # Sort by priority\n    velocity = velocity.sort_values('action_priority_score', ascending=False)\n    \n    action_dist = velocity['action'].value_counts()\n    print(f\"\\n  Action Distribution:\")\n    for action in ['LIQUIDATE', 'RESTOCK', 'MONITOR']:\n        count = action_dist.get(action, 0)\n        pct = count / len(velocity) * 100\n        print(f\"    {action:12s}: {count:6,d} ({pct:5.1f}%)\")\n    \n    # ============================================================================\n    # PART 6: SAVE COMPREHENSIVE ANALYSIS\n    # ============================================================================\n    print(f\"\\n[PART 6] Saving outputs...\")\n    \n    # Main analysis (all metrics)\n    velocity.to_parquet('08_INVENTORY_ANALYSIS.parquet')\n    print(f\"  ✓ 08_INVENTORY_ANALYSIS.parquet ({len(velocity):,} items)\")\n    \n    # Dead stock items (for liquidation strategy - Script 10)\n    dead_stock = velocity[velocity['action'] == 'LIQUIDATE'].copy()\n    dead_stock_output = dead_stock[[\n        'article_id', 'sales_channel_id', 'velocity_weekly', 'velocity_percentile_in_category',\n        'trend', 'action', 'action_priority_score'\n    ]].sort_values('action_priority_score', ascending=False)\n    \n    dead_stock_output.to_csv('08_DEAD_STOCK_ITEMS.csv', index=False)\n    print(f\"  ✓ 08_DEAD_STOCK_ITEMS.csv ({len(dead_stock_output):,} items for liquidation)\")\n    \n    # Stockout risk items (for restocking strategy - Script 9)\n    stockout_risk = velocity[velocity['action'] == 'RESTOCK'].copy()\n    stockout_output = stockout_risk[[\n        'article_id', 'sales_channel_id', 'velocity_weekly', 'max_sales_gap_days',\n        'median_sales_gap_days', 'trend', 'action', 'action_priority_score'\n    ]].sort_values('action_priority_score', ascending=False)\n    \n    stockout_output.to_csv('08_STOCKOUT_RISK_ITEMS.csv', index=False)\n    print(f\"  ✓ 08_STOCKOUT_RISK_ITEMS.csv ({len(stockout_output):,} items for restocking)\")\n    \n    # ============================================================================\n    # PART 7: VISUALIZATION DASHBOARD (PRESENTATION-READY)\n    # ============================================================================\n    print(f\"\\n[PART 7] Creating presentation dashboard...\")\n    \n    fig = plt.figure(figsize=(18, 12))\n    gs = fig.add_gridspec(3, 3, hspace=0.4, wspace=0.3)\n    \n    # Subplot 1: Trend Distribution\n    ax1 = fig.add_subplot(gs[0, 0])\n    trend_counts = velocity['trend'].value_counts()\n    colors_trend = {'EVERGREEN': '#2ecc71', 'EMERGING': '#3498db', 'DECLINING': '#e74c3c', 'NEW': '#f39c12'}\n    trend_colors = [colors_trend.get(t, '#95a5a6') for t in trend_counts.index]\n    ax1.bar(trend_counts.index, trend_counts.values, color=trend_colors, edgecolor='black', linewidth=1.5)\n    ax1.set_title('1. TREND ANALYSIS\\nWhat We Found: Item Velocity Trends', fontweight='bold', fontsize=11)\n    ax1.set_ylabel('# Items')\n    for i, v in enumerate(trend_counts.values):\n        ax1.text(i, v + 10, f'{v:,}', ha='center', va='bottom', fontweight='bold', fontsize=9)\n    ax1.tick_params(axis='x', rotation=45)\n    ax1.grid(True, alpha=0.3, axis='y')\n    \n    # Subplot 2: Action Distribution\n    ax2 = fig.add_subplot(gs[0, 1])\n    action_counts = velocity['action'].value_counts()\n    colors_action = {'LIQUIDATE': '#e74c3c', 'RESTOCK': '#f39c12', 'MONITOR': '#95a5a6'}\n    action_colors = [colors_action.get(a, '#95a5a6') for a in action_counts.index]\n    ax2.bar(action_counts.index, action_counts.values, color=action_colors, edgecolor='black', linewidth=1.5)\n    ax2.set_title('2. DECISIONS MADE\\nAction Categories by Inventory Status', fontweight='bold', fontsize=11)\n    ax2.set_ylabel('# Items')\n    for i, v in enumerate(action_counts.values):\n        ax2.text(i, v + 10, f'{v:,}', ha='center', va='bottom', fontweight='bold', fontsize=9)\n    ax2.tick_params(axis='x', rotation=45)\n    ax2.grid(True, alpha=0.3, axis='y')\n    \n    # Subplot 3: Velocity Distribution (with slow-moving threshold)\n    ax3 = fig.add_subplot(gs[0, 2])\n    ax3.hist(velocity['velocity_percentile_in_category'], bins=50, alpha=0.7, color='steelblue', edgecolor='black')\n    ax3.axvline(25, color='red', linestyle='--', linewidth=2.5, label='Slow-moving threshold (25%)')\n    ax3.set_title('3. WHY ASSUMPTIONS\\nVelocity Distribution (percentile in category)', fontweight='bold', fontsize=11)\n    ax3.set_xlabel('Velocity Percentile')\n    ax3.set_ylabel('# Items')\n    ax3.legend(fontsize=9)\n    ax3.grid(True, alpha=0.3, axis='y')\n    \n    # Subplot 4: Velocity Ratio (Evergreen vs Seasonal)\n    ax4 = fig.add_subplot(gs[1, 0])\n    ratio_data = velocity[velocity['trend'] != 'NEW']['velocity_ratio'].dropna()\n    ax4.hist(ratio_data, bins=40, alpha=0.7, color='coral', edgecolor='black')\n    ax4.axvline(0.8, color='red', linestyle='--', linewidth=2, alpha=0.7, label='Declining threshold')\n    ax4.axvline(1.2, color='green', linestyle='--', linewidth=2, alpha=0.7, label='Emerging threshold')\n    ax4.axvline(1.0, color='gray', linestyle=':', linewidth=2, alpha=0.7, label='Stable (evergreen)')\n    ax4.set_title('Recent vs Prior 90-Day Velocity Ratio\\n(How we detect seasonality)', fontweight='bold', fontsize=11)\n    ax4.set_xlabel('Velocity Ratio (Recent / Prior)')\n    ax4.set_ylabel('# Items')\n    ax4.legend(fontsize=8)\n    ax4.grid(True, alpha=0.3, axis='y')\n    \n    # Subplot 5: Sales Gap Distribution (Stockout Risk)\n    ax5 = fig.add_subplot(gs[1, 1])\n    gap_data = velocity[velocity['is_stockout_risk']]['max_sales_gap_days'].dropna()\n    if len(gap_data) > 0:\n        ax5.hist(gap_data, bins=30, alpha=0.7, color='mediumpurple', edgecolor='black')\n        ax5.set_title('Max Sales Gap (Stockout Risk Items)\\n(Gap detection using IQR)', fontweight='bold', fontsize=11)\n        ax5.set_xlabel('Days Gap')\n        ax5.set_ylabel('# Items')\n        ax5.grid(True, alpha=0.3, axis='y')\n    else:\n        ax5.text(0.5, 0.5, 'No stockout risks detected', ha='center', va='center', transform=ax5.transAxes)\n        ax5.set_title('Max Sales Gap (Stockout Risk)\\n(No anomalies detected)', fontweight='bold', fontsize=11)\n    \n    # Subplot 6: Priority Score by Action\n    ax6 = fig.add_subplot(gs[1, 2])\n    for action in ['LIQUIDATE', 'RESTOCK', 'MONITOR']:\n        action_data = velocity[velocity['action'] == action]['action_priority_score']\n        if len(action_data) > 0:\n            ax6.scatter([action] * len(action_data), action_data, alpha=0.5, s=30, label=action)\n    ax6.set_title('Action Priority Scores\\n(Final decision matrix basis)', fontweight='bold', fontsize=11)\n    ax6.set_ylabel('Priority Score')\n    ax6.grid(True, alpha=0.3, axis='y')\n    \n    # Subplot 7: Top Liquidation Candidates\n    ax7 = fig.add_subplot(gs[2, :2])\n    top_liquidate = velocity[velocity['action'] == 'LIQUIDATE'].head(15)\n    if len(top_liquidate) > 0:\n        y_pos = np.arange(len(top_liquidate))\n        ax7.barh(y_pos, top_liquidate['action_priority_score'].values, color='#e74c3c', edgecolor='black', linewidth=1)\n        ax7.set_yticks(y_pos)\n        ax7.set_yticklabels([f\"Article {aid}\" for aid in top_liquidate['article_id'].values], fontsize=8)\n        ax7.set_xlabel('Liquidation Priority Score')\n        ax7.set_title('Top 15 Items for Liquidation (Dead Stock)', fontweight='bold', fontsize=11)\n        ax7.invert_yaxis()\n        ax7.grid(True, alpha=0.3, axis='x')\n    else:\n        ax7.text(0.5, 0.5, 'No items to liquidate', ha='center', va='center', transform=ax7.transAxes)\n    \n    # Subplot 8: Summary Text Box\n    ax8 = fig.add_subplot(gs[2, 2])\n    ax8.axis('off')\n    \n    summary_text = f\"\"\"\nINVENTORY SUMMARY\n\nTotal Items: {len(velocity):,}\n\nTrends:\n• Evergreen: {(velocity['trend']=='EVERGREEN').sum():,}\n• Emerging: {(velocity['trend']=='EMERGING').sum():,}\n• Declining: {(velocity['trend']=='DECLINING').sum():,}\n\nActions:\n• Liquidate: {(velocity['action']=='LIQUIDATE').sum():,}\n• Restock: {(velocity['action']=='RESTOCK').sum():,}\n• Monitor: {(velocity['action']=='MONITOR').sum():,}\n\nRisks:\n• Slow-moving: {velocity['is_slow_moving'].sum():,}\n• Stockout risk: {velocity['is_stockout_risk'].sum():,}\n\"\"\"\n    \n    ax8.text(0.1, 0.95, summary_text, transform=ax8.transAxes, fontsize=10,\n            verticalalignment='top', fontfamily='monospace',\n            bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))\n    \n    plt.suptitle('INVENTORY PROFILING: WHAT → FIND → DECIDE → WHY → MATRIX',\n                 fontsize=14, fontweight='bold', y=0.995)\n    \n    plt.savefig('08_INVENTORY_DASHBOARD.png', dpi=150, bbox_inches='tight')\n    plt.close()\n    \n    print(f\"  ✓ Dashboard saved: 08_INVENTORY_DASHBOARD.png\")\n    \n    # ============================================================================\n    # FINAL SUMMARY & NARRATIVE\n    # ============================================================================\n    elapsed = (datetime.now() - start_time).total_seconds()\n    \n    print(f\"\\n{'=' * 80}\")\n    print(f\"✓✓✓ SCRIPT 08 COMPLETE: INVENTORY PROFILING & OPTIMIZATION\")\n    print(f\"{'=' * 80}\")\n    print(f\"\\nEXECUTION TIME: {elapsed:.1f} seconds ({elapsed/60:.1f} minutes)\")\n    print(f\"\\nNARRATIVE FOR PRESENTATION:\\n\")\n    \n    print(f\"WHAT WE DID:\")\n    print(f\"  1. Analyzed sales velocity trends (recent 90d vs prior 90d)\")\n    print(f\"  2. Detected supply chain gaps using IQR-based outlier detection\")\n    print(f\"  3. Identified slow-moving items using category-level percentiles\")\n    print(f\"  4. Created prescriptive actions (LIQUIDATE, RESTOCK, MONITOR)\")\n    print(f\"\\nWHAT WE FOUND:\")\n    print(f\"  • Total items analyzed: {len(velocity):,}\")\n    print(f\"  • Evergreen items (stable velocity): {(velocity['trend']=='EVERGREEN').sum():,} ({(velocity['trend']=='EVERGREEN').sum()/len(velocity)*100:.1f}%)\")\n    print(f\"  • Emerging items (trending up): {(velocity['trend']=='EMERGING').sum():,} ({(velocity['trend']=='EMERGING').sum()/len(velocity)*100:.1f}%)\")\n    print(f\"  • Declining items (seasonal dip): {(velocity['trend']=='DECLINING').sum():,} ({(velocity['trend']=='DECLINING').sum()/len(velocity)*100:.1f}%)\")\n    print(f\"  • Dead stock items (slow-moving): {velocity['is_slow_moving'].sum():,} ({velocity['is_slow_moving'].sum()/len(velocity)*100:.1f}%)\")\n    print(f\"  • Stockout risk items: {velocity['is_stockout_risk'].sum():,} ({velocity['is_stockout_risk'].sum()/len(velocity)*100:.1f}%)\")\n    \n    print(f\"\\nDECISIONS MADE:\")\n    print(f\"  • LIQUIDATE: {(velocity['action']=='LIQUIDATE').sum():,} items (free up capital, prevent dead stock)\")\n    print(f\"  • RESTOCK: {(velocity['action']=='RESTOCK').sum():,} items (address supply gaps, recover lost sales)\")\n    print(f\"  • MONITOR: {(velocity['action']=='MONITOR').sum():,} items (track trends, maintain status quo)\")\n    \n    print(f\"\\nWHY THESE ASSUMPTIONS:\")\n    print(f\"  • Trend detection: Rolling 90-day windows capture seasonality without full year overhead\")\n    print(f\"  • Stockout detection: IQR-based (±1.5×IQR) adapts to each item's normal variation\")\n    print(f\"  • Dead stock: Bottom 25% in category (category-specific, not global)\")\n    print(f\"  • Priority scoring: Combines velocity percentile + supply gap severity\")\n    \n    print(f\"\\nFINAL DECISION MATRIX:\")\n    print(f\"\\n  Decision Tree:\")\n    print(f\"  IF slow-moving AND (declining OR evergreen):\")\n    print(f\"    → LIQUIDATE (dead stock risk)\")\n    print(f\"  ELSE IF stockout_risk AND high_velocity:\")\n    print(f\"    → RESTOCK (supply chain issue, lost sales)\")\n    print(f\"  ELSE:\")\n    print(f\"    → MONITOR (stable or contextual)\")\n    \n    print(f\"\\nOUTPUTS CREATED:\")\n    print(f\"  1. 08_INVENTORY_ANALYSIS.parquet - All {len(velocity):,} items with metrics\")\n    print(f\"  2. 08_DEAD_STOCK_ITEMS.csv - {(velocity['action']=='LIQUIDATE').sum():,} items for liquidation\")\n    print(f\"  3. 08_STOCKOUT_RISK_ITEMS.csv - {(velocity['action']=='RESTOCK').sum():,} items for restocking\")\n    print(f\"  4. 08_INVENTORY_DASHBOARD.png - Presentation visualization\")\n    \n    print(f\"\\nREADY FOR DOWNSTREAM:\")\n    print(f\"  • Script 09: Multi-channel analysis (can filter by action/trend)\")\n    print(f\"  • Script 10: Markdown logic (uses LIQUIDATE items)\")\n    print(f\"  • Script 11: Retention strategy (can context with inventory status)\")\n    print(f\"  • Script 12: Decision matrix (all metrics available)\\n\")\n\nexcept Exception as e:\n    print(f\"\\n❌ Error: {e}\")\n    import traceback\n    traceback.print_exc()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T14:14:47.801785Z","iopub.execute_input":"2026-09-03T14:14:47.802241Z","iopub.status.idle":"2026-09-03T14:19:48.526060Z","shell.execute_reply.started":"2026-09-03T14:14:47.802210Z","shell.execute_reply":"2026-09-03T14:19:48.524873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSCRIPT 09: MULTI-CHANNEL EFFICIENCY ANALYSIS (FULL VERSION)\n===========================================================\nRobust implementation with:\n1. Channel summary metrics\n2. Repeat purchase analysis (all metrics)\n3. SKU performance by channel (bulletproof approach)\n4. Dead stock by channel (Script 08 integration)\n5. Cross-channel customer behavior\n6. RFM × Channel cross-tab (if time permits)\n7. Complete visualization dashboard\n\nUses pure Python loops for SKU analysis to avoid pandas issues.\nTime estimate: 13-15 minutes\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\" * 80)\nprint(\"SCRIPT 09: MULTI-CHANNEL EFFICIENCY ANALYSIS (FULL)\")\nprint(\"=\" * 80)\n\ntry:\n    # ============================================================================\n    # PART 0: LOAD DATA\n    # ============================================================================\n    print(\"\\n[PART 0] Loading data...\")\n    script_start = datetime.now()\n    \n    master = pd.read_parquet('master_with_features.parquet')\n    rfm_segments = pd.read_parquet('06_RFM_SEGMENTS.parquet')\n    \n    # Try to load inventory\n    try:\n        inventory = pd.read_parquet('08_INVENTORY_ANALYSIS.parquet')\n        inventory_available = True\n        print(f\"  ✓ Master: {master.shape}\")\n        print(f\"  ✓ RFM segments: {rfm_segments.shape}\")\n        print(f\"  ✓ Inventory: {inventory.shape}\")\n    except:\n        inventory_available = False\n        print(f\"  ✓ Master: {master.shape}\")\n        print(f\"  ✓ RFM segments: {rfm_segments.shape}\")\n        print(f\"  ℹ️  Inventory not available yet\")\n    \n    max_date = master['t_dat'].max()\n    \n    # ============================================================================\n    # PART 1: TIME-BASED SPLIT (ALIGNED WITH RFM)\n    # ============================================================================\n    print(\"\\n[PART 1] Setting up training period...\")\n    \n    validation_start = max_date - pd.Timedelta(days=90)\n    training_end = validation_start - pd.Timedelta(days=1)\n    \n    training_master = master[master['t_dat'] <= training_end].copy()\n    \n    print(f\"  Training period: {training_master['t_dat'].min().date()} to {training_end.date()}\")\n    print(f\"  Transactions: {len(training_master):,}\")\n    \n    # Create channel name\n    if 'channel_name' not in training_master.columns:\n        training_master['channel_name'] = training_master['sales_channel_id'].map({1: 'Online', 2: 'Store'})\n    \n    # ============================================================================\n    # PART 2: CHANNEL SUMMARY\n    # ============================================================================\n    print(\"\\n[PART 2] Computing channel summary...\")\n    \n    channel_summary = training_master.groupby('channel_name').agg({\n        'customer_id': 'nunique',\n        'article_id': 'nunique',\n        't_dat': 'count',\n        'price': ['sum', 'mean', 'median']\n    }).round(2)\n    \n    channel_summary.columns = ['Unique_Customers', 'Unique_SKUs', 'Total_Transactions',\n                               'Total_Revenue', 'Avg_Price', 'Median_Price']\n    \n    channel_summary['Revenue_per_Customer'] = (channel_summary['Total_Revenue'] / \n                                               channel_summary['Unique_Customers']).round(2)\n    channel_summary['Transactions_per_Customer'] = (channel_summary['Total_Transactions'] / \n                                                    channel_summary['Unique_Customers']).round(2)\n    \n    print(f\"\\n  Channel Performance:\")\n    for channel in channel_summary.index:\n        row = channel_summary.loc[channel]\n        print(f\"    {channel}: {int(row['Unique_Customers']):,} customers, ${row['Total_Revenue']:,.0f} revenue\")\n    \n    channel_summary.to_csv('09_CHANNEL_SUMMARY.csv')\n    print(f\"  ✓ 09_CHANNEL_SUMMARY.csv\")\n    \n    # ============================================================================\n    # PART 3: REPEAT PURCHASE ANALYSIS\n    # ============================================================================\n    print(\"\\n[PART 3] Analyzing repeat purchases...\")\n    \n    customer_channel_txns = training_master.groupby(['customer_id', 'channel_name']).size().reset_index(name='txn_count')\n    customer_channel_txns['is_repeater'] = (customer_channel_txns['txn_count'] >= 2).astype(int)\n    customer_channel_txns['is_one_time'] = (customer_channel_txns['txn_count'] == 1).astype(int)\n    \n    repeat_summary = customer_channel_txns.groupby('channel_name').agg({\n        'customer_id': 'count',\n        'is_repeater': 'sum',\n        'is_one_time': 'sum',\n        'txn_count': ['mean', 'median', 'std']\n    }).round(2)\n    \n    repeat_summary.columns = ['Total_Customers', 'Repeaters_Count', 'One_Time_Count',\n                              'Avg_Transactions', 'Median_Transactions', 'Std_Dev']\n    \n    repeat_summary['Repeat_Rate_%'] = (repeat_summary['Repeaters_Count'] / \n                                       repeat_summary['Total_Customers'] * 100).round(1)\n    repeat_summary['One_Time_Rate_%'] = (repeat_summary['One_Time_Count'] / \n                                         repeat_summary['Total_Customers'] * 100).round(1)\n    \n    print(f\"  Repeat Purchase Summary:\")\n    for channel in repeat_summary.index:\n        row = repeat_summary.loc[channel]\n        print(f\"    {channel}: {row['Repeat_Rate_%']:.1f}% repeat, avg {row['Avg_Transactions']:.2f} txns\")\n    \n    repeat_summary.to_csv('09_REPEAT_RATE_BY_CHANNEL.csv')\n    print(f\"  ✓ 09_REPEAT_RATE_BY_CHANNEL.csv\")\n    \n    # ============================================================================\n    # PART 4: SKU PERFORMANCE BY CHANNEL (BULLETPROOF - PURE PYTHON)\n    # ============================================================================\n    print(\"\\n[PART 4] Analyzing SKU performance by channel...\")\n    \n    # Build SKU data using pure Python dict\n    sku_dict = {}\n    \n    for idx, row in training_master.iterrows():\n        article = int(row['article_id'])\n        channel = row['channel_name']\n        key = (article, channel)\n        \n        if key not in sku_dict:\n            sku_dict[key] = {'units': 0, 'customers': set(), 'revenue': 0.0}\n        \n        sku_dict[key]['units'] += 1\n        sku_dict[key]['customers'].add(row['customer_id'])\n        sku_dict[key]['revenue'] += float(row['price'])\n    \n    # Convert to list of dicts\n    sku_perf_list = []\n    for (article, channel), data in sku_dict.items():\n        sku_perf_list.append({\n            'article_id': article,\n            'channel_name': channel,\n            'units': data['units'],\n            'customers': len(data['customers']),\n            'revenue': round(data['revenue'], 2)\n        })\n    \n    sku_channel_perf = pd.DataFrame(sku_perf_list)\n    \n    # Now aggregate: for each article, calculate totals\n    sku_summary_list = []\n    for article in sku_channel_perf['article_id'].unique():\n        article_data = sku_channel_perf[sku_channel_perf['article_id'] == article]\n        \n        online_row = article_data[article_data['channel_name'] == 'Online']\n        store_row = article_data[article_data['channel_name'] == 'Store']\n        \n        online_units = int(online_row['units'].sum()) if len(online_row) > 0 else 0\n        store_units = int(store_row['units'].sum()) if len(store_row) > 0 else 0\n        total_units = online_units + store_units\n        \n        online_pct = float((online_units / total_units * 100)) if total_units > 0 else 0.0\n        store_pct = float((store_units / total_units * 100)) if total_units > 0 else 0.0\n        \n        sku_summary_list.append({\n            'article_id': int(article),\n            'online_units': online_units,\n            'store_units': store_units,\n            'total_units': total_units,\n            'online_pct': online_pct,\n            'store_pct': store_pct\n        })\n    \n    sku_channel_output = pd.DataFrame(sku_summary_list).sort_values('total_units', ascending=False)\n    sku_channel_output.to_csv('09_SKU_PERFORMANCE_BY_CHANNEL.csv', index=False)\n    print(f\"  ✓ 09_SKU_PERFORMANCE_BY_CHANNEL.csv\")\n    print(f\"    Top 5 items:\")\n    \n    # Print using explicit type conversion\n    for i in range(min(5, len(sku_channel_output))):\n        art_id = int(sku_channel_output.iloc[i]['article_id'])\n        tot_u = int(sku_channel_output.iloc[i]['total_units'])\n        on_p = float(sku_channel_output.iloc[i]['online_pct'])\n        st_p = float(sku_channel_output.iloc[i]['store_pct'])\n        print(f\"      Article {art_id}: {tot_u:,} units ({on_p:.0f}% online, {st_p:.0f}% store)\")\n    \n    # ============================================================================\n    # PART 5: DEAD STOCK BY CHANNEL (Script 08 integration)\n    # ============================================================================\n    if inventory_available:\n        print(\"\\n[PART 5] Analyzing dead stock by channel...\")\n        \n        dead_stock_items = inventory[inventory['action'] == 'LIQUIDATE']['article_id'].unique()\n        dead_stock_master = training_master[training_master['article_id'].isin(dead_stock_items)]\n        \n        dead_stock_by_channel = dead_stock_master.groupby('channel_name').agg({\n            'article_id': 'nunique',\n            'customer_id': 'nunique',\n            't_dat': 'count',\n            'price': 'sum'\n        }).reset_index()\n        \n        dead_stock_by_channel.columns = ['channel_name', 'dead_stock_skus', 'customers', 'units', 'revenue']\n        dead_stock_by_channel['revenue_per_unit'] = (dead_stock_by_channel['revenue'] / \n                                                     dead_stock_by_channel['units']).round(2)\n        \n        dead_stock_by_channel.to_csv('09_DEAD_STOCK_BY_CHANNEL.csv', index=False)\n        print(f\"  ✓ 09_DEAD_STOCK_BY_CHANNEL.csv\")\n        for _, row in dead_stock_by_channel.iterrows():\n            print(f\"    {row['channel_name']}: {int(row['dead_stock_skus']):,} items, ${row['revenue']:,.0f}\")\n    else:\n        print(\"\\n[PART 5] Dead stock analysis skipped (Script 08 not available yet)\")\n    \n    # ============================================================================\n    # PART 6: CROSS-CHANNEL CUSTOMERS\n    # ============================================================================\n    print(\"\\n[PART 6] Analyzing cross-channel behavior...\")\n    \n    customer_channels = training_master.groupby('customer_id')['channel_name'].apply(set).reset_index()\n    customer_channels['channel_count'] = customer_channels['channel_name'].apply(len)\n    customer_channels['channels_used'] = customer_channels['channel_name'].apply(lambda x: ', '.join(sorted(x)))\n    \n    cross_channel_summary = pd.DataFrame({\n        'Customer_Segment': ['Single Channel (Online)', 'Single Channel (Store)', 'Multi-Channel'],\n        'Count': [\n            (customer_channels['channels_used'] == 'Online').sum(),\n            (customer_channels['channels_used'] == 'Store').sum(),\n            (customer_channels['channel_count'] > 1).sum()\n        ]\n    })\n    \n    cross_channel_summary['Percentage'] = (cross_channel_summary['Count'] / \n                                          customer_channels.shape[0] * 100).round(1)\n    \n    cross_channel_summary.to_csv('09_CROSS_CHANNEL_CUSTOMERS.csv', index=False)\n    print(f\"  ✓ 09_CROSS_CHANNEL_CUSTOMERS.csv\")\n    for _, row in cross_channel_summary.iterrows():\n        print(f\"    {row['Customer_Segment']}: {int(row['Count']):,} ({row['Percentage']:.1f}%)\")\n    \n    # ============================================================================\n    # PART 7: RFM × CHANNEL (OPTIONAL - TIME PERMITTING)\n    # ============================================================================\n    elapsed_so_far = (datetime.now() - script_start).total_seconds() / 60\n    time_remaining = 15 - elapsed_so_far\n    \n    print(f\"\\n[PART 7] Time checkpoint: {elapsed_so_far:.1f}min used, {time_remaining:.1f}min left...\")\n    \n    if time_remaining > 2:\n        print(\"  ✓ Including RFM×Channel analysis\")\n        \n        training_with_rfm = training_master.merge(\n            rfm_segments[['customer_id', 'segment']],\n            on='customer_id',\n            how='left'\n        )\n        \n        rfm_channel_crosstab = pd.crosstab(\n            training_with_rfm['segment'],\n            training_with_rfm['channel_name'],\n            margins=True\n        )\n        \n        rfm_channel_pct = pd.crosstab(\n            training_with_rfm['segment'],\n            training_with_rfm['channel_name'],\n            normalize='index'\n        ) * 100\n        \n        rfm_channel_crosstab.to_csv('09_RFM_CHANNEL_CROSS_TAB.csv')\n        rfm_channel_pct.to_csv('09_RFM_CHANNEL_PREFERENCES.csv')\n        \n        print(f\"  ✓ 09_RFM_CHANNEL_CROSS_TAB.csv\")\n        print(f\"  ✓ 09_RFM_CHANNEL_PREFERENCES.csv\")\n        rfm_included = True\n    else:\n        print(f\"  ⏭️  Skipping RFM×Channel (only {time_remaining:.1f}min remaining)\")\n        rfm_included = False\n    \n    # ============================================================================\n    # PART 8: VISUALIZATION\n    # ============================================================================\n    print(\"\\n[PART 8] Creating dashboard...\")\n    \n    fig = plt.figure(figsize=(18, 12))\n    gs = fig.add_gridspec(3, 3, hspace=0.4, wspace=0.3)\n    \n    # Plot 1: Channel Revenue\n    ax1 = fig.add_subplot(gs[0, 0])\n    channel_summary['Total_Revenue'].plot(kind='bar', ax=ax1, color=['#3498db', '#e74c3c'], edgecolor='black', linewidth=1.5)\n    ax1.set_title('Total Revenue by Channel', fontweight='bold', fontsize=11)\n    ax1.set_ylabel('Revenue ($)')\n    ax1.tick_params(axis='x', rotation=0)\n    ax1.grid(True, alpha=0.3, axis='y')\n    \n    # Plot 2: Repeat Rate\n    ax2 = fig.add_subplot(gs[0, 1])\n    repeat_summary['Repeat_Rate_%'].plot(kind='bar', ax=ax2, color=['#3498db', '#e74c3c'], edgecolor='black', linewidth=1.5)\n    ax2.set_title('Repeat Purchase Rate (%)', fontweight='bold', fontsize=11)\n    ax2.set_ylabel('Repeat Rate (%)')\n    ax2.tick_params(axis='x', rotation=0)\n    ax2.grid(True, alpha=0.3, axis='y')\n    \n    # Plot 3: Avg Transactions\n    ax3 = fig.add_subplot(gs[0, 2])\n    repeat_summary['Avg_Transactions'].plot(kind='bar', ax=ax3, color=['#3498db', '#e74c3c'], edgecolor='black', linewidth=1.5)\n    ax3.set_title('Avg Transactions per Customer', fontweight='bold', fontsize=11)\n    ax3.set_ylabel('# Transactions')\n    ax3.tick_params(axis='x', rotation=0)\n    ax3.grid(True, alpha=0.3, axis='y')\n    \n    # Plot 4: Revenue per Customer\n    ax4 = fig.add_subplot(gs[1, 0])\n    channel_summary['Revenue_per_Customer'].plot(kind='bar', ax=ax4, color=['#3498db', '#e74c3c'], edgecolor='black', linewidth=1.5)\n    ax4.set_title('Revenue per Customer ($)', fontweight='bold', fontsize=11)\n    ax4.set_ylabel('Revenue ($)')\n    ax4.tick_params(axis='x', rotation=0)\n    ax4.grid(True, alpha=0.3, axis='y')\n    \n    # Plot 5: Cross-Channel Distribution\n    ax5 = fig.add_subplot(gs[1, 1])\n    ax5.pie(cross_channel_summary['Count'].values, labels=cross_channel_summary['Customer_Segment'].values, \n            autopct='%1.1f%%', colors=['#3498db', '#e74c3c', '#2ecc71'])\n    ax5.set_title('Customer Channel Usage', fontweight='bold', fontsize=11)\n    \n    # Plot 6: SKU Distribution\n    ax6 = fig.add_subplot(gs[1, 2])\n    ax6.text(0.5, 0.7, f\"Total SKUs: {len(sku_channel_output):,}\", \n             ha='center', va='top', transform=ax6.transAxes, fontsize=12, fontweight='bold')\n    ax6.text(0.5, 0.5, f\"Online: {int(sku_channel_output['online_units'].sum()):,} units\\nStore: {int(sku_channel_output['store_units'].sum()):,} units\", \n             ha='center', va='center', transform=ax6.transAxes, fontsize=11, fontfamily='monospace',\n             bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))\n    ax6.axis('off')\n    ax6.set_title('SKU Unit Distribution', fontweight='bold', fontsize=11)\n    \n    # Plot 7: One-time vs Repeat\n    ax7 = fig.add_subplot(gs[2, 0])\n    one_time = repeat_summary['Total_Customers'] - repeat_summary['Repeaters_Count']\n    x_pos = np.arange(len(repeat_summary))\n    width = 0.35\n    ax7.bar(x_pos - width/2, one_time.values, width, label='One-Time', color='#95a5a6', edgecolor='black', linewidth=1)\n    ax7.bar(x_pos + width/2, repeat_summary['Repeaters_Count'].values, width, label='Repeaters', color='#2ecc71', edgecolor='black', linewidth=1)\n    ax7.set_title('Customer Types by Channel', fontweight='bold', fontsize=11)\n    ax7.set_ylabel('# Customers')\n    ax7.set_xticks(x_pos)\n    ax7.set_xticklabels(repeat_summary.index)\n    ax7.legend(fontsize=9)\n    ax7.grid(True, alpha=0.3, axis='y')\n    \n    # Plot 8: Unique Customers\n    ax8 = fig.add_subplot(gs[2, 1])\n    channel_summary['Unique_Customers'].plot(kind='bar', ax=ax8, color=['#3498db', '#e74c3c'], edgecolor='black', linewidth=1.5)\n    ax8.set_title('Unique Customers by Channel', fontweight='bold', fontsize=11)\n    ax8.set_ylabel('# Customers')\n    ax8.tick_params(axis='x', rotation=0)\n    ax8.grid(True, alpha=0.3, axis='y')\n    \n    # Plot 9: Summary\n    ax9 = fig.add_subplot(gs[2, 2])\n    ax9.axis('off')\n    summary_txt = f\"\"\"SUMMARY\n    \nOnline:\n• Rev: ${channel_summary.loc['Online', 'Total_Revenue']:,.0f}\n• Repeat: {repeat_summary.loc['Online', 'Repeat_Rate_%']:.1f}%\n\nStore:\n• Rev: ${channel_summary.loc['Store', 'Total_Revenue']:,.0f}\n• Repeat: {repeat_summary.loc['Store', 'Repeat_Rate_%']:.1f}%\n\"\"\"\n    ax9.text(0.1, 0.9, summary_txt, transform=ax9.transAxes, fontsize=10,\n            verticalalignment='top', fontfamily='monospace',\n            bbox=dict(boxstyle='round', facecolor='lightblue', alpha=0.5))\n    \n    plt.suptitle('MULTI-CHANNEL EFFICIENCY ANALYSIS', fontsize=14, fontweight='bold')\n    plt.savefig('09_CHANNEL_EFFICIENCY_DASHBOARD.png', dpi=150, bbox_inches='tight')\n    plt.close()\n    print(f\"  ✓ 09_CHANNEL_EFFICIENCY_DASHBOARD.png\")\n    \n    # ============================================================================\n    # FINAL SUMMARY\n    # ============================================================================\n    elapsed_total = (datetime.now() - script_start).total_seconds() / 60\n    \n    print(f\"\\n{'=' * 80}\")\n    print(f\"✓✓✓ SCRIPT 09 COMPLETE\")\n    print(f\"{'=' * 80}\")\n    print(f\"\\nExecution time: {elapsed_total:.1f} minutes\")\n    print(f\"\\nOUTPUTS CREATED:\")\n    print(f\"  ✓ 09_CHANNEL_SUMMARY.csv\")\n    print(f\"  ✓ 09_REPEAT_RATE_BY_CHANNEL.csv\")\n    print(f\"  ✓ 09_SKU_PERFORMANCE_BY_CHANNEL.csv\")\n    print(f\"  ✓ 09_CROSS_CHANNEL_CUSTOMERS.csv\")\n    if inventory_available:\n        print(f\"  ✓ 09_DEAD_STOCK_BY_CHANNEL.csv\")\n    if rfm_included:\n        print(f\"  ✓ 09_RFM_CHANNEL_CROSS_TAB.csv\")\n        print(f\"  ✓ 09_RFM_CHANNEL_PREFERENCES.csv\")\n    print(f\"  ✓ 09_CHANNEL_EFFICIENCY_DASHBOARD.png\")\n    print(f\"\\nREADY FOR SCRIPT 10 (Markdown Logic)\\n\")\n\nexcept Exception as e:\n    print(f\"\\n❌ Error: {e}\")\n    import traceback\n    traceback.print_exc()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T14:49:09.081518Z","iopub.execute_input":"2026-09-03T14:49:09.082012Z","iopub.status.idle":"2026-09-03T15:01:28.247665Z","shell.execute_reply.started":"2026-09-03T14:49:09.081976Z","shell.execute_reply":"2026-09-03T15:01:28.246704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ============================================================================\n# # SCRIPT 09: MULTI-CHANNEL EFFICIENCY ANALYSIS (30 min)\n# # ============================================================================\n# def script_09_channel_efficiency():\n#     \"\"\"\n#     Compare online vs. store performance metrics.\n#     RUN IN PARALLEL with script_08.\n#     \"\"\"\n#     print(\"\\n\" + \"=\" * 70)\n#     print(\"SCRIPT 09: MULTI-CHANNEL EFFICIENCY ANALYSIS\")\n#     print(\"=\" * 70)\n    \n#     try:\n#         print(\"Loading master dataset...\")\n#         master = pd.read_parquet('master_with_features.parquet')\n#         rfm = pd.read_parquet('rfm_features.parquet')\n        \n#         # Estimated margins\n#         master['estimated_margin'] = master['price'] * master['channel_name'].map({\n#             'Online': 0.15, 'Store': 0.25\n#         })\n        \n#         # Channel-level aggregation\n#         print(\"Computing channel metrics...\")\n#         channel_summary = master.groupby('channel_name').agg({\n#             'customer_id': 'nunique',\n#             'article_id': ['nunique', 'count'],\n#             'price': ['sum', 'mean', 'median'],\n#             'estimated_margin': 'sum',\n#             't_dat': 'count'\n#         }).round(2)\n        \n#         channel_summary.columns = ['Unique_Customers', 'Unique_SKUs', 'Total_Units','Total_Revenue', \n#                                    'Avg_Transaction', 'Median_Price',  \n#                                    'Total_Est_Margin', 'Transaction_Count']\n        \n#         # Add metrics\n#         channel_summary['Revenue_per_Customer'] = (channel_summary['Total_Revenue'] / \n#                                                    channel_summary['Unique_Customers']).round(2)\n#         channel_summary['Profit_per_Customer'] = (channel_summary['Total_Est_Margin'] / \n#                                                   channel_summary['Unique_Customers']).round(2)\n#         channel_summary['Margin_%'] = ((channel_summary['Total_Est_Margin'] / \n#                                         channel_summary['Total_Revenue']) * 100).round(1)\n        \n#         # Repeat purchase rate by channel\n#         repeat_by_channel = master.groupby('customer_id').agg({\n#             'sales_channel_id': lambda x: x.unique()\n#         }).reset_index()\n#         repeat_by_channel['primary_channel'] = repeat_by_channel['sales_channel_id'].apply(\n#             lambda x: 1 if len(x) > 0 and 1 in x else 2\n#         )\n#         repeat_by_channel['is_multichannel'] = repeat_by_channel['sales_channel_id'].apply(len) > 1\n        \n#         print(\"✓ Channel efficiency analysis complete\")\n#         print(\"\\nChannel Performance Summary:\")\n#         print(channel_summary)\n#         print(f\"\\nMulti-channel customers: {repeat_by_channel['is_multichannel'].sum():,}\")\n        \n#         # Save\n#         channel_summary.to_csv('08_CHANNEL_EFFICIENCY.csv')\n        \n#         return channel_summary\n    \n#     except Exception as e:\n#         print(f\"❌ Error in script 09: {e}\")\n#         return None\n\n# channel_eff = script_09_channel_efficiency()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:12:41.465836Z","iopub.execute_input":"2026-09-03T09:12:41.466291Z","iopub.status.idle":"2026-09-03T09:13:09.126812Z","shell.execute_reply.started":"2026-09-03T09:12:41.466248Z","shell.execute_reply":"2026-09-03T09:13:09.125868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSCRIPT 10: MARKDOWN LOGIC FRAMEWORK\n===================================\nCreates prescriptive markdown recommendations based on:\n1. Category-specific tier logic (luxury never marked down, basics can tolerate discounts)\n2. Seasonal detection (off-season items must clear before next season)\n3. RFM revenue contribution analysis (strategic insight into segment value)\n4. Velocity percentile + time held (objective triggers, not arbitrary costs)\n\nKey principle: Transparent assumptions, no fictional financial modeling\n\nOutput:\n- 10_MARKDOWN_RECOMMENDATIONS.csv (prescriptive labels)\n- 10_CATEGORY_RFM_REVENUE_MIX.csv (strategic insight)\n- 10_MARKDOWN_DISTRIBUTION.csv (summary by tier)\n- 10_MARKDOWN_DASHBOARD.png (visualization)\n\nTime estimate: 12-15 minutes\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\" * 80)\nprint(\"SCRIPT 10: MARKDOWN LOGIC FRAMEWORK\")\nprint(\"=\" * 80)\n\ntry:\n    # ============================================================================\n    # PART 0: LOAD DATA\n    # ============================================================================\n    print(\"\\n[PART 0] Loading data...\")\n    script_start = datetime.now()\n    \n    inventory = pd.read_parquet('08_INVENTORY_ANALYSIS.parquet')\n    master = pd.read_parquet('master_with_features.parquet')\n    rfm_segments = pd.read_parquet('06_RFM_SEGMENTS.parquet')\n    \n    print(f\"  ✓ Inventory analysis: {inventory.shape}\")\n    print(f\"  ✓ Master: {master.shape}\")\n    print(f\"  ✓ RFM segments: {rfm_segments.shape}\")\n    \n    # ============================================================================\n    # PART 1: GET PRODUCT METADATA\n    # ============================================================================\n    print(\"\\n[PART 1] Loading product metadata...\")\n    \n    # Try to get category from master first, then articles.csv as fallback\n    article_category = None\n    has_categories = False\n    \n    try:\n        # Try master first (faster)\n        if 'product_type_name' in master.columns:\n            article_category = master[['article_id', 'product_type_name']].drop_duplicates()\n            print(f\"  ✓ Categories from master: {article_category['product_type_name'].nunique()} categories\")\n            has_categories = True\n    except:\n        pass\n    \n    if not has_categories:\n        try:\n            # Fallback to articles.csv\n            articles = pd.read_csv('/kaggle/input/datasets/shantanusmankar/h-and-m-data/articles.csv')\n            if 'product_type_name' in articles.columns:\n                article_category = articles[['article_id', 'product_type_name']].drop_duplicates()\n                print(f\"  ✓ Categories from articles.csv: {article_category['product_type_name'].nunique()} categories\")\n                has_categories = True\n        except:\n            print(f\"  ⚠️  Could not load product categories\")\n    \n    if not has_categories:\n        print(f\"  ⚠️  Categories unavailable - will use generic classification\")\n    \n    # ============================================================================\n    # PART 2: CLASSIFY ITEMS BY CATEGORY & LUXURY STATUS\n    # ============================================================================\n    print(\"\\n[PART 2] Classifying items by category...\")\n    \n    # Merge metadata if available\n    if has_categories and article_category is not None:\n        markdown_data = inventory.merge(article_category, on='article_id', how='left')\n    else:\n        markdown_data = inventory.copy()\n    \n    # Ensure product_type_name exists (fill missing with 'Unknown')\n    if 'product_type_name' not in markdown_data.columns:\n        markdown_data['product_type_name'] = 'Unknown'\n    else:\n        markdown_data['product_type_name'] = markdown_data['product_type_name'].fillna('Unknown')\n    \n    # Define luxury categories (common luxury items in apparel/footwear)\n    luxury_keywords = ['luxury', 'premium', 'designer', 'high-end', 'exclusive']\n    \n    def classify_category_tier(cat_name):\n        \"\"\"Classify category into business tier\"\"\"\n        if pd.isna(cat_name):\n            return 'Unknown'\n        \n        cat_lower = str(cat_name).lower()\n        \n        # Luxury items get special treatment\n        if any(keyword in cat_lower for keyword in luxury_keywords):\n            return 'Luxury'\n        \n        # Basics (typically lower margin, can tolerate discounts)\n        basics_keywords = ['basics', 'essential', 'plain', 't-shirt', 'sock', 'underwear', 'basic']\n        if any(keyword in cat_lower for keyword in basics_keywords):\n            return 'Basics'\n        \n        # Fast fashion (seasonal, medium-high price point)\n        if any(keyword in cat_lower for keyword in ['dress', 'jacket', 'pants', 'shirt', 'skirt']):\n            return 'Fast_Fashion'\n        \n        # Default\n        return 'Standard'\n    \n    markdown_data['category_tier'] = markdown_data['product_type_name'].apply(classify_category_tier)\n    \n    print(f\"  Category tier distribution:\")\n    for tier in markdown_data['category_tier'].unique():\n        count = (markdown_data['category_tier'] == tier).sum()\n        pct = count / len(markdown_data) * 100\n        print(f\"    {tier}: {count:,} items ({pct:.1f}%)\")\n    \n    # ============================================================================\n    # PART 3: SEASONAL STATUS & MARKDOWN ELIGIBILITY\n    # ============================================================================\n    print(\"\\n[PART 3] Identifying seasonal items...\")\n    \n    # Items with trend = DECLINING or marked as off-season are candidates for aggressive clearing\n    markdown_data['is_seasonal_item'] = markdown_data['trend'].isin(['DECLINING', 'NEW'])\n    markdown_data['is_off_season'] = markdown_data['trend'] == 'DECLINING'\n    \n    seasonal_count = markdown_data['is_seasonal_item'].sum()\n    off_season_count = markdown_data['is_off_season'].sum()\n    \n    print(f\"  Seasonal items (DECLINING trend): {off_season_count:,}\")\n    print(f\"  New items (NEW trend): {(markdown_data['trend'] == 'NEW').sum():,}\")\n    print(f\"  Evergreen items: {(markdown_data['trend'] == 'EVERGREEN').sum():,}\")\n    \n    # ============================================================================\n    # PART 4: MARKDOWN TIER LOGIC (CATEGORY-SPECIFIC, SEASONAL-AWARE)\n    # ============================================================================\n    print(\"\\n[PART 4] Applying markdown tier logic...\")\n    \n    def get_markdown_tier(row):\n        \"\"\"\n        Apply markdown recommendation based on:\n        1. Category tier (Luxury never discounted)\n        2. Velocity percentile (how slow is it?)\n        3. Seasonal status (must clear off-season)\n        4. Days since sale (proxy for how old)\n        \n        TRANSPARENT ASSUMPTIONS:\n        - Luxury items: Brand value > margin. Hold indefinitely rather than discount.\n        - Basics: Can absorb discounts, customers price-sensitive\n        - Fast Fashion: Seasonal, must clear when out of season\n        - Off-season items: Must clear before next season cycle\n        \"\"\"\n        \n        category = row['category_tier']\n        velocity_pct = row['velocity_percentile_in_category']\n        days_gap = row['max_sales_gap_days'] if pd.notna(row['max_sales_gap_days']) else 0\n        is_off_season = row['is_off_season']\n        action = row['action']\n        \n        # LUXURY RULE: Never markdown, regardless of velocity\n        if category == 'Luxury':\n            return 'No_Action_Luxury_Hold'\n        \n        # OFF-SEASON RULE: Must clear before next season\n        if is_off_season:\n            if velocity_pct < 20 and days_gap > 14:\n                return 'Tier_3_Aggressive_Clear_-30%'\n            elif velocity_pct < 35 and days_gap > 7:\n                return 'Tier_2_Moderate_Clear_-20%'\n            else:\n                return 'Tier_1_Light_Discount_-10%'\n        \n        # EVERGREEN ITEMS: Category-specific logic\n        if category == 'Basics':\n            # Basics can tolerate discounts (price-sensitive segment)\n            if velocity_pct < 10:\n                return 'Tier_3_Clearance_-25%'\n            elif velocity_pct < 25:\n                return 'Tier_2_Markdown_-15%'\n            elif action == 'LIQUIDATE':\n                return 'Tier_1_Monitor_-10%'\n            else:\n                return 'No_Action'\n        \n        elif category == 'Fast_Fashion':\n            # Fast fashion: mid-tier discounts\n            if velocity_pct < 10:\n                return 'Tier_3_Clearance_-20%'\n            elif velocity_pct < 25:\n                return 'Tier_2_Markdown_-15%'\n            elif action == 'LIQUIDATE':\n                return 'Tier_1_Monitor_-10%'\n            else:\n                return 'No_Action'\n        \n        else:  # Standard\n            # Middle ground\n            if velocity_pct < 10:\n                return 'Tier_3_Clearance_-20%'\n            elif velocity_pct < 25:\n                return 'Tier_2_Markdown_-10%'\n            elif action == 'LIQUIDATE':\n                return 'Tier_1_Monitor_-5%'\n            else:\n                return 'No_Action'\n    \n    markdown_data['markdown_recommendation'] = markdown_data.apply(get_markdown_tier, axis=1)\n    \n    # Sort by priority (high velocity percentile = more urgent)\n    # Use np.where for vectorized operation on Series\n    markdown_data['priority_score'] = (100 - markdown_data['velocity_percentile_in_category']) * \\\n                                       np.where(markdown_data['is_off_season'], 1.0, 0.5)\n    markdown_data = markdown_data.sort_values('priority_score', ascending=False)\n    \n    print(f\"\\n  Markdown distribution:\")\n    for tier in markdown_data['markdown_recommendation'].unique():\n        count = (markdown_data['markdown_recommendation'] == tier).sum()\n        pct = count / len(markdown_data) * 100\n        print(f\"    {tier}: {count:,} items ({pct:.1f}%)\")\n    \n    # ============================================================================\n    # PART 5: RFM REVENUE CONTRIBUTION ANALYSIS (STRATEGIC INSIGHT)\n    # ============================================================================\n    print(\"\\n[PART 5] Analyzing RFM revenue contribution by category...\")\n    \n    # Merge RFM segments into master\n    master_with_rfm = master.merge(rfm_segments[['customer_id', 'segment']], on='customer_id', how='left')\n    \n    # Add product categories\n    if has_categories and article_category is not None:\n        master_with_rfm = master_with_rfm.merge(article_category, on='article_id', how='left')\n    \n    # Ensure product_type_name exists\n    if 'product_type_name' not in master_with_rfm.columns:\n        master_with_rfm['product_type_name'] = 'Unknown'\n    else:\n        master_with_rfm['product_type_name'] = master_with_rfm['product_type_name'].fillna('Unknown')\n    \n    # Calculate revenue by category × RFM segment\n    rfm_revenue = master_with_rfm.groupby(['product_type_name', 'segment']).agg({\n        'price': 'sum',\n        'customer_id': 'nunique',\n        't_dat': 'count'\n    }).reset_index()\n    \n    rfm_revenue.columns = ['product_type_name', 'segment', 'revenue', 'unique_customers', 'transactions']\n    \n    # Calculate % of revenue by segment within each category\n    rfm_revenue['pct_revenue_in_category'] = rfm_revenue.groupby('product_type_name')['revenue'].transform(\n        lambda x: (x / x.sum() * 100).round(1)\n    )\n    \n    rfm_revenue_summary = rfm_revenue.pivot_table(\n        index='product_type_name',\n        columns='segment',\n        values='pct_revenue_in_category',\n        fill_value=0\n    )\n    \n    rfm_revenue_summary.to_csv('10_CATEGORY_RFM_REVENUE_MIX.csv')\n    print(f\"  ✓ 10_CATEGORY_RFM_REVENUE_MIX.csv\")\n    print(f\"\\n  Revenue contribution by RFM segment (sample):\")\n    if len(rfm_revenue_summary) > 0:\n        for cat in rfm_revenue_summary.index[:3]:\n            champions_pct = rfm_revenue_summary.loc[cat, 'Champions'] if 'Champions' in rfm_revenue_summary.columns else 0\n            print(f\"    {cat}: Champions={champions_pct:.0f}% of revenue\")\n    \n    # ============================================================================\n    # PART 6: SAVE OUTPUTS\n    # ============================================================================\n    print(\"\\n[PART 6] Saving outputs...\")\n    \n    # Main recommendations\n    output_cols = ['article_id', 'sales_channel_id', 'product_type_name', 'category_tier', \n                   'velocity_percentile_in_category', 'trend', 'is_off_season', 'max_sales_gap_days',\n                   'action', 'markdown_recommendation', 'priority_score']\n    \n    # Only include columns that exist\n    output_cols = [col for col in output_cols if col in markdown_data.columns]\n    \n    markdown_output = markdown_data[output_cols].sort_values('priority_score', ascending=False)\n    markdown_output.to_csv('10_MARKDOWN_RECOMMENDATIONS.csv', index=False)\n    print(f\"  ✓ 10_MARKDOWN_RECOMMENDATIONS.csv ({len(markdown_output):,} items)\")\n    \n    # Distribution summary\n    markdown_dist = markdown_data['markdown_recommendation'].value_counts().reset_index()\n    markdown_dist.columns = ['markdown_tier', 'count']\n    markdown_dist['percentage'] = (markdown_dist['count'] / len(markdown_data) * 100).round(1)\n    markdown_dist.to_csv('10_MARKDOWN_DISTRIBUTION.csv', index=False)\n    print(f\"  ✓ 10_MARKDOWN_DISTRIBUTION.csv\")\n    \n    # Also save parquet for downstream use\n    markdown_output.to_parquet('10_MARKDOWN_LOGIC.parquet')\n    print(f\"  ✓ 10_MARKDOWN_LOGIC.parquet\")\n    \n    # ============================================================================\n    # PART 7: VISUALIZATION DASHBOARD\n    # ============================================================================\n    print(\"\\n[PART 7] Creating visualization...\")\n    \n    fig = plt.figure(figsize=(16, 10))\n    gs = fig.add_gridspec(2, 3, hspace=0.35, wspace=0.3)\n    \n    # Plot 1: Markdown tier distribution\n    ax1 = fig.add_subplot(gs[0, 0])\n    tier_order = ['No_Action_Luxury_Hold', 'No_Action', 'Tier_1_Monitor_-5%', 'Tier_1_Monitor_-10%', \n                  'Tier_1_Light_Discount_-10%', 'Tier_2_Markdown_-10%', 'Tier_2_Markdown_-15%', \n                  'Tier_2_Moderate_Clear_-20%', 'Tier_3_Clearance_-20%', 'Tier_3_Clearance_-25%', \n                  'Tier_3_Aggressive_Clear_-30%']\n    tier_counts = markdown_data['markdown_recommendation'].value_counts()\n    tier_counts = tier_counts.reindex([t for t in tier_order if t in tier_counts.index])\n    \n    colors = ['#2ecc71' if 'No_Action' in t else '#f39c12' if 'Tier_1' in t else '#e67e22' if 'Tier_2' in t else '#e74c3c' \n              for t in tier_counts.index]\n    ax1.barh(range(len(tier_counts)), tier_counts.values, color=colors, edgecolor='black', linewidth=1)\n    ax1.set_yticks(range(len(tier_counts)))\n    ax1.set_yticklabels(tier_counts.index, fontsize=9)\n    ax1.set_xlabel('# Items')\n    ax1.set_title('Markdown Tier Distribution', fontweight='bold', fontsize=11)\n    ax1.grid(True, alpha=0.3, axis='x')\n    \n    # Plot 2: Category tier breakdown\n    ax2 = fig.add_subplot(gs[0, 1])\n    cat_tier_counts = markdown_data['category_tier'].value_counts()\n    colors_cat = ['#e74c3c' if t == 'Luxury' else '#3498db' for t in cat_tier_counts.index]\n    ax2.bar(cat_tier_counts.index, cat_tier_counts.values, color=colors_cat, edgecolor='black', linewidth=1.5)\n    ax2.set_ylabel('# Items')\n    ax2.set_title('Items by Category Tier', fontweight='bold', fontsize=11)\n    ax2.grid(True, alpha=0.3, axis='y')\n    \n    # Plot 3: Seasonal breakdown\n    ax3 = fig.add_subplot(gs[0, 2])\n    seasonal_dist = pd.DataFrame({\n        'Status': ['Off-Season', 'Evergreen', 'Emerging', 'New'],\n        'Count': [\n            (markdown_data['trend'] == 'DECLINING').sum(),\n            (markdown_data['trend'] == 'EVERGREEN').sum(),\n            (markdown_data['trend'] == 'EMERGING').sum(),\n            (markdown_data['trend'] == 'NEW').sum()\n        ]\n    })\n    ax3.pie(seasonal_dist['Count'].values, labels=seasonal_dist['Status'].values, autopct='%1.1f%%',\n            colors=['#e74c3c', '#2ecc71', '#3498db', '#f39c12'])\n    ax3.set_title('Item Trend Distribution', fontweight='bold', fontsize=11)\n    \n    # Plot 4: Markdown by category tier\n    ax4 = fig.add_subplot(gs[1, 0])\n    markdown_by_cat = markdown_data.groupby('category_tier')['markdown_recommendation'].apply(\n        lambda x: (x.str.contains('Tier_3', case=False).sum() / len(x) * 100)\n    ).sort_values(ascending=False)\n    ax4.bar(markdown_by_cat.index, markdown_by_cat.values, color=['#e74c3c', '#f39c12', '#3498db'],\n            edgecolor='black', linewidth=1.5)\n    ax4.set_ylabel('% in Tier 3 (Aggressive)')\n    ax4.set_title('Aggressive Markdown % by Category', fontweight='bold', fontsize=11)\n    ax4.grid(True, alpha=0.3, axis='y')\n    \n    # Plot 5: Velocity vs Markdown\n    ax5 = fig.add_subplot(gs[1, 1])\n    scatter_data = markdown_data[markdown_data['category_tier'] != 'Luxury'].sample(min(1000, len(markdown_data)))\n    tier_colors = {'Tier_3_Clearance_-25%': '#e74c3c', 'Tier_3_Clearance_-20%': '#e74c3c',\n                   'Tier_3_Aggressive_Clear_-30%': '#c0392b', 'Tier_2_Markdown_-15%': '#f39c12',\n                   'No_Action': '#95a5a6'}\n    scatter_colors = [tier_colors.get(t, '#3498db') for t in scatter_data['markdown_recommendation']]\n    ax5.scatter(scatter_data['velocity_percentile_in_category'], \n               scatter_data['max_sales_gap_days'].fillna(0), \n               c=scatter_colors, alpha=0.5, s=20, edgecolors='black', linewidth=0.5)\n    ax5.axvline(25, color='red', linestyle='--', alpha=0.5, label='Slow-moving threshold (25%)')\n    ax5.set_xlabel('Velocity Percentile')\n    ax5.set_ylabel('Days Since Sale')\n    ax5.set_title('Velocity vs Days Since Sale', fontweight='bold', fontsize=11)\n    ax5.grid(True, alpha=0.3)\n    ax5.legend(fontsize=8)\n    \n    # Plot 6: Summary box\n    ax6 = fig.add_subplot(gs[1, 2])\n    ax6.axis('off')\n    \n    no_action_count = (markdown_data['markdown_recommendation'] == 'No_Action_Luxury_Hold').sum()\n    tier1_count = markdown_data['markdown_recommendation'].str.contains('Tier_1', case=False).sum()\n    tier2_count = markdown_data['markdown_recommendation'].str.contains('Tier_2', case=False).sum()\n    tier3_count = markdown_data['markdown_recommendation'].str.contains('Tier_3', case=False).sum()\n    no_action_regular = (markdown_data['markdown_recommendation'] == 'No_Action').sum()\n    \n    summary_txt = f\"\"\"MARKDOWN SUMMARY\n\nNo Markdown (Luxury): {no_action_count:,}\nNo Action (Healthy): {no_action_regular:,}\n\nTier 1 (Monitor -5-10%): {tier1_count:,}\nTier 2 (Markdown -10-20%): {tier2_count:,}\nTier 3 (Aggressive -20-30%): {tier3_count:,}\n\nTotal Items: {len(markdown_data):,}\n\nKEY PRINCIPLE:\nLuxury items never marked down.\nCategory-specific rules apply.\nOff-season items prioritized for\naggressive clearing.\n\"\"\"\n    ax6.text(0.05, 0.95, summary_txt, transform=ax6.transAxes, fontsize=9,\n            verticalalignment='top', fontfamily='monospace',\n            bbox=dict(boxstyle='round', facecolor='lightyellow', alpha=0.7))\n    \n    plt.suptitle('MARKDOWN LOGIC FRAMEWORK\\nCategory-Specific, Seasonal-Aware, RFM-Informed',\n                 fontsize=13, fontweight='bold', y=0.995)\n    \n    plt.savefig('10_MARKDOWN_DASHBOARD.png', dpi=150, bbox_inches='tight')\n    plt.close()\n    print(f\"  ✓ 10_MARKDOWN_DASHBOARD.png\")\n    \n    # ============================================================================\n    # FINAL SUMMARY & ASSUMPTIONS\n    # ============================================================================\n    elapsed = (datetime.now() - script_start).total_seconds() / 60\n    \n    print(f\"\\n{'=' * 80}\")\n    print(f\"✓✓✓ SCRIPT 10 COMPLETE: MARKDOWN LOGIC\")\n    print(f\"{'=' * 80}\")\n    print(f\"\\nExecution time: {elapsed:.1f} minutes\")\n    \n    print(f\"\\nKEY DECISIONS:\")\n    print(f\"  1. LUXURY RULE: Never markdown. Brand value > margin. Hold indefinitely.\")\n    print(f\"  2. SEASONAL RULE: Off-season items prioritized for aggressive clearing.\")\n    print(f\"  3. CATEGORY LOGIC: Basics > Fast Fashion > Standard (discount tolerance)\")\n    print(f\"  4. VELOCITY TRIGGER: Bottom 10% = Tier 3, Bottom 25% = Tier 1\")\n    print(f\"  5. RFM INSIGHT: Revenue mix by segment analyzed (informs strategy, not pricing)\")\n    \n    print(f\"\\nTRANSPARENT ASSUMPTIONS:\")\n    print(f\"  ✓ No holding cost estimates (insufficient data)\")\n    print(f\"  ✓ No elasticity modeling (no price-response data)\")\n    print(f\"  ✓ No fictional financial projections (avoid contrived scenarios)\")\n    print(f\"  ✓ Category tiers based on domain logic, not data\")\n    print(f\"  ✓ RFM breakdown for strategic context only\")\n    \n    print(f\"\\nOUTPUTS:\")\n    print(f\"  ✓ 10_MARKDOWN_RECOMMENDATIONS.csv ({len(markdown_output):,} items)\")\n    print(f\"  ✓ 10_MARKDOWN_DISTRIBUTION.csv (tier summary)\")\n    print(f\"  ✓ 10_CATEGORY_RFM_REVENUE_MIX.csv (strategic insight)\")\n    print(f\"  ✓ 10_MARKDOWN_LOGIC.parquet (for downstream use)\")\n    print(f\"  ✓ 10_MARKDOWN_DASHBOARD.png (visualization)\")\n    \n    print(f\"\\nREADY FOR SCRIPT 11 (Retention Tiers)\\n\")\n\nexcept Exception as e:\n    print(f\"\\n❌ Error: {e}\")\n    import traceback\n    traceback.print_exc()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T15:38:56.700506Z","iopub.execute_input":"2026-09-03T15:38:56.700847Z","iopub.status.idle":"2026-09-03T15:39:44.734233Z","shell.execute_reply.started":"2026-09-03T15:38:56.700820Z","shell.execute_reply":"2026-09-03T15:39:44.732805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSCRIPT 11: RETENTION INTERVENTIONS & STRATEGIC GUIDELINES\n==========================================================\nCreates RFM × Churn Risk matrix with strategic intervention guidelines.\n\nKey principle: Transparent strategic logic, no invented financial modeling.\n\nWe DON'T have:\n- Intervention cost data\n- Retention rate by intervention type\n- Elasticity/price-response data\n- Quantitative ROI models\n\nWe DO provide:\n- Strategic intervention guidelines by segment\n- Segment-level analysis (size, concentration, risk exposure)\n- Reasoning for each recommendation\n- Clear data gaps for Final Report\n\nOutput:\n- 11_RETENTION_INTERVENTIONS.csv (recommendations by customer)\n- 11_RFM_CHURN_MATRIX.csv (segment-level analysis)\n- 11_INTERVENTION_PRIORITY.csv (which segments to prioritize)\n- 11_RETENTION_DASHBOARD.png (visualization)\n\nTime estimate: 12-15 minutes\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\" * 80)\nprint(\"SCRIPT 11: RETENTION INTERVENTIONS & STRATEGIC GUIDELINES\")\nprint(\"=\" * 80)\n\ntry:\n    # ============================================================================\n    # PART 0: LOAD DATA\n    # ============================================================================\n    print(\"\\n[PART 0] Loading data...\")\n    script_start = datetime.now()\n    \n    churn = pd.read_parquet('07_CHURN_SCORES.parquet')\n    rfm_segments = pd.read_parquet('06_RFM_SEGMENTS.parquet')\n    master = pd.read_parquet('master_with_features.parquet')\n    \n    print(f\"  ✓ Churn scores: {churn.shape}\")\n    print(f\"  ✓ RFM segments: {rfm_segments.shape}\")\n    print(f\"  ✓ Master: {master.shape}\")\n    \n    # ============================================================================\n    # PART 1: USE RFM SEGMENTS (ALREADY IN CHURN DATA)\n    # ============================================================================\n    print(\"\\n[PART 1] Preparing retention data...\")\n    \n    # Churn dataframe already has 'segment' from Script 07\n    retention_data = churn.copy()\n    \n    print(f\"  ✓ Retention data ready: {len(retention_data):,} customers\")\n    print(f\"  Columns: {retention_data.columns.tolist()}\")\n    print(f\"  Segment values: {retention_data['segment'].nunique()}\")\n    \n    # Ensure segment exists and handle NaN\n    if 'segment' not in retention_data.columns:\n        raise ValueError(\"'segment' column not found in churn data\")\n    \n    retention_data['segment'] = retention_data['segment'].fillna('Unknown')\n    \n    print(f\"  RFM segments distribution:\")\n    for seg in retention_data['segment'].unique():\n        count = (retention_data['segment'] == seg).sum()\n        pct = count / len(retention_data) * 100\n        print(f\"    {seg}: {count:,} ({pct:.1f}%)\")\n    \n    # ============================================================================\n    # PART 2: CLASSIFY CHURN RISK LEVEL\n    # ============================================================================\n    print(\"\\n[PART 2] Classifying churn risk levels...\")\n    \n    # Determine churn risk thresholds using quantiles\n    high_risk_threshold = retention_data['churn_probability'].quantile(0.67)  # Top 33%\n    medium_risk_threshold = retention_data['churn_probability'].quantile(0.33)  # Middle 33%\n    \n    def classify_churn_risk(prob):\n        \"\"\"Classify churn probability into risk tiers\"\"\"\n        if prob >= high_risk_threshold:\n            return 'HIGH_RISK'\n        elif prob >= medium_risk_threshold:\n            return 'MEDIUM_RISK'\n        else:\n            return 'LOW_RISK'\n    \n    retention_data['churn_risk_tier'] = retention_data['churn_probability'].apply(classify_churn_risk)\n    \n    print(f\"  Churn risk distribution:\")\n    for risk in ['LOW_RISK', 'MEDIUM_RISK', 'HIGH_RISK']:\n        count = (retention_data['churn_risk_tier'] == risk).sum()\n        avg_churn_prob = retention_data[retention_data['churn_risk_tier'] == risk]['churn_probability'].mean()\n        print(f\"    {risk}: {count:,} customers (avg churn prob: {avg_churn_prob:.2%})\")\n    \n    # ============================================================================\n    # PART 3: RFM × CHURN RISK MATRIX - STRATEGIC GUIDELINES\n    # ============================================================================\n    print(\"\\n[PART 3] Creating RFM × Churn Risk intervention matrix...\")\n    \n    def get_intervention_strategy(rfm_segment, churn_risk):\n        \"\"\"\n        Strategic intervention guidelines based on RFM segment and churn risk.\n        \n        PRINCIPLE: Different segments need different strategies.\n        - Champions: Preserve loyalty, prevent loss (high value)\n        - Loyal: Deepen engagement, reward consistency\n        - Middle: Convert to loyal or let go (depends on churn risk)\n        - At-Risk: Win-back efforts, retention offers\n        - Lost: Only if high-value signals in past behavior\n        \n        IMPORTANT CAVEAT:\n        These are qualitative guidelines. We DON'T have data on:\n        - Effectiveness of each intervention type\n        - Customer response rates by segment\n        - Intervention costs\n        - ROI by strategy\n        \n        Implementation and exact tactics are deferred to Final Report.\n        \"\"\"\n        \n        matrix = {\n            'Champions': {\n                'HIGH_RISK': {\n                    'action': 'VIP_Retention_Program',\n                    'rationale': 'Highest value segment at risk. Preserve relationship at all costs.',\n                    'priority': 'CRITICAL'\n                },\n                'MEDIUM_RISK': {\n                    'action': 'Premium_Engagement',\n                    'rationale': 'High value, monitor for churn signals. Deepen loyalty.',\n                    'priority': 'HIGH'\n                },\n                'LOW_RISK': {\n                    'action': 'VIP_Nurture',\n                    'rationale': 'Stable high-value customers. Continue premium treatment.',\n                    'priority': 'MEDIUM'\n                }\n            },\n            'Loyal': {\n                'HIGH_RISK': {\n                    'action': 'Loyalty_Rescue',\n                    'rationale': 'Consistent customers showing churn signals. Investigate why.',\n                    'priority': 'HIGH'\n                },\n                'MEDIUM_RISK': {\n                    'action': 'Engagement_Program',\n                    'rationale': 'Consolidate loyalty. Reward repeat behavior.',\n                    'priority': 'MEDIUM'\n                },\n                'LOW_RISK': {\n                    'action': 'Loyalty_Rewards',\n                    'rationale': 'Stable loyal base. Reinforce through recognition.',\n                    'priority': 'MEDIUM'\n                }\n            },\n            'Middle': {\n                'HIGH_RISK': {\n                    'action': 'Convert_or_Release',\n                    'rationale': 'Medium value at high risk. Try conversion once, then let go.',\n                    'priority': 'LOW'\n                },\n                'MEDIUM_RISK': {\n                    'action': 'Engagement_Experiment',\n                    'rationale': 'Test engagement tactics. Low investment, selective outreach.',\n                    'priority': 'LOW'\n                },\n                'LOW_RISK': {\n                    'action': 'Monitor',\n                    'rationale': 'Stable but medium value. Low-touch monitoring.',\n                    'priority': 'LOW'\n                }\n            },\n            'At_Risk': {\n                'HIGH_RISK': {\n                    'action': 'Win_Back_Campaign',\n                    'rationale': 'Already at-risk segment now showing churn signals. Last-ditch effort.',\n                    'priority': 'MEDIUM'\n                },\n                'MEDIUM_RISK': {\n                    'action': 'Retention_Offer',\n                    'rationale': 'May still be saveable. Targeted retention intervention.',\n                    'priority': 'LOW'\n                },\n                'LOW_RISK': {\n                    'action': 'No_Action',\n                    'rationale': 'At-risk but not showing churn signals currently. Passive monitoring.',\n                    'priority': 'NONE'\n                }\n            },\n            'Lost': {\n                'HIGH_RISK': {\n                    'action': 'No_Action',\n                    'rationale': 'Already lost and now inactive. Return unlikely.',\n                    'priority': 'NONE'\n                },\n                'MEDIUM_RISK': {\n                    'action': 'No_Action',\n                    'rationale': 'Already lost segment. Opportunity cost too high.',\n                    'priority': 'NONE'\n                },\n                'LOW_RISK': {\n                    'action': 'No_Action',\n                    'rationale': 'Lost segment. Resources better spent on others.',\n                    'priority': 'NONE'\n                }\n            },\n            'Unknown': {\n                'HIGH_RISK': {\n                    'action': 'Monitor_and_Classify',\n                    'rationale': 'Unclassified segment at high churn risk. Investigate RFM status.',\n                    'priority': 'MEDIUM'\n                },\n                'MEDIUM_RISK': {\n                    'action': 'Monitor',\n                    'rationale': 'Unclassified segment. Low-touch monitoring.',\n                    'priority': 'LOW'\n                },\n                'LOW_RISK': {\n                    'action': 'No_Action',\n                    'rationale': 'Unclassified but low churn risk. Passive.',\n                    'priority': 'NONE'\n                }\n            }\n        }\n        \n        if rfm_segment in matrix and churn_risk in matrix[rfm_segment]:\n            return matrix[rfm_segment][churn_risk]\n        else:\n            return {\n                'action': 'No_Action',\n                'rationale': 'Unclassified segment.',\n                'priority': 'NONE'\n            }\n    \n    # Apply matrix\n    retention_data['intervention_action'] = retention_data.apply(\n        lambda row: get_intervention_strategy(row['segment'], row['churn_risk_tier'])['action'],\n        axis=1\n    )\n    \n    retention_data['intervention_rationale'] = retention_data.apply(\n        lambda row: get_intervention_strategy(row['segment'], row['churn_risk_tier'])['rationale'],\n        axis=1\n    )\n    \n    retention_data['intervention_priority'] = retention_data.apply(\n        lambda row: get_intervention_strategy(row['segment'], row['churn_risk_tier'])['priority'],\n        axis=1\n    )\n    \n    print(f\"\\n  Intervention distribution:\")\n    for action in retention_data['intervention_action'].unique():\n        count = (retention_data['intervention_action'] == action).sum()\n        pct = count / len(retention_data) * 100\n        print(f\"    {action}: {count:,} ({pct:.1f}%)\")\n    \n    # ============================================================================\n    # PART 4: SEGMENT-LEVEL ANALYSIS\n    # ============================================================================\n    print(\"\\n[PART 4] Analyzing segment-level metrics...\")\n    \n    # Create RFM × Churn matrix summary\n    segment_analysis = retention_data.groupby(['segment', 'churn_risk_tier']).agg({\n        'customer_id': 'count',\n        'churn_probability': ['mean', 'std'],\n        'total_spend': ['sum', 'mean'],\n        'frequency': 'mean',\n        'recency_days': 'mean'\n    }).reset_index()\n    \n    segment_analysis.columns = ['segment', 'churn_risk', 'customer_count', 'avg_churn_prob', \n                                'std_churn_prob', 'total_revenue', 'avg_revenue', \n                                'avg_frequency', 'avg_recency_days']\n    \n    # Calculate % of total customers\n    segment_analysis['pct_of_customers'] = (segment_analysis['customer_count'] / len(retention_data) * 100).round(1)\n    \n    # Calculate % of total revenue (revenue concentration)\n    total_revenue = retention_data['total_spend'].sum()\n    segment_analysis['pct_of_revenue'] = (segment_analysis['total_revenue'] / total_revenue * 100).round(1)\n    \n    segment_analysis = segment_analysis.sort_values('total_revenue', ascending=False)\n    \n    segment_analysis.to_csv('11_RFM_CHURN_MATRIX.csv', index=False)\n    print(f\"  ✓ 11_RFM_CHURN_MATRIX.csv\")\n    print(f\"\\n  High-priority segments (revenue at risk):\")\n    \n    high_priority = retention_data[retention_data['intervention_priority'].isin(['CRITICAL', 'HIGH'])]\n    high_priority_summary = high_priority.groupby('segment').agg({\n        'customer_id': 'count',\n        'total_spend': 'sum',\n        'churn_probability': 'mean'\n    }).sort_values('total_spend', ascending=False)\n    \n    for seg in high_priority_summary.index:\n        count = int(high_priority_summary.loc[seg, 'customer_id'])\n        revenue = high_priority_summary.loc[seg, 'total_spend']\n        avg_churn = high_priority_summary.loc[seg, 'churn_probability']\n        print(f\"    {seg}: {count:,} customers, ${revenue:,.0f} revenue, {avg_churn:.1%} avg churn risk\")\n    \n    # ============================================================================\n    # PART 5: INTERVENTION PRIORITY RANKING\n    # ============================================================================\n    print(\"\\n[PART 5] Creating intervention priority ranking...\")\n    \n    priority_ranking = retention_data.groupby('intervention_action').agg({\n        'customer_id': 'count',\n        'total_spend': 'sum',\n        'churn_probability': 'mean',\n        'intervention_priority': lambda x: x.iloc[0]  # Get priority (same for all in group)\n    }).reset_index()\n    \n    priority_ranking.columns = ['intervention_action', 'customer_count', 'revenue_at_risk', \n                               'avg_churn_prob', 'priority_level']\n    \n    priority_ranking['pct_of_atrisk_customers'] = (priority_ranking['customer_count'] / \n                                                   retention_data[retention_data['intervention_priority'] != 'NONE'].shape[0] * 100).round(1)\n    \n    priority_ranking = priority_ranking.sort_values('revenue_at_risk', ascending=False)\n    priority_ranking.to_csv('11_INTERVENTION_PRIORITY.csv', index=False)\n    print(f\"  ✓ 11_INTERVENTION_PRIORITY.csv\")\n    print(f\"\\n  Intervention priorities (by revenue exposure):\")\n    for _, row in priority_ranking.iterrows():\n        print(f\"    {row['intervention_action']}: {int(row['customer_count']):,} customers, \" +\n              f\"${row['revenue_at_risk']:,.0f} revenue at risk\")\n    \n    # ============================================================================\n    # PART 6: SAVE INDIVIDUAL RECOMMENDATIONS\n    # ============================================================================\n    print(\"\\n[PART 6] Saving individual recommendations...\")\n    \n    output_cols = ['customer_id', 'segment', 'churn_probability', 'churn_risk_tier',\n                   'total_spend', 'frequency', 'recency_days', \n                   'intervention_action', 'intervention_priority']\n    \n    output_cols = [col for col in output_cols if col in retention_data.columns]\n    \n    retention_output = retention_data[output_cols].sort_values('churn_probability', ascending=False)\n    retention_output.to_csv('11_RETENTION_INTERVENTIONS.csv', index=False)\n    retention_output.to_parquet('11_RETENTION_INTERVENTIONS.parquet')\n    print(f\"  ✓ 11_RETENTION_INTERVENTIONS.csv ({len(retention_output):,} customers)\")\n    print(f\"  ✓ 11_RETENTION_INTERVENTIONS.parquet\")\n    \n    # ============================================================================\n    # PART 7: VISUALIZATION DASHBOARD\n    # ============================================================================\n    print(\"\\n[PART 7] Creating visualization dashboard...\")\n    \n    fig = plt.figure(figsize=(16, 10))\n    gs = fig.add_gridspec(2, 3, hspace=0.35, wspace=0.3)\n    \n    # Plot 1: Churn risk distribution\n    ax1 = fig.add_subplot(gs[0, 0])\n    risk_order = ['LOW_RISK', 'MEDIUM_RISK', 'HIGH_RISK']\n    risk_counts = retention_data['churn_risk_tier'].value_counts()\n    # Ensure all risk levels present, fill missing with 0\n    risk_counts = pd.Series([risk_counts.get(r, 0) for r in risk_order], index=risk_order)\n    colors_risk = ['#2ecc71', '#f39c12', '#e74c3c']\n    bars = ax1.bar(risk_counts.index, risk_counts.values, color=colors_risk, edgecolor='black', linewidth=1.5)\n    ax1.set_ylabel('# Customers', fontsize=10)\n    ax1.set_title('Churn Risk Distribution', fontweight='bold', fontsize=11)\n    ax1.grid(True, alpha=0.3, axis='y')\n    # Add value labels on bars\n    for bar, val in zip(bars, risk_counts.values):\n        height = bar.get_height()\n        if height > 0:  # Only label non-zero bars\n            ax1.text(bar.get_x() + bar.get_width()/2., height,\n                    f'{int(val):,}', ha='center', va='bottom', fontweight='bold', fontsize=9)\n    \n    # Plot 2: RFM segment distribution\n    ax2 = fig.add_subplot(gs[0, 1])\n    rfm_counts = retention_data['segment'].value_counts().sort_values(ascending=False)\n    colors_rfm = ['#3498db', '#2ecc71', '#f39c12', '#e67e22', '#e74c3c', '#95a5a6']\n    bars = ax2.bar(range(len(rfm_counts)), rfm_counts.values, color=colors_rfm[:len(rfm_counts)], \n                   edgecolor='black', linewidth=1.5)\n    ax2.set_xticks(range(len(rfm_counts)))\n    ax2.set_xticklabels(rfm_counts.index, rotation=45, ha='right', fontsize=9)\n    ax2.set_ylabel('# Customers', fontsize=10)\n    ax2.set_title('RFM Segment Distribution', fontweight='bold', fontsize=11)\n    ax2.grid(True, alpha=0.3, axis='y')\n    # Add value labels\n    for bar, val in zip(bars, rfm_counts.values):\n        height = bar.get_height()\n        ax2.text(bar.get_x() + bar.get_width()/2., height,\n                f'{int(val):,}', ha='center', va='bottom', fontweight='bold', fontsize=8)\n    \n    # Plot 3: Intervention priority breakdown\n    ax3 = fig.add_subplot(gs[0, 2])\n    priority_order = ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'NONE']\n    priority_counts = retention_data['intervention_priority'].value_counts()\n    # Ensure all priority levels present\n    priority_counts = pd.Series([priority_counts.get(p, 0) for p in priority_order], index=priority_order)\n    colors_priority = {'CRITICAL': '#c0392b', 'HIGH': '#e74c3c', 'MEDIUM': '#f39c12', 'LOW': '#3498db', 'NONE': '#95a5a6'}\n    bars = ax3.barh(priority_counts.index, priority_counts.values, \n            color=[colors_priority[p] for p in priority_counts.index], edgecolor='black', linewidth=1.5)\n    ax3.set_xlabel('# Customers', fontsize=10)\n    ax3.set_title('Intervention Priority Levels', fontweight='bold', fontsize=11)\n    ax3.grid(True, alpha=0.3, axis='x')\n    # Add value labels\n    for bar, val in zip(bars, priority_counts.values):\n        if val > 0:\n            ax3.text(val, bar.get_y() + bar.get_height()/2., f'{int(val):,}', \n                    va='center', ha='left', fontweight='bold', fontsize=8)\n    \n    # Plot 4: Heatmap - RFM × Churn Risk (customer count)\n    ax4 = fig.add_subplot(gs[1, 0])\n    heatmap_data = retention_data.groupby(['segment', 'churn_risk_tier']).size().unstack(fill_value=0)\n    segment_order = ['Champions', 'Loyal', 'Middle', 'At_Risk', 'Lost', 'Unknown']\n    heatmap_data = heatmap_data.reindex([s for s in segment_order if s in heatmap_data.index]).fillna(0)\n    sns.heatmap(heatmap_data, annot=True, fmt='d', cmap='RdYlGn_r', ax=ax4, cbar_kws={'label': '# Customers'})\n    ax4.set_title('RFM × Churn Risk Matrix (Customer Count)', fontweight='bold', fontsize=11)\n    ax4.set_ylabel('RFM Segment')\n    ax4.set_xlabel('Churn Risk Tier')\n    \n    # Plot 5: Heatmap - Revenue concentration\n    ax5 = fig.add_subplot(gs[1, 1])\n    revenue_heatmap = retention_data.groupby(['segment', 'churn_risk_tier'])['total_spend'].sum().unstack(fill_value=0)\n    segment_order = ['Champions', 'Loyal', 'Middle', 'At_Risk', 'Lost', 'Unknown']\n    revenue_heatmap = revenue_heatmap.reindex([s for s in segment_order if s in revenue_heatmap.index]).fillna(0)\n    sns.heatmap(revenue_heatmap, annot=True, fmt='.0f', cmap='YlOrRd', ax=ax5, cbar_kws={'label': 'Revenue ($)'})\n    ax5.set_title('RFM × Churn Risk: Revenue at Risk ($)', fontweight='bold', fontsize=11)\n    ax5.set_ylabel('RFM Segment')\n    ax5.set_xlabel('Churn Risk Tier')\n    \n    # Plot 6: Summary box\n    ax6 = fig.add_subplot(gs[1, 2])\n    ax6.axis('off')\n    \n    critical_count = (retention_data['intervention_priority'] == 'CRITICAL').sum()\n    high_count = (retention_data['intervention_priority'] == 'HIGH').sum()\n    critical_revenue = retention_data[retention_data['intervention_priority'] == 'CRITICAL']['total_spend'].sum()\n    high_revenue = retention_data[retention_data['intervention_priority'] == 'HIGH']['total_spend'].sum()\n    \n    summary_txt = f\"\"\"RETENTION SUMMARY\n\nCRITICAL Priority:\n• Customers: {critical_count:,}\n• Revenue: ${critical_revenue:,.0f}\n\nHIGH Priority:\n• Customers: {high_count:,}\n• Revenue: ${high_revenue:,.0f}\n\nKEY DATA GAPS:\n✗ No intervention costs\n✗ No effectiveness rates\n✗ No elasticity data\n✗ No ROI models\n\n(See Final Report for\nstrategic framework &\nimplementation guidance)\n\"\"\"\n    ax6.text(0.05, 0.95, summary_txt, transform=ax6.transAxes, fontsize=9,\n            verticalalignment='top', fontfamily='monospace',\n            bbox=dict(boxstyle='round', facecolor='lightyellow', alpha=0.7))\n    \n    plt.suptitle('RETENTION INTERVENTIONS & STRATEGIC GUIDELINES\\nRFM × Churn Risk Analysis',\n                 fontsize=13, fontweight='bold', y=0.995)\n    \n    plt.savefig('11_RETENTION_DASHBOARD.png', dpi=150, bbox_inches='tight')\n    plt.close()\n    print(f\"  ✓ 11_RETENTION_DASHBOARD.png\")\n    \n    # ============================================================================\n    # FINAL SUMMARY\n    # ============================================================================\n    elapsed = (datetime.now() - script_start).total_seconds() / 60\n    \n    print(f\"\\n{'=' * 80}\")\n    print(f\"✓✓✓ SCRIPT 11 COMPLETE: RETENTION INTERVENTIONS\")\n    print(f\"{'=' * 80}\")\n    print(f\"\\nExecution time: {elapsed:.1f} minutes\")\n    \n    print(f\"\\nKEY INSIGHTS:\")\n    print(f\"  • CRITICAL priority interventions: {critical_count:,} customers, ${critical_revenue:,.0f}\")\n    print(f\"  • HIGH priority interventions: {high_count:,} customers, ${high_revenue:,.0f}\")\n    print(f\"  • Total customers analyzed: {len(retention_data):,}\")\n    \n    print(f\"\\nSTRATEGIC FRAMEWORK:\")\n    print(f\"  ✓ RFM × Churn Risk matrix defines strategic guidelines\")\n    print(f\"  ✓ Champions at risk = VIP retention (highest priority)\")\n    print(f\"  ✓ Loyal at risk = Rescue attempts (high priority)\")\n    print(f\"  ✓ Middle/At-Risk = Selective interventions (lower priority)\")\n    print(f\"  ✓ Lost = No action (opportunity cost too high)\")\n    \n    print(f\"\\nTRANSPARENT DATA GAPS:\")\n    print(f\"  ✗ No intervention cost data → Can't calculate ROI\")\n    print(f\"  ✗ No retention rate by intervention → Can't model effectiveness\")\n    print(f\"  ✗ No elasticity data → Can't predict customer response\")\n    print(f\"  ✗ No historical campaign data → Can't benchmark results\")\n    print(f\"  → See Final Report for strategic implementation recommendations\")\n    \n    print(f\"\\nOUTPUTS:\")\n    print(f\"  ✓ 11_RETENTION_INTERVENTIONS.csv ({len(retention_output):,} customers)\")\n    print(f\"  ✓ 11_RFM_CHURN_MATRIX.csv (segment-level analysis)\")\n    print(f\"  ✓ 11_INTERVENTION_PRIORITY.csv (priority ranking)\")\n    print(f\"  ✓ 11_RETENTION_DASHBOARD.png (visualization)\")\n    print(f\"\\nREADY FOR SCRIPT 12 (Decision Matrix)\\n\")\n\nexcept Exception as e:\n    print(f\"\\n❌ Error: {e}\")\n    import traceback\n    traceback.print_exc()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T16:28:30.006284Z","iopub.execute_input":"2026-09-03T16:28:30.006756Z","iopub.status.idle":"2026-09-03T16:29:30.026088Z","shell.execute_reply.started":"2026-09-03T16:28:30.006723Z","shell.execute_reply":"2026-09-03T16:29:30.025116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ============================================================================\n# # SCRIPT 11: RETENTION INTERVENTIONS (45 min)\n# # ============================================================================\n# def script_11_retention_interventions():\n#     \"\"\"\n#     Prescribe retention strategies by customer segment.\n#     RUN AFTER script_07 completes.\n#     \"\"\"\n#     print(\"\\n\" + \"=\" * 70)\n#     print(\"SCRIPT 11: RETENTION INTERVENTIONS\")\n#     print(\"=\" * 70)\n    \n#     try:\n#         print(\"Loading churn and RFM data...\")\n#         churn = pd.read_parquet('06_CHURN_SCORES.parquet')\n        \n#         # CLV calculation\n#         print(\"Computing CLV...\")\n#         clv_calc = churn.copy()\n#         clv_calc['avg_order_value'] = clv_calc['total_spend'] / (clv_calc['frequency'] + 1)  # +1 to avoid div by 0\n#         clv_calc['repeat_rate'] = (clv_calc['frequency'] - 1) / (clv_calc['frequency'] + 1)\n#         clv_calc['clv_12m'] = clv_calc['avg_order_value'] * (1 + clv_calc['repeat_rate'] * 2)  # 12-month projection\n        \n#         # Top 20%\n#         clv_threshold = clv_calc['clv_12m'].quantile(0.8)\n#         clv_calc['is_top_20'] = clv_calc['clv_12m'] > clv_threshold\n        \n#         # Retention intervention tiers\n#         def retention_tier_logic(row):\n#             if row['churn_segment'] != 'HIGH_VALUE_AT_RISK':\n#                 return 'No_Action'\n#             elif row['clv_12m'] > 500:\n#                 return 'Tier_1_Exclusive_Access'\n#             elif 300 <= row['clv_12m'] <= 500:\n#                 return 'Tier_2_Loyalty_Discount_-10%'\n#             else:\n#                 return 'Tier_3_Newsletter_Reeng'\n        \n#         clv_calc['retention_intervention'] = clv_calc.apply(retention_tier_logic, axis=1)\n        \n#         # Intervention costs\n#         intervention_costs = {\n#             'Tier_1_Exclusive_Access': 50,\n#             'Tier_2_Loyalty_Discount_-10%': 20,\n#             'Tier_3_Newsletter_Reeng': 5,\n#             'No_Action': 0\n#         }\n        \n#         clv_calc['intervention_cost'] = clv_calc['retention_intervention'].map(intervention_costs)\n#         clv_calc['roi_multiple'] = clv_calc['clv_12m'] / (clv_calc['intervention_cost'] + 1)\n        \n#         # Save\n#         clv_calc.to_parquet('10_RETENTION_INTERVENTIONS.parquet')\n        \n#         print(\"✓ Retention interventions defined\")\n#         print(\"\\nIntervention distribution:\")\n#         print(clv_calc['retention_intervention'].value_counts())\n#         print(f\"\\nFinancial impact:\")\n#         print(f\"  Total intervention cost: ${clv_calc['intervention_cost'].sum():,.0f}\")\n#         at_risk = clv_calc[clv_calc['retention_intervention'] != 'No_Action']\n#         print(f\"  Total CLV at risk: ${at_risk['clv_12m'].sum():,.0f}\")\n#         print(f\"  Expected ROI if 30% retained: ${(at_risk['clv_12m'] * 0.3).sum() - clv_calc['intervention_cost'].sum():,.0f}\")\n        \n#         return clv_calc\n    \n#     except Exception as e:\n#         print(f\"❌ Error in script 11: {e}\")\n#         return None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T09:13:09.142312Z","iopub.execute_input":"2026-09-03T09:13:09.142779Z","iopub.status.idle":"2026-09-03T09:13:09.176363Z","shell.execute_reply.started":"2026-09-03T09:13:09.142747Z","shell.execute_reply":"2026-09-03T09:13:09.175236Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nSCRIPT 12: DECISION MATRIX VISUALIZATION\n========================================\nCreates two complementary decision matrices:\n\n1. SKU-LEVEL MATRIX (Inventory × Item Velocity)\n   - X-axis: Sales velocity percentile (from Script 08)\n   - Y-axis: Average churn risk of customers buying this item\n   - Bubble size: Item count or revenue\n   - Shows which inventory items need which actions\n\n2. SEGMENT-LEVEL MATRIX (RFM × Churn Risk)\n   - X-axis: RFM segment (Champions → Lost)\n   - Y-axis: Churn risk tier (Low → High)\n   - Shows which customer segments drive decisions\n   - Maps to retention interventions (Script 11)\n\nBoth matrices tie back to:\n- Markdown logic (Script 10)\n- Retention interventions (Script 11)\n- Inventory profiling (Script 08)\n\nOutput:\n- 12_SKU_DECISION_MATRIX.png (operational decisions)\n- 12_SEGMENT_DECISION_MATRIX.png (strategic decisions)\n- 12_DECISION_MATRIX_SUMMARY.csv (data backing the matrices)\n\nTime estimate: 15-20 minutes\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\" * 80)\nprint(\"SCRIPT 12: DECISION MATRIX VISUALIZATION\")\nprint(\"=\" * 80)\n\ntry:\n    # ============================================================================\n    # PART 0: LOAD DATA\n    # ============================================================================\n    print(\"\\n[PART 0] Loading data from all prior scripts...\")\n    script_start = datetime.now()\n    \n    inventory = pd.read_parquet('08_INVENTORY_ANALYSIS.parquet')\n    churn = pd.read_parquet('07_CHURN_SCORES.parquet')\n    markdown = pd.read_parquet('10_MARKDOWN_LOGIC.parquet')\n    retention = pd.read_parquet('11_RETENTION_INTERVENTIONS.parquet')\n    master = pd.read_parquet('master_with_features.parquet')\n    \n    print(f\"  ✓ Inventory: {inventory.shape}\")\n    print(f\"  ✓ Churn scores: {churn.shape}\")\n    print(f\"  ✓ Markdown logic: {markdown.shape}\")\n    print(f\"  ✓ Retention interventions: {retention.shape}\")\n    print(f\"  ✓ Master: {master.shape}\")\n    \n    # ============================================================================\n    # PART 1: SKU-LEVEL DECISION MATRIX\n    # ============================================================================\n    print(\"\\n[PART 1] Building SKU-level decision matrix...\")\n    \n    # For each SKU, calculate:\n    # - Velocity percentile (from inventory)\n    # - Average churn risk of customers who buy it (from master + churn)\n    \n    # Get customers per SKU and their churn risk\n    sku_customer_churn = master[['article_id', 'customer_id']].merge(\n        churn[['customer_id', 'churn_probability', 'segment']],\n        on='customer_id',\n        how='left'\n    )\n    \n    # Aggregate by SKU\n    sku_metrics = sku_customer_churn.groupby('article_id').agg({\n        'customer_id': 'nunique',\n        'churn_probability': 'mean'\n    }).reset_index()\n    \n    sku_metrics.columns = ['article_id', 'unique_customers', 'avg_churn_prob']\n    \n    # Merge with inventory velocity data\n    if 'velocity_percentile_in_category' in inventory.columns:\n        vel_col = 'velocity_percentile_in_category'\n    elif 'velocity_percentile' in inventory.columns:\n        vel_col = 'velocity_percentile'\n    else:\n        vel_col = None\n        print(f\"  ⚠️  No velocity percentile column. Available: {inventory.columns.tolist()}\")\n    \n    if vel_col:\n        sku_inventory = inventory[['article_id', vel_col, 'action']].drop_duplicates()\n        sku_matrix_data = sku_metrics.merge(sku_inventory, on='article_id', how='left')\n        \n        # Define quadrants\n        vel_median = sku_matrix_data[vel_col].median()\n        churn_median = sku_matrix_data['avg_churn_prob'].median()\n        \n        def assign_sku_quadrant(row):\n            \"\"\"Assign SKU to quadrant based on velocity and churn\"\"\"\n            vel = row[vel_col]\n            churn = row['avg_churn_prob']\n            \n            if vel >= vel_median and churn < churn_median:\n                return 'CASH_COWS'\n            elif vel >= vel_median and churn >= churn_median:\n                return 'QUESTION_MARKS'\n            elif vel < vel_median and churn < churn_median:\n                return 'DOGS'\n            else:\n                return 'LIQUIDATION_PRIORITY'\n        \n        sku_matrix_data['quadrant'] = sku_matrix_data.apply(assign_sku_quadrant, axis=1)\n        \n        print(f\"  SKU quadrant distribution:\")\n        for quad in ['CASH_COWS', 'QUESTION_MARKS', 'DOGS', 'LIQUIDATION_PRIORITY']:\n            count = (sku_matrix_data['quadrant'] == quad).sum()\n            rev = sku_matrix_data[sku_matrix_data['quadrant'] == quad]['unique_customers'].sum()\n            print(f\"    {quad}: {count:,} SKUs, {rev:,} unique customers\")\n        \n        sku_matrix_data.to_csv('12_DECISION_MATRIX_SKU_DATA.csv', index=False)\n        print(f\"  ✓ 12_DECISION_MATRIX_SKU_DATA.csv\")\n    else:\n        print(f\"  ⚠️  Cannot build SKU matrix without velocity data\")\n        sku_matrix_data = None\n    \n    # ============================================================================\n    # PART 2: SEGMENT-LEVEL DECISION MATRIX\n    # ============================================================================\n    print(\"\\n[PART 2] Building segment-level decision matrix...\")\n    \n    # Create RFM × Churn Risk matrix\n    segment_matrix_data = retention.groupby(['segment', 'churn_risk_tier']).agg({\n        'customer_id': 'count',\n        'total_spend': 'sum',\n        'churn_probability': 'mean'\n    }).reset_index()\n    \n    segment_matrix_data.columns = ['segment', 'churn_risk_tier', 'customer_count', 'revenue', 'avg_churn_prob']\n    \n    print(f\"  Segment-churn combinations: {len(segment_matrix_data)}\")\n    print(f\"  Segment distribution:\")\n    for seg in segment_matrix_data['segment'].unique():\n        count = segment_matrix_data[segment_matrix_data['segment'] == seg]['customer_count'].sum()\n        print(f\"    {seg}: {count:,} customers\")\n    \n    segment_matrix_data.to_csv('12_DECISION_MATRIX_SEGMENT_DATA.csv', index=False)\n    print(f\"  ✓ 12_DECISION_MATRIX_SEGMENT_DATA.csv\")\n    \n    # ============================================================================\n    # PART 3: VISUALIZATION - SKU MATRIX\n    # ============================================================================\n    if sku_matrix_data is not None:\n        print(\"\\n[PART 3] Creating SKU-level decision matrix visualization...\")\n        \n        fig, ax = plt.subplots(figsize=(14, 10))\n        \n        # Define quadrant colors and labels\n        quad_colors = {\n            'CASH_COWS': '#2ecc71',\n            'QUESTION_MARKS': '#f39c12',\n            'DOGS': '#95a5a6',\n            'LIQUIDATION_PRIORITY': '#e74c3c'\n        }\n        \n        quad_edges = {\n            'CASH_COWS': '#27ae60',\n            'QUESTION_MARKS': '#d68910',\n            'DOGS': '#7f8c8d',\n            'LIQUIDATION_PRIORITY': '#c0392b'\n        }\n        \n        quad_labels = {\n            'CASH_COWS': 'CASH COWS\\n(High Velocity + Low Churn)',\n            'QUESTION_MARKS': 'QUESTION MARKS\\n(High Velocity + High Churn)',\n            'DOGS': 'DOGS\\n(Low Velocity + Low Churn)',\n            'LIQUIDATION_PRIORITY': 'LIQUIDATION PRIORITY\\n(Low Velocity + High Churn)'\n        }\n        \n        # Plot each SKU as bubble\n        for quad, color in quad_colors.items():\n            quad_data = sku_matrix_data[sku_matrix_data['quadrant'] == quad]\n            if len(quad_data) > 0:\n                ax.scatter(quad_data[vel_col], quad_data['avg_churn_prob'] * 100,\n                          s=quad_data['unique_customers'] * 0.5,\n                          alpha=0.6, color=color, edgecolors=quad_edges[quad],\n                          linewidth=1.5, label=quad_labels[quad], zorder=3)\n        \n        # Add quadrant dividing lines\n        ax.axhline(y=churn_median * 100, color='black', linestyle='--', linewidth=1.5, alpha=0.4, zorder=2)\n        ax.axvline(x=vel_median, color='black', linestyle='--', linewidth=1.5, alpha=0.4, zorder=2)\n        \n        # Labels and styling\n        ax.set_xlabel('Sales Velocity Percentile (%) →', fontsize=12, fontweight='bold')\n        ax.set_ylabel('Avg Churn Risk of Buyers (%) →', fontsize=12, fontweight='bold')\n        ax.set_title('SKU-LEVEL DECISION MATRIX\\n(Bubble size = # unique customers buying this SKU)',\n                    fontsize=14, fontweight='bold', pad=20)\n        ax.set_xlim(-5, 105)\n        ax.set_ylim(-5, 105)\n        ax.grid(True, alpha=0.2, linestyle=':')\n        ax.set_facecolor('#f8f9fa')\n        ax.legend(loc='upper left', fontsize=9, framealpha=0.95)\n        \n        # Add action recommendations box\n        actions_text = \"\"\"RECOMMENDED ACTIONS (from Scripts 08, 10, 11):\n        \nCASH COWS: Premium management\n  → Full price, limited markdown (Script 10)\n  → Loyalty rewards for buyers (Script 11)\n  → Maintain stock, optimize placement\n        \nQUESTION MARKS: Investigate churn drivers\n  → -10% to -15% markdown (Script 10)\n  → Target retention interventions (Script 11)\n  → Why are these customers at high churn risk?\n        \nDOGS: Discontinue or bundle\n  → -20% markdown + bundle with Cash Cows (Script 10)\n  → Low engagement - deprioritize resources\n        \nLIQUIDATION: Aggressive clearance\n  → -25% to -30% markdown (Script 10)\n  → Win-back campaigns (Script 11)\n  → Clear before end of season\"\"\"\n        \n        ax.text(0.02, 0.02, actions_text, transform=ax.transAxes, fontsize=8,\n               verticalalignment='bottom', horizontalalignment='left',\n               bbox=dict(boxstyle='round', facecolor='lightyellow', alpha=0.8, pad=1),\n               family='monospace')\n        \n        plt.tight_layout()\n        plt.savefig('12_SKU_DECISION_MATRIX.png', dpi=150, bbox_inches='tight')\n        plt.close()\n        print(f\"  ✓ 12_SKU_DECISION_MATRIX.png\")\n    \n    # ============================================================================\n    # PART 4: VISUALIZATION - SEGMENT MATRIX\n    # ============================================================================\n    print(\"\\n[PART 4] Creating segment-level decision matrix visualization...\")\n    \n    fig, ax = plt.subplots(figsize=(14, 10))\n    \n    # Create pivot for heatmap\n    segment_order = ['Champions', 'Loyal', 'Middle', 'At_Risk', 'Lost', 'Unknown']\n    risk_order = ['LOW_RISK', 'MEDIUM_RISK', 'HIGH_RISK']\n    \n    # Heatmap data: customer count\n    heatmap_count = segment_matrix_data.pivot_table(\n        index='segment',\n        columns='churn_risk_tier',\n        values='customer_count',\n        fill_value=0\n    )\n    \n    heatmap_count = heatmap_count.reindex([s for s in segment_order if s in heatmap_count.index])\n    heatmap_count = heatmap_count[[r for r in risk_order if r in heatmap_count.columns]]\n    \n    # Plot heatmap\n    sns.heatmap(heatmap_count, annot=True, fmt=',.0f', cmap='RdYlGn_r', ax=ax,\n               cbar_kws={'label': '# Customers'}, linewidths=1, linecolor='black')\n    \n    ax.set_title('SEGMENT-LEVEL DECISION MATRIX\\n(RFM Segment × Churn Risk Tier)',\n                fontsize=14, fontweight='bold', pad=20)\n    ax.set_xlabel('Churn Risk Tier →', fontsize=12, fontweight='bold')\n    ax.set_ylabel('RFM Segment →', fontsize=12, fontweight='bold')\n    \n    # Add text annotations for strategies\n    strategies = {\n        ('Champions', 'HIGH_RISK'): 'VIP SAVE',\n        ('Champions', 'MEDIUM_RISK'): 'Premium\\nEngage',\n        ('Champions', 'LOW_RISK'): 'VIP Nurture',\n        ('Loyal', 'HIGH_RISK'): 'Loyalty\\nRescue',\n        ('Loyal', 'MEDIUM_RISK'): 'Engagement',\n        ('Loyal', 'LOW_RISK'): 'Reward',\n        ('Middle', 'HIGH_RISK'): 'Convert\\nor Release',\n        ('Middle', 'MEDIUM_RISK'): 'Test',\n        ('At_Risk', 'HIGH_RISK'): 'Win-Back',\n        ('At_Risk', 'MEDIUM_RISK'): 'Offer',\n        ('Lost', 'HIGH_RISK'): 'Let Go',\n    }\n    \n    # Add strategy labels to cells\n    for (seg, risk), strategy in strategies.items():\n        if seg in heatmap_count.index and risk in heatmap_count.columns:\n            x = list(heatmap_count.columns).index(risk)\n            y = list(heatmap_count.index).index(seg)\n            ax.text(x + 0.5, y + 0.7, strategy, ha='center', va='center',\n                   fontsize=7, fontweight='bold', color='white',\n                   bbox=dict(boxstyle='round,pad=0.3', facecolor='black', alpha=0.6))\n    \n    plt.tight_layout()\n    plt.savefig('12_SEGMENT_DECISION_MATRIX.png', dpi=150, bbox_inches='tight')\n    plt.close()\n    print(f\"  ✓ 12_SEGMENT_DECISION_MATRIX.png\")\n    \n    # ============================================================================\n    # PART 5: SUMMARY INSIGHTS\n    # ============================================================================\n    print(\"\\n[PART 5] Generating summary insights...\")\n    \n    if sku_matrix_data is not None:\n        cash_cows = sku_matrix_data[sku_matrix_data['quadrant'] == 'CASH_COWS']\n        question_marks = sku_matrix_data[sku_matrix_data['quadrant'] == 'QUESTION_MARKS']\n        dogs = sku_matrix_data[sku_matrix_data['quadrant'] == 'DOGS']\n        liquidation = sku_matrix_data[sku_matrix_data['quadrant'] == 'LIQUIDATION_PRIORITY']\n        \n        print(f\"\\n  SKU QUADRANT SUMMARY:\")\n        print(f\"    CASH COWS: {len(cash_cows):,} SKUs, {cash_cows['unique_customers'].sum():,} customers\")\n        print(f\"    QUESTION MARKS: {len(question_marks):,} SKUs, {question_marks['unique_customers'].sum():,} customers\")\n        print(f\"    DOGS: {len(dogs):,} SKUs, {dogs['unique_customers'].sum():,} customers\")\n        print(f\"    LIQUIDATION PRIORITY: {len(liquidation):,} SKUs, {liquidation['unique_customers'].sum():,} customers\")\n    \n    # Critical segments\n    critical_segments = retention[retention['intervention_priority'] == 'CRITICAL']\n    high_segments = retention[retention['intervention_priority'] == 'HIGH']\n    \n    print(f\"\\n  SEGMENT PRIORITY SUMMARY:\")\n    print(f\"    CRITICAL interventions: {len(critical_segments):,} customers, ${critical_segments['total_spend'].sum():,.0f} revenue at risk\")\n    print(f\"    HIGH interventions: {len(high_segments):,} customers, ${high_segments['total_spend'].sum():,.0f} revenue at risk\")\n    \n    # ============================================================================\n    # FINAL SUMMARY\n    # ============================================================================\n    elapsed = (datetime.now() - script_start).total_seconds() / 60\n    \n    print(f\"\\n{'=' * 80}\")\n    print(f\"✓✓✓ SCRIPT 12 COMPLETE: DECISION MATRICES\")\n    print(f\"{'=' * 80}\")\n    print(f\"\\nExecution time: {elapsed:.1f} minutes\")\n    \n    print(f\"\\nOUTPUTS:\")\n    print(f\"  ✓ 12_SKU_DECISION_MATRIX.png (operational decisions)\")\n    print(f\"  ✓ 12_SEGMENT_DECISION_MATRIX.png (strategic decisions)\")\n    print(f\"  ✓ 12_DECISION_MATRIX_SKU_DATA.csv (SKU data)\")\n    print(f\"  ✓ 12_DECISION_MATRIX_SEGMENT_DATA.csv (segment data)\")\n    \n    print(f\"\\nKEY INSIGHTS:\")\n    print(f\"  • Two complementary views: SKUs (operational) + Segments (strategic)\")\n    print(f\"  • Actions tied directly to Scripts 08, 10, 11 recommendations\")\n    print(f\"  • Markdown tiers follow velocity & churn signals\")\n    print(f\"  • Retention strategies tied to RFM × Churn Risk\")\n    \n    print(f\"\\nREADY FOR FINAL REPORT\\n\")\n\nexcept Exception as e:\n    print(f\"\\n❌ Error: {e}\")\n    import traceback\n    traceback.print_exc()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T16:34:03.410597Z","iopub.execute_input":"2026-09-03T16:34:03.411552Z","iopub.status.idle":"2026-09-03T16:34:40.464566Z","shell.execute_reply.started":"2026-09-03T16:34:03.411514Z","shell.execute_reply":"2026-09-03T16:34:40.463697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nmaster = pd.read_parquet('master_with_features.parquet')\nrfm = pd.read_parquet('06_RFM_SEGMENTS.parquet')\n\nnum_cols = ['price','age','recency_days','frequency','total_spend']\nstats = pd.concat([\n    master[[c for c in ['price','age'] if c in master.columns]].describe().T,\n    rfm[[c for c in ['recency_days','frequency','total_spend'] if c in rfm.columns]].describe().T\n])\nstats['median'] = stats['50%']\nstats['IQR'] = stats['75%'] - stats['25%']\nstats = stats[['count','mean','median','std','min','25%','50%','75%','max','IQR']].round(4)\nprint(stats.to_string())\nstats.to_csv('DESCRIPTIVE_STATS.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T17:00:22.977510Z","iopub.execute_input":"2026-09-03T17:00:22.978036Z","iopub.status.idle":"2026-09-03T17:00:41.546839Z","shell.execute_reply.started":"2026-09-03T17:00:22.977997Z","shell.execute_reply":"2026-09-03T17:00:41.545652Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rfm.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T17:07:00.476797Z","iopub.execute_input":"2026-09-03T17:07:00.477211Z","iopub.status.idle":"2026-09-03T17:07:00.592652Z","shell.execute_reply.started":"2026-09-03T17:07:00.477178Z","shell.execute_reply":"2026-09-03T17:07:00.591717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nPATCH: FIX TWO LOGIC COLLAPSES\n==============================\nFix 1: Category tiers all resolved to 'Standard' (keyword list did not match\n       real H&M product_type_name values). Rebuild tiers from garment_group_name\n       + price percentile, which exist in the data.\n\nFix 2: Zero LOW_RISK customers. Churn probabilities are bimodal at 0/1 due to\n       recency leakage, so quantile cutoffs degenerate. Re-tier on recency_days,\n       the actual underlying signal, giving three populated tiers.\n\nRegenerates: 10_MARKDOWN_RECOMMENDATIONS.csv, 10_MARKDOWN_LOGIC.parquet,\n             10_CATEGORY_TIER_SUMMARY.csv,\n             11_RETENTION_INTERVENTIONS.csv/.parquet,\n             11_RFM_CHURN_MATRIX.csv, plus both dashboards.\n\nRuntime: ~5 minutes\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"=\" * 80)\nprint(\"PATCH: FIXING CATEGORY TIERS AND CHURN RISK TIERS\")\nprint(\"=\" * 80)\nt0 = datetime.now()\n\n# ============================================================================\n# FIX 1: CATEGORY TIERS\n# ============================================================================\nprint(\"\\n[FIX 1] Rebuilding category tiers from real H&M columns...\")\n\nmarkdown = pd.read_parquet('10_MARKDOWN_LOGIC.parquet')\nmaster = pd.read_parquet('master_with_features.parquet')\n\n# Load the columns that actually exist in H&M articles\narticles = pd.read_csv(\n    '/kaggle/input/datasets/shantanusmankar/h-and-m-data/articles.csv',\n    usecols=['article_id', 'product_type_name', 'product_group_name',\n             'garment_group_name', 'index_group_name']\n)\n\nprint(f\"  Real garment groups: {articles['garment_group_name'].nunique()}\")\nprint(f\"  Sample: {articles['garment_group_name'].dropna().unique()[:8].tolist()}\")\n\n# Price percentile per article (proxy for premium positioning)\narticle_price = master.groupby('article_id')['price'].mean().reset_index()\narticle_price.columns = ['article_id', 'mean_price']\narticle_price['price_pctile'] = article_price['mean_price'].rank(pct=True) * 100\n\narticles = articles.merge(article_price, on='article_id', how='left')\n\nBASICS_GROUPS = {\n    'Jersey Basic', 'Socks and Tights', 'Under-, Nightwear',\n    'Unknown', 'Special Offers'\n}\nFASHION_GROUPS = {\n    'Jersey Fancy', 'Dresses Ladies', 'Blouses', 'Dressed', 'Skirts',\n    'Dresses/Skirts girls', 'Knitwear', 'Outdoor', 'Swimwear'\n}\n\ndef tier_category(row):\n    \"\"\"Assign a business tier using columns that exist in the data.\n\n    PREMIUM is defined as the top decile of average selling price. H&M is a\n    fast-fashion retailer with no luxury line, so 'Luxury' is intentionally\n    absent: the never-markdown rule instead attaches to PREMIUM.\n    \"\"\"\n    grp = row['garment_group_name']\n    pct = row['price_pctile']\n\n    if pd.notna(pct) and pct >= 90:\n        return 'Premium'\n    if grp in BASICS_GROUPS:\n        return 'Basics'\n    if grp in FASHION_GROUPS:\n        return 'Fast_Fashion'\n    return 'Standard'\n\narticles['category_tier_new'] = articles.apply(tier_category, axis=1)\n\nprint(f\"\\n  New category tier distribution (article level):\")\nfor tier, cnt in articles['category_tier_new'].value_counts().items():\n    print(f\"    {tier}: {cnt:,} articles ({cnt/len(articles)*100:.1f}%)\")\n\n# Attach to markdown table\nmarkdown = markdown.drop(columns=[c for c in ['category_tier', 'product_type_name',\n                                              'garment_group_name'] if c in markdown.columns])\nmarkdown = markdown.merge(\n    articles[['article_id', 'product_type_name', 'garment_group_name',\n              'category_tier_new', 'price_pctile']],\n    on='article_id', how='left'\n)\nmarkdown = markdown.rename(columns={'category_tier_new': 'category_tier'})\nmarkdown['category_tier'] = markdown['category_tier'].fillna('Standard')\n\n# Re-apply markdown logic with working tiers\ndef markdown_tier(row):\n    cat = row['category_tier']\n    vel = row['velocity_percentile_in_category']\n    gap = row['max_sales_gap_days'] if pd.notna(row['max_sales_gap_days']) else 0\n    off_season = bool(row['is_off_season'])\n    action = row['action']\n\n    # PREMIUM: protect price positioning, never discount\n    if cat == 'Premium':\n        return 'No_Action_Premium_Hold'\n\n    if off_season:\n        if vel < 20 and gap > 14:\n            return 'Tier_3_Aggressive_Clear_-30%'\n        elif vel < 35 and gap > 7:\n            return 'Tier_2_Moderate_Clear_-20%'\n        else:\n            return 'Tier_1_Light_Discount_-10%'\n\n    if cat == 'Basics':\n        if vel < 10:\n            return 'Tier_3_Clearance_-25%'\n        elif vel < 25:\n            return 'Tier_2_Markdown_-15%'\n        elif action == 'LIQUIDATE':\n            return 'Tier_1_Monitor_-10%'\n        return 'No_Action'\n\n    if cat == 'Fast_Fashion':\n        if vel < 10:\n            return 'Tier_3_Clearance_-20%'\n        elif vel < 25:\n            return 'Tier_2_Markdown_-15%'\n        elif action == 'LIQUIDATE':\n            return 'Tier_1_Monitor_-10%'\n        return 'No_Action'\n\n    if vel < 10:\n        return 'Tier_3_Clearance_-20%'\n    elif vel < 25:\n        return 'Tier_2_Markdown_-10%'\n    elif action == 'LIQUIDATE':\n        return 'Tier_1_Monitor_-5%'\n    return 'No_Action'\n\nmarkdown['markdown_recommendation'] = markdown.apply(markdown_tier, axis=1)\nmarkdown['priority_score'] = (100 - markdown['velocity_percentile_in_category']) * \\\n                             np.where(markdown['is_off_season'], 1.0, 0.5)\nmarkdown = markdown.sort_values('priority_score', ascending=False)\n\nprint(f\"\\n  New markdown distribution:\")\nfor tier, cnt in markdown['markdown_recommendation'].value_counts().items():\n    print(f\"    {tier}: {cnt:,} ({cnt/len(markdown)*100:.1f}%)\")\n\n# Category tier summary table for the report\ncat_summary = markdown.groupby('category_tier').agg(\n    items=('article_id', 'count'),\n    mean_velocity_pctile=('velocity_percentile_in_category', 'mean'),\n    pct_off_season=('is_off_season', 'mean'),\n).reset_index()\ncat_summary['mean_velocity_pctile'] = cat_summary['mean_velocity_pctile'].round(1)\ncat_summary['pct_off_season'] = (cat_summary['pct_off_season'] * 100).round(1)\ncat_summary.to_csv('10_CATEGORY_TIER_SUMMARY.csv', index=False)\n\nmarkdown.to_csv('10_MARKDOWN_RECOMMENDATIONS.csv', index=False)\nmarkdown.to_parquet('10_MARKDOWN_LOGIC.parquet')\nprint(f\"  ✓ 10_MARKDOWN_RECOMMENDATIONS.csv / .parquet\")\nprint(f\"  ✓ 10_CATEGORY_TIER_SUMMARY.csv\")\n\n# ============================================================================\n# FIX 2: CHURN RISK TIERS\n# ============================================================================\nprint(\"\\n[FIX 2] Re-tiering churn risk on recency (probabilities are degenerate)...\")\n\nretention = pd.read_parquet('11_RETENTION_INTERVENTIONS.parquet')\n\n# Diagnose the degeneracy for the report\nprob_at_zero = (retention['churn_probability'] < 0.01).mean() * 100\nprob_at_one = (retention['churn_probability'] > 0.99).mean() * 100\nprint(f\"  Churn probability mass at ~0.0: {prob_at_zero:.1f}%\")\nprint(f\"  Churn probability mass at ~1.0: {prob_at_one:.1f}%\")\nprint(f\"  → Bimodal. Quantile cutoffs cannot produce three tiers.\")\n\n# Re-tier on recency_days, the signal the model actually learned.\n# Terciles guarantee three populated, interpretable tiers.\nr33 = retention['recency_days'].quantile(0.33)\nr67 = retention['recency_days'].quantile(0.67)\nprint(f\"  Recency cutoffs: LOW < {r33:.0f}d, MEDIUM {r33:.0f}-{r67:.0f}d, HIGH > {r67:.0f}d\")\n\nretention['churn_risk_tier'] = np.select(\n    [retention['recency_days'] <= r33,\n     retention['recency_days'] <= r67],\n    ['LOW_RISK', 'MEDIUM_RISK'],\n    default='HIGH_RISK'\n)\n\nprint(f\"\\n  New churn risk distribution:\")\nfor tier in ['LOW_RISK', 'MEDIUM_RISK', 'HIGH_RISK']:\n    sub = retention[retention['churn_risk_tier'] == tier]\n    print(f\"    {tier}: {len(sub):,} customers (mean recency {sub['recency_days'].mean():.0f}d)\")\n\n# Re-apply the intervention matrix\nMATRIX = {\n    'Champions': {'HIGH_RISK': ('VIP_Retention_Program', 'CRITICAL'),\n                  'MEDIUM_RISK': ('Premium_Engagement', 'HIGH'),\n                  'LOW_RISK': ('VIP_Nurture', 'MEDIUM')},\n    'Loyal': {'HIGH_RISK': ('Loyalty_Rescue', 'HIGH'),\n              'MEDIUM_RISK': ('Engagement_Program', 'MEDIUM'),\n              'LOW_RISK': ('Loyalty_Rewards', 'MEDIUM')},\n    'Middle': {'HIGH_RISK': ('Convert_or_Release', 'LOW'),\n               'MEDIUM_RISK': ('Engagement_Experiment', 'LOW'),\n               'LOW_RISK': ('Monitor', 'LOW')},\n    'At-Risk': {'HIGH_RISK': ('Win_Back_Campaign', 'MEDIUM'),\n                'MEDIUM_RISK': ('Retention_Offer', 'LOW'),\n                'LOW_RISK': ('No_Action', 'NONE')},\n    'At_Risk': {'HIGH_RISK': ('Win_Back_Campaign', 'MEDIUM'),\n                'MEDIUM_RISK': ('Retention_Offer', 'LOW'),\n                'LOW_RISK': ('No_Action', 'NONE')},\n    'Lost': {'HIGH_RISK': ('No_Action', 'NONE'),\n             'MEDIUM_RISK': ('No_Action', 'NONE'),\n             'LOW_RISK': ('No_Action', 'NONE')},\n}\n\ndef lookup(seg, risk, idx):\n    return MATRIX.get(seg, {}).get(risk, ('Monitor', 'LOW'))[idx]\n\nretention['intervention_action'] = [\n    lookup(s, r, 0) for s, r in zip(retention['segment'], retention['churn_risk_tier'])\n]\nretention['intervention_priority'] = [\n    lookup(s, r, 1) for s, r in zip(retention['segment'], retention['churn_risk_tier'])\n]\n\nprint(f\"\\n  New intervention priority distribution:\")\nfor p in ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'NONE']:\n    cnt = (retention['intervention_priority'] == p).sum()\n    print(f\"    {p}: {cnt:,}\")\n\nretention.to_csv('11_RETENTION_INTERVENTIONS.csv', index=False)\nretention.to_parquet('11_RETENTION_INTERVENTIONS.parquet')\n\nseg_matrix = retention.groupby(['segment', 'churn_risk_tier']).agg(\n    customer_count=('customer_id', 'count'),\n    revenue=('total_spend', 'sum'),\n    mean_recency=('recency_days', 'mean'),\n).reset_index()\nseg_matrix.to_csv('11_RFM_CHURN_MATRIX.csv', index=False)\nprint(f\"  ✓ 11_RETENTION_INTERVENTIONS.csv / .parquet\")\nprint(f\"  ✓ 11_RFM_CHURN_MATRIX.csv\")\n\n# ============================================================================\n# REGENERATE AFFECTED CHARTS\n# ============================================================================\nprint(\"\\n[CHARTS] Regenerating affected panels...\")\n\nSEG_ORDER = ['Champions', 'Loyal', 'Middle', 'At-Risk', 'At_Risk', 'Lost']\nRISK_ORDER = ['LOW_RISK', 'MEDIUM_RISK', 'HIGH_RISK']\n\nfig, axes = plt.subplots(1, 3, figsize=(18, 5.5))\n\n# Category tiers (now populated)\nct = markdown['category_tier'].value_counts()\naxes[0].bar(ct.index, ct.values,\n            color=['#8e44ad' if t == 'Premium' else '#3498db' for t in ct.index],\n            edgecolor='black', linewidth=1.5)\naxes[0].set_title('Items by Category Tier (corrected)', fontweight='bold')\naxes[0].set_ylabel('# Items')\naxes[0].tick_params(axis='x', rotation=20)\naxes[0].grid(True, alpha=0.3, axis='y')\nfor i, v in enumerate(ct.values):\n    axes[0].text(i, v, f'{int(v):,}', ha='center', va='bottom',\n                 fontweight='bold', fontsize=9)\n\n# Churn risk (now three tiers)\nrc = pd.Series([(retention['churn_risk_tier'] == r).sum() for r in RISK_ORDER],\n               index=RISK_ORDER)\naxes[1].bar(rc.index, rc.values, color=['#2ecc71', '#f39c12', '#e74c3c'],\n            edgecolor='black', linewidth=1.5)\naxes[1].set_title('Churn Risk Distribution (corrected)', fontweight='bold')\naxes[1].set_ylabel('# Customers')\naxes[1].grid(True, alpha=0.3, axis='y')\nfor i, v in enumerate(rc.values):\n    axes[1].text(i, v, f'{int(v):,}', ha='center', va='bottom',\n                 fontweight='bold', fontsize=9)\n\n# RFM x risk heatmap\nhm = retention.groupby(['segment', 'churn_risk_tier']).size().unstack(fill_value=0)\nhm = hm.reindex([s for s in SEG_ORDER if s in hm.index])\nhm = hm[[r for r in RISK_ORDER if r in hm.columns]]\nsns.heatmap(hm, annot=True, fmt=',d', cmap='RdYlGn_r', ax=axes[2],\n            linewidths=1, linecolor='black', cbar_kws={'label': '# Customers'})\naxes[2].set_title('RFM Segment × Churn Risk (corrected)', fontweight='bold')\naxes[2].set_xlabel('Churn Risk Tier')\naxes[2].set_ylabel('RFM Segment')\n\nplt.suptitle('CORRECTED SEGMENTATION PANELS', fontsize=14, fontweight='bold')\nplt.tight_layout()\nplt.savefig('13_CORRECTED_PANELS.png', dpi=150, bbox_inches='tight')\nplt.close()\nprint(f\"  ✓ 13_CORRECTED_PANELS.png\")\n\nprint(f\"\\n{'=' * 80}\")\nprint(f\"PATCH COMPLETE — {(datetime.now() - t0).total_seconds()/60:.1f} min\")\nprint(f\"{'=' * 80}\")\nprint(\"\\nPaste the console output above into the chat for the report.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-03T17:06:04.008163Z","iopub.execute_input":"2026-09-03T17:06:04.008723Z","iopub.status.idle":"2026-09-03T17:06:36.222252Z","shell.execute_reply.started":"2026-09-03T17:06:04.008687Z","shell.execute_reply":"2026-09-03T17:06:36.221288Z"}},"outputs":[],"execution_count":null}]}