{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import joblib\n\nprint(\"💾 Saving Best Pipeline...\")\n\n# Save cluster assignments\nlookup_df = pd.DataFrame({'article_id': df_res['article_id'], 'cluster': best_labels})\nlookup_df.to_csv(\"best_clusters.csv\", index=False)\nprint(\"  ✓ Saved cluster assignments\")\n\n# Save Scaler (Always needed for new data)\njoblib.dump(scaler, \"scaler.joblib\")\nprint(\"  ✓ Saved StandardScaler\")\n\n# Save PCA model\nimport pickle\nwith open(\"best_pca.pkl\", \"wb\") as f:\n    if hasattr(pca, 'get'): \n        # If GPU model, convert to sklearn equivalent\n        from sklearn.decomposition import PCA as skPCA\n        sk_pca = skPCA(n_components=32)\n        sk_pca.fit(X_scaled)\n        pickle.dump(sk_pca, f)\n    else:\n        pickle.dump(pca, f)\nprint(\"  ✓ Saved PCA model\")\n\nprint(\"\\n✅ All files saved successfully!\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. IMPORTS AND SETUP\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Scikit-learn imports\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder\nfrom sklearn.decomposition import PCA, TruncatedSVD\nfrom sklearn.manifold import TSNE\nfrom sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering\nfrom sklearn.metrics import silhouette_score, davies_bouldin_score, calinski_harabasz_score\nfrom sklearn.model_selection import train_test_split\n\n# Deep Learning imports\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, Model\n\n# RAPIDS cuML for GPU acceleration (if available)\ntry:\n    import cuml\n    from cuml.cluster import KMeans as cuKMeans\n    from cuml.cluster import DBSCAN as cuDBSCAN\n    from cuml.decomposition import PCA as cuPCA\n    RAPIDS_AVAILABLE = True\n    print(\"✓ RAPIDS GPU acceleration available\")\nexcept ImportError:\n    RAPIDS_AVAILABLE = False\n    print(\"⚠ RAPIDS not available - using CPU fallback\")\n    cuKMeans = KMeans\n    cuDBSCAN = DBSCAN\n    cuPCA = PCA\n\n# Configuration\nsns.set(style=\"whitegrid\")\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)\ntf.random.set_seed(RANDOM_STATE)\n\n# Path configuration (Adjust based on environment)\nDATA_PATH = Path(\"../input/h-and-m-personalized-fashion-recommendations\")\nif not DATA_PATH.exists():\n    DATA_PATH = Path(\".\")","metadata":{"execution":{"iopub.status.busy":"2026-02-03T22:58:56.766979Z","iopub.execute_input":"2026-02-03T22:58:56.767208Z","iopub.status.idle":"2026-02-03T22:59:21.009022Z","shell.execute_reply.started":"2026-02-03T22:58:56.767186Z","shell.execute_reply":"2026-02-03T22:59:21.008245Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_data():\n    \"\"\"Load and optimize data types for memory efficiency.\"\"\"\n    print(\"\\n[1/6] Loading Data...\")\n    \n    # Load Articles (Content Features)\n    articles = pd.read_csv(\n        DATA_PATH / \"articles.csv\",\n        dtype={'article_id': 'int32'}\n    )\n    \n    # Load Transactions (Behavioral Features)\n    # Only load necessary columns to save RAM\n    transactions = pd.read_csv(\n        DATA_PATH / \"transactions_train.csv\",\n        dtype={'article_id': 'int32', 'price': 'float32', 'sales_channel_id': 'int8'},\n        parse_dates=['t_dat']\n    )\n    \n    print(f\"  Articles shape: {articles.shape}\")\n    print(f\"  Transactions shape: {transactions.shape}\")\n    \n    return articles, transactions\n\ndef explore_data(df, title=\"Dataset\"):\n    \"\"\"Standard exploration function (Professor style).\"\"\"\n    print(f\"\\n--- Exploration: {title} ---\")\n    print(f\"Shape: {df.shape}\")\n    print(\"\\nMissing Values:\")\n    print(df.isnull().sum()[df.isnull().sum() > 0])\n    print(\"\\nDuplicate Rows:\", df.duplicated().sum())\n    display(df.head(3))","metadata":{"execution":{"iopub.status.busy":"2026-02-03T22:59:28.358252Z","iopub.execute_input":"2026-02-03T22:59:28.358877Z","iopub.status.idle":"2026-02-03T22:59:28.364837Z","shell.execute_reply.started":"2026-02-03T22:59:28.358848Z","shell.execute_reply":"2026-02-03T22:59:28.364137Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. DATA LOADING AND EXPLORATION\n","metadata":{}},{"cell_type":"code","source":"def load_data():\n    \"\"\"Load and optimize data types for memory efficiency.\"\"\"\n    print(\"\\n[1/6] Loading Data...\")\n    \n    # Load Articles (Content Features)\n    articles = pd.read_csv(\n        DATA_PATH / \"articles.csv\",\n        dtype={'article_id': 'int32'}\n    )\n    \n    # Load Transactions (Behavioral Features)\n    # Only load necessary columns to save RAM\n    transactions = pd.read_csv(\n        DATA_PATH / \"transactions_train.csv\",\n        dtype={'article_id': 'int32', 'price': 'float32', 'sales_channel_id': 'int8'},\n        parse_dates=['t_dat']\n    )\n    \n    print(f\"  Articles shape: {articles.shape}\")\n    print(f\"  Transactions shape: {transactions.shape}\")\n    \n    return articles, transactions\n\ndef explore_data(df, title=\"Dataset\"):\n    \"\"\"Standard exploration function (Professor style).\"\"\"\n    print(f\"\\n--- Exploration: {title} ---\")\n    print(f\"Shape: {df.shape}\")\n    print(\"\\nMissing Values:\")\n    print(df.isnull().sum()[df.isnull().sum() > 0])\n    print(\"\\nDuplicate Rows:\", df.duplicated().sum())\n    display(df.head(3))","metadata":{"execution":{"iopub.status.busy":"2026-02-03T22:59:39.495512Z","iopub.execute_input":"2026-02-03T22:59:39.495798Z","iopub.status.idle":"2026-02-03T22:59:39.501687Z","shell.execute_reply.started":"2026-02-03T22:59:39.495776Z","shell.execute_reply":"2026-02-03T22:59:39.500995Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. FEATURE ENGINEERING (THE \"SMART\" PART)\n","metadata":{}},{"cell_type":"code","source":"def engineer_features(articles, transactions, use_ohe=False):\n    \"\"\"\n    Engineers features with option for One-Hot Encoding (Better for complex models).\n    \n    Combines:\n    - Content Features: Metadata about articles (product type, color, department, etc.)\n    - Behavioral Features: Patterns from transactions (price stats, velocity, sales channel)\n    \"\"\"\n    print(f\"\\n[2/6] Engineering Features (OHE={use_ohe})...\")\n\n    # --- 1. Content Features ---\n    # Select columns that aren't too high-cardinality (avoid exploding RAM)\n    ohe_columns = [\n        'product_group_name', 'graphical_appearance_name',\n        'perceived_colour_value_name', 'index_group_name', \n        'section_name', 'garment_group_name'\n    ]\n    \n    # Columns that have too many values (keep as Label Encode to save RAM)\n    le_columns = ['prod_name', 'product_type_name', 'department_name', 'detail_desc']\n\n    if use_ohe:\n        # One-Hot Encoding (The \"Right\" Way for neural networks)\n        df_features = pd.get_dummies(articles[['article_id'] + ohe_columns], columns=ohe_columns, dtype='int8')\n        \n        # Label Encode the rest\n        temp_le = articles[['article_id'] + le_columns].copy()\n        for col in le_columns:\n            le = LabelEncoder()\n            temp_le[col] = le.fit_transform(temp_le[col].astype(str))\n        \n        df_features = df_features.merge(temp_le, on='article_id')\n        del temp_le\n    else:\n        # Label Encoding (The \"Memory Safe\" Way)\n        df_features = articles[['article_id']].copy()\n        all_cats = ohe_columns + le_columns\n        for col in all_cats:\n            le = LabelEncoder()\n            df_features[col] = le.fit_transform(articles[col].astype(str))\n\n    # --- 2. Behavioral Features ---\n    # Price Stats (Is it luxury or basic?)\n    print(\"  -> Calculating Price Stats...\")\n    price_stats = transactions.groupby('article_id')['price'].agg(['mean', 'max', 'std']).reset_index()\n    price_stats.columns = ['article_id', 'price_mean', 'price_max', 'price_std']\n    price_stats['price_std'] = price_stats['price_std'].fillna(0)\n    \n    df_features = df_features.merge(price_stats, on='article_id', how='left')\n    del price_stats; gc.collect()\n\n    # Velocity (Last 90 days) - Is it popular?\n    print(\"  -> Calculating Velocity...\")\n    max_date = transactions['t_dat'].max()\n    recent_trans = transactions[transactions['t_dat'] > (max_date - pd.Timedelta(days=90))]\n    velocity = recent_trans.groupby('article_id').size().reset_index(name='recent_sales_count')\n    del recent_trans; gc.collect()\n    \n    df_features = df_features.merge(velocity, on='article_id', how='left')\n    del velocity; gc.collect()\n\n    # Channel Stats - Customer demographics\n    print(\"  -> Calculating Channel Stats...\")\n    channel_stats = transactions.groupby('article_id')['sales_channel_id'].mean().reset_index(name='avg_sales_channel')\n    df_features = df_features.merge(channel_stats, on='article_id', how='left')\n    del channel_stats; gc.collect()\n\n    # --- 3. Better Null Handling ---\n    # Instead of Mean(), use -1 to indicate \"No Data\" (Distinct from average)\n    df_features = df_features.fillna(-1)\n\n    print(f\"  Feature Engineering Complete. Shape: {df_features.shape}\")\n    return df_features","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:00:12.9001Z","iopub.execute_input":"2026-02-03T23:00:12.90063Z","iopub.status.idle":"2026-02-03T23:00:12.909876Z","shell.execute_reply.started":"2026-02-03T23:00:12.900604Z","shell.execute_reply":"2026-02-03T23:00:12.909171Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. PREPROCESSING PIPELINE\n","metadata":{}},{"cell_type":"code","source":"def preprocess_for_clustering(df):\n    \"\"\"\n    Standardize features before clustering.\n    \n    Why standardization is important:\n    - K-Means is distance-based, so features on different scales affect results\n    - StandardScaler ensures all features have mean=0 and std=1\n    - This prevents high-variance features from dominating the clustering\n    \"\"\"\n    print(\"\\n[3/6] Preprocessing...\")\n    \n    # Drop ID for training (we only need numerical features)\n    X = df.drop('article_id', axis=1).values\n    \n    # Scale (StandardScaler is robust to outliers)\n    scaler = StandardScaler()\n    X_scaled = scaler.fit_transform(X)\n    \n    print(f\"  Feature Matrix Shape: {X_scaled.shape}\")\n    print(f\"  Features standardized (mean≈0, std≈1)\")\n    \n    return X_scaled, scaler","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:00:41.083136Z","iopub.execute_input":"2026-02-03T23:00:41.083482Z","iopub.status.idle":"2026-02-03T23:00:41.088235Z","shell.execute_reply.started":"2026-02-03T23:00:41.083455Z","shell.execute_reply":"2026-02-03T23:00:41.087444Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. DIMENSIONALITY REDUCTION (PCA)\n\n**Pourquoi ?** Les données haute-dimensionnelle sont difficiles à visualiser et à analyser. PCA réduit la dimensionnalité en gardant 95% de la variance.\n","metadata":{}},{"cell_type":"markdown","source":"## 6. CLUSTERING ALGORITHMS (COMPARISON)\n\n**Objectif :** Comparer plusieurs algorithmes de clustering pour trouver le meilleur groupement d'articles.\n\n**Algorithmes testés :**\n1. **K-Means** : Partitionne les données en K groupes distincts (rapide et interprétable)\n2. **DBSCAN** : Détecte les clusters basés sur la densité (flexibilité de forme)\n\n**Métriques d'évaluation :**\n- **Silhouette Score** : Mesure la cohésion des clusters (-1 à 1, plus élevé = mieux)\n- **Davies-Bouldin Index** : Mesure la séparation entre clusters (0 à ∞, plus bas = mieux)\n","metadata":{}},{"cell_type":"code","source":"def run_clustering_algorithms(X_latent, n_clusters=15):\n    \"\"\"\n    Run multiple algorithms to find the best grouping.\n    \n    Parameters:\n    - X_latent: Feature matrix (already scaled and reduced)\n    - n_clusters: Number of clusters to create\n    \n    Returns:\n    - Dictionary with clustering results from each algorithm\n    \"\"\"\n    print(f\"\\n[5/6] Running Clustering Algorithms (K={n_clusters})...\")\n    results = {}\n    \n    # --- Algorithm A: K-Means (GPU/CPU) ---\n    print(\"  -> Running KMeans...\")\n    kmeans = cuKMeans(n_clusters=n_clusters, random_state=RANDOM_STATE, n_init=10)\n    labels_kmeans = kmeans.fit_predict(X_latent)\n    results['kmeans'] = labels_kmeans\n    \n    # --- Algorithm B: DBSCAN (Density) ---\n    print(\"  -> Running DBSCAN...\")\n    dbscan = cuDBSCAN(eps=0.5, min_samples=5)\n    labels_dbscan = dbscan.fit_predict(X_latent)\n    results['dbscan'] = labels_dbscan\n    \n    # Evaluation Metrics\n    print(\"\\n--- Evaluation Metrics (Latent Space) ---\")\n    \n    # Sample for speed\n    sil = silhouette_score(X_latent[:10000], labels_kmeans[:10000])\n    db = davies_bouldin_score(X_latent[:10000], labels_kmeans[:10000])\n    \n    print(f\"K-Means Silhouette Score: {sil:.4f} (Higher is better)\")\n    print(f\"K-Means Davies-Bouldin: {db:.4f} (Lower is better)\")\n    \n    return results","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:00:46.819301Z","iopub.execute_input":"2026-02-03T23:00:46.819839Z","iopub.status.idle":"2026-02-03T23:00:46.825286Z","shell.execute_reply.started":"2026-02-03T23:00:46.81981Z","shell.execute_reply":"2026-02-03T23:00:46.824535Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. VISUALIZATION & RECOMMENDATION SYSTEM\n\n**Fonction principale :** Utiliser les clusters pour construire un système de recommandation basé sur la similarité.\n\n**Approche :**\n1. **t-SNE** : Visualiser les clusters en 2D\n2. **Euclidean Distance** : Calculer la similarité entre articles dans l'espace latent\n3. **Top-K Recommendations** : Retourner les N articles les plus similaires\n","metadata":{}},{"cell_type":"code","source":"def visualize_clusters(X_latent, labels, title=\"Cluster Visualization\"):\n    \"\"\"\n    Visualize high-dimensional clusters using t-SNE.\n    \n    t-SNE is a dimensionality reduction technique that preserves local structure,\n    making it ideal for visualizing cluster separation.\n    \"\"\"\n    print(\"\\n[6/6] Visualizing Clusters (t-SNE)...\")\n    \n    # Sample for speed (t-SNE is computationally expensive)\n    idx = np.random.choice(len(X_latent), size=min(5000, len(X_latent)), replace=False)\n    X_sample = X_latent[idx]\n    labels_sample = labels[idx]\n    \n    tsne = TSNE(n_components=2, random_state=RANDOM_STATE, n_iter=1000)\n    X_embedded = tsne.fit_transform(X_sample)\n    \n    plt.figure(figsize=(12, 8))\n    scatter = plt.scatter(X_embedded[:,0], X_embedded[:,1], c=labels_sample, cmap='tab20', s=20, alpha=0.6)\n    plt.colorbar(scatter, label='Cluster ID')\n    plt.title(f't-SNE Projection: {title}', fontsize=14)\n    plt.xlabel('t-SNE Component 1')\n    plt.ylabel('t-SNE Component 2')\n    plt.grid(True, alpha=0.3)\n    plt.show()\n\ndef display_image(article_id, ax=None, title=None):\n    \"\"\"\n    Display an image for a given article ID using matplotlib.\n    \n    Handles path structure: 012/0123456789.jpg\n    \"\"\"\n    str_id = str(article_id).zfill(10)\n    folder = str_id[:3]\n    path = DATA_PATH / f\"images/{folder}/{str_id}.jpg\"\n    \n    if not path.exists():\n        if ax:\n            ax.text(0.5, 0.5, \"Image Not Found\", ha='center', va='center')\n            ax.axis('off')\n            if title: ax.set_title(title, fontsize=8)\n        return\n\n    try:\n        import matplotlib.image as mpimg\n        img = mpimg.imread(str(path))\n        if ax:\n            ax.imshow(img)\n            ax.axis('off')\n            if title: ax.set_title(title, fontsize=8)\n        else:\n            plt.imshow(img)\n            plt.axis('off')\n            if title: plt.title(title)\n            plt.show()\n    except Exception as e:\n        if ax:\n            ax.text(0.5, 0.5, \"Error Loading\", ha='center', va='center')\n            ax.axis('off')\n\nfrom sklearn.metrics.pairwise import euclidean_distances\n\ndef recommend_similar_items(article_id, df_features, X_latent, top_k=5):\n    \"\"\"\n    Recommender Logic: Distance-Based Similarity in Latent Space\n    \n    Steps:\n    1. Find the latent vector for the input article\n    2. Calculate Euclidean distance to all other items\n    3. Return the K items with smallest distance\n    \n    Why latent space?\n    - Captures both content and behavioral patterns\n    - Much more meaningful than raw feature distances\n    \"\"\"\n    try:\n        # 1. Find the index of the source article\n        idx = df_features[df_features['article_id'] == article_id].index[0]\n        \n        # 2. Get the vector for this specific item\n        source_vector = X_latent[idx].reshape(1, -1)\n        \n        # 3. Calculate Euclidean Distance to ALL items\n        dists = euclidean_distances(source_vector, X_latent).flatten()\n        \n        # 4. Get indices of smallest distances (skip first = self)\n        closest_indices = dists.argsort()[1:top_k+1]\n        \n        # 5. Retrieve Article IDs\n        recommended_ids = df_features.iloc[closest_indices]['article_id'].values\n        \n        return recommended_ids\n\n    except IndexError:\n        print(f\"Error: Article ID {article_id} not found in processed data.\")\n        return []\n\ndef recommend_and_visualize(article_id, df_features, X_latent, articles_meta, top_k=5):\n    \"\"\"\n    Visualize the source item and its recommendations side-by-side.\n    \"\"\"\n    recs = recommend_similar_items(article_id, df_features, X_latent, top_k)\n\n    if len(recs) == 0:\n        return\n\n    # Setup Plot\n    n_cols = top_k + 1\n    fig, axes = plt.subplots(1, n_cols, figsize=(3 * n_cols, 4))\n    if n_cols == 1: axes = [axes]\n\n    # 1. Plot Source\n    try:\n        source_info = articles_meta[articles_meta['article_id'] == article_id].iloc[0]\n        title_text = f\"SOURCE:\\n{source_info['prod_name'][:15]}...\\n({source_info['product_type_name']})\"\n        display_image(article_id, axes[0], title=title_text)\n    except:\n        display_image(article_id, axes[0], title=\"Source (Metadata Missing)\")\n\n    # 2. Plot Recommendations\n    for i, rec_id in enumerate(recs):\n        try:\n            rec_info = articles_meta[articles_meta['article_id'] == rec_id].iloc[0]\n            title_text = f\"REC #{i+1}:\\n{rec_info['prod_name'][:15]}...\\n({rec_info['product_type_name']})\"\n            display_image(rec_id, axes[i+1], title=title_text)\n        except IndexError:\n            display_image(rec_id, axes[i+1], title=f\"REC #{i+1}\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:01:34.205337Z","iopub.execute_input":"2026-02-03T23:01:34.206015Z","iopub.status.idle":"2026-02-03T23:01:34.219397Z","shell.execute_reply.started":"2026-02-03T23:01:34.205986Z","shell.execute_reply":"2026-02-03T23:01:34.218772Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ============================================================================\n# MAIN PIPELINE - EXÉCUTION COMPLÈTE\n# ============================================================================\n\nCette section exécute le pipeline complet de chargement, prétraitement, réduction de dimensionnalité, clustering et recommandation.\n\n**Flux :**\n1. Charger les données (articles + transactions)\n2. Ingénierie des caractéristiques (content + behavioral)\n3. Prétraitement (standardisation)\n4. Réduction dimensionnelle (PCA)\n5. Clustering (K-Means + DBSCAN)\n6. Recommandations","metadata":{}},{"cell_type":"markdown","source":"### Étape 1 : Charger les données brutes","metadata":{}},{"cell_type":"code","source":"# Load Raw Data\nprint(\"⏳ Loading Raw Data...\")\narticles_df, trans_df = load_data()\nprint(\"✅ Data Loaded Successfully.\")","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:01:38.738905Z","iopub.execute_input":"2026-02-03T23:01:38.739556Z","iopub.status.idle":"2026-02-03T23:04:20.672042Z","shell.execute_reply.started":"2026-02-03T23:01:38.739529Z","shell.execute_reply":"2026-02-03T23:04:20.671389Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Étape 2 : Ingénierie des caractéristiques et prétraitement\n\n**Pourquoi cette étape ?**\n- Les modèles de clustering ont besoin de bonnes features\n- Nous combinons métadonnées (contenu) + signaux transactionnels (comportement)\n- La gestion mémoire est critique avec les grandes données","metadata":{}},{"cell_type":"code","source":"# 1. Load Data\n\n\n# 2. Engineer Features\nprint(\"⚙️  Engineering Features...\")\ndf_res = engineer_features(articles_df, trans_df, use_ohe=False)\n\n# 3. Free memory by deleting transactions\nprint(\"🗑️  Deleting raw Transactions to free RAM...\")\ndel trans_df\ngc.collect()\n\n# 4. Scale Data\nprint(\"⚖️  Scaling Data...\")\nX_scaled, scaler = preprocess_for_clustering(df_res)\n\nprint(f\"\\n✅ Ready for training. X_scaled shape: {X_scaled.shape}\")","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:04:38.971734Z","iopub.execute_input":"2026-02-03T23:04:38.97239Z","iopub.status.idle":"2026-02-03T23:04:43.578334Z","shell.execute_reply.started":"2026-02-03T23:04:38.972341Z","shell.execute_reply":"2026-02-03T23:04:43.577591Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Étape 3 : Réduction dimensionnelle (PCA)\n\n**PCA (Principal Component Analysis)** :\n- Réduit 40+ features à 32 dimensions\n- Garde ~95% de la variance\n- Accélère le clustering et améliore la visualisation","metadata":{}},{"cell_type":"code","source":"# Perform Dimensionality Reduction using PCA\nprint(\"📉 Applying PCA Dimensionality Reduction...\")\n\nn_features = X_scaled.shape[1]\npca = cuPCA(n_components=min(32, n_features))\nX_latent = pca.fit_transform(X_scaled)\n\n# Convert to numpy if needed (in case cuPCA returns GPU array)\nif hasattr(X_latent, 'get'): \n    X_latent = X_latent.get()\n\nprint(f\"✅ PCA Reduction complete.\")\nprint(f\"  Original shape: {X_scaled.shape}\")\nprint(f\"  Reduced shape: {X_latent.shape}\")\nprint(f\"  Explained variance: ~{pca.explained_variance_ratio_.sum():.2%}\")","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:04:58.101132Z","iopub.execute_input":"2026-02-03T23:04:58.101472Z","iopub.status.idle":"2026-02-03T23:04:59.432575Z","shell.execute_reply.started":"2026-02-03T23:04:58.101449Z","shell.execute_reply":"2026-02-03T23:04:59.431952Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Clustering Battle: Compare K-Means with different K values\nfrom sklearn.metrics import silhouette_score, davies_bouldin_score\n\n# Configuration\nn_clusters_list = [10, 15, 20]\nresults_log = []\nmodels = {}\n\nprint(\"🏆 STARTING CLUSTERING BATTLE\\n\")\n\n# --- 1. K-Means Loop ---\nfor k in n_clusters_list:\n    print(f\"  -> Testing K-Means (K={k})...\")\n    kmeans = cuKMeans(n_clusters=k, random_state=42, n_init=10)\n    labels = kmeans.fit_predict(X_latent)\n    \n    # Convert to numpy if needed\n    if hasattr(labels, 'to_numpy'): \n        labels = labels.to_numpy()\n    \n    # Score (Use a sample for speed)\n    sil = silhouette_score(X_latent[:10000], labels[:10000])\n    db = davies_bouldin_score(X_latent[:10000], labels[:10000])\n    \n    results_log.append({\n        \"Algorithm\": \"K-Means\",\n        \"Params\": f\"K={k}\",\n        \"Silhouette\": sil,\n        \"Davies-Bouldin\": db\n    })\n    models[f\"kmeans_{k}\"] = labels\n\n# --- 2. Display Comparison Table ---\nresults_df = pd.DataFrame(results_log).sort_values(\"Silhouette\", ascending=False)\nprint(\"\\n🏆 LEADERBOARD 🏆\")\ndisplay(results_df)\n\n# --- 3. Select Best Model ---\nbest_k_row = results_df[results_df['Algorithm'] == 'K-Means'].iloc[0]\nbest_model_name = f\"kmeans_{best_k_row['Params'].split('=')[1]}\"\nbest_labels = models[best_model_name]\n\nprint(f\"\\n✅ Selected Best Model: {best_model_name}\")\n\n# Visualize the winner\nvisualize_clusters(X_latent, best_labels, title=f\"Winner: {best_model_name}\")","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:05:07.39329Z","iopub.execute_input":"2026-02-03T23:05:07.394088Z","iopub.status.idle":"2026-02-03T23:05:43.136033Z","shell.execute_reply.started":"2026-02-03T23:05:07.394059Z","shell.execute_reply":"2026-02-03T23:05:43.135255Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Étape 4 : Test du système de recommandation","metadata":{}},{"cell_type":"code","source":"# Test Recommendation System on Random Article\nprint(\"🎯 Testing Recommendation System\\n\")\n\n# Pick a random item\nrandom_id = df_res['article_id'].sample(1).iloc[0]\nprint(f\"Testing Random Article ID: {random_id}\\n\")\n\n# Visualize Recommendations\nrecommend_and_visualize(\n    article_id=random_id,\n    df_features=df_res,\n    X_latent=X_latent,     \n    articles_meta=articles_df,\n    top_k=5\n)","metadata":{"execution":{"iopub.status.busy":"2026-02-03T23:05:43.137297Z","iopub.execute_input":"2026-02-03T23:05:43.137618Z","iopub.status.idle":"2026-02-03T23:05:44.322097Z","shell.execute_reply.started":"2026-02-03T23:05:43.137595Z","shell.execute_reply":"2026-02-03T23:05:44.321283Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def recommend_kmeans(article_id, df_features, kmeans_model, top_k=5):\n \n    try:\n        idx = df_features[df_features['article_id'] == article_id].index[0]\n        \n        cluster_label = kmeans_model.labels_[idx]\n        \n        cluster_indices = np.where(kmeans_model.labels_ == cluster_label)[0]\n        cluster_indices = cluster_indices[cluster_indices != idx]\n        \n        recommended_ids = df_features.iloc[cluster_indices]['article_id'].values[:top_k]\n        return recommended_ids\n    \n    except IndexError:\n        print(f\"Error: Article ID {article_id} not found.\")\n        return []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T23:07:02.475155Z","iopub.execute_input":"2026-02-03T23:07:02.475599Z","iopub.status.idle":"2026-02-03T23:07:02.4807Z","shell.execute_reply.started":"2026-02-03T23:07:02.475572Z","shell.execute_reply":"2026-02-03T23:07:02.480035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def recommend_and_visualize(article_id, df_features, X_latent, articles_meta, top_k=5, kmeans_model=None, use_kmeans=False):\n    \"\"\"\n    Visualise l'article source et ses recommandations.\n    Si use_kmeans=True, utilise KMeans simple, sinon distance-based.\n    \"\"\"\n    if use_kmeans and kmeans_model is not None:\n        recs = recommend_kmeans(article_id, df_features, kmeans_model, top_k)\n    else:\n        recs = recommend_similar_items(article_id, df_features, X_latent, top_k)\n    \n    if len(recs) == 0:\n        return\n\n    # Setup Plot\n    n_cols = top_k + 1\n    fig, axes = plt.subplots(1, n_cols, figsize=(3 * n_cols, 4))\n    if n_cols == 1: axes = [axes] # Handle single plot case\n\n    # 1. Plot Source\n    try:\n        source_info = articles_meta[articles_meta['article_id'] == article_id].iloc[0]\n        title_text = f\"SOURCE:\\n{source_info['prod_name'][:15]}...\\n({source_info['product_type_name']})\"\n        display_image(article_id, axes[0], title=title_text)\n    except:\n         display_image(article_id, axes[0], title=\"Source (Metadata Missing)\")\n\n    # 2. Plot Recommendations\n    for i, rec_id in enumerate(recs):\n        try:\n\n            rec_info = articles_meta[articles_meta['article_id'] == rec_id].iloc[0]\n            title_text = f\"REC #{i+1}:\\n{rec_info['prod_name'][:15]}...\\n({rec_info['product_type_name']})\"\n            display_image(rec_id, axes[i+1], title=title_text)\n        except IndexError:\n            display_image(rec_id, axes[i+1], title=f\"REC #{i+1}\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T23:10:15.876534Z","iopub.execute_input":"2026-02-03T23:10:15.877228Z","iopub.status.idle":"2026-02-03T23:10:15.884344Z","shell.execute_reply.started":"2026-02-03T23:10:15.877204Z","shell.execute_reply":"2026-02-03T23:10:15.883601Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Distance-based classique\nrandom_id = df_res['article_id'].sample(1).iloc[0]\nrecommend_and_visualize(article_id=715786001, df_features=df_res, X_latent=X_latent, articles_meta=articles_df, top_k=5)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T23:07:36.010042Z","iopub.execute_input":"2026-02-03T23:07:36.010558Z","iopub.status.idle":"2026-02-03T23:07:37.167845Z","shell.execute_reply.started":"2026-02-03T23:07:36.01053Z","shell.execute_reply":"2026-02-03T23:07:37.167025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kmeans_model = KMeans(n_clusters=15, random_state=RANDOM_STATE, n_init=10)\nkmeans_model.fit(X_latent)\n\n# Les labels de cluster pour chaque article\ncluster_labels = kmeans_model.labels_\nprint(\"Clusters assignés :\", np.unique(cluster_labels))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T23:09:45.916684Z","iopub.execute_input":"2026-02-03T23:09:45.916989Z","iopub.status.idle":"2026-02-03T23:09:49.496074Z","shell.execute_reply.started":"2026-02-03T23:09:45.916968Z","shell.execute_reply":"2026-02-03T23:09:49.495275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# KMeans simple\nrecommend_and_visualize(article_id=715786001, df_features=df_res, X_latent=X_latent, articles_meta=articles_df, top_k=5,\n                        kmeans_model=kmeans_model, use_kmeans=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-03T23:12:53.081424Z","iopub.execute_input":"2026-02-03T23:12:53.082214Z","iopub.status.idle":"2026-02-03T23:12:54.181468Z","shell.execute_reply.started":"2026-02-03T23:12:53.082182Z","shell.execute_reply":"2026-02-03T23:12:54.180719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"### Étape 5 : Sauvegarde du pipeline","metadata":{"execution":{"iopub.execute_input":"2026-01-29T20:42:21.583395Z","iopub.status.busy":"2026-01-29T20:42:21.583055Z","iopub.status.idle":"2026-01-29T20:42:21.675753Z","shell.execute_reply":"2026-01-29T20:42:21.675008Z","shell.execute_reply.started":"2026-01-29T20:42:21.58337Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ipywidgets import interact\n\ndef dynamic_recommend(category):\n    \"\"\"Generate recommendations for a random article in the selected category.\"\"\"\n    # Find all articles in this category\n    category_ids = articles_df[articles_df['product_group_name'] == category]['article_id']\n\n    # Filter by IDs that exist in our processed features\n    available_ids = category_ids[category_ids.isin(df_res['article_id'])]\n\n    if len(available_ids) > 0:\n        # Pick random article from category\n        target_id = available_ids.sample(1).iloc[0]\n        print(f\"🛍️  Showing recommendations for a {category} item (ID: {target_id})\")\n        print(f\"Total items in category: {len(available_ids)}\\n\")\n        \n        # Generate recommendations\n        recommend_and_visualize(\n            article_id=target_id, \n            df_features=df_res, \n            X_latent=X_latent,\n            articles_meta=articles_df,\n            top_k=5\n        )\n    else:\n        print(f\"❌ No processed data found for category: {category}\")\n\n# Interactive widget\nprint(\"📱 INTERACTIVE RECOMMENDATION WIDGET\\n\")\ninteract(dynamic_recommend, category=sorted(articles_df['product_group_name'].unique()));","metadata":{"execution":{"iopub.execute_input":"2026-01-29T20:37:43.855961Z","iopub.status.busy":"2026-01-29T20:37:43.855259Z","iopub.status.idle":"2026-01-29T20:37:45.578699Z","shell.execute_reply":"2026-01-29T20:37:45.578101Z","shell.execute_reply.started":"2026-01-29T20:37:43.85593Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def generate_dashboard(df_features, X_latent, labels, articles_df):\n    \"\"\"Generate comprehensive diagnostic dashboard.\"\"\"\n    print(\"📊 GENERATING FINAL DASHBOARD... (This may take a moment)\")\n    \n    # Setup the layout\n    fig = plt.figure(figsize=(20, 15))\n    gs = fig.add_gridspec(3, 2)\n    \n    # --- 1. FEATURE DISTRIBUTION (Top Left) ---\n    ax1 = fig.add_subplot(gs[0, 0])\n    \n    # Focus on behavioral features\n    key_features = ['price_mean', 'price_std', 'recent_sales_count', 'avg_sales_channel']\n    \n    # Normalize for visualization\n    plot_data = df_features[key_features].copy()\n    for col in plot_data.columns:\n        if plot_data[col].min() >= 0:\n            plot_data[col] = np.log1p(plot_data[col])\n            \n    sns.boxplot(data=plot_data, ax=ax1, palette=\"Set2\")\n    ax1.set_title(\"Distribution of Behavioral Features (Log Scaled)\", fontsize=14)\n    ax1.set_ylabel(\"Log(Value)\")\n\n    # --- 2. CLUSTER SIZES (Top Right) ---\n    ax2 = fig.add_subplot(gs[0, 1])\n    unique_labels, counts = np.unique(labels, return_counts=True)\n    \n    sns.barplot(x=unique_labels, y=counts, ax=ax2, palette=\"viridis\")\n    ax2.set_title(\"Cluster Sizes\", fontsize=14)\n    ax2.set_xlabel(\"Cluster ID\")\n    ax2.set_ylabel(\"Number of Articles\")\n\n    # --- 3. CLUSTER PROFILES HEATMAP (Middle Row) ---\n    ax3 = fig.add_subplot(gs[1, :])\n    \n    # Add cluster labels temporarily\n    temp_df = df_features[key_features].copy()\n    temp_df['Cluster'] = labels[:len(temp_df)]\n    \n    # Calculate average feature value per cluster\n    cluster_profiles = temp_df.groupby('Cluster').mean()\n    cluster_profiles_norm = (cluster_profiles - cluster_profiles.mean()) / cluster_profiles.std()\n    \n    sns.heatmap(cluster_profiles_norm, cmap=\"RdBu_r\", center=0, annot=True, fmt=\".1f\", ax=ax3)\n    ax3.set_title(\"Cluster Profiles (Red = High, Blue = Low)\", fontsize=14)\n    ax3.set_ylabel(\"Cluster ID\")\n    \n    # --- 4. DATA SUMMARY (Bottom Row) ---\n    ax4 = fig.add_subplot(gs[2, :])\n    ax4.axis('off')\n    \n    summary_text = (\n        f\"FINAL DATA STRUCTURE:\\n\"\n        f\"{'='*50}\\n\"\n        f\"• Total Articles: {df_features.shape[0]:,}\\n\"\n        f\"• Total Features (before PCA): {df_features.shape[1]}\\n\"\n        f\"• Latent Dimensions (after PCA): {X_latent.shape[1]}\\n\"\n        f\"• Number of Clusters: {len(np.unique(labels))}\\n\"\n        f\"• Recommendation Type: Distance-based (Euclidean)\\n\"\n    )\n    \n    ax4.text(0.05, 0.5, summary_text, fontsize=13, family='monospace', va='center',\n             bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.3))\n\n    plt.tight_layout()\n    plt.show()\n\n# Generate Dashboard\ngenerate_dashboard(df_res, X_latent, best_labels, articles_df)","metadata":{"execution":{"iopub.execute_input":"2026-01-29T20:24:24.396154Z","iopub.status.busy":"2026-01-29T20:24:24.39534Z","iopub.status.idle":"2026-01-29T20:24:26.695227Z","shell.execute_reply":"2026-01-29T20:24:26.694504Z","shell.execute_reply.started":"2026-01-29T20:24:24.396125Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def full_diagnostic_report(df_features, X_latent, labels):\n    \"\"\"Generate comprehensive diagnostic report with 5 visualizations.\"\"\"\n    print(\"🔬 GENERATING COMPREHENSIVE DIAGNOSTIC REPORT...\")\n    \n    # Create a 3-row, 3-column layout\n    fig = plt.figure(figsize=(24, 20))\n    gs = fig.add_gridspec(3, 3)\n    \n    # --- 1. CORRELATION MATRIX (Top Left) ---\n    ax1 = fig.add_subplot(gs[0, 0])\n    \n    numeric_df = df_features.select_dtypes(include=[np.number])\n    corr = numeric_df.corr()\n    \n    # Mask upper triangle\n    mask = np.triu(np.ones_like(corr, dtype=bool))\n    \n    sns.heatmap(corr, mask=mask, cmap=\"coolwarm\", center=0,\n                square=True, linewidths=.5, cbar_kws={\"shrink\": .5}, ax=ax1)\n    ax1.set_title(\"Feature Correlation Matrix\", fontsize=14, fontweight='bold')\n\n    # --- 2. LATENT SPACE SCATTER (Top Middle) ---\n    ax2 = fig.add_subplot(gs[0, 1])\n    \n    idx = np.random.choice(len(X_latent), size=min(10000, len(X_latent)), replace=False)\n    \n    scatter = ax2.scatter(X_latent[idx, 0], X_latent[idx, 1], \n                         c=labels[idx], cmap='tab20', alpha=0.6, s=10)\n    ax2.set_title(\"Latent Space: Dim 1 vs Dim 2\", fontsize=14, fontweight='bold')\n    ax2.set_xlabel(\"Latent Feature 1\")\n    ax2.set_ylabel(\"Latent Feature 2\")\n    fig.colorbar(scatter, ax=ax2, label='Cluster')\n\n    # --- 3. FEATURE IMPORTANCE / VARIANCE (Top Right) ---\n    ax3 = fig.add_subplot(gs[0, 2])\n    \n    variances = numeric_df.var().sort_values(ascending=False).head(15)\n    \n    sns.barplot(x=variances.values, y=variances.index, ax=ax3, palette=\"magma\")\n    ax3.set_title(\"Top 15 High-Variance Features\", fontsize=14, fontweight='bold')\n    ax3.set_xlabel(\"Variance\")\n\n    # --- 4. STATISTICS TABLE (Middle Row - spans 3 columns) ---\n    ax4 = fig.add_subplot(gs[1, :])\n    ax4.axis('off')\n    \n    desc = df_features.describe().T\n    desc = desc[['mean', 'std', 'min', '50%', 'max']].round(2)\n    \n    table = ax4.table(cellText=desc.values,\n                      colLabels=desc.columns,\n                      rowLabels=desc.index,\n                      cellLoc = 'center', \n                      loc='center')\n    \n    table.auto_set_font_size(False)\n    table.set_fontsize(8)\n    table.scale(1, 1)\n    ax4.set_title(\"Statistical Summary of All Features\", fontsize=14, fontweight='bold', pad=20)\n\n    # --- 5. CLUSTER PROFILES HEATMAP (Bottom Row - spans 3 columns) ---\n    ax5 = fig.add_subplot(gs[2, :])\n    \n    plot_df = numeric_df.copy()\n    plot_df['Cluster'] = labels[:len(plot_df)]\n    \n    cluster_means = plot_df.groupby('Cluster').mean()\n    cluster_means_norm = (cluster_means - cluster_means.mean()) / cluster_means.std()\n    \n    sns.heatmap(cluster_means_norm, cmap=\"RdBu_r\", center=0, ax=ax5, cbar_kws={\"label\": \"Std Dev\"})\n    ax5.set_title(\"Cluster Profiles - How each cluster differs from average\", fontsize=14, fontweight='bold')\n    ax5.set_xlabel(\"Feature Name\")\n    ax5.set_ylabel(\"Cluster ID\")\n\n    plt.tight_layout()\n    plt.show()\n\n# Run Diagnostics\nfull_diagnostic_report(df_res, X_latent, best_labels)","metadata":{"execution":{"iopub.execute_input":"2026-01-29T20:25:55.559362Z","iopub.status.busy":"2026-01-29T20:25:55.558804Z","iopub.status.idle":"2026-01-29T20:25:57.719644Z","shell.execute_reply":"2026-01-29T20:25:57.718846Z","shell.execute_reply.started":"2026-01-29T20:25:55.559334Z"},"trusted":true},"outputs":[],"execution_count":null}]}