{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\n# List all files under the input directory, but ignore image files\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if filename.endswith('.csv'):\n            print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:37:45.623123Z","iopub.execute_input":"2025-01-09T14:37:45.623752Z","iopub.status.idle":"2025-01-09T14:39:04.580397Z","shell.execute_reply.started":"2025-01-09T14:37:45.623701Z","shell.execute_reply":"2025-01-09T14:39:04.579653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\n\n# Define the dataset path\nbase_path = Path('/kaggle/input/h-and-m-personalized-fashion-recommendations') \n\n# Load the files\narticles = pd.read_csv(base_path / 'articles.csv')\ncustomers = pd.read_csv(base_path / 'customers.csv')\ntransactions = pd.read_csv(base_path / 'transactions_train.csv')\n\n# Display basic information\nprint(articles.head())\nprint(customers.head())\nprint(transactions.head())\n\nprint(f\"Number of rows in articles.csv: {articles.shape[0]}\") \nprint(f\"Number of rows in customers.csv: {customers.shape[0]}\") \nprint(f\"Number of rows in transactions_train.csv: {transactions.shape[0]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:39:04.581501Z","iopub.execute_input":"2025-01-09T14:39:04.581841Z","iopub.status.idle":"2025-01-09T14:40:11.126022Z","shell.execute_reply.started":"2025-01-09T14:39:04.581820Z","shell.execute_reply":"2025-01-09T14:40:11.125313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datetime import datetime, timedelta\n\n# Convert t_dat to datetime\ntransactions['t_dat'] = pd.to_datetime(transactions['t_dat'])\n\n# Filter for the last 6 months (adjust as needed for size)\nmax_date = transactions['t_dat'].max()\nmin_date = max_date - timedelta(days=180)  # 6 months\n\nrecent_transactions = transactions[transactions['t_dat'] >= min_date]\n\nprint(f\"Filtered to recent transactions: {recent_transactions.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:40:11.127126Z","iopub.execute_input":"2025-01-09T14:40:11.127441Z","iopub.status.idle":"2025-01-09T14:40:14.642036Z","shell.execute_reply.started":"2025-01-09T14:40:11.127419Z","shell.execute_reply":"2025-01-09T14:40:14.641244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Identify top customers and articles based on activity\ntop_articles = recent_transactions['article_id'].value_counts().nlargest(2000).index\ntop_customers = recent_transactions['customer_id'].value_counts().nlargest(20000).index\n\n# Step 2: Filter transactions that involve only the top customers and articles\nfiltered_transactions = recent_transactions[\n    (recent_transactions['article_id'].isin(top_articles)) & \n    (recent_transactions['customer_id'].isin(top_customers))\n]\n# Step 4: Further refine by ensuring connected components\nconnected_articles = filtered_transactions['article_id'].unique()\nconnected_customers = filtered_transactions['customer_id'].unique()\n\nfinal_filtered_transactions = recent_transactions[\n    (recent_transactions['article_id'].isin(connected_articles)) & \n    (recent_transactions['customer_id'].isin(connected_customers))\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:40:33.965584Z","iopub.execute_input":"2025-01-09T14:40:33.965884Z","iopub.status.idle":"2025-01-09T14:40:37.408199Z","shell.execute_reply.started":"2025-01-09T14:40:33.965864Z","shell.execute_reply":"2025-01-09T14:40:37.407244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 5: Display final dataset statistics\nretention_rate = len(final_filtered_transactions) / len(recent_transactions)\nprint(f\"Retention Rate: {retention_rate * 100:.2f}%\")\nprint(f\"Final transactions shape: {final_filtered_transactions.shape}\")\n\n# Save the filtered transactions for model training\nfinal_filtered_transactions.to_csv('filtered_transactions.csv', index=False)\n# Check missing values\nprint(articles.isnull().sum())\nprint(customers.isnull().sum())\nprint(transactions.isnull().sum())\n\n# Check for duplicates in transactions\nprint(transactions.duplicated().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:40:50.461903Z","iopub.execute_input":"2025-01-09T14:40:50.462186Z","iopub.status.idle":"2025-01-09T14:41:09.490656Z","shell.execute_reply.started":"2025-01-09T14:40:50.462166Z","shell.execute_reply":"2025-01-09T14:41:09.489799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Number of transactions per customer\ncustomer_counts = final_filtered_transactions['customer_id'].value_counts()\nprint(customer_counts.describe())\n\n# Plot the distribution\nplt.figure(figsize=(10, 6))\ncustomer_counts.hist(bins=50)\nplt.title('Distribution of Transactions per Customer')\nplt.xlabel('Number of Transactions')\nplt.ylabel('Number of Customers')\nplt.show()\n\n# Number of purchases per article\narticle_counts = final_filtered_transactions['article_id'].value_counts()\nprint(article_counts.describe())\n\n# Plot the distribution\nplt.figure(figsize=(10, 6))\narticle_counts.hist(bins=50)\nplt.title('Distribution of Purchases per Article')\nplt.xlabel('Number of Purchases')\nplt.ylabel('Number of Articles')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:41:09.491505Z","iopub.execute_input":"2025-01-09T14:41:09.491724Z","iopub.status.idle":"2025-01-09T14:41:10.056664Z","shell.execute_reply.started":"2025-01-09T14:41:09.491706Z","shell.execute_reply":"2025-01-09T14:41:10.055903Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\n# Create user-item matrix\nuser_item_matrix = final_filtered_transactions.pivot_table(index='customer_id', columns='article_id', aggfunc='size', fill_value=0)\n\n# Calculate sparsity\nnon_zero_entries = np.count_nonzero(user_item_matrix)\ntotal_entries = user_item_matrix.size\nsparsity = 1 - (non_zero_entries / total_entries)\n\nprint(f\"Sparsity of the dataset: {sparsity:.4f}\")\nprint(f\"Unique customers: {final_filtered_transactions['customer_id'].nunique()}\")\nprint(f\"Unique articles: {final_filtered_transactions['article_id'].nunique()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:41:14.855221Z","iopub.execute_input":"2025-01-09T14:41:14.855547Z","iopub.status.idle":"2025-01-09T14:41:16.071595Z","shell.execute_reply.started":"2025-01-09T14:41:14.855521Z","shell.execute_reply":"2025-01-09T14:41:16.070730Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datetime import timedelta\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.optim as optim\n\n# Load data (replace with actual paths)\nfinal_filtered_transactions = pd.read_csv(\"/kaggle/working/filtered_transactions.csv\")\n\n# Convert t_dat to datetime\nfinal_filtered_transactions['t_dat'] = pd.to_datetime(final_filtered_transactions['t_dat'])\n\n# Time-based split: Last 7 days as the test set\nsplit_date = final_filtered_transactions['t_dat'].max() - timedelta(days=7)\ntrain_data = final_filtered_transactions[final_filtered_transactions['t_dat'] <= split_date]\ntest_data = final_filtered_transactions[final_filtered_transactions['t_dat'] > split_date]\n\n# Random split for validation\ntrain_data, val_data = train_test_split(train_data, test_size=0.1, random_state=42)\n\nprint(f\"Training data shape: {train_data.shape}\")\nprint(f\"Validation data shape: {val_data.shape}\")\nprint(f\"Test data shape: {test_data.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:41:19.219010Z","iopub.execute_input":"2025-01-09T14:41:19.219355Z","iopub.status.idle":"2025-01-09T14:41:23.174633Z","shell.execute_reply.started":"2025-01-09T14:41:19.219308Z","shell.execute_reply":"2025-01-09T14:41:23.173711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dataset Class\nclass CollabDataset(Dataset):\n    def __init__(self, data):\n        self.customers = torch.tensor(data['customer_id'].values, dtype=torch.long)\n        self.articles = torch.tensor(data['article_id'].values, dtype=torch.long)\n        self.targets = torch.tensor(data['price'].values, dtype=torch.float)\n\n    def __len__(self):\n        return len(self.customers)\n\n    def __getitem__(self, idx):\n        return self.customers[idx], self.articles[idx], self.targets[idx]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:41:25.735002Z","iopub.execute_input":"2025-01-09T14:41:25.735358Z","iopub.status.idle":"2025-01-09T14:41:25.740659Z","shell.execute_reply.started":"2025-01-09T14:41:25.735308Z","shell.execute_reply":"2025-01-09T14:41:25.739691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data['customer_id'].index","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:41:26.395977Z","iopub.execute_input":"2025-01-09T14:41:26.396280Z","iopub.status.idle":"2025-01-09T14:41:26.401895Z","shell.execute_reply.started":"2025-01-09T14:41:26.396253Z","shell.execute_reply":"2025-01-09T14:41:26.401133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data['customer_id'].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:41:27.502667Z","iopub.execute_input":"2025-01-09T14:41:27.502944Z","iopub.status.idle":"2025-01-09T14:41:27.508087Z","shell.execute_reply.started":"2025-01-09T14:41:27.502924Z","shell.execute_reply":"2025-01-09T14:41:27.507229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.tensor(train_data['customer_id'].values, dtype=torch.long)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:41:29.051664Z","iopub.execute_input":"2025-01-09T14:41:29.051974Z","iopub.status.idle":"2025-01-09T14:41:29.086003Z","shell.execute_reply.started":"2025-01-09T14:41:29.051949Z","shell.execute_reply":"2025-01-09T14:41:29.084833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data Preparation\ntrain_dataset = CollabDataset(train_data)\nval_dataset = CollabDataset(val_data)\ntest_dataset = CollabDataset(test_data)\n\nbatch_size = 512\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-09T14:41:30.542067Z","iopub.execute_input":"2025-01-09T14:41:30.542477Z","iopub.status.idle":"2025-01-09T14:41:30.564148Z","shell.execute_reply.started":"2025-01-09T14:41:30.542440Z","shell.execute_reply":"2025-01-09T14:41:30.562872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Collaborative Filtering Model\nclass CollabModel(nn.Module):\n    def __init__(self, num_customers, num_articles, embedding_size):\n        super(CollabModel, self).__init__()\n        self.customer_embedding = nn.Embedding(num_customers, embedding_size)\n        self.article_embedding = nn.Embedding(num_articles, embedding_size)\n        self.dropout = nn.Dropout(p=0.3)\n        self.fc = nn.Linear(embedding_size, 1)\n\n    def forward(self, customers, articles):\n        customer_emb = self.dropout(self.customer_embedding(customers))\n        article_emb = self.dropout(self.article_embedding(articles))\n        interaction = customer_emb * article_emb\n        return self.fc(interaction).squeeze()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.171437Z","iopub.status.idle":"2024-12-31T16:27:32.171684Z","shell.execute_reply":"2024-12-31T16:27:32.171584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize Model\nnum_customers = len(train_data['customer_id'].astype('category').cat.categories)\nnum_articles = len(train_data['article_id'].astype('category').cat.categories)\nembedding_size = 50\nmodel = CollabModel(num_customers, num_articles, embedding_size)\n\n# Training Setup\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = model.to(device)\ncriterion = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=1e-9)\n\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nscheduler = ReduceLROnPlateau(optimizer, mode='min', patience=3, factor=0.7)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.172419Z","iopub.status.idle":"2024-12-31T16:27:32.172694Z","shell.execute_reply":"2024-12-31T16:27:32.172585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training Loop\nepochs = 10\nfor epoch in range(epochs):\n    model.train()\n    total_loss = 0\n    for customers, articles, targets in train_loader:\n        customers, articles, targets = customers.to(device), articles.to(device), targets.to(device)\n        predictions = model(customers, articles)\n        loss = criterion(predictions, targets)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item()\n\n    model.eval()\n    val_loss = 0\n    with torch.no_grad():\n        for customers, articles, targets in val_loader:\n            customers, articles, targets = customers.to(device), articles.to(device), targets.to(device)\n            predictions = model(customers, articles)\n            val_loss += criterion(predictions, targets).item() / len(targets)\n    scheduler.step(val_loss)\n\n    print(f\"Epoch {epoch+1}/{epochs}, Training Loss: {total_loss/len(train_loader):.4f}, Validation Loss: {val_loss:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.173193Z","iopub.status.idle":"2024-12-31T16:27:32.173420Z","shell.execute_reply":"2024-12-31T16:27:32.173326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Learning Rate Finder\ndef lr_finder(model, optimizer, criterion, dataloader, start_lr=1e-7, end_lr=1, num_iter=50):\n    model.train()\n    lrs = []\n    losses = []\n    lr = start_lr\n    optimizer.param_groups[0]['lr'] = lr\n    gamma = (end_lr / start_lr) ** (1 / num_iter)  # Multiplicative factor for learning rate\n\n    for i, (customers, articles, targets) in enumerate(dataloader):\n        if i >= num_iter:\n            break\n        customers, articles, targets = customers.to(device), articles.to(device), targets.to(device)\n        optimizer.zero_grad()\n        outputs = model(customers, articles)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n\n        lrs.append(lr)\n        losses.append(loss.item())\n        lr *= gamma\n        optimizer.param_groups[0]['lr'] = lr\n\n    return lrs, losses\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.174155Z","iopub.status.idle":"2024-12-31T16:27:32.174401Z","shell.execute_reply":"2024-12-31T16:27:32.174302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Evaluation\nmodel.eval()\nval_loss = 0\nall_predictions = []\nall_targets = []\nwith torch.no_grad():\n    for customers, articles, targets in val_loader:\n        customers, articles, targets = customers.to(device), articles.to(device), targets.to(device)\n        predictions = model(customers, articles).cpu().numpy()\n        all_predictions.extend(predictions)\n        all_targets.extend(targets.cpu().numpy())\n        batch_loss = criterion(torch.tensor(predictions), targets.cpu()).item() / len(targets)\n        val_loss += batch_loss\n\nprint(f\"Validation Loss: {val_loss:.4f}\")\nrmse = mean_squared_error(all_targets, all_predictions, squared=False)\nmae = mean_absolute_error(all_targets, all_predictions)\nprint(f\"RMSE: {rmse:.4f}, MAE: {mae:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.175309Z","iopub.status.idle":"2024-12-31T16:27:32.175545Z","shell.execute_reply":"2024-12-31T16:27:32.175447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Average test loss\nval_loss /= len(val_loader)\n\n# Calculate RMSE and MAE\nrmse = mean_squared_error(all_targets, all_predictions, squared=False)\nmae = mean_absolute_error(all_targets, all_predictions)\n\nprint(f\"Test Loss: {val_loss:.4f}\")\nprint(f\"RMSE: {rmse:.4f}\")\nprint(f\"MAE: {mae:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.176193Z","iopub.status.idle":"2024-12-31T16:27:32.176481Z","shell.execute_reply":"2024-12-31T16:27:32.176375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Scatter plot: Predictions vs. True Values\nplt.figure(figsize=(8, 6))\nsns.scatterplot(x=all_targets, y=all_predictions, alpha=0.7)\nplt.plot([min(all_targets), max(all_targets)], [min(all_targets), max(all_targets)], color='red', linestyle='--')\nplt.title(\"Predictions vs True Values\")\nplt.xlabel(\"True Values\")\nplt.ylabel(\"Predicted Values\")\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.177031Z","iopub.status.idle":"2024-12-31T16:27:32.177291Z","shell.execute_reply":"2024-12-31T16:27:32.177188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Residuals calculation\nresiduals = np.array(all_targets) - np.array(all_predictions)\n\n# Plot residuals\nplt.figure(figsize=(8, 6))\nsns.histplot(residuals, kde=True, bins=50, color=\"blue\")\nplt.title(\"Distribution of Residuals\")\nplt.xlabel(\"Residuals (True - Predicted)\")\nplt.ylabel(\"Frequency\")\nplt.grid()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.178124Z","iopub.status.idle":"2024-12-31T16:27:32.178490Z","shell.execute_reply":"2024-12-31T16:27:32.178330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# MAPE calculation\nmape = np.mean(np.abs((np.array(all_targets) - np.array(all_predictions)) / np.array(all_targets))) * 100\nprint(f\"Mean Absolute Percentage Error (MAPE): {mape:.2f}%\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.179277Z","iopub.status.idle":"2024-12-31T16:27:32.179603Z","shell.execute_reply":"2024-12-31T16:27:32.179447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import r2_score\n\n# R² calculation\nr2 = r2_score(all_targets, all_predictions)\nprint(f\"R² (Coefficient of Determination): {r2:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.180514Z","iopub.status.idle":"2024-12-31T16:27:32.181110Z","shell.execute_reply":"2024-12-31T16:27:32.180665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cumulative distribution function of absolute errors\nabsolute_errors = np.abs(np.array(all_targets) - np.array(all_predictions))\n\n# Plot CDF\nplt.figure(figsize=(8, 6))\nsns.ecdfplot(absolute_errors, color=\"green\")\nplt.title(\"CDF of Absolute Errors\")\nplt.xlabel(\"Absolute Error\")\nplt.ylabel(\"Cumulative Probability\")\nplt.grid()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.181945Z","iopub.status.idle":"2024-12-31T16:27:32.182228Z","shell.execute_reply":"2024-12-31T16:27:32.182125Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize RMSE and MAE relative to the range of the target variable\n\n# Convert the list to a NumPy array\nall_targets_array = np.array(all_targets)\nall_predictions_array =  np.array(all_predictions)\n# Now you can calculate the target range\ntarget_range = all_targets_array.max() - all_targets_array.min()\nnormalized_rmse = rmse / target_range\nnormalized_mae = mae / target_range\n\n\nprint(f\"Normalized RMSE: {normalized_rmse:.4f} (Relative to Target Range)\")\nprint(f\"Normalized MAE: {normalized_mae:.4f} (Relative to Target Range)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.183052Z","iopub.status.idle":"2024-12-31T16:27:32.183364Z","shell.execute_reply":"2024-12-31T16:27:32.183231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compute test metrics\ntest_mse = mean_squared_error(all_targets, all_predictions)\ntest_mae = mean_absolute_error(all_targets, all_predictions)\ntest_rmse = np.sqrt(test_mse)\ntest_r2 = r2_score(all_targets, all_predictions)\n\nprint(f\"Test Metrics:\")\nprint(f\"MAE: {test_mae:.4f}, RMSE: {test_rmse:.4f}, R²: {test_r2:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.184002Z","iopub.status.idle":"2024-12-31T16:27:32.184240Z","shell.execute_reply":"2024-12-31T16:27:32.184145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate residuals\nresiduals = all_targets_array - all_predictions_array\nabsolute_residuals = np.abs(residuals)\n\n# Threshold for outliers (e.g., top 5% of errors)\nthreshold = np.percentile(absolute_residuals, 95)\noutliers = np.where(absolute_residuals > threshold)[0]\n\n# Print details about outliers\nprint(f\"Number of outliers: {len(outliers)}\")\nprint(f\"Threshold for outliers: {threshold:.4f}\")\nprint(f\"Outlier True Values: {all_targets_array[outliers]}\")\nprint(f\"Outlier Predictions: {all_predictions_array[outliers]}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.184929Z","iopub.status.idle":"2024-12-31T16:27:32.185220Z","shell.execute_reply":"2024-12-31T16:27:32.185065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8, 6))\nplt.scatter(all_targets, residuals, alpha=0.5)\nplt.axhline(0, color='red', linestyle='--', label='Zero Error')\nplt.xlabel(\"True Values\")\nplt.ylabel(\"Residuals\")\nplt.title(\"Residuals vs True Values\")\nplt.legend()\nplt.grid()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.185899Z","iopub.status.idle":"2024-12-31T16:27:32.186134Z","shell.execute_reply":"2024-12-31T16:27:32.186033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example threshold (e.g., acceptable MAE is 0.01)\nthreshold = 0.01\nwithin_threshold = np.sum(absolute_residuals <= threshold) / len(absolute_residuals) * 100\nprint(f\"Percentage of predictions within the threshold ({threshold}): {within_threshold:.2f}%\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.186677Z","iopub.status.idle":"2024-12-31T16:27:32.187000Z","shell.execute_reply":"2024-12-31T16:27:32.186892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sort errors by magnitude\nsorted_indices = np.argsort(absolute_residuals)[::-1]\ntop_errors = sorted_indices[:10]\n\n# Print top-error cases\nfor idx in top_errors:\n    print(f\"True Value: {all_targets[idx]:.4f}, Predicted: {all_predictions[idx]:.4f}, Error: {absolute_residuals[idx]:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.187628Z","iopub.status.idle":"2024-12-31T16:27:32.187905Z","shell.execute_reply":"2024-12-31T16:27:32.187782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n# Map customer_id and article_id to indices used in embeddings\ncustomer_mapping = {id_: idx for idx, id_ in enumerate(train_data['customer_id'].astype('category').cat.categories)}\narticle_mapping = {id_: idx for idx, id_ in enumerate(train_data['article_id'].astype('category').cat.categories)}\n\nval_data['mapped_customer_id'] = val_data['customer_id'].map(customer_mapping)\nval_data['mapped_article_id'] = val_data['article_id'].map(article_mapping)\n\n# Filter users who exist in validation data\nvalid_users = val_data['mapped_customer_id'].dropna().unique().astype(int)\n\n# Select a sample of users for evaluation\nsample_user_ids = valid_users[:100]  # Take first 5 users as an example\n\n# Recommendation function with scaling\ndef recommend_products_scaled(model, user_id, num_articles, min_price, max_price, top_n=5):\n    model.eval()\n    user_id_tensor = torch.tensor([user_id] * num_articles, device=device)\n    article_ids_tensor = torch.arange(num_articles, device=device)\n\n    with torch.no_grad():\n        scores = model(user_id_tensor, article_ids_tensor).cpu().numpy()\n    \n    # Scale scores to match price range\n    scores = scores * (max_price - min_price) + min_price\n    top_articles = np.argsort(scores)[-top_n:][::-1]\n    return top_articles.tolist(), scores\n\n# Evaluate recommendations and create a heatmap\nheatmap_data = []\narticle_names = {idx: f\"Article {id_}\" for id_, idx in article_mapping.items()}  # Create article labels\n\nfor user_id in sample_user_ids:\n    # Get actual and recommended articles\n    actual_articles = val_data[val_data['mapped_customer_id'] == user_id]['mapped_article_id'].tolist()\n    recommended_articles, scores = recommend_products_scaled(\n        model, user_id, len(article_mapping), \n        val_data['price'].min(), val_data['price'].max()\n    )\n\n    # Record results\n    row = []\n    for article in recommended_articles:\n        row.append(1 if article in actual_articles else 0)\n    heatmap_data.append(row)\n\n# Convert to DataFrame for heatmap\nheatmap_df = pd.DataFrame(heatmap_data, columns=[article_names[a] for a in recommended_articles],\n                          index=[f\"User {u}\" for u in sample_user_ids])\n\n# Plot heatmap\nplt.figure(figsize=(10, 6))\nsns.heatmap(heatmap_df, annot=True, cmap=\"YlGnBu\", cbar=False)\nplt.title(\"Recommendation Overlap (1=Match, 0=No Match)\")\nplt.xlabel(\"Recommended Articles\")\nplt.ylabel(\"Users\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T16:27:32.188407Z","iopub.status.idle":"2024-12-31T16:27:32.188712Z","shell.execute_reply":"2024-12-31T16:27:32.188583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dummy Dataset Example\nclass RecommendationDataset(Dataset):\n    def __init__(self, user_ids, article_ids, prices):\n        self.user_ids = torch.tensor(user_ids, dtype=torch.long)\n        self.article_ids = torch.tensor(article_ids, dtype=torch.long)\n        self.prices = torch.tensor(prices, dtype=torch.float32)\n\n    def __len__(self):\n        return len(self.prices)\n\n    def __getitem__(self, idx):\n        return self.user_ids[idx], self.article_ids[idx], self.prices[idx]\n\n# Collaborative Filtering Model\nclass CollaborativeFiltering(nn.Module):\n    def __init__(self, num_users, num_articles, embedding_size=50):\n        super(CollaborativeFiltering, self).__init__()\n        self.user_embedding = nn.Embedding(num_users, embedding_size)\n        self.article_embedding = nn.Embedding(num_articles, embedding_size)\n        self.fc = nn.Sequential(\n            nn.Linear(embedding_size, 128),  # Embedding size should match the input size here\n            nn.ReLU(),\n            nn.Linear(128, 1)\n        )\n        self.dropout = nn.Dropout(0.2)\n        \n    def forward(self, user_ids, article_ids):\n        user_emb = self.user_embedding(user_ids)  # Shape: [batch_size, embedding_size]\n        article_emb = self.article_embedding(article_ids)  # Shape: [batch_size, embedding_size]\n        interaction = user_emb * article_emb  # Element-wise multiplication\n        interaction = self.dropout(interaction)\n        interaction_sum = interaction.sum(dim=1)  # Summing along embedding dimension\n        return self.fc(interaction_sum)  # Ensure compatibility with the input size of self.fc\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T16:24:10.401367Z","iopub.execute_input":"2024-12-22T16:24:10.401794Z","iopub.status.idle":"2024-12-22T16:24:10.410662Z","shell.execute_reply.started":"2024-12-22T16:24:10.401756Z","shell.execute_reply":"2024-12-22T16:24:10.40954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load Dataset\nnum_users = 1000\nnum_articles = 500\nnum_samples = 10000\nnp.random.seed(42)\n\nuser_ids = np.random.randint(0, num_users, size=num_samples)\narticle_ids = np.random.randint(0, num_articles, size=num_samples)\nprices = np.random.randint(0, 6, size=num_samples)  # prices between 0 and 5\n\ntrain_users, val_users, train_articles, val_articles, train_prices, val_prices = train_test_split(\n    user_ids, article_ids, prices, test_size=0.2, random_state=42\n)\n\ntrain_dataset = RecommendationDataset(train_users, train_articles, train_prices)\nval_dataset = RecommendationDataset(val_users, val_articles, val_prices)\n\ntrain_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)\n\n# Model, Loss, Optimizer\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = CollaborativeFiltering(num_users, num_articles, embedding_size=50).to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nscheduler = ReduceLROnPlateau(optimizer, mode='min', patience=2, factor=0.5)\n\n# Training Function\ndef train_epoch(model, data_loader, criterion, optimizer):\n    model.train()\n    running_loss = 0.0\n    for user_ids, article_ids, prices in data_loader:\n        user_ids, article_ids, prices = user_ids.to(device), article_ids.to(device), prices.to(device)\n        optimizer.zero_grad()\n        predictions = model(user_ids, article_ids).squeeze()\n        loss = criterion(predictions, prices)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item() * len(user_ids)\n    return running_loss / len(data_loader.dataset)\n\n# Evaluation Function\ndef evaluate(model, data_loader, criterion):\n    model.eval()\n    running_loss = 0.0\n    all_predictions, all_targets = [], []\n    with torch.no_grad():\n        for user_ids, article_ids, prices in data_loader:\n            user_ids, article_ids, prices = user_ids.to(device), article_ids.to(device), prices.to(device)\n            predictions = model(user_ids, article_ids).squeeze()\n            loss = criterion(predictions, prices)\n            running_loss += loss.item() * len(user_ids)\n            all_predictions.append(predictions.cpu().numpy())\n            all_targets.append(prices.cpu().numpy())\n    return running_loss / len(data_loader.dataset), np.concatenate(all_predictions), np.concatenate(all_targets)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T16:24:12.175744Z","iopub.execute_input":"2024-12-22T16:24:12.176088Z","iopub.status.idle":"2024-12-22T16:24:12.19435Z","shell.execute_reply.started":"2024-12-22T16:24:12.17606Z","shell.execute_reply":"2024-12-22T16:24:12.192981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training Loop\nnum_epochs = 10\nfor epoch in range(num_epochs):\n    train_loss = train_epoch(model, train_loader, criterion, optimizer)\n    val_loss, val_predictions, val_targets = evaluate(model, val_loader, criterion)\n    scheduler.step(val_loss)\n    print(f\"Epoch {epoch + 1}/{num_epochs} - Train Loss: {train_loss:.4f} - Val Loss: {val_loss:.4f}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T16:24:14.634299Z","iopub.execute_input":"2024-12-22T16:24:14.634661Z","iopub.status.idle":"2024-12-22T16:24:14.684366Z","shell.execute_reply.started":"2024-12-22T16:24:14.63463Z","shell.execute_reply":"2024-12-22T16:24:14.682805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Debugging: Training Predictions\ntrain_loss, train_predictions, train_targets = evaluate(model, train_loader, criterion)\nprint(\"Training Targets vs Predictions:\")\nplt.scatter(train_targets, train_predictions, alpha=0.5)\nplt.xlabel(\"True prices (Train)\")\nplt.ylabel(\"Predicted prices (Train)\")\nplt.title(\"Training Predictions vs True prices\")\nplt.show()\n\n# Debugging: Validation Predictions\nprint(\"Validation Targets vs Predictions:\")\nplt.scatter(val_targets, val_predictions, alpha=0.5)\nplt.xlabel(\"True prices (Validation)\")\nplt.ylabel(\"Predicted prices (Validation)\")\nplt.title(\"Validation Predictions vs True prices\")\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Recommendations\ndef recommend_products(model, user_id, num_articles, top_n=5):\n    model.eval()\n    user_id_tensor = torch.tensor([user_id] * num_articles, device=device)\n    article_ids_tensor = torch.arange(num_articles, device=device)\n\n    with torch.no_grad():\n        scores = model(user_id_tensor, article_ids_tensor).cpu().numpy()\n\n    top_articles = np.argsort(scores)[-top_n:][::-1]\n    return top_articles.tolist()\n\n# Example Recommendations\nuser_id = 0\nrecommended_articles = recommend_products(model, user_id, num_articles, top_n=5)\nprint(f\"Recommended articles for user {user_id}: {recommended_articles}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}