{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7042674,"sourceType":"datasetVersion","datasetId":4024942}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.16"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Suppress warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nfrom torchvision import models, transforms\nimport timm\n\nfrom sklearn.cluster import KMeans\n\nSEED = 6174\nnp.random.seed(SEED)\n\n# Set the folder path for data\nfolder_path = \"/kaggle/input/open-problems-single-cell-perturbations\"","metadata":{"execution":{"iopub.execute_input":"2023-11-25T21:58:01.569334Z","iopub.status.busy":"2023-11-25T21:58:01.569072Z","iopub.status.idle":"2023-11-25T21:58:02.563339Z","shell.execute_reply":"2023-11-25T21:58:02.562609Z","shell.execute_reply.started":"2023-11-25T21:58:01.569313Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train = pd.read_parquet(f'{folder_path}/de_train.parquet')\ngenes = de_train.columns[5:]\nid_map = pd.read_csv (f'{folder_path}/id_map.csv')\nid_map = id_map.reindex(id_map.columns.tolist() + genes.tolist(), axis=1)\n\nsm_lincs_id = de_train.set_index('sm_name')[\"sm_lincs_id\"].to_dict()\nsm_name_to_smiles = de_train.set_index('sm_name')['SMILES'].to_dict()\n\nid_map['sm_lincs_id'] = id_map['sm_name'].map(sm_lincs_id)\nid_map['SMILES'] = id_map['sm_name'].map(sm_name_to_smiles)\n\nde_train","metadata":{"execution":{"iopub.execute_input":"2023-11-25T21:58:31.572088Z","iopub.status.busy":"2023-11-25T21:58:31.571320Z","iopub.status.idle":"2023-11-25T21:58:32.789111Z","shell.execute_reply":"2023-11-25T21:58:32.788604Z","shell.execute_reply.started":"2023-11-25T21:58:31.572064Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Drop cell_type = 'T cells CD4+', 'T cells CD8+'\n#de_train.drop(de_train[de_train['cell_type'] == 'T cells CD4+'].index, inplace = True)\n#de_train.drop(de_train[de_train['cell_type'] == 'T cells CD8+'].index, inplace = True)\n\n#de_train","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_sm_names = de_train[de_train[\"cell_type\"]==\"B cells\"][\"sm_name\"].to_list()\nall_de_train = de_train[de_train[\"sm_name\"].isin(all_sm_names)]\n#de_train.drop(all_de_train.index, inplace=True)\n\nall_de_train","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_statistic(df, group_col, cols, stat_func, stat_name):\n    stat_df = df.groupby(group_col)[cols].apply(stat_func)\n    stat_df.columns = [f'{group_col}_{stat_name}_{col}' for col in cols]\n    return stat_df.reset_index().astype({group_col: str})\n\ncell_type_mean = calculate_statistic(all_de_train, \"cell_type\", genes, lambda x: x.mean(), 'mean')\ncell_type_std = calculate_statistic(all_de_train, \"cell_type\", genes, lambda x: x.std(), 'std')\ncell_type_min = calculate_statistic(all_de_train, \"cell_type\", genes, lambda x: x.min(), 'min')\ncell_type_max = calculate_statistic(all_de_train, \"cell_type\", genes, lambda x: x.max(), 'max')\ncell_type_median = calculate_statistic(all_de_train, \"cell_type\", genes, lambda x: x.median(), 'median')\ncell_type_skew = calculate_statistic(all_de_train, \"cell_type\", genes, lambda x: x.skew(), 'skew')\ncell_type_kurt = calculate_statistic(all_de_train, \"cell_type\", genes, lambda x: x.kurt(), 'kurt')\ncell_type_ratio_mean_std = calculate_statistic(all_de_train, \"cell_type\", genes, lambda x: x.mean()/x.std(), 'ratio_mean_std')\n\nsm_name_mean = calculate_statistic(de_train, \"sm_name\", genes, lambda x: x.mean(), 'mean')\nsm_name_std = calculate_statistic(de_train, \"sm_name\", genes, lambda x: x.std(), 'std')\nsm_name_min = calculate_statistic(de_train, \"sm_name\", genes, lambda x: x.min(), 'min')\nsm_name_max = calculate_statistic(de_train, \"sm_name\", genes, lambda x: x.max(), 'max')\nsm_name_median = calculate_statistic(de_train, \"sm_name\", genes, lambda x: x.median(), 'median')\nsm_name_skew = calculate_statistic(de_train, \"sm_name\", genes, lambda x: x.skew(), 'skew')\nsm_name_kurt = calculate_statistic(de_train, \"sm_name\", genes, lambda x: x.kurt(), 'kurt')\nsm_name_ratio_mean_std = calculate_statistic(de_train, \"sm_name\", genes, lambda x: x.mean()/x.std(), 'ratio_mean_std')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def split_sign(text):\n#    text = text.replace(')(', ' ')\n#    text = text.replace('(' , ' ')\n#    text = text.replace(')' , ' ')\n#    return text.split(\" \")\n\n#de_train['_SMILES'] = [split_sign(text) for text in de_train['SMILES'].values]\n\n#sign = []\n#for row in de_train['_SMILES'].values:\n#    for ele in row:\n#        sign.append(ele)\n        \n#sign_list = list(set(sign))\n\n#data = np.zeros((len(de_train), len(sign_list)), dtype=int)\n#de_features = pd.DataFrame(data=data, columns=sign_list)\n\n#for sign in sign_list:\n#    for i in range(len(de_train)):\n#        row = de_train['_SMILES'].values[i]\n\n#        for ele in row:\n#            if ele == sign:\n#                de_features[sign][i] += 1\n\n                \n#id_map['_SMILES'] = [split_sign(text) for text in id_map['SMILES'].values]\n\n#sign = []\n#for row in id_map['_SMILES'].values:\n#    for ele in row:\n#        sign.append(ele)\n        \n#sign_list = list(set(sign))\n\n#data = np.zeros((len(id_map), len(sign_list)), dtype=int)\n#test_features = pd.DataFrame(data=data, columns=sign_list)\n\n#for sign in sign_list:\n#    for i in range(len(id_map)):\n#        row = id_map['_SMILES'].values[i]\n\n#        for ele in row:\n#            if ele == sign:\n#                test_features[sign][i] += 1\n                \n#uncommon = [f for f in de_features if f not in test_features]\n#de_features = de_features.drop(columns=uncommon)\n\n#de_features = de_features.sort_index(axis = 1)\n#test_features = test_features.sort_index(axis = 1)\n\n#print(\"Columns Check\", list(de_features.columns) == list(test_features.columns))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Connect de_train and diff_all_de by cell_type\nfeatures = [\"cell_type\", \"sm_name\"]\nde_train.reset_index(inplace=True)\n\nX = de_train[features].copy()\nX = pd.merge(X, cell_type_mean, on='cell_type', how='left')\nX = pd.merge(X, cell_type_std, on='cell_type', how='left')\nX = pd.merge(X, cell_type_min, on='cell_type', how='left')\nX = pd.merge(X, cell_type_max, on='cell_type', how='left')\nX = pd.merge(X, cell_type_median, on='cell_type', how='left')\nX = pd.merge(X, cell_type_skew, on='cell_type', how='left')\n\nX = pd.merge(X, sm_name_mean, on='sm_name', how='left')\nX = pd.merge(X, sm_name_std, on='sm_name', how='left')\nX = pd.merge(X, sm_name_min, on='sm_name', how='left')\nX = pd.merge(X, sm_name_max, on='sm_name', how='left')\nX = pd.merge(X, sm_name_median, on='sm_name', how='left')\nX = pd.merge(X, sm_name_skew, on='sm_name', how='left')\nX.drop(features, axis=1, inplace=True)\nX","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = de_train[genes].copy()\ny","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_sub = id_map[features].copy()\nX_sub = pd.merge(X_sub, cell_type_mean, on='cell_type', how='left')\nX_sub = pd.merge(X_sub, cell_type_std, on='cell_type', how='left')\nX_sub = pd.merge(X_sub, cell_type_min, on='cell_type', how='left')\nX_sub = pd.merge(X_sub, cell_type_max, on='cell_type', how='left')\nX_sub = pd.merge(X_sub, cell_type_median, on='cell_type', how='left')\nX_sub = pd.merge(X_sub, cell_type_skew, on='cell_type', how='left')\n\nX_sub = pd.merge(X_sub, sm_name_mean, on='sm_name', how='left')\nX_sub = pd.merge(X_sub, sm_name_std, on='sm_name', how='left')\nX_sub = pd.merge(X_sub, sm_name_min, on='sm_name', how='left')\nX_sub = pd.merge(X_sub, sm_name_max, on='sm_name', how='left')\nX_sub = pd.merge(X_sub, sm_name_median, on='sm_name', how='left')\nX_sub = pd.merge(X_sub, sm_name_skew, on='sm_name', how='left')\n\nX_sub.drop(features, axis=1, inplace=True)\nX_sub","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del cell_type_mean, cell_type_std, cell_type_min, cell_type_max, cell_type_median, cell_type_skew, cell_type_kurt, cell_type_ratio_mean_std\ndel sm_name_mean, sm_name_std, sm_name_min, sm_name_max, sm_name_median, sm_name_skew, sm_name_kurt, sm_name_ratio_mean_std","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get index of cell_type is NK cells\nindex = de_train[de_train[\"cell_type\"] == \"NK cells\"].index\nindex = index.difference(all_de_train.index)\n\nX_val = X.loc[index]\ny_val = y.loc[index]\n\n# Drop cell_type = 'NK cells'\nX_train = X.drop(index)\ny_train = y.drop(index)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torchvision import models\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import TruncatedSVD\nfrom scipy.ndimage import zoom\n\ndef convert_to_image_format(data, output_size=(224, 224)):\n\t\tsquare_side = int(math.ceil(math.sqrt(data.shape[1])))\n\t\tpadding_size = square_side ** 2 - data.shape[1]\n\t\t\n\t\tdata_padded = np.pad(data, ((0, 0), (0, padding_size)), 'constant', constant_values=0)\n\t\t\n\t\t# We are reshaping to [N, H, W, C] because the final torch tensor needs to be [N, C, H, W]\n\t\tdata_reshaped = data_padded.reshape(-1, square_side, square_side, 1)\n\t\t\n\t\t# Expand the last dimension to three channels by repeating the data\n\t\tdata_rgb = np.repeat(data_reshaped, 3, -1)\n\t\t\n\t\treturn data_rgb\n\n#X_scaler = StandardScaler()\n#y_scaler = StandardScaler()\n\n## Assuming de_features and genes are defined elsewhere in your script\n#X_scaled = X_scaler.fit_transform(X_train.values)\n#y_scaled = y_scaler.fit_transform(y_train.values)\n\nn_components = 100\ny_svd = TruncatedSVD(n_components=n_components, random_state=6174)\ny_truncated = y_svd.fit_transform(y_train.values)\n\n# Convert scaled data to image format\nX_img = convert_to_image_format(X_train.values)\n\n# Convert to tensors\nX_tensor = torch.tensor(X_img, dtype=torch.float32).permute(0, 3, 1, 2)\ny_tensor = torch.tensor(y_truncated, dtype=torch.float32)\n\n# KFold setup\nnum_epochs = 50  # Define the number of epochs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_tensor[0].permute(1, 2, 0))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##from torchvision import models\n##models.vit_h_14(weights=models.ViT_H_14_Weights.IMAGENET1K_SWAG_E2E_V1)\n\n#timm.create_model('eva02_large_patch14_448.mim_m38m_ft_in22k_in1k', pretrained=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom sklearn.model_selection import KFold\n\ntorch.cuda.empty_cache()\n\ndef get_model():\n\t\t#model = timm.create_model('eva02_large_patch14_448.mim_m38m_ft_in22k_in1k', pretrained=True)\n\t\t#model = models.resnet152(pretrained=True)\n\t\tmodel = models.regnet_y_400mf(pretrained=True)\n\t\t\n\t\tnum_ftrs = model.fc.in_features\n\t\tmodel.fc = nn.Linear(num_ftrs, y_tensor.shape[1])\n\t\t\n\t\t#num_ftrs = model.head.in_features\n\t\t#model.head = nn.Linear(num_ftrs, len(genes))\n\n\t\t#num_ftrs = model.classifier[6].in_features\n\t\t#model.classifier[6] = nn.Linear(num_ftrs, len(genes))\n\n\t\t#num_ftrs = model.head.in_features\n\t\t#model.head = nn.Linear(num_ftrs, len(genes))\n\n\t\treturn model.cuda()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nkf = KFold(n_splits=5, shuffle=True, random_state=6174)\nfor fold, (train_index, val_index) in enumerate(kf.split(X_tensor)):\n\t\tprint(f'Fold {fold+1}')\n\n\t\t# Split data into training and validation sets\n\t\tX_train_tensor, y_train_tensor = X_tensor[train_index], y_tensor[train_index]\n\t\tX_val_tensor, y_val_tensor = X_tensor[val_index], y_tensor[val_index]\n\t\t\n\t\t# Move data to GPU\n\t\tX_train_tensor, y_train_tensor = X_train_tensor.cuda(), y_train_tensor.cuda()\n\t\tX_val_tensor, y_val_tensor = X_val_tensor.cuda(), y_val_tensor.cuda()\n\n\t\t# DataLoader setup\n\t\ttrain_dataset = TensorDataset(X_train_tensor, y_train_tensor)\n\t\tval_dataset = TensorDataset(X_val_tensor, y_val_tensor)\n\t\ttrain_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n\t\tval_loader = DataLoader(val_dataset, batch_size=16)\n\n\t\t# Model, loss, optimizer setup\n\t\tmodel = get_model()\n\t\tcriterion = nn.MSELoss()\n\t\toptimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=1e-4)\n\t\tscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=5, verbose=True)\n\n\t\t# Training loop for each fold\n\t\tbest_val_loss = float('inf')\n\t\tearly_stopping_patience = 10\n\t\tearly_stopping_counter = 0\n\t\tgradient_clip_value = 1\n\t\t\n\t\tfor epoch in range(num_epochs):\n\t\t\t\tmodel.train()\n\t\t\t\tfor inputs, targets in train_loader:\n\t\t\t\t\t\toptimizer.zero_grad()\n\t\t\t\t\t\toutputs = model(torchvision.transforms.Resize(448)(inputs))\n\t\t\t\t\t\t#outputs = model(inputs)\n\t\t\t\t\t\tloss = criterion(outputs, targets)\n\t\t\t\t\t\tloss.backward()\n\t\t\t\t\t\ttorch.nn.utils.clip_grad_norm_(model.parameters(), gradient_clip_value)  # Gradient clipping\n\t\t\t\t\t\toptimizer.step()\n\n\t\t\t\t# Validation loop\n\t\t\t\tmodel.eval()\n\t\t\t\tvalid_loss_squared_sum = 0  # For MRRMSE\n\n\t\t\t\twith torch.no_grad():\n\t\t\t\t\t\tfor inputs, targets in val_loader:\n\t\t\t\t\t\t\t\toutputs = model(torchvision.transforms.Resize(448)(inputs))\n\t\t\t\t\t\t\t\tloss = criterion(outputs, targets)\n\t\t\t\t\t\t\t\tvalid_loss_squared_sum += loss.item() * inputs.size(0)  # Sum of squared errors\n\t\t\t\t\t\tvalid_mrrmse = np.sqrt(valid_loss_squared_sum / len(val_loader.dataset))\n\t\t\t\t\t\tprint(f'Epoch {epoch}, MRRMSE: {valid_mrrmse}')\n\n\t\t\t\t\t\t# Update the learning rate scheduler based on validation loss\n\t\t\t\t\t\tscheduler.step(valid_mrrmse)\n\n\t\t\t\t\t\tif valid_mrrmse < best_val_loss:\n\t\t\t\t\t\t\t\tbest_val_loss = valid_mrrmse\n\t\t\t\t\t\t\t\tearly_stopping_counter = 0\n\t\t\t\t\t\t\t\ttorch.save(model.state_dict(), f'best_model_fold_{fold+1}.pth')  # Save best model checkpoint\n\t\t\t\t\t\telse:\n\t\t\t\t\t\t\t\tearly_stopping_counter += 1\n\t\t\t\t\t\t\t\tif early_stopping_counter >= early_stopping_patience:\n\t\t\t\t\t\t\t\t\t\tprint(\"Early stopping triggered\")\n\t\t\t\t\t\t\t\t\t\tprint()\n\t\t\t\t\t\t\t\t\t\tbreak\n\t\tprint()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\ndef predict(model, X_data, y_svd=None, is_image_format=False):\n    \"\"\"\n    Preprocess the features if needed, predict using the trained model, and inverse transform the predictions.\n    \"\"\"\n    if not is_image_format:\n        #X_scaled = X_scaler.transform(X_data)\n        X_img = convert_to_image_format(X_data.values)\n        X_tensor = torch.tensor(X_img, dtype=torch.float32).permute(0, 3, 1, 2).cuda()\n    else:\n        X_tensor = X_data\n\n    model.eval()\n    with torch.no_grad():\n        test_outputs = model(torchvision.transforms.Resize(448)(X_tensor))\n        \n    test_predictions = test_outputs.cpu().numpy()\n    if y_svd is not None:\n        test_predictions = y_svd.inverse_transform(test_predictions)\n    \n    return test_predictions\n\nensemble_cv_predictions = []\n\n# Load models and collect their predictions for the validation and test set\nfor fold in range(1, 6):\n    # Load the pre-trained model\n    model = get_model()\n    checkpoint_path = f'best_model_fold_{fold}.pth'\n    model.load_state_dict(torch.load(checkpoint_path))\n    model = model.cuda()\n    \n    # Make predictions on the validation set\n    # X_val_tensor is already in image format, no need for scaling\n    val_predictions = predict(model, X_val, y_svd=y_svd)\n    ensemble_cv_predictions.append(val_predictions)\n    \n  \n# Calculate the average prediction across all folds\nensemble_cv_predictions = np.mean(ensemble_cv_predictions, axis=0)\n\n# We can also calculate the ensemble CV score if 'y_val_tensor' holds the true validation labels\nensemble_mrrmse = np.sqrt(mean_squared_error(y_val, ensemble_cv_predictions))\nprint(f'Ensemble CV MRRMSE: {ensemble_mrrmse}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train again by using all data\n\n# Convert scaled data to image format\nX_img = convert_to_image_format(X.values)\n\ny_svd = TruncatedSVD(n_components=n_components, random_state=6174)\ny_truncated = y_svd.fit_transform(y.values)\n\n# Convert to tensors\nX_tensor = torch.tensor(X_img, dtype=torch.float32).permute(0, 3, 1, 2)\ny_tensor = torch.tensor(y_truncated, dtype=torch.float32)\n\n# KFold setup\nnum_epochs = 50  # Define the number of epochs\n\nkf = KFold(n_splits=5, shuffle=True, random_state=6174)\nfor fold, (train_index, val_index) in enumerate(kf.split(X_tensor)):\n\t\tprint(f'Fold {fold+1}')\n\n\t\t# Split data into training and validation sets\n\t\tX_train_tensor, y_train_tensor = X_tensor[train_index], y_tensor[train_index]\n\t\tX_val_tensor, y_val_tensor = X_tensor[val_index], y_tensor[val_index]\n\n\t\t# Move data to GPU\n\t\tX_train_tensor, y_train_tensor = X_train_tensor.cuda(), y_train_tensor.cuda()\n\t\tX_val_tensor, y_val_tensor = X_val_tensor.cuda(), y_val_tensor.cuda()\n\n\t\t# DataLoader setup\n\t\ttrain_dataset = TensorDataset(X_train_tensor, y_train_tensor)\n\t\tval_dataset = TensorDataset(X_val_tensor, y_val_tensor)\n\t\ttrain_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n\t\tval_loader = DataLoader(val_dataset, batch_size=16)\n\n\t\t# Model, loss, optimizer setup\n\t\tmodel = get_model()\n\t\tcriterion = nn.MSELoss()\n\t\toptimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=1e-4)\n\t\tscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=5, verbose=True)\n\n\t\t# Training loop for each fold\n\t\tbest_val_loss = float('inf')\n\t\tearly_stopping_patience = 10\n\t\tearly_stopping_counter = 0\n\t\tgradient_clip_value = 1\n\n\t\tfor epoch in range(num_epochs):\n\t\t\t\tmodel.train()\n\t\t\t\tfor inputs, targets in train_loader:\n\t\t\t\t\t\toptimizer.zero_grad()\n\t\t\t\t\t\toutputs = model(torchvision.transforms.Resize(448)(inputs))\n\t\t\t\t\t\tloss = criterion(outputs, targets)\n\t\t\t\t\t\tloss.backward()\n\t\t\t\t\t\ttorch.nn.utils.clip_grad_norm_(model.parameters(), gradient_clip_value)  # Gradient clipping\n\t\t\t\t\t\toptimizer.step()\n\n\t\t\t\t# Validation loop\n\t\t\t\tmodel.eval()\n\t\t\t\tvalid_loss_squared_sum = 0  # For MRRMSE\n\n\t\t\t\twith torch.no_grad():\n\t\t\t\t\t\tfor inputs, targets in val_loader:\n\t\t\t\t\t\t\t\toutputs = model(torchvision.transforms.Resize(448)(inputs))\n\t\t\t\t\t\t\t\tloss = criterion(outputs, targets)\n\t\t\t\t\t\t\t\tvalid_loss_squared_sum += loss.item() * inputs.size(0)  # Sum of squared errors\n\t\t\t\t\t\tvalid_mrrmse = np.sqrt(valid_loss_squared_sum / len(val_loader.dataset))\n\t\t\t\t\t\tprint(f'Epoch {epoch}, MRRMSE: {valid_mrrmse}')\n\n\t\t\t\t\t\t# Update the learning rate scheduler based on validation loss\n\t\t\t\t\t\tscheduler.step(valid_mrrmse)\n\n\t\t\t\t\t\tif valid_mrrmse < best_val_loss:\n\t\t\t\t\t\t\t\tbest_val_loss = valid_mrrmse\n\t\t\t\t\t\t\t\tearly_stopping_counter = 0\n\t\t\t\t\t\t\t\ttorch.save(model.state_dict(), f'best_model_fold_{fold+1}.pth')  # Save best model checkpoint\n\t\t\t\t\t\telse:\n\t\t\t\t\t\t\t\tearly_stopping_counter += 1\n\t\t\t\t\t\t\t\tif early_stopping_counter >= early_stopping_patience:\n\t\t\t\t\t\t\t\t\t\tprint(\"Early stopping triggered\")\n\t\t\t\t\t\t\t\t\t\tprint()\n\t\t\t\t\t\t\t\t\t\tbreak\n\t\tprint()\n\n\nensemble_sub_predictions = []\nfor fold in range(1, 6):\n\t\t# Load the pre-trained model\n\t\tmodel = get_model()\n\t\tcheckpoint_path = f'best_model_fold_{fold}.pth'\n\t\tmodel.load_state_dict(torch.load(checkpoint_path))\n\t\tmodel = model.cuda()\n\t\t\n\t\t# Make predictions on the test set\n\t\t# X_sub_tensor is already in image format, no need for scaling\n\t\tsub_predictions = predict(model, X_sub, y_svd=y_svd)\n\t\tensemble_sub_predictions.append(sub_predictions)\n\t\n# Calculate the average prediction across all folds\nensemble_sub_predictions = np.mean(ensemble_sub_predictions, axis=0)\n\n# Save the predictions to a CSV file\nid_map.loc[:, genes] = ensemble_sub_predictions\nid_map = id_map.loc[:, [\"id\"] + genes.to_list()]\nid_map.to_csv('submission.csv', index=False)\nid_map","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(20, 10))\nplt.plot(id_map[genes].values[0])\nplt.legend()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tensor.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- sub_0: m-nb002 v25, Ridge, CV: 1.11, LB: 0.611\n- sub_1: m-nb006 v5, py-boost GradientBoosting, CV: 0.977, LB: 0.591\n- sub_2: m-nb014 v12, LinearSVM with flat genes, CV: 1.848, LB: 0.768\n- sub_3: m-nb015 v4, regnet_y_32gf, CV: 1.691, LB: 0.630\n- sub_4: s-nb001 v4, LinearRegression, CV: 1.135, LB: 0.627","metadata":{}},{"cell_type":"code","source":"\n#sub_0 = id_map.copy()\n#sub_1 = pd.read_csv('/kaggle/input/opscp-subs/OPSCP-m-nb006 - Version 5.csv')\n#sub_2 = pd.read_csv('/kaggle/input/opscp-subs/opscp-m-nb014 - Version 12.csv')\n#sub_3 = pd.read_csv('/kaggle/input/opscp-subs/opscp-m-nb015 - Version 4.csv')\n#sub_4 = pd.read_csv('/kaggle/input/opscp-subs/opscp-s-nb001 - Version 4.csv')\n\n\n#diff_1_0 = sub_1[genes].values - sub_0[genes].values\n#diff_2_0 = sub_2[genes].values - sub_0[genes].values\n#diff_3_0 = sub_3[genes].values - sub_0[genes].values\n#diff_4_0 = sub_4[genes].values - sub_0[genes].values\n\n\n#plt.figure(figsize=(20, 10))\n#plt.plot(diff_1_0[0], label='diff_1_0', alpha=0.5)\n#plt.plot(diff_2_0[0], label='diff_2_0', alpha=0.5)\n#plt.plot(diff_3_0[0], label='diff_3_0', alpha=0.5)\n#plt.plot(diff_4_0[0], label='diff_4_0', alpha=0.5)\n#plt.legend()\n#plt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Ensemble\n#id_map = pd.read_csv(f'{folder_path}/id_map.csv')\n#id_map.loc[:, genes] = (\n#    0.25 * sub_0[genes].values +\n#    0.3 * sub_1[genes].values +\n#    0.1 * sub_2[genes].values +\n#    0.2 * sub_3[genes].values +\n#    0.15 * sub_4[genes].values\n#)\n#id_map = id_map.loc[:, [\"id\"] + genes.to_list()]\n#id_map.to_csv('submission.csv', index=False)\n\n#pd.read_csv('submission.csv')\n\n## Plot it\n#plt.figure(figsize=(20, 10))\n#plt.plot(id_map[genes].values[0], label='ensemble', alpha=0.5)\n#plt.plot(sub_0[genes].values[0], label='sub_0', alpha=0.2)\n#plt.plot(sub_1[genes].values[0], label='sub_1', alpha=0.2)\n#plt.plot(sub_2[genes].values[0], label='sub_2', alpha=0.2)\n#plt.plot(sub_3[genes].values[0], label='sub_3', alpha=0.2)\n#plt.plot(sub_4[genes].values[0], label='sub_4', alpha=0.2)\n#plt.legend()\n#plt.show()\n\n","metadata":{},"execution_count":null,"outputs":[]}]}