{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-24T20:53:02.519220Z","iopub.execute_input":"2024-09-24T20:53:02.519731Z","iopub.status.idle":"2024-09-24T20:53:02.528663Z","shell.execute_reply.started":"2024-09-24T20:53:02.519686Z","shell.execute_reply":"2024-09-24T20:53:02.527355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Processing","metadata":{}},{"cell_type":"code","source":"!pip install rdkit","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:02.543627Z","iopub.execute_input":"2024-09-24T20:53:02.544067Z","iopub.status.idle":"2024-09-24T20:53:18.277938Z","shell.execute_reply.started":"2024-09-24T20:53:02.544017Z","shell.execute_reply":"2024-09-24T20:53:18.276449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_parquet('../input/open-problems-single-cell-perturbations/de_train.parquet')\ndf.tail()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:18.280733Z","iopub.execute_input":"2024-09-24T20:53:18.281228Z","iopub.status.idle":"2024-09-24T20:53:19.520044Z","shell.execute_reply.started":"2024-09-24T20:53:18.281173Z","shell.execute_reply":"2024-09-24T20:53:19.518800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:19.521748Z","iopub.execute_input":"2024-09-24T20:53:19.522257Z","iopub.status.idle":"2024-09-24T20:53:19.531647Z","shell.execute_reply.started":"2024-09-24T20:53:19.522204Z","shell.execute_reply":"2024-09-24T20:53:19.530257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loader","metadata":{}},{"cell_type":"markdown","source":"# Building Experts","metadata":{}},{"cell_type":"markdown","source":"# final trial","metadata":{}},{"cell_type":"code","source":"feature_cols =['cell_type','sm_name']\ntarget_cols = ['cell_type','sm_name','sm_lincs_id','SMILES','control']\ntargets = df.drop(columns=target_cols)\nfeatures = pd.DataFrame(df,columns=feature_cols)\nfeatures","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:19.534956Z","iopub.execute_input":"2024-09-24T20:53:19.536100Z","iopub.status.idle":"2024-09-24T20:53:19.604610Z","shell.execute_reply.started":"2024-09-24T20:53:19.536041Z","shell.execute_reply":"2024-09-24T20:53:19.603337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from rdkit import Chem\n# from rdkit.Chem import AllChem\n# import numpy as np\n# from rdkit import DataStructs\n# from sklearn.preprocessing import OneHotEncoder, StandardScaler\n# import pandas as pd\n\n# def extract_morgan_fingerprint(smiles, radius=2, nBits=2048):\n#     if smiles is None or smiles == '':\n#         return None\n    \n#     mol = Chem.MolFromSmiles(smiles)\n#     if mol is None:\n#         return None\n    \n#     # Generate Morgan fingerprint using MorganGenerator\n#     morgan_gen = AllChem.GetMorganGenerator(radius=radius, fpSize=nBits)\n#     fp = morgan_gen.GetFingerprint(mol)\n    \n#     # Convert to numpy array\n#     fp_array = np.zeros((nBits,))\n#     DataStructs.ConvertToNumpyArray(fp, fp_array)\n    \n#     # Convert numpy array to list\n#     fp_list = fp_array.tolist()\n    \n#     return fp_list\n\n# def process_data(df, categorical_features):\n#     # Extract Morgan fingerprints\n#     morgan_fp_list = df['SMILES'].apply(extract_morgan_fingerprint)\n#     morgan_fp_array = np.stack(morgan_fp_list)\n\n#     # Create a DataFrame with Morgan fingerprints\n#     reduced_fp_df = pd.DataFrame(morgan_fp_array, columns=[f'MorganFP_{i}' for i in range(morgan_fp_array.shape[1])])\n    \n#     # Normalize the fingerprints\n#     scaler = StandardScaler()\n#     reduced_fp_norm = pd.DataFrame(scaler.fit_transform(reduced_fp_df), \n#                                    columns=reduced_fp_df.columns, \n#                                    index=reduced_fp_df.index)\n    \n#     # One-hot encode categorical features\n#     df_categorical = df[categorical_features]\n#     encoder = OneHotEncoder(sparse=False)\n#     encoded_features = encoder.fit_transform(df_categorical)\n#     encoded_df = pd.DataFrame(encoded_features, \n#                               columns=encoder.get_feature_names_out(categorical_features),\n#                               index=df.index)\n    \n#     # Combine normalized fingerprints and encoded categorical features\n#     final_df = pd.concat([reduced_fp_norm, encoded_df], axis=1)\n    \n#     # Add any other numerical features if needed\n#     # numerical_features = ['feature1', 'feature2', ...]  # Add your numerical feature names here\n#     # final_df = pd.concat([final_df, df[numerical_features]], axis=1)\n    \n#     print(f\"Shape of final dataframe: {final_df.shape}\")\n#     return final_df, scaler, encoder\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:19.606298Z","iopub.execute_input":"2024-09-24T20:53:19.606738Z","iopub.status.idle":"2024-09-24T20:53:19.614727Z","shell.execute_reply.started":"2024-09-24T20:53:19.606692Z","shell.execute_reply":"2024-09-24T20:53:19.613193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from rdkit import Chem\nfrom rdkit.Chem import AllChem\nimport numpy as np\nfrom rdkit import DataStructs\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, TensorDataset\n\n# Define the autoencoder model\nclass Autoencoder(nn.Module):\n    def __init__(self, input_size, hidden_size):\n        super(Autoencoder, self).__init__()\n        # Encoder\n        self.encoder = nn.Sequential(\n            nn.Linear(input_size, hidden_size),\n            nn.ReLU()\n        )\n        # Decoder\n        self.decoder = nn.Sequential(\n            nn.Linear(hidden_size, input_size),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return encoded, decoded\n\n# Function to train the autoencoder\ndef train_autoencoder(data, input_size, hidden_size, num_epochs=50, batch_size=32, learning_rate=0.001):\n    autoencoder = Autoencoder(input_size=input_size, hidden_size=hidden_size)\n    criterion = nn.MSELoss()\n    optimizer = torch.optim.Adam(autoencoder.parameters(), lr=learning_rate)\n\n    dataset = TensorDataset(torch.tensor(data, dtype=torch.float32))\n    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)\n\n    for epoch in range(num_epochs):\n        for batch in dataloader:\n            inputs = batch[0]\n            _, reconstructed = autoencoder(inputs)\n\n            loss = criterion(reconstructed, inputs)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{num_epochs}], Loss: {loss.item():.4f}')\n\n    return autoencoder\n\n# Function to extract Morgan fingerprint\ndef extract_morgan_fingerprint(smiles, radius=2, nBits=2048):\n    if smiles is None or smiles == '':\n        return None\n    \n    mol = Chem.MolFromSmiles(smiles)\n    if mol is None:\n        return None\n    \n    morgan_gen = AllChem.GetMorganGenerator(radius=radius, fpSize=nBits)\n    fp = morgan_gen.GetFingerprint(mol)\n    \n    fp_array = np.zeros((nBits,))\n    DataStructs.ConvertToNumpyArray(fp, fp_array)\n    \n    return fp_array\n\n# Process data\ndef process_data(df, categorical_features, hidden_size=128):\n    # Extract Morgan fingerprints\n    morgan_fp_list = df['SMILES'].apply(extract_morgan_fingerprint)\n    morgan_fp_array = np.stack(morgan_fp_list)\n\n    # Normalize the fingerprints\n    scaler = StandardScaler()\n    morgan_fp_scaled = scaler.fit_transform(morgan_fp_array)\n\n    # Train autoencoder to compress fingerprints\n    input_size = morgan_fp_scaled.shape[1]\n    autoencoder = train_autoencoder(morgan_fp_scaled, input_size, hidden_size)\n\n    # Get compressed fingerprints\n    with torch.no_grad():\n        compressed_fp = autoencoder.encoder(torch.tensor(morgan_fp_scaled, dtype=torch.float32)).numpy()\n\n    compressed_fp_df = pd.DataFrame(compressed_fp, columns=[f'CompressedFP_{i}' for i in range(hidden_size)])\n\n    # One-hot encode categorical features\n    df_categorical = df[categorical_features]\n    encoder = OneHotEncoder(sparse=False)\n    encoded_features = encoder.fit_transform(df_categorical)\n    encoded_df = pd.DataFrame(encoded_features, \n                              columns=encoder.get_feature_names_out(categorical_features),\n                              index=df.index)\n\n    # Combine compressed fingerprints and encoded categorical features\n    final_df = pd.concat([compressed_fp_df, encoded_df], axis=1)\n\n    print(f\"Shape of final dataframe: {final_df.shape}\")\n    \n    # Return four values now\n    return final_df, scaler, encoder, autoencoder\n\n# Example usage\ncategorical_features = ['cell_type', 'sm_name']\n\n# Assuming `df` is your input dataframe\nfinal_df, scaler, encoder, autoencoder = process_data(df, categorical_features, hidden_size=128)\n\n# Now you have `final_df`, `scaler`, `encoder`, and `autoencoder`\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:09:51.448830Z","iopub.execute_input":"2024-09-24T21:09:51.449271Z","iopub.status.idle":"2024-09-24T21:09:56.398605Z","shell.execute_reply.started":"2024-09-24T21:09:51.449231Z","shell.execute_reply":"2024-09-24T21:09:56.397430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# categorical_features = ['cell_type', 'sm_name']\n\n# # Process data\n# final_df, scaler, encoder = process_data(df, categorical_features)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:24.724371Z","iopub.execute_input":"2024-09-24T20:53:24.724858Z","iopub.status.idle":"2024-09-24T20:53:24.729701Z","shell.execute_reply.started":"2024-09-24T20:53:24.724807Z","shell.execute_reply":"2024-09-24T20:53:24.728615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage\ncategorical_features = ['cell_type', 'sm_name']\n\n# Assuming `df` is your input dataframe\nfinal_df, scaler, encoder, autoencoder = process_data(df, categorical_features, hidden_size=128)\n\n# Now you have `final_df`, `scaler`, `encoder`, and `autoencoder`\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:10:00.517696Z","iopub.execute_input":"2024-09-24T21:10:00.518173Z","iopub.status.idle":"2024-09-24T21:10:06.020167Z","shell.execute_reply.started":"2024-09-24T21:10:00.518128Z","shell.execute_reply":"2024-09-24T21:10:06.019233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df.shape, df.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:10:12.862724Z","iopub.execute_input":"2024-09-24T21:10:12.863179Z","iopub.status.idle":"2024-09-24T21:10:12.870457Z","shell.execute_reply.started":"2024-09-24T21:10:12.863136Z","shell.execute_reply":"2024-09-24T21:10:12.869388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df.columns","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:10:19.839066Z","iopub.execute_input":"2024-09-24T21:10:19.839958Z","iopub.status.idle":"2024-09-24T21:10:19.847625Z","shell.execute_reply.started":"2024-09-24T21:10:19.839911Z","shell.execute_reply":"2024-09-24T21:10:19.846418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data for PyTorch\nimport torch\nX = torch.tensor(final_df.values, dtype=torch.float32)\ny = torch.tensor(targets.values, dtype=torch.float32)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:10:38.473276Z","iopub.execute_input":"2024-09-24T21:10:38.474054Z","iopub.status.idle":"2024-09-24T21:10:38.503811Z","shell.execute_reply.started":"2024-09-24T21:10:38.474005Z","shell.execute_reply":"2024-09-24T21:10:38.502599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df.shape, targets.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:10:41.831476Z","iopub.execute_input":"2024-09-24T21:10:41.832485Z","iopub.status.idle":"2024-09-24T21:10:41.839881Z","shell.execute_reply.started":"2024-09-24T21:10:41.832435Z","shell.execute_reply":"2024-09-24T21:10:41.838601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset, TensorDataset\n# class dataset(Dataset):\n#     def __init__(self,X_train,y_train):\n#         self.X_train=X_train\n#         self.y_train=y_train\n#     def __len__(self):\n#         return len(self.X_train)\n#     def __getitem__(self,idx):\n#         return self.X_train[idx], self.y_train[idx]\n    # Create DataLoader\ndataset = TensorDataset(X, y)\ntrain_size = int(0.8 * len(dataset))\nval_size = len(dataset) - train_size\ntrain_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:10:44.505560Z","iopub.execute_input":"2024-09-24T21:10:44.506004Z","iopub.status.idle":"2024-09-24T21:10:44.514569Z","shell.execute_reply.started":"2024-09-24T21:10:44.505960Z","shell.execute_reply":"2024-09-24T21:10:44.513206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom sklearn.metrics import mean_absolute_error\n\nclass Expert(nn.Module):\n    def __init__(self, input_size, output_size, hidden_size=500, dropout_rate=0.3):\n        super(Expert, self).__init__()\n        self.fc0 = nn.Linear(input_size, hidden_size)\n        self.dropout = nn.Dropout(dropout_rate)\n        self.fc1 = nn.Linear(hidden_size, output_size)\n    \n    def forward(self, x):\n        h1 = F.relu(self.fc0(x))\n        h1 = self.dropout(h1)\n        return self.fc1(h1)\n\nclass GatingNetwork(nn.Module):\n    def __init__(self, input_size, num_experts):\n        super(GatingNetwork, self).__init__()\n        self.linear1 = nn.Linear(input_size, 64)\n        self.linear2 = nn.Linear(64, num_experts)\n    \n    def forward(self, x):\n        x = F.relu(self.linear1(x))\n        return F.softmax(self.linear2(x), dim=-1)\n\nclass MixtureOfExperts(nn.Module):\n    def __init__(self, input_size, output_size, num_experts=5):\n        super(MixtureOfExperts, self).__init__()\n        self.experts = nn.ModuleList([Expert(input_size, output_size) for _ in range(num_experts)])\n        self.gating_network = GatingNetwork(input_size, num_experts)\n    \n    def forward(self, x):\n        expert_outputs = torch.stack([expert(x) for expert in self.experts], dim=1)\n        gating_weights = self.gating_network(x).unsqueeze(-1)\n        return torch.sum(expert_outputs * gating_weights, dim=1) \n\n# RMSE_rowwise_loss function\n\ndef RMSE_rowwise_loss(y_pred, y_true):\n    return torch.sqrt(torch.mean((y_pred - y_true)**2, dim=1)).mean()\n\nclass EarlyStopping:\n    def __init__(self, patience=10, min_delta=0):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.best_loss = None\n        self.counter = 0\n\n    def __call__(self, val_loss):\n        if self.best_loss is None:\n            self.best_loss = val_loss\n        elif val_loss > self.best_loss - self.min_delta:\n            self.counter += 1\n            if self.counter >= self.patience:\n                return True\n        else:\n            self.best_loss = val_loss\n            self.counter = 0\n        return False","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:11:00.651662Z","iopub.execute_input":"2024-09-24T21:11:00.652095Z","iopub.status.idle":"2024-09-24T21:11:00.670872Z","shell.execute_reply.started":"2024-09-24T21:11:00.652048Z","shell.execute_reply":"2024-09-24T21:11:00.669502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\nimport matplotlib.pyplot as plt\n\ndef train_model(model, train_loader, val_loader, epochs=200, lr=0.001):\n    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-5)\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)\n    early_stopping = EarlyStopping(patience=10)\n    train_losses = []\n    val_losses = []\n    \n    for epoch in range(epochs):\n        model.train()\n        train_loss = 0\n        for batch, (X, y) in enumerate(train_loader):\n            optimizer.zero_grad()\n            output = model(X)\n            loss = RMSE_rowwise_loss(output, y)\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item()\n        \n        train_losses.append(train_loss / len(train_loader))\n        \n        model.eval()\n        val_loss = 0\n        with torch.no_grad():\n            for X, y in val_loader:\n                output = model(X)\n                val_loss += RMSE_rowwise_loss(output, y).item()\n        \n        val_losses.append(val_loss / len(val_loader))\n        \n        if epoch % 10 == 0:\n            print(f\"Epoch {epoch}: Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}\")\n        \n        # Step the learning rate scheduler at the end of each epoch\n        scheduler.step()\n        \n        # Check early stopping condition\n        if early_stopping(val_loss / len(val_loader)):\n            print(\"Early stopping triggered\")\n            break\n\n    \n    # Plot the losses\n    plt.figure(figsize=(10, 5))\n    plt.plot(train_losses, label='Train Loss')\n    plt.plot(val_losses, label='Val Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.title('Training and Validation Losses')\n    plt.show()\n\n    return train_losses, val_losses","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:11:02.816667Z","iopub.execute_input":"2024-09-24T21:11:02.817793Z","iopub.status.idle":"2024-09-24T21:11:02.832990Z","shell.execute_reply.started":"2024-09-24T21:11:02.817722Z","shell.execute_reply":"2024-09-24T21:11:02.831532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize and train the model\ninput_size = X.shape[1]  # Updated input size including reduced Morgan fingerprints and one-hot encoded categories\noutput_size = y.shape[1]  # Your original output size\nnum_experts = 5\n\nprint(f\"Model input size: {input_size}\")\nprint(f\"Model output size: {output_size}\")\n\nmodel = MixtureOfExperts(input_size, output_size, num_experts)\ntrain_losses, val_losses = train_model(model, train_loader, val_loader)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:11:05.025482Z","iopub.execute_input":"2024-09-24T21:11:05.025967Z","iopub.status.idle":"2024-09-24T21:13:07.091223Z","shell.execute_reply.started":"2024-09-24T21:11:05.025924Z","shell.execute_reply":"2024-09-24T21:13:07.090003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/id_map.csv\")\nid_map","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:14:24.431142Z","iopub.execute_input":"2024-09-24T21:14:24.432097Z","iopub.status.idle":"2024-09-24T21:14:24.451872Z","shell.execute_reply.started":"2024-09-24T21:14:24.432027Z","shell.execute_reply":"2024-09-24T21:14:24.450724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:14:27.145768Z","iopub.execute_input":"2024-09-24T21:14:27.146263Z","iopub.status.idle":"2024-09-24T21:14:27.183604Z","shell.execute_reply.started":"2024-09-24T21:14:27.146219Z","shell.execute_reply":"2024-09-24T21:14:27.182427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smiles_mapping = df[['sm_name', 'SMILES']].drop_duplicates()\nsmiles_mapping","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:15:09.364948Z","iopub.execute_input":"2024-09-24T21:15:09.365442Z","iopub.status.idle":"2024-09-24T21:15:09.381935Z","shell.execute_reply.started":"2024-09-24T21:15:09.365397Z","shell.execute_reply":"2024-09-24T21:15:09.380628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata = pd.DataFrame(id_map,columns=feature_cols) \ntestdata = testdata.merge(smiles_mapping, on='sm_name', how='left')\ntestdata.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:15:12.256352Z","iopub.execute_input":"2024-09-24T21:15:12.256838Z","iopub.status.idle":"2024-09-24T21:15:12.274703Z","shell.execute_reply.started":"2024-09-24T21:15:12.256793Z","shell.execute_reply":"2024-09-24T21:15:12.273556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def reduce_fingerprint_dimensionality(fingerprints, pca):\n#     reduced_fingerprints = pca.transform(fingerprints)\n#     return reduced_fingerprints\n\n# def process_test_data(df, categorical_features, pca, scaler, encoder):\n#     # Extract Morgan fingerprints\n#     morgan_fp_list = df['SMILES'].apply(extract_morgan_fingerprint)\n#     morgan_fp_array = np.stack(morgan_fp_list)\n    \n#     # Reduce dimensionality using pre-fitted PCA\n#     reduced_fingerprints = reduce_fingerprint_dimensionality(morgan_fp_array, pca)\n    \n#     # Create a DataFrame with reduced fingerprints\n#     reduced_fp_df = pd.DataFrame(reduced_fingerprints, columns=[f'MorganFP_PCA_{i}' for i in range(pca.n_components_)])\n    \n#     # Normalize using the pre-fitted scaler\n#     reduced_fp_norm = pd.DataFrame(scaler.transform(reduced_fp_df), \n#                                    columns=reduced_fp_df.columns, \n#                                    index=reduced_fp_df.index)\n    \n#     # One-hot encode using the pre-fitted encoder\n#     df_categorical = df[categorical_features]\n#     encoded_features = encoder.transform(df_categorical)\n#     encoded_df = pd.DataFrame(encoded_features, \n#                               columns=encoder.get_feature_names_out(categorical_features),\n#                               index=df.index)\n    \n#     # Combine normalized fingerprints and encoded categorical features\n#     final_df = pd.concat([reduced_fp_norm, encoded_df], axis=1)\n    \n#     # Add any other numerical features if needed\n#     # numerical_features = ['feature1', 'feature2', ...]  # Add your numerical feature names here\n#     # final_df = pd.concat([final_df, df[numerical_features]], axis=1)\n    \n#     print(f\"Shape of final dataframe: {final_df.shape}\")\n#     return final_df","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:40.140565Z","iopub.status.idle":"2024-09-24T20:53:40.141808Z","shell.execute_reply.started":"2024-09-24T20:53:40.141475Z","shell.execute_reply":"2024-09-24T20:53:40.141509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def process_test_data(df, categorical_features, scaler, encoder):\n#     # Extract Morgan fingerprints\n#     morgan_fp_list = df['SMILES'].apply(extract_morgan_fingerprint)\n#     morgan_fp_array = np.stack(morgan_fp_list)\n    \n#     # Create a DataFrame with the original Morgan fingerprints\n#     reduced_fp_df = pd.DataFrame(morgan_fp_array, columns=[f'MorganFP_{i}' for i in range(morgan_fp_array.shape[1])])\n    \n#     # Normalize using the pre-fitted scaler\n#     reduced_fp_norm = pd.DataFrame(scaler.transform(reduced_fp_df), \n#                                    columns=reduced_fp_df.columns, \n#                                    index=reduced_fp_df.index)\n    \n#     # One-hot encode using the pre-fitted encoder\n#     df_categorical = df[categorical_features]\n#     encoded_features = encoder.transform(df_categorical)\n#     encoded_df = pd.DataFrame(encoded_features, \n#                               columns=encoder.get_feature_names_out(categorical_features),\n#                               index=df.index)\n    \n#     # Combine normalized fingerprints and encoded categorical features\n#     final_df = pd.concat([reduced_fp_norm, encoded_df], axis=1)\n    \n#     # Add any other numerical features if needed\n#     # numerical_features = ['feature1', 'feature2', ...]  # Add your numerical feature names here\n#     # final_df = pd.concat([final_df, df[numerical_features]], axis=1)\n    \n#     print(f\"Shape of final dataframe: {final_df.shape}\")\n#     return final_df\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:40.143414Z","iopub.status.idle":"2024-09-24T20:53:40.144516Z","shell.execute_reply.started":"2024-09-24T20:53:40.144200Z","shell.execute_reply":"2024-09-24T20:53:40.144233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Process test data\ndef process_test_data(df, categorical_features, scaler, encoder, autoencoder):\n    # Extract Morgan fingerprints\n    morgan_fp_list = df['SMILES'].apply(extract_morgan_fingerprint)\n    morgan_fp_array = np.stack(morgan_fp_list)\n\n    # Normalize using the fitted scaler\n    morgan_fp_scaled = scaler.transform(morgan_fp_array)\n\n    # Compress fingerprints using the trained autoencoder\n    with torch.no_grad():\n        compressed_fp = autoencoder.encoder(torch.tensor(morgan_fp_scaled, dtype=torch.float32)).numpy()\n\n    compressed_fp_df = pd.DataFrame(compressed_fp, columns=[f'CompressedFP_{i}' for i in range(compressed_fp.shape[1])])\n\n    # One-hot encode categorical features using the fitted encoder\n    df_categorical = df[categorical_features]\n    encoded_features = encoder.transform(df_categorical)\n    encoded_df = pd.DataFrame(encoded_features, \n                              columns=encoder.get_feature_names_out(categorical_features),\n                              index=df.index)\n\n    # Combine compressed fingerprints and encoded categorical features\n    final_df = pd.concat([compressed_fp_df, encoded_df], axis=1)\n\n    print(f\"Shape of test dataframe: {final_df.shape}\")\n    return final_df\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:15:18.588600Z","iopub.execute_input":"2024-09-24T21:15:18.589035Z","iopub.status.idle":"2024-09-24T21:15:18.599210Z","shell.execute_reply.started":"2024-09-24T21:15:18.588993Z","shell.execute_reply":"2024-09-24T21:15:18.598074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assume df_test is your testing dataset\n# Process the testing data using the fitted scaler, encoder, and autoencoder from training\ntest_final_df = process_test_data(testdata, categorical_features, scaler, encoder, autoencoder)\n\n# Convert the processed test DataFrame to a tensor for model testing\ntest_X = torch.tensor(test_final_df.values, dtype=torch.float32)\n\n# Load the model and set it to evaluation mode\n# model = MixtureOfExperts(input_size, output_size, num_experts)\n# model.load_state_dict(torch.load('mixture_of_experts_model.pth'))\nmodel.eval()\n\n# Test the model\nwith torch.no_grad():\n    test_output = model(test_X)\n    print(test_output)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:16:09.879403Z","iopub.execute_input":"2024-09-24T21:16:09.880209Z","iopub.status.idle":"2024-09-24T21:16:10.389211Z","shell.execute_reply.started":"2024-09-24T21:16:09.880163Z","shell.execute_reply":"2024-09-24T21:16:10.388064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv\")\nsample_submission.columns","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:16:13.675360Z","iopub.execute_input":"2024-09-24T21:16:13.675830Z","iopub.status.idle":"2024-09-24T21:16:16.611936Z","shell.execute_reply.started":"2024-09-24T21:16:13.675786Z","shell.execute_reply":"2024-09-24T21:16:16.610789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_columns = sample_submission.columns\nsample_columns= sample_columns[1:]\nsubmission_df = pd.DataFrame(test_output.detach().numpy(), columns=sample_columns)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:16:19.340109Z","iopub.execute_input":"2024-09-24T21:16:19.340838Z","iopub.status.idle":"2024-09-24T21:16:19.347201Z","shell.execute_reply.started":"2024-09-24T21:16:19.340791Z","shell.execute_reply":"2024-09-24T21:16:19.345841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.insert(0, 'id', range(255))","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:16:23.749991Z","iopub.execute_input":"2024-09-24T21:16:23.750460Z","iopub.status.idle":"2024-09-24T21:16:23.760281Z","shell.execute_reply.started":"2024-09-24T21:16:23.750414Z","shell.execute_reply":"2024-09-24T21:16:23.758919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:40.157452Z","iopub.status.idle":"2024-09-24T20:53:40.157987Z","shell.execute_reply.started":"2024-09-24T20:53:40.157720Z","shell.execute_reply":"2024-09-24T20:53:40.157748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2024-09-24T20:53:40.159446Z","iopub.status.idle":"2024-09-24T20:53:40.160259Z","shell.execute_reply.started":"2024-09-24T20:53:40.159973Z","shell.execute_reply":"2024-09-24T20:53:40.160004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T21:16:29.396786Z","iopub.execute_input":"2024-09-24T21:16:29.397659Z","iopub.status.idle":"2024-09-24T21:16:39.375393Z","shell.execute_reply.started":"2024-09-24T21:16:29.397613Z","shell.execute_reply":"2024-09-24T21:16:39.374316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}