{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"},{"sourceId":8275617,"sourceType":"datasetVersion","datasetId":4914065}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Original Notes:\n\nIn this notebook we will train a deep learning model using all the data available !\n* preprocessing : I encoded the smiles of all the train & test set and saved it [here](https://www.kaggle.com/datasets/ahmedelfazouan/belka-enc-dataset) , this may take up to 1 hour on TPU.\n* Training & Inference : I used a simple 1dcnn model trained on 20 epochs.\n\nHow to improve :\n* Try a different architecture : I'm able to get an LB score of 0.604 with minor changes on this architecture.\n* Try another model like Transformer, or LSTM.\n* Train for more epochs.\n* Add more features like a one hot encoding of bb2 or bb3.\n* And of course ensembling with GBDT models.\n\nHuge thanks to @ahmedelfazouan, check out his original notebook!\n\nhttps://www.kaggle.com/code/ahmedelfazouan/belka-1dcnn-starter-with-all-data/notebook","metadata":{}},{"cell_type":"markdown","source":"I saw some people asked for a pytorch version of the 1dcnn notebook under the comment section, so here is a simple py version that I used. :)\n\nI could only run this notebook locally, so highly recommend checking other people's implementation on how to reduce the memory usage!\n\nNotes: the embedding layer in pytorch is different than tensorflow, in which it doesn't have the mask_zero option, so I had to change the num of embedding to 37 to make it work. Please let me know if there's a better way to implement it!","metadata":{}},{"cell_type":"code","source":"!pip install fastparquet -q","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install --quiet \"ipython[notebook]>=8.0.0, <8.12.0\" \"lightning>=2.0.0rc0\" \"setuptools==67.4.0\" \"torch>=1.8.1, <1.14.0\" \"torchvision\" \"pytorch-lightning>=1.4, <2.0.0\" \"torchmetrics>=0.7, <0.12\"\n! pip install -U torch_xla -q","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport random\nimport joblib\nimport pandas as pd\n# import polars as pd\nfrom tqdm import tqdm\n\nimport numpy as np\nimport torch\nfrom torch.utils.data import TensorDataset, Dataset, DataLoader\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\nfrom sklearn.model_selection import StratifiedKFold\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_lightning.callbacks.lr_monitor import LearningRateMonitor","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n\n    PREPROCESS = False\n    EPOCHS = 30 #20\n    BATCH_SIZE = 4096\n    LR = 1e-3\n    WD = 0.05\n\n    NBR_FOLDS = 15\n    SELECTED_FOLDS = [0]\n\n    SEED = 2024","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow as tf\nimport torch\ndef set_seeds(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    #tf.random.set_seed(seed)\n    torch.manual_seed(seed)\n    np.random.seed(seed)\n\nset_seeds(seed=CFG.SEED)","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"if CFG.PREPROCESS:\n    enc = {'l': 1, 'y': 2, '@': 3, '3': 4, 'H': 5, 'S': 6, 'F': 7, 'C': 8, 'r': 9, 's': 10, '/': 11, 'c': 12, 'o': 13,\n           '+': 14, 'I': 15, '5': 16, '(': 17, '2': 18, ')': 19, '9': 20, 'i': 21, '#': 22, '6': 23, '8': 24, '4': 25, '=': 26,\n           '1': 27, 'O': 28, '[': 29, 'D': 30, 'B': 31, ']': 32, 'N': 33, '7': 34, 'n': 35, '-': 36}\n    train_raw = pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet')\n    smiles = train_raw[train_raw['protein_name']=='BRD4']['molecule_smiles'].values\n    assert (smiles!=train_raw[train_raw['protein_name']=='HSA']['molecule_smiles'].values).sum() == 0\n    assert (smiles!=train_raw[train_raw['protein_name']=='sEH']['molecule_smiles'].values).sum() == 0\n    def encode_smile(smile):\n        tmp = [enc[i] for i in smile]\n        tmp = tmp + [0]*(142-len(tmp))\n        return np.array(tmp).astype(np.uint8)\n\n    smiles_enc = joblib.Parallel(n_jobs=96)(joblib.delayed(encode_smile)(smile) for smile in tqdm(smiles))\n    smiles_enc = np.stack(smiles_enc)\n    train = pd.DataFrame(smiles_enc, columns = [f'enc{i}' for i in range(142)])\n    train['bind1'] = train_raw[train_raw['protein_name']=='BRD4']['binds'].values\n    train['bind2'] = train_raw[train_raw['protein_name']=='HSA']['binds'].values\n    train['bind3'] = train_raw[train_raw['protein_name']=='sEH']['binds'].values\n    train.to_parquet('train_enc.parquet')\n\n    test_raw = pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet')\n    smiles = test_raw['molecule_smiles'].values\n\n    smiles_enc = joblib.Parallel(n_jobs=96)(joblib.delayed(encode_smile)(smile) for smile in tqdm(smiles))\n    smiles_enc = np.stack(smiles_enc)\n    test = pd.DataFrame(smiles_enc, columns = [f'enc{i}' for i in range(142)])\n    test.to_parquet('test_enc.parquet')\n\nelse:\n    train = pd.read_parquet('/kaggle/input/belka-enc-dataset/train_enc.parquet')\n    test = pd.read_parquet('/kaggle/input/belka-enc-dataset/test_enc.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"class MyModel(pl.LightningModule):\n    def __init__(self, input_dim=142, input_dim_embedding=37, hidden_dim=128, num_filters=32, output_dim=3, lr=1e-3, weight_decay=1e-6):\n        super(MyModel, self).__init__()\n        self.save_hyperparameters()\n\n        self.embedding = nn.Embedding(num_embeddings=self.hparams.input_dim_embedding, embedding_dim=self.hparams.hidden_dim, padding_idx=0)\n        self.conv1 = nn.Conv1d(in_channels=self.hparams.hidden_dim, out_channels=self.hparams.num_filters, kernel_size=3, stride=1, padding=0)\n        self.conv2 = nn.Conv1d(in_channels=self.hparams.num_filters, out_channels=self.hparams.num_filters*2, kernel_size=3, stride=1, padding=0)\n        self.conv3 = nn.Conv1d(in_channels=self.hparams.num_filters*2, out_channels=self.hparams.num_filters*3, kernel_size=3, stride=1, padding=0)\n        self.global_max_pool = nn.AdaptiveMaxPool1d(1)\n        self.fc1 = nn.Linear(self.hparams.num_filters*3, 1024)\n        self.dropout = nn.Dropout(0.1)\n        self.fc2 = nn.Linear(1024, 1024)\n        self.fc3 = nn.Linear(1024, 512)\n        self.output = nn.Linear(512, self.hparams.output_dim)\n\n    def forward(self, x):\n        x = self.embedding(x).permute(0,2,1)\n        x = F.relu(self.conv1(x))\n        x = F.relu(self.conv2(x))\n        x = F.relu(self.conv3(x))\n        x = self.global_max_pool(x).squeeze(2)\n        x = F.relu(self.fc1(x))\n        x = self.dropout(x)\n        x = F.relu(self.fc2(x))\n        x = self.dropout(x)\n        x = F.relu(self.fc3(x))\n        x = self.dropout(x)\n        x = self.output(x)\n        return x\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        logits = self(x)\n        loss = F.binary_cross_entropy_with_logits(logits, y)\n        self.log('train_loss', loss)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        logits = self(x)\n        loss = F.binary_cross_entropy_with_logits(logits, y)\n        self.log('val_loss', loss)\n        return loss\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=self.hparams.lr, weight_decay=self.hparams.weight_decay)\n        return optimizer","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train & Inference","metadata":{}},{"cell_type":"code","source":"FEATURES = [f'enc{i}' for i in range(142)]\nTARGETS = ['bind1', 'bind2', 'bind3']\nskf = StratifiedKFold(n_splits=CFG.NBR_FOLDS, shuffle=True, random_state=42)\nall_preds = []\n\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(train, train[TARGETS].sum(1))):\n    if fold not in CFG.SELECTED_FOLDS:\n        continue\n\n    # Convert pandas dataframes to PyTorch tensors\n    X_train = torch.tensor(train.loc[train_idx, FEATURES].values, dtype=torch.int)\n    y_train = torch.tensor(train.loc[train_idx, TARGETS].values, dtype=torch.float16)\n    X_val = torch.tensor(train.loc[valid_idx, FEATURES].values, dtype=torch.int)\n    y_val = torch.tensor(train.loc[valid_idx, TARGETS].values, dtype=torch.float16)\n    \n    # Create TensorDatasets\n    train_dataset = TensorDataset(X_train, y_train)\n    valid_dataset = TensorDataset(X_val, y_val)\n    \n    # Create DataLoaders\n    train_loader = DataLoader(train_dataset, batch_size=CFG.BATCH_SIZE, shuffle=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CFG.BATCH_SIZE)\n        \n    model = MyModel(lr=CFG.LR, weight_decay=CFG.WD)\n\n    early_stop_callback = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5, verbose=True)\n    checkpoint_callback = ModelCheckpoint(monitor=\"val_loss\", dirpath=\"./\", filename=f\"model-{fold}\", save_top_k=1, mode=\"min\")\n    lr_monitor = LearningRateMonitor(logging_interval='epoch')\n\n    trainer = pl.Trainer(\n        max_epochs=CFG.EPOCHS,\n        callbacks=[early_stop_callback, checkpoint_callback, lr_monitor],\n        devices=1,\n        accelerator=\"tpu\",  # Adjust based on your hardware\n        enable_progress_bar=True,\n    )\n\n    trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=valid_loader)\n\n    model = MyModel.load_from_checkpoint(checkpoint_callback.best_model_path)\n    oof = model(X_val)\n    print('fold :', fold, 'CV score =', APS(y_val, oof, average='micro'))\n\n    preds = model(test)\n    all_preds.append(preds)\n\npreds = np.mean(all_preds, 0)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T01:49:42.048758Z","iopub.execute_input":"2024-05-04T01:49:42.048979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"tst = pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet')\ntst['binds'] = 0\ntst.loc[tst['protein_name']=='BRD4', 'binds'] = preds[(tst['protein_name']=='BRD4').values, 0]\ntst.loc[tst['protein_name']=='HSA', 'binds'] = preds[(tst['protein_name']=='HSA').values, 1]\ntst.loc[tst['protein_name']=='sEH', 'binds'] = preds[(tst['protein_name']=='sEH').values, 2]\ntst[['id', 'binds']].to_csv('submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}