{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.8.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"},{"sourceId":8275617,"sourceType":"datasetVersion","datasetId":4914065}],"dockerImageVersionId":30514,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook we will train a deep learning model using all the data available !\n* preprocessing : I encoded the smiles of all the train & test set and saved it [here](https://www.kaggle.com/datasets/ahmedelfazouan/belka-enc-dataset) , this may take up to 1 hour on TPU.\n* Training & Inference : I used a simple 1dcnn model trained on 20 epochs.\n\nHow to improve :\n* Try a different architecture : I'm able to get an LB score of 0.604 with minor changes on this architecture.\n* Try another model like Transformer, or LSTM.\n* Train for more epochs.\n* Add more features like a one hot encoding of bb2 or bb3.\n* And of course ensembling with GBDT models.","metadata":{}},{"cell_type":"code","source":"!pip install fastparquet -q","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-05-13T02:27:37.962574Z","iopub.execute_input":"2024-05-13T02:27:37.962911Z","iopub.status.idle":"2024-05-13T02:27:44.070382Z","shell.execute_reply.started":"2024-05-13T02:27:37.962881Z","shell.execute_reply":"2024-05-13T02:27:44.069501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport random\nimport joblib\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import average_precision_score as APS\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import Sequential\nimport tensorflow_addons as tfa","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-13T02:27:44.071989Z","iopub.execute_input":"2024-05-13T02:27:44.072228Z","iopub.status.idle":"2024-05-13T02:28:23.500161Z","shell.execute_reply.started":"2024-05-13T02:27:44.072205Z","shell.execute_reply":"2024-05-13T02:28:23.499395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    debug = False\n    PREPROCESS = False\n    EPOCHS = 2 if debug else 20\n    BATCH_SIZE = 4096\n    LR = 5e-4\n    WD = 1e-4\n    NBR_FOLDS = 10\n    SEED = 42","metadata":{"execution":{"iopub.status.busy":"2024-05-13T02:28:23.501175Z","iopub.execute_input":"2024-05-13T02:28:23.501694Z","iopub.status.idle":"2024-05-13T02:28:23.505843Z","shell.execute_reply.started":"2024-05-13T02:28:23.501667Z","shell.execute_reply":"2024-05-13T02:28:23.505158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seeds(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n    np.random.seed(seed)\n\nset_seeds(seed=CFG.SEED)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-05-13T02:28:23.507654Z","iopub.execute_input":"2024-05-13T02:28:23.507909Z","iopub.status.idle":"2024-05-13T02:28:23.519501Z","shell.execute_reply.started":"2024-05-13T02:28:23.507887Z","shell.execute_reply":"2024-05-13T02:28:23.518684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"Running on TPU\")\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\nexcept tf.errors.NotFoundError:\n    print(\"Not on TPU\")","metadata":{"execution":{"iopub.status.busy":"2024-05-13T02:28:23.520438Z","iopub.execute_input":"2024-05-13T02:28:23.520677Z","iopub.status.idle":"2024-05-13T02:28:33.103877Z","shell.execute_reply.started":"2024-05-13T02:28:23.520655Z","shell.execute_reply":"2024-05-13T02:28:33.103117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"if CFG.PREPROCESS:\n    enc = {'l': 1, 'y': 2, '@': 3, '3': 4, 'H': 5, 'S': 6, 'F': 7, 'C': 8, 'r': 9, 's': 10, '/': 11, 'c': 12, 'o': 13,\n           '+': 14, 'I': 15, '5': 16, '(': 17, '2': 18, ')': 19, '9': 20, 'i': 21, '#': 22, '6': 23, '8': 24, '4': 25, '=': 26,\n           '1': 27, 'O': 28, '[': 29, 'D': 30, 'B': 31, ']': 32, 'N': 33, '7': 34, 'n': 35, '-': 36}\n    train_raw = pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet')\n    smiles = train_raw[train_raw['protein_name']=='BRD4']['molecule_smiles'].values\n    assert (smiles!=train_raw[train_raw['protein_name']=='HSA']['molecule_smiles'].values).sum() == 0\n    assert (smiles!=train_raw[train_raw['protein_name']=='sEH']['molecule_smiles'].values).sum() == 0\n    def encode_smile(smile):\n        tmp = [enc[i] for i in smile]\n        tmp = tmp + [0]*(142-len(tmp))\n        return np.array(tmp).astype(np.uint8)\n\n    smiles_enc = joblib.Parallel(n_jobs=96)(joblib.delayed(encode_smile)(smile) for smile in tqdm(smiles))\n    smiles_enc = np.stack(smiles_enc)\n    train = pd.DataFrame(smiles_enc, columns = [f'enc{i}' for i in range(142)])\n    train['bind1'] = train_raw[train_raw['protein_name']=='BRD4']['binds'].values\n    train['bind2'] = train_raw[train_raw['protein_name']=='HSA']['binds'].values\n    train['bind3'] = train_raw[train_raw['protein_name']=='sEH']['binds'].values\n    train.to_parquet('train_enc.parquet')\n\n    test_raw = pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet')\n    smiles = test_raw['molecule_smiles'].values\n\n    smiles_enc = joblib.Parallel(n_jobs=96)(joblib.delayed(encode_smile)(smile) for smile in tqdm(smiles))\n    smiles_enc = np.stack(smiles_enc)\n    test = pd.DataFrame(smiles_enc, columns = [f'enc{i}' for i in range(142)])\n    test.to_parquet('test_enc.parquet')\nelse:\n    train = pd.read_parquet('/kaggle/input/belka-enc-dataset/train_enc.parquet')\n    test = pd.read_parquet('/kaggle/input/belka-enc-dataset/test_enc.parquet')\n    \nif CFG.debug:\n    train = train.iloc[:100000]\n    test = test.iloc[:100000]","metadata":{"execution":{"iopub.status.busy":"2024-05-13T02:28:33.104824Z","iopub.execute_input":"2024-05-13T02:28:33.105064Z","iopub.status.idle":"2024-05-13T02:30:21.637194Z","shell.execute_reply.started":"2024-05-13T02:28:33.105043Z","shell.execute_reply":"2024-05-13T02:30:21.636235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"def my_model():\n    with strategy.scope():\n        INP_LEN = 142\n        NUM_FILTERS = 64\n        hidden_dim = 64\n\n        inputs = layers.Input(shape=(INP_LEN,), dtype='int32')\n        x = layers.Embedding(input_dim=36, output_dim=hidden_dim, input_length=INP_LEN, mask_zero=True)(inputs)\n        x = layers.Dropout(0.2)(x)\n        # (142, 64)\n        x = layers.Conv1D(filters=NUM_FILTERS*4, kernel_size=3, activation=layers.LeakyReLU(alpha=0.01), padding='valid',  strides=1)(x)\n        x = layers.MaxPool1D(2)(x)\n        # (256, 32)\n        x = layers.Conv1D(filters=NUM_FILTERS*2, kernel_size=3, activation=layers.LeakyReLU(alpha=0.01), padding='valid',  strides=1)(x)\n        x = layers.MaxPool1D(2)(x)\n        # (128, 16)\n        x = layers.Conv1D(filters=NUM_FILTERS, kernel_size=3, activation=layers.LeakyReLU(alpha=0.01), padding='valid',  strides=1)(x)\n        hidden0 = layers.Flatten()(x)\n        # (1024)\n\n        hidden1 = Sequential([\n            layers.Dropout(0.3),\n            tfa.layers.WeightNormalization(layers.Dense(1024, activation=layers.LeakyReLU(alpha=0.01))),\n        ])(layers.Concatenate()([hidden0]))\n\n        hidden2 = Sequential([\n            layers.Dropout(0.4),\n            tfa.layers.WeightNormalization(layers.Dense(1024, activation=layers.LeakyReLU(alpha=0.01))),\n        ])(layers.Concatenate()([hidden0, hidden1]))\n\n        hidden3 = Sequential([\n            layers.Dropout(0.5),\n            tfa.layers.WeightNormalization(layers.Dense(1024, activation=layers.LeakyReLU(alpha=0.01))),\n        ])(layers.Concatenate()([hidden0, hidden1, hidden2]))\n\n        outputs = layers.Dense(3, activation='sigmoid')(hidden3)\n\n        model = tf.keras.models.Model(inputs=inputs, outputs=outputs)\n        learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay(\n            CFG.LR,\n            len(train_dataset) * CFG.EPOCHS,\n            end_learning_rate=1e-7,\n            power=0.5,\n        )\n        optimizer = tf.keras.optimizers.AdamW(learning_rate=learning_rate_fn, weight_decay=CFG.WD)\n        loss = 'binary_crossentropy'\n        weighted_metrics = [tf.keras.metrics.AUC(curve='PR', name='avg_precision')]\n        model.compile(\n            loss=loss,\n            optimizer=optimizer,\n            weighted_metrics=weighted_metrics,\n        )\n        return model","metadata":{"execution":{"iopub.status.busy":"2024-05-13T03:02:06.544268Z","iopub.execute_input":"2024-05-13T03:02:06.544689Z","iopub.status.idle":"2024-05-13T03:02:06.558504Z","shell.execute_reply.started":"2024-05-13T03:02:06.544664Z","shell.execute_reply":"2024-05-13T03:02:06.557482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train & Inference","metadata":{}},{"cell_type":"code","source":"FEATURES = [f'enc{i}' for i in range(142)]\nTARGETS = ['bind1', 'bind2', 'bind3']\nskf = StratifiedKFold(n_splits=CFG.NBR_FOLDS, shuffle=True, random_state=CFG.SEED)\n\nall_preds = []\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(train, train[TARGETS].sum(1))):\n    \n    # Create a TensorFlow dataset from the NumPy arrays\n    train_dataset = tf.data.Dataset.from_tensor_slices((train.loc[train_idx, FEATURES].values, train.loc[train_idx, TARGETS].values))\n    # Shuffle the dataset and batch it\n    train_dataset = train_dataset.shuffle(buffer_size=len(train_idx), reshuffle_each_iteration=True)\n    train_dataset = train_dataset.batch(batch_size=CFG.BATCH_SIZE, drop_remainder=True)\n    # Prefetch the dataset for better performance\n    train_dataset = train_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n\n    # Create a TensorFlow dataset from the NumPy arrays\n    valid_dataset = tf.data.Dataset.from_tensor_slices((train.loc[valid_idx, FEATURES].values, train.loc[valid_idx, TARGETS].values))\n    # Shuffle the dataset and batch it\n    valid_dataset = valid_dataset.batch(batch_size=CFG.BATCH_SIZE)\n    # Prefetch the dataset for better performance\n    valid_dataset = valid_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n    \n    es = tf.keras.callbacks.EarlyStopping(patience=5, monitor=\"val_loss\", mode='min', verbose=1)\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        monitor='val_loss', filepath=f\"model-{fold}.h5\",\n        save_best_only=True, save_weights_only=True,\n        mode='min'\n    )\n    model = my_model()\n    history = model.fit(\n        train_dataset,\n        validation_data=valid_dataset,\n        epochs=CFG.EPOCHS,\n        callbacks=[checkpoint, es],\n        verbose=1,\n    )\n    model.load_weights(f\"model-{fold}.h5\")\n    oof = model.predict(valid_dataset)\n    print('fold :', fold, 'CV score =', APS(train.loc[valid_idx, TARGETS].values, oof, average='micro'))\n\n    # Create a TensorFlow dataset from the NumPy arrays\n    test_dataset = tf.data.Dataset.from_tensor_slices((test.loc[:, FEATURES].values,))\n    # Shuffle the dataset and batch it\n    test_dataset = test_dataset.batch(batch_size=CFG.BATCH_SIZE)\n    # Prefetch the dataset for better performance\n    test_dataset = test_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n    \n    preds = model.predict(test_dataset)\n    all_preds.append(preds)\n    \n    # for holdout strategy (9:1)\n    break\n\npreds = np.mean(all_preds, 0)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T03:02:06.857998Z","iopub.execute_input":"2024-05-13T03:02:06.858312Z","iopub.status.idle":"2024-05-13T03:14:12.352966Z","shell.execute_reply.started":"2024-05-13T03:02:06.858289Z","shell.execute_reply":"2024-05-13T03:14:12.351387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-13T03:14:12.353771Z","iopub.status.idle":"2024-05-13T03:14:12.354116Z","shell.execute_reply.started":"2024-05-13T03:14:12.353950Z","shell.execute_reply":"2024-05-13T03:14:12.353966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"tst = pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet')\ntst['binds'] = 0\ntst.loc[tst['protein_name']=='BRD4', 'binds'] = preds[(tst['protein_name']=='BRD4').values, 0]\ntst.loc[tst['protein_name']=='HSA', 'binds'] = preds[(tst['protein_name']=='HSA').values, 1]\ntst.loc[tst['protein_name']=='sEH', 'binds'] = preds[(tst['protein_name']=='sEH').values, 2]\ntst[['id', 'binds']].to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T02:48:11.511043Z","iopub.status.idle":"2024-05-13T02:48:11.511334Z","shell.execute_reply.started":"2024-05-13T02:48:11.511193Z","shell.execute_reply":"2024-05-13T02:48:11.511207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}