{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.8.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"},{"sourceId":8275617,"sourceType":"datasetVersion","datasetId":4914065}],"dockerImageVersionId":30514,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook we will train a deep learning model using all the data available !\n* preprocessing : I encoded the smiles of all the train & test set and saved it [here](https://www.kaggle.com/datasets/ahmedelfazouan/belka-enc-dataset) , this may take up to 1 hour on TPU.\n* Training & Inference : I used a simple 1dcnn model trained on 20 epochs.\n\nHow to improve :\n* Try a different architecture : I'm able to get an LB score of 0.604 with minor changes on this architecture.\n* Try another model like Transformer, or LSTM.\n* Train for more epochs.\n* Add more features like a one hot encoding of bb2 or bb3.\n* And of course ensembling with GBDT models.","metadata":{}},{"cell_type":"code","source":"!pip install fastparquet -q","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport random\nimport joblib\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import average_precision_score as APS","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-01T12:01:48.750797Z","iopub.execute_input":"2024-05-01T12:01:48.751195Z","iopub.status.idle":"2024-05-01T12:01:56.110275Z","shell.execute_reply.started":"2024-05-01T12:01:48.751166Z","shell.execute_reply":"2024-05-01T12:01:56.108921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n\n    PREPROCESS = False\n    EPOCHS = 20\n    BATCH_SIZE = 4096\n    LR = 1e-3\n    WD = 0.05\n\n    NBR_FOLDS = 15\n    SELECTED_FOLDS = [0]\n\n    SEED = 2024","metadata":{"execution":{"iopub.status.busy":"2024-05-01T12:00:28.621165Z","iopub.execute_input":"2024-05-01T12:00:28.621503Z","iopub.status.idle":"2024-05-01T12:00:28.625731Z","shell.execute_reply.started":"2024-05-01T12:00:28.621462Z","shell.execute_reply":"2024-05-01T12:00:28.625098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ndef set_seeds(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n    np.random.seed(seed)\n\nset_seeds(seed=CFG.SEED)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T12:00:28.626462Z","iopub.execute_input":"2024-05-01T12:00:28.626712Z","iopub.status.idle":"2024-05-01T12:01:07.833616Z","shell.execute_reply.started":"2024-05-01T12:00:28.626692Z","shell.execute_reply":"2024-05-01T12:01:07.832835Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"Running on TPU\")\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\nexcept tf.errors.NotFoundError:\n    print(\"Not on TPU\")","metadata":{"execution":{"iopub.status.busy":"2024-05-01T12:01:07.834565Z","iopub.execute_input":"2024-05-01T12:01:07.834981Z","iopub.status.idle":"2024-05-01T12:01:16.336925Z","shell.execute_reply.started":"2024-05-01T12:01:07.834957Z","shell.execute_reply":"2024-05-01T12:01:16.335732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"if CFG.PREPROCESS:\n    enc = {'l': 1, 'y': 2, '@': 3, '3': 4, 'H': 5, 'S': 6, 'F': 7, 'C': 8, 'r': 9, 's': 10, '/': 11, 'c': 12, 'o': 13,\n           '+': 14, 'I': 15, '5': 16, '(': 17, '2': 18, ')': 19, '9': 20, 'i': 21, '#': 22, '6': 23, '8': 24, '4': 25, '=': 26,\n           '1': 27, 'O': 28, '[': 29, 'D': 30, 'B': 31, ']': 32, 'N': 33, '7': 34, 'n': 35, '-': 36}\n    train_raw = pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet')\n    smiles = train_raw[train_raw['protein_name']=='BRD4']['molecule_smiles'].values\n    assert (smiles!=train_raw[train_raw['protein_name']=='HSA']['molecule_smiles'].values).sum() == 0\n    assert (smiles!=train_raw[train_raw['protein_name']=='sEH']['molecule_smiles'].values).sum() == 0\n    def encode_smile(smile):\n        tmp = [enc[i] for i in smile]\n        tmp = tmp + [0]*(142-len(tmp))\n        return np.array(tmp).astype(np.uint8)\n\n    smiles_enc = joblib.Parallel(n_jobs=96)(joblib.delayed(encode_smile)(smile) for smile in tqdm(smiles))\n    smiles_enc = np.stack(smiles_enc)\n    train = pd.DataFrame(smiles_enc, columns = [f'enc{i}' for i in range(142)])\n    train['bind1'] = train_raw[train_raw['protein_name']=='BRD4']['binds'].values\n    train['bind2'] = train_raw[train_raw['protein_name']=='HSA']['binds'].values\n    train['bind3'] = train_raw[train_raw['protein_name']=='sEH']['binds'].values\n    train.to_parquet('train_enc.parquet')\n\n    test_raw = pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet')\n    smiles = test_raw['molecule_smiles'].values\n\n    smiles_enc = joblib.Parallel(n_jobs=96)(joblib.delayed(encode_smile)(smile) for smile in tqdm(smiles))\n    smiles_enc = np.stack(smiles_enc)\n    test = pd.DataFrame(smiles_enc, columns = [f'enc{i}' for i in range(142)])\n    test.to_parquet('test_enc.parquet')\n\nelse:\n    train = pd.read_parquet('/kaggle/input/belka-enc-dataset/train_enc.parquet')\n    test = pd.read_parquet('/kaggle/input/belka-enc-dataset/test_enc.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-05-01T12:02:00.527603Z","iopub.execute_input":"2024-05-01T12:02:00.528028Z","iopub.status.idle":"2024-05-01T12:03:43.962629Z","shell.execute_reply.started":"2024-05-01T12:02:00.527996Z","shell.execute_reply":"2024-05-01T12:03:43.961562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"def my_model():\n    with strategy.scope():\n        INP_LEN = 142\n        NUM_FILTERS = 32\n        hidden_dim = 128\n\n        inputs = tf.keras.layers.Input(shape=(INP_LEN,), dtype='int32')\n        x = tf.keras.layers.Embedding(input_dim=36, output_dim=hidden_dim, input_length=INP_LEN, mask_zero=True)(inputs)\n\n\n        x = tf.keras.layers.Reshape((INP_LEN, 1, hidden_dim))(x)  # shape becomes (batch_size, INP_LEN, 1, hidden_dim)\n\n        x = tf.keras.layers.ConvLSTM1D(filters=NUM_FILTERS, kernel_size=3, activation='tanh', padding='valid', strides=1, return_sequences=True)(x)\n        x = tf.keras.layers.ConvLSTM1D(filters=NUM_FILTERS2, kernel_size=3, activation='tanh', padding='valid', strides=1, return_sequences=True)(x)\n        x = tf.keras.layers.ConvLSTM1D(filters=NUM_FILTERS3, kernel_size=3, activation='elu', padding='valid', strides=1)(x)\n\n\n        x = tf.keras.layers.GlobalMaxPooling1D()(x)\n\n        x = tf.keras.layers.Dense(1024, activation='tanh')(x)\n        x = tf.keras.layers.Dropout(0.1)(x)\n        x = tf.keras.layers.Dense(1024, activation='tanh')(x)\n        x = tf.keras.layers.Dropout(0.1)(x)\n        x = tf.keras.layers.Dense(512, activation='elu')(x)\n        x = tf.keras.layers.Dropout(0.1)(x)\n\n        outputs = tf.keras.layers.Dense(3, activation='sigmoid')(x)\n\n        model = tf.keras.models.Model(inputs = inputs, outputs = outputs)\n        optimizer = tf.keras.optimizers.Adam(learning_rate=CFG.LR, weight_decay = CFG.WD)\n        loss = 'binary_crossentropy'\n        weighted_metrics = [tf.keras.metrics.AUC(curve='PR', name = 'avg_precision')]\n        model.compile(\n        loss=loss,\n        optimizer=optimizer,\n        weighted_metrics=weighted_metrics,\n        )\n        return model","metadata":{"execution":{"iopub.status.busy":"2024-05-01T12:03:43.964291Z","iopub.execute_input":"2024-05-01T12:03:43.96486Z","iopub.status.idle":"2024-05-01T12:03:43.97628Z","shell.execute_reply.started":"2024-05-01T12:03:43.964832Z","shell.execute_reply":"2024-05-01T12:03:43.975456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train & Inference","metadata":{}},{"cell_type":"code","source":"FEATURES = [f'enc{i}' for i in range(142)]\nTARGETS = ['bind1', 'bind2', 'bind3']\nskf = StratifiedKFold(n_splits = CFG.NBR_FOLDS, shuffle = True, random_state = 42)\n\nall_preds = []\nfor fold,(train_idx, valid_idx) in enumerate(skf.split(train, train[TARGETS].sum(1))):\n    \n    if fold not in CFG.SELECTED_FOLDS:\n        continue;\n    \n    X_train = train.loc[train_idx, FEATURES]\n    y_train = train.loc[train_idx, TARGETS]\n    X_val = train.loc[valid_idx, FEATURES]\n    y_val = train.loc[valid_idx, TARGETS]\n\n    es = tf.keras.callbacks.EarlyStopping(patience=5, monitor=\"val_loss\", mode='min', verbose=1)\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(monitor='val_loss', filepath=f\"model-{fold}.h5\",\n                                                        save_best_only=True, save_weights_only=True,\n                                                    mode='min')\n    reduce_lr_loss = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.05, patience=5, verbose=1)\n    model = my_model()\n    history = model.fit(\n            X_train, y_train,\n            validation_data=(X_val, y_val),\n            epochs=CFG.EPOCHS,\n            callbacks=[checkpoint, reduce_lr_loss, es],\n            batch_size=CFG.BATCH_SIZE,\n            verbose=1,\n        )\n    model.load_weights(f\"model-{fold}.h5\")\n    oof = model.predict(X_val, batch_size = 2*CFG.BATCH_SIZE)\n    print('fold :', fold, 'CV score =', APS(y_val, oof, average = 'micro'))\n    \n    preds = model.predict(test, batch_size = 2*CFG.BATCH_SIZE)\n    all_preds.append(preds)\n\npreds = np.mean(all_preds, 0)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T12:03:43.977193Z","iopub.execute_input":"2024-05-01T12:03:43.977427Z","iopub.status.idle":"2024-05-01T12:07:11.897795Z","shell.execute_reply.started":"2024-05-01T12:03:43.977407Z","shell.execute_reply":"2024-05-01T12:07:11.896478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"tst = pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet')\ntst['binds'] = 0\ntst.loc[tst['protein_name']=='BRD4', 'binds'] = preds[(tst['protein_name']=='BRD4').values, 0]\ntst.loc[tst['protein_name']=='HSA', 'binds'] = preds[(tst['protein_name']=='HSA').values, 1]\ntst.loc[tst['protein_name']=='sEH', 'binds'] = preds[(tst['protein_name']=='sEH').values, 2]\ntst[['id', 'binds']].to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2024-05-01T12:07:11.898726Z","iopub.status.idle":"2024-05-01T12:07:11.899062Z","shell.execute_reply.started":"2024-05-01T12:07:11.898916Z","shell.execute_reply":"2024-05-01T12:07:11.89893Z"},"trusted":true},"execution_count":null,"outputs":[]}]}