{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.8.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":67356,"databundleVersionId":8006601,"sourceType":"competition"},{"sourceId":8275617,"sourceType":"datasetVersion","datasetId":4914065},{"sourceId":8819963,"sourceType":"datasetVersion","datasetId":5214038}],"dockerImageVersionId":30514,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initialization\n\nThis notebook is forked and inspired from the [BELKA 1DCNN Starter](https://www.kaggle.com/code/ahmedelfazouan/belka-1dcnn-starter-with-all-data/notebook).\n\nThis original notebook attempted to encoded the smiles of all the train & test set and saved it locally, this may take up to 1 hour on TPU for each fold. While the original notebook only utilize one fold, there are only 1 fold training, due to the limit of computational power. Therefore, I pre-tained one model for each fold and store the model weights locally so that the prediction results could be combined to make better LB score. The models are stored [here](https://www.kaggle.com/datasets/hugowjd/1dcnn-models-for-belka-competition).\n\nThe encoded data is stored [here](https://www.kaggle.com/datasets/ahmedelfazouan/belka-enc-dataset) , \n\nHow to improve :\n* Change the fold size (better machine may deal with smaller fold number)\n* Try another model like Transformer, or LSTM.\n* Train for more epochs for each folds.\n* Add more features like a one hot encoding of bb2 or bb3.\n* And of ensembling with other models.\n\nVersion Information\n* Version 9: 15 folds with 15 epochs each. Model-0 to Model-14 are weights for 15-epochs pretrained model weights\n* Version 24: 15 folds with 20 epochs each. Model-15 to Model-29 are weights for 20-epochs pretrained model weights","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/1dcnn-models-for-belka-competition'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:36:25.997666Z","iopub.execute_input":"2024-06-25T19:36:25.998130Z","iopub.status.idle":"2024-06-25T19:36:26.418135Z","shell.execute_reply.started":"2024-06-25T19:36:25.998094Z","shell.execute_reply":"2024-06-25T19:36:26.417332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install fastparquet -q","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-06-25T19:36:30.712538Z","iopub.execute_input":"2024-06-25T19:36:30.713073Z","iopub.status.idle":"2024-06-25T19:36:36.800463Z","shell.execute_reply.started":"2024-06-25T19:36:30.713033Z","shell.execute_reply":"2024-06-25T19:36:36.799364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport random\nimport joblib\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import average_precision_score as APS\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-25T19:36:36.802424Z","iopub.execute_input":"2024-06-25T19:36:36.802688Z","iopub.status.idle":"2024-06-25T19:36:37.457899Z","shell.execute_reply.started":"2024-06-25T19:36:36.802662Z","shell.execute_reply":"2024-06-25T19:36:37.456748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n\n    PREPROCESS = False\n    PRETRAINED = True   # True -> use already trained model weights / False -> train new weights\n    EPOCHS = 20\n    BATCH_SIZE = 4096\n    LR = 1e-3\n    WD = 0.05\n    # Number of folds\n    NBR_FOLDS = 15\n\n    # Only the first fold selected\n    SELECTED_FOLDS = [10,11,12,13,14]\n    EXIST_MODELS = range(30)\n\n    SEED = 2222","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:36:37.458823Z","iopub.execute_input":"2024-06-25T19:36:37.459149Z","iopub.status.idle":"2024-06-25T19:36:37.464903Z","shell.execute_reply.started":"2024-06-25T19:36:37.459120Z","shell.execute_reply":"2024-06-25T19:36:37.463983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ndef set_seeds(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n    np.random.seed(seed)\n\nset_seeds(seed=CFG.SEED)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-06-25T19:36:37.466759Z","iopub.execute_input":"2024-06-25T19:36:37.467031Z","iopub.status.idle":"2024-06-25T19:37:16.176301Z","shell.execute_reply.started":"2024-06-25T19:36:37.467008Z","shell.execute_reply":"2024-06-25T19:37:16.175084Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=\"local\") # \"local\" for 1VM TPU\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"Running on TPU\")\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\nexcept tf.errors.NotFoundError:\n    print(\"Not on TPU\")","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:37:16.177481Z","iopub.execute_input":"2024-06-25T19:37:16.177922Z","iopub.status.idle":"2024-06-25T19:37:25.047016Z","shell.execute_reply.started":"2024-06-25T19:37:16.177898Z","shell.execute_reply":"2024-06-25T19:37:25.046186Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"if CFG.PREPROCESS:\n    enc = {'l': 1, 'y': 2, '@': 3, '3': 4, 'H': 5, 'S': 6, 'F': 7, 'C': 8, 'r': 9, 's': 10, '/': 11, 'c': 12, 'o': 13,\n           '+': 14, 'I': 15, '5': 16, '(': 17, '2': 18, ')': 19, '9': 20, 'i': 21, '#': 22, '6': 23, '8': 24, '4': 25, '=': 26,\n           '1': 27, 'O': 28, '[': 29, 'D': 30, 'B': 31, ']': 32, 'N': 33, '7': 34, 'n': 35, '-': 36}\n    train_raw = pd.read_parquet('/kaggle/input/leash-BELKA/train.parquet')\n    smiles = train_raw[train_raw['protein_name']=='BRD4']['molecule_smiles'].values\n    assert (smiles!=train_raw[train_raw['protein_name']=='HSA']['molecule_smiles'].values).sum() == 0\n    assert (smiles!=train_raw[train_raw['protein_name']=='sEH']['molecule_smiles'].values).sum() == 0\n    def encode_smile(smile):\n        tmp = [enc[i] for i in smile]\n        tmp = tmp + [0]*(142-len(tmp))\n        return np.array(tmp).astype(np.uint8)\n\n    smiles_enc = joblib.Parallel(n_jobs=96)(joblib.delayed(encode_smile)(smile) for smile in tqdm(smiles))\n    smiles_enc = np.stack(smiles_enc)\n    train = pd.DataFrame(smiles_enc, columns = [f'enc{i}' for i in range(142)])\n    train['bind1'] = train_raw[train_raw['protein_name']=='BRD4']['binds'].values\n    train['bind2'] = train_raw[train_raw['protein_name']=='HSA']['binds'].values\n    train['bind3'] = train_raw[train_raw['protein_name']=='sEH']['binds'].values\n    train.to_parquet('train_enc.parquet')\n\n    test_raw = pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet')\n    smiles = test_raw['molecule_smiles'].values\n\n    smiles_enc = joblib.Parallel(n_jobs=96)(joblib.delayed(encode_smile)(smile) for smile in tqdm(smiles))\n    smiles_enc = np.stack(smiles_enc)\n    test = pd.DataFrame(smiles_enc, columns = [f'enc{i}' for i in range(142)])\n    test.to_parquet('test_enc.parquet')\n\nelse:\n    train = pd.read_parquet('/kaggle/input/belka-enc-dataset/train_enc.parquet')\n    test = pd.read_parquet('/kaggle/input/belka-enc-dataset/test_enc.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:37:25.048223Z","iopub.execute_input":"2024-06-25T19:37:25.048505Z","iopub.status.idle":"2024-06-25T19:39:13.741411Z","shell.execute_reply.started":"2024-06-25T19:37:25.048479Z","shell.execute_reply":"2024-06-25T19:39:13.740245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Shape of Train set:\", train.shape)\ntrain.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:39:13.742504Z","iopub.execute_input":"2024-06-25T19:39:13.743010Z","iopub.status.idle":"2024-06-25T19:39:13.764276Z","shell.execute_reply.started":"2024-06-25T19:39:13.742984Z","shell.execute_reply":"2024-06-25T19:39:13.763506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Shape of Test set:\", test.shape)\ntest.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:39:13.765209Z","iopub.execute_input":"2024-06-25T19:39:13.765478Z","iopub.status.idle":"2024-06-25T19:39:13.778689Z","shell.execute_reply.started":"2024-06-25T19:39:13.765453Z","shell.execute_reply":"2024-06-25T19:39:13.777989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"# 1D-CNN model\ndef OneDCNN_model():\n    with strategy.scope():\n        INP_LEN = 142\n        NUM_FILTERS = 32\n        hidden_dim = 128\n\n        inputs = tf.keras.layers.Input(shape=(INP_LEN,), dtype='int32')\n        x = tf.keras.layers.Embedding(input_dim=36, output_dim=hidden_dim, input_length=INP_LEN, mask_zero = True)(inputs)\n        x = tf.keras.layers.Conv1D(filters=NUM_FILTERS, kernel_size=3,  activation='relu', padding='valid',  strides=1)(x)\n        x = tf.keras.layers.Conv1D(filters=NUM_FILTERS*2, kernel_size=3,  activation='relu', padding='valid',  strides=1)(x)\n        x = tf.keras.layers.Conv1D(filters=NUM_FILTERS*3, kernel_size=3,  activation='relu', padding='valid',  strides=1)(x)\n        x = tf.keras.layers.GlobalMaxPooling1D()(x)\n\n        x = tf.keras.layers.Dense(1024, activation='relu')(x)\n        x = tf.keras.layers.Dropout(0.1)(x)\n        x = tf.keras.layers.Dense(1024, activation='relu')(x)\n        x = tf.keras.layers.Dropout(0.1)(x)\n        x = tf.keras.layers.Dense(512, activation='relu')(x)\n        x = tf.keras.layers.Dropout(0.1)(x)\n\n        outputs = tf.keras.layers.Dense(3, activation='sigmoid')(x)\n\n        model = tf.keras.models.Model(inputs = inputs, outputs = outputs)\n        optimizer = tf.keras.optimizers.Adam(learning_rate=CFG.LR, weight_decay = CFG.WD)\n        loss = 'binary_crossentropy'\n        weighted_metrics = [tf.keras.metrics.AUC(curve='PR', name = 'avg_precision')]\n        model.compile(\n        loss=loss,\n        optimizer=optimizer,\n        weighted_metrics=weighted_metrics,\n        )\n        return model","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:39:13.780765Z","iopub.execute_input":"2024-06-25T19:39:13.781032Z","iopub.status.idle":"2024-06-25T19:39:13.792472Z","shell.execute_reply.started":"2024-06-25T19:39:13.781010Z","shell.execute_reply":"2024-06-25T19:39:13.791746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"%time\nFEATURES = [f'enc{i}' for i in range(142)] # The first 142 encoded columns as features\nTARGETS = ['bind1', 'bind2', 'bind3'] # Three types of binds as targets\nskf = StratifiedKFold(n_splits = CFG.NBR_FOLDS, shuffle = True, random_state = 42) \n                                                                                    \nif CFG.PRETRAINED:\n    print(\"Model Pretrained. Skip Training process.\")\nelse:\n    for fold,(train_idx, valid_idx) in enumerate(skf.split(train, train[TARGETS].sum(1))):\n        if fold in CFG.SELECTED_FOLDS:\n            print(f\"Working on fold {fold}\")\n            X_train = train.loc[train_idx, FEATURES]\n            y_train = train.loc[train_idx, TARGETS]\n            X_val = train.loc[valid_idx, FEATURES]\n            y_val = train.loc[valid_idx, TARGETS]\n\n            es = tf.keras.callbacks.EarlyStopping(patience=5, monitor=\"val_loss\", mode='min', verbose=1)\n            checkpoint = tf.keras.callbacks.ModelCheckpoint(monitor='val_loss', filepath=f\"model-{fold}.h5\",\n                                                                save_best_only=True, save_weights_only=True,\n                                                            mode='min')\n            reduce_lr_loss = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.05, patience=5, verbose=1)\n            model = OneDCNN_model()\n            history = model.fit(\n                X_train, y_train,\n                validation_data=(X_val, y_val),\n                epochs=CFG.EPOCHS,\n                callbacks=[checkpoint, reduce_lr_loss, es],\n                batch_size=CFG.BATCH_SIZE,\n                verbose=1,\n            )\n            train_loss = history.history['loss']\n            val_loss = history.history.get('val_loss', [])  # Use get to avoid errors if validation loss is not available\n\n            train_prec = history.history['avg_precision']\n            val_prec = history.history.get('val_avg_precision', [])  # Similarly for validation accuracy\n            plt.figure(figsize=(12, 6))\n\n            plt.subplot(1, 2, 1)\n            plt.plot(train_loss, label='Train')\n            if val_loss:\n                plt.plot(val_loss, label='Validation')\n            plt.title('Model Loss')\n            plt.ylabel('Loss')\n            plt.xlabel('Epoch')\n            plt.legend(loc='upper right')\n\n            # Plot training & validation accuracy values\n            plt.subplot(1, 2, 2)\n            plt.plot(train_prec, label='Train')\n            if val_prec:\n                plt.plot(val_prec, label='Validation')\n            plt.title('Model Accuracy')\n            plt.ylabel('Accuracy')\n            plt.xlabel('Epoch')\n            plt.legend(loc='lower right')\n\n            plt.tight_layout()\n            plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:39:13.793299Z","iopub.execute_input":"2024-06-25T19:39:13.793524Z","iopub.status.idle":"2024-06-25T19:39:13.806349Z","shell.execute_reply.started":"2024-06-25T19:39:13.793505Z","shell.execute_reply":"2024-06-25T19:39:13.805627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"all_preds = []\nif CFG.PRETRAINED:\n    for fold in CFG.EXIST_MODELS:\n        model = OneDCNN_model()\n        print(f\"loading weight model-{fold}.h5\")\n        model.load_weights(f\"/kaggle/input/1dcnn-models-for-belka-competition/model-{fold}.h5\")\n        preds = model.predict(test, batch_size = 2*CFG.BATCH_SIZE)\n        all_preds.append(preds)\nelse:\n    for fold in CFG.SELECTED_FOLDS:\n        model = OneDCNN_model()\n        print(f\"loading weight model-{fold}.h5\")\n        model.load_weights(f\"model-{fold}.h5\")\n        preds = model.predict(test, batch_size = 2*CFG.BATCH_SIZE)\n        all_preds.append(preds)\npreds = np.mean(all_preds, 0)","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:39:33.695635Z","iopub.execute_input":"2024-06-25T19:39:33.696063Z","iopub.status.idle":"2024-06-25T19:42:50.011698Z","shell.execute_reply.started":"2024-06-25T19:39:33.696032Z","shell.execute_reply":"2024-06-25T19:42:50.010610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Shape of Predictions: {len(preds)} * {len(preds[0])}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:43:30.088876Z","iopub.execute_input":"2024-06-25T19:43:30.089266Z","iopub.status.idle":"2024-06-25T19:43:30.094622Z","shell.execute_reply.started":"2024-06-25T19:43:30.089237Z","shell.execute_reply":"2024-06-25T19:43:30.093765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot","metadata":{}},{"cell_type":"code","source":"\n# Assuming 'history' is the object returned from model.fit()\nif CFG.PRETRAINED:\n    print(\"Pretrained -> no plot\")\nelse:\n    train_loss = history.history['loss']\n    val_loss = history.history.get('val_loss', [])  # Use get to avoid errors if validation loss is not available\n\n    train_prec = history.history['avg_precision']\n    val_prec = history.history.get('val_avg_precision', [])  # Similarly for validation accuracy\n    plt.figure(figsize=(12, 6))\n\n    plt.subplot(1, 2, 1)\n    plt.plot(train_loss, label='Train')\n    if val_loss:\n        plt.plot(val_loss, label='Validation')\n    plt.title('Model Loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(loc='upper right')\n\n    # Plot training & validation accuracy values\n    plt.subplot(1, 2, 2)\n    plt.plot(train_prec, label='Train')\n    if val_prec:\n        plt.plot(val_prec, label='Validation')\n    plt.title('Model Accuracy')\n    plt.ylabel('Accuracy')\n    plt.xlabel('Epoch')\n    plt.legend(loc='lower right')\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:43:31.825575Z","iopub.execute_input":"2024-06-25T19:43:31.825920Z","iopub.status.idle":"2024-06-25T19:43:33.326205Z","shell.execute_reply.started":"2024-06-25T19:43:31.825894Z","shell.execute_reply":"2024-06-25T19:43:33.325198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"tst = pd.read_parquet('/kaggle/input/leash-BELKA/test.parquet')\ntst['binds'] = 0\ntst.loc[tst['protein_name']=='BRD4', 'binds'] = preds[(tst['protein_name']=='BRD4').values, 0]\ntst.loc[tst['protein_name']=='HSA', 'binds'] = preds[(tst['protein_name']=='HSA').values, 1]\ntst.loc[tst['protein_name']=='sEH', 'binds'] = preds[(tst['protein_name']=='sEH').values, 2]\ntst[['id', 'binds']].to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2024-06-25T19:43:34.766372Z","iopub.execute_input":"2024-06-25T19:43:34.766729Z","iopub.status.idle":"2024-06-25T19:43:41.727215Z","shell.execute_reply.started":"2024-06-25T19:43:34.766697Z","shell.execute_reply":"2024-06-25T19:43:41.726244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}