{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA-MICCAI Brain Tumor Radiogenomic Classification","metadata":{}},{"cell_type":"markdown","source":"Thanks to previous great Notebooks.\n\n1. [[TF]: 3D & 2D Model for Brain Tumor Classification][1]\n\n2. [【Brain Tumor】EDA for starter(日本語version)][2]\n\n3. [Efficientnet3D with one MRI type][3]\n\n4. [🧠Brain Tumor 3D [Training]][4]\n\n---\n\n[1]: https://www.kaggle.com/ipythonx/tf-3d-2d-model-for-brain-tumor-classification\n\n[2]: https://www.kaggle.com/chumajin/brain-tumor-eda-for-starter-version\n\n[3]: https://www.kaggle.com/rluethy/efficientnet3d-with-one-mri-type\n\n[4]: https://www.kaggle.com/ammarnassanalhajali/brain-tumor-3d-training/data","metadata":{}},{"cell_type":"markdown","source":"# 0. Settings","metadata":{}},{"cell_type":"code","source":"# Import dependencies \nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \n%matplotlib inline\n\nimport os, sys, glob, gc \nimport math, re, random, time\nfrom tqdm import tqdm \nimport cv2, pydicom\n\nfrom sklearn.model_selection import StratifiedKFold \n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:47.735404Z","iopub.execute_input":"2021-12-02T09:44:47.735743Z","iopub.status.idle":"2021-12-02T09:44:47.743950Z","shell.execute_reply.started":"2021-12-02T09:44:47.735707Z","shell.execute_reply":"2021-12-02T09:44:47.742873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Params\nconfig = {\n    'data_path': '../input/rsna-miccai-brain-tumor-radiogenomic-classification',\n    'model_path': '../input/keras-3d-efficientnet-imagenet-weights-b0b7/efficientnet3d_keras/efficientnet-b0_inp_channel_3_tch_0_top_False.h5',\n    'input_path': '../input', \n    'output_path': './',\n    'num_3d': 16,\n    'img_size': 64,\n    'n_gradients': 16,\n    'nfolds': 5, \n    'batch_size': 16,\n    'learning_rate': 1e-4,\n    'num_epochs': 10\n}\n\nAUTO = tf.data.AUTOTUNE\n\n# For reproducible results    \ndef seed_all(s):\n    random.seed(s)\n    np.random.seed(s)\n    tf.random.set_seed(s)\n    os.environ['TF_CUDNN_DETERMINISTIC'] = '1'\n    os.environ['PYTHONHASHSEED'] = str(s) \nglobal_seed = 42\nseed_all(global_seed)\n\ninput_modality = [\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\"]\nmodality_list = [\"FLAIR\", \"T1w\", \"T2w\"] \n\ntrain_folder = os.path.join(config['data_path'], 'train')\ntest_folder = os.path.join(config['data_path'], 'test')\nsample_submission_path = os.path.join(config['data_path'], 'sample_submission.csv')\n\ntrain_df = pd.read_csv(os.path.join(config['data_path'], 'train_labels.csv')); print(train_df.shape)\nsample_df = pd.read_csv(sample_submission_path); print(sample_df.shape)\ntest_df = sample_df.copy(); print(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:47.745501Z","iopub.execute_input":"2021-12-02T09:44:47.746112Z","iopub.status.idle":"2021-12-02T09:44:47.784309Z","shell.execute_reply.started":"2021-12-02T09:44:47.746076Z","shell.execute_reply":"2021-12-02T09:44:47.783500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. EDA","metadata":{}},{"cell_type":"markdown","source":"## 1.1 Train DataFrame","metadata":{}},{"cell_type":"code","source":"# Getting each folder paths of BraTS21ID\n\ntrain_df['imfolder'] = ['{:05d}'.format(s) for s in train_df['BraTS21ID']]\ntrain_df['path'] = [os.path.join(train_folder, s) for s in train_df['imfolder']]\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:47.786876Z","iopub.execute_input":"2021-12-02T09:44:47.787194Z","iopub.status.idle":"2021-12-02T09:44:47.807437Z","shell.execute_reply.started":"2021-12-02T09:44:47.787168Z","shell.execute_reply":"2021-12-02T09:44:47.806729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Counting the files in FLAIR folder\n\n#input_modality = [\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\"] \ninput_modality = [\"FLAIR\"] \nfor modality in input_modality:   \n    modality_count = []\n    for i in range(len(train_df)):\n        sample_folder = train_df['path'].iloc[i]\n        modality_folder = os.path.join(sample_folder, modality)\n        if os.path.exists(modality_folder):\n            modality_count.append(len(os.listdir(modality_folder)))\n        else:\n            modality_count.append(0)\n        \n    train_df[f'{modality}_count'] = modality_count    \n    \ntrain_df = train_df.query(\"FLAIR_count >= 16\").reset_index()\n    \ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:47.809000Z","iopub.execute_input":"2021-12-02T09:44:47.809318Z","iopub.status.idle":"2021-12-02T09:44:48.491649Z","shell.execute_reply.started":"2021-12-02T09:44:47.809285Z","shell.execute_reply":"2021-12-02T09:44:48.490782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# k-fold (n=5) for cross-validation (I conducted hold-out validation in this notebook, though.)\n\nskf = StratifiedKFold(n_splits=config['nfolds'], shuffle=True, random_state=global_seed)\n\nfor index, (train_index, val_index) in enumerate(skf.split(X=train_df.index, y=train_df.MGMT_value)):\n    train_df.loc[val_index, 'fold'] = index\n    \nprint(train_df.groupby(['fold', train_df.MGMT_value]).size())","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:48.492909Z","iopub.execute_input":"2021-12-02T09:44:48.493394Z","iopub.status.idle":"2021-12-02T09:44:48.507277Z","shell.execute_reply.started":"2021-12-02T09:44:48.493357Z","shell.execute_reply":"2021-12-02T09:44:48.506248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2 Test DataFrame","metadata":{}},{"cell_type":"code","source":"test_df['imfolder'] = ['{:05d}'.format(s) for s in test_df['BraTS21ID']]\ntest_df['path'] = [os.path.join(test_folder, s) for s in test_df['imfolder']]\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:48.510360Z","iopub.execute_input":"2021-12-02T09:44:48.510604Z","iopub.status.idle":"2021-12-02T09:44:48.531127Z","shell.execute_reply.started":"2021-12-02T09:44:48.510582Z","shell.execute_reply":"2021-12-02T09:44:48.530331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#input_modality = [\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\"] \ninput_modality = [\"FLAIR\"] \n\nfor modality in input_modality:   \n    modality_count = []\n    for i in range(len(test_df)):\n        sample_folder = test_df['path'].iloc[i]\n        modality_folder = os.path.join(sample_folder, modality)\n        if os.path.exists(modality_folder):\n            modality_count.append(len(os.listdir(modality_folder)))\n        else:\n            modality_count.append(0)\n        \n    test_df[f'{modality}_count'] = modality_count    \n    \ntest_df = test_df.query(\"FLAIR_count >= 16\").reset_index()\n\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:48.533402Z","iopub.execute_input":"2021-12-02T09:44:48.533641Z","iopub.status.idle":"2021-12-02T09:44:48.648295Z","shell.execute_reply.started":"2021-12-02T09:44:48.533619Z","shell.execute_reply":"2021-12-02T09:44:48.647353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. DataLoader","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Train Dataset","metadata":{}},{"cell_type":"code","source":"def get_img_path_3d(df, index, mri_type='FLAIR'):\n    patient_id = df['BraTS21ID'][index]\n    patient_path = df['path'][index]\n    modality_path = os.path.join(patient_path, mri_type)\n    total_img_num = df[f'{mri_type}_count'][index]\n    \n    files = sorted(glob.glob(f\"{modality_path}/*.dcm\"), \n                   key=lambda var:[int(x) if x.isdigit() else x for x in re.findall(r'[^0-9]|[0-9]+', var)])\n    \n    mid_num = total_img_num // 2\n    num_3d2 = config['num_3d'] // 2\n    start_idx = max(0, mid_num - num_3d2)\n    end_idx = min(len(files), mid_num + num_3d2)\n    \n    target_file_paths = files[start_idx:end_idx]\n    \n    return target_file_paths\n\n@tf.function\ndef preprocessing_img(img):\n    #img = img / tf.math.reduce_max(img)\n    img = tf.expand_dims(img, -1)\n    img = tf.image.resize(img, [config['img_size'], config['img_size']])\n    img = tf.expand_dims(img, -2)\n    return img\n\n    \nclass ImageGenerator(tf.keras.utils.Sequence):\n    def __init__(self, df, mri_type='FLAIR'):\n        self.df = df\n        self.mri_type = mri_type\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        paths = get_img_path_3d(self.df, index)\n        img_list = []\n        for path in paths:\n            dicom = pydicom.read_file(path)\n            img = dicom.pixel_array\n            img = tf.convert_to_tensor(img, dtype=tf.float32)\n            img = preprocessing_img(img)\n            img_list.append(img)\n        img_3d = tf.concat(img_list, axis=-2)\n        return img_3d\n    \n    \ndef parse(x):\n    result = tf.io.parse_tensor(x, out_type=tf.float32)\n    result = tf.reshape(result, [config['img_size'], config['img_size'], config['num_3d'], 1])\n    return result\n\n\ndef build_3d_train_dataloader(train_df, p_fold=0):\n    p_train = train_df.query(f'fold != {p_fold}').reset_index(drop=True)\n    p_valid = train_df.query(f'fold == {p_fold}').reset_index(drop=True)\n\n    AUTOTUNE = tf.data.experimental.AUTOTUNE\n\n    train_datasets = []\n    for mode, df in zip(['train', 'valid'], [p_train, p_valid]):\n        i_g = ImageGenerator(df)\n        img_ds = tf.data.Dataset.from_generator(lambda: map(tuple, i_g),\n                                                output_types=(tf.float32),\n                                                output_shapes=(tf.TensorShape([config['img_size'], config['img_size'], config['num_3d'], 1])),\n                                                 )\n        \n        serial_ds = img_ds.map(tf.io.serialize_tensor)\n\n        if not os.path.exists(f'{mode}-{p_fold}-img.tfrec'):\n            img_tfrec = tf.data.experimental.TFRecordWriter(f'{mode}-{p_fold}-img.tfrec')\n            img_tfrec.write(serial_ds)\n        serial_ds = tf.data.TFRecordDataset(f'{mode}-{p_fold}-img.tfrec')\n        serial_ds = serial_ds.map(parse, num_parallel_calls=AUTOTUNE)\n\n        labels = df['MGMT_value']\n        label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(labels, tf.int32))\n\n        ds = tf.data.Dataset.zip((img_ds, label_ds))\n        \n        ds = ds.cache(filename=f'./cache.tf-{mode}-{p_fold}-data')\n        if mode == 'train':\n            train_count = len(df)\n            ds = ds.shuffle(buffer_size=train_count)\n        ds = ds.batch(config['batch_size'], drop_remainder=True)\n        ds = ds.prefetch(buffer_size=AUTOTUNE)\n        train_datasets.append(ds)\n\n    return train_datasets","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:48.650779Z","iopub.execute_input":"2021-12-02T09:44:48.651091Z","iopub.status.idle":"2021-12-02T09:44:48.677609Z","shell.execute_reply.started":"2021-12-02T09:44:48.651060Z","shell.execute_reply":"2021-12-02T09:44:48.676762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Building Dataset\np_fold = 0\n\ntrain_datasets = build_3d_train_dataloader(train_df, p_fold=p_fold)\ntrain_ds = train_datasets[0]\nvalid_ds = train_datasets[1]\n\nfor d, l in train_ds.take(1):\n    print('Train Data shape: ', d.shape)\n    print('Train Label shape: ', l.shape)\n    \nfor d, l in valid_ds.take(1):\n    print('Valid Data shape: ', d.shape)\n    print('Valid Label shape: ', l.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:48.679936Z","iopub.execute_input":"2021-12-02T09:44:48.680368Z","iopub.status.idle":"2021-12-02T09:44:48.794801Z","shell.execute_reply.started":"2021-12-02T09:44:48.680318Z","shell.execute_reply":"2021-12-02T09:44:48.793983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Test Dataset","metadata":{}},{"cell_type":"code","source":"# TestDataset without Labels\ndef build_3d_test_dataloader(test_df):\n    AUTOTUNE = tf.data.experimental.AUTOTUNE\n\n    i_g = ImageGenerator(test_df)\n    img_ds = tf.data.Dataset.from_generator(lambda: map(tuple, i_g),\n                                         output_types=(tf.float32),\n                                         output_shapes=(tf.TensorShape([config['img_size'], config['img_size'], config['num_3d'], 1])),\n                                                 )\n    serial_ds = img_ds.map(tf.io.serialize_tensor)\n\n    if not os.path.exists('test-img.tfrec'):\n        img_tfrec = tf.data.experimental.TFRecordWriter('test-img.tfrec')\n        img_tfrec.write(serial_ds)\n    serial_ds = tf.data.TFRecordDataset('test-img.tfrec')\n    test_ds = serial_ds.map(parse, num_parallel_calls=AUTOTUNE)\n\n    test_ds = test_ds.cache(filename='./cache.tf-test-data')\n    test_ds = test_ds.batch(config['batch_size'], drop_remainder=False)\n    test_ds = test_ds.prefetch(buffer_size=AUTOTUNE)\n\n    return test_ds","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:48.796166Z","iopub.execute_input":"2021-12-02T09:44:48.796507Z","iopub.status.idle":"2021-12-02T09:44:48.804985Z","shell.execute_reply.started":"2021-12-02T09:44:48.796473Z","shell.execute_reply":"2021-12-02T09:44:48.804016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = build_3d_test_dataloader(test_df)\n\nfor d in test_ds.take(1):\n    print('Test Data shape: ', d.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:48.806497Z","iopub.execute_input":"2021-12-02T09:44:48.806890Z","iopub.status.idle":"2021-12-02T09:44:48.861782Z","shell.execute_reply.started":"2021-12-02T09:44:48.806854Z","shell.execute_reply":"2021-12-02T09:44:48.860875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Train","metadata":{}},{"cell_type":"markdown","source":"## 3.1 Model","metadata":{}},{"cell_type":"code","source":"def get_3d_model(width=config['img_size'], height=config['img_size'], depth=config['num_3d']):\n    \"\"\"Build a 3D convolutional neural network model.\"\"\"\n\n    inputs = keras.Input((width, height, depth, 1))\n    \n    x = layers.Conv3D(filters=32, kernel_size=3, padding='same', activation=\"relu\")(inputs)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Conv3D(filters=32, kernel_size=3, padding='same', activation=\"relu\")(inputs)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Conv3D(filters=64, kernel_size=3, padding='same', activation=\"relu\")(inputs)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.01)(x)\n    \n    x = layers.Conv3D(filters=128, kernel_size=3, padding='same', activation=\"relu\")(x)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.02)(x)\n\n    x = layers.Conv3D(filters=256, kernel_size=3, padding='same', activation=\"relu\")(x)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.03)(x)\n\n    x = layers.Conv3D(filters=512, kernel_size=3, padding='same', activation=\"relu\")(x)\n    x = layers.MaxPool3D(pool_size=2)(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.04)(x)\n\n    x = layers.GlobalAveragePooling3D()(x)\n    x = layers.Dense(units=1024, activation=\"relu\")(x)\n    x = layers.Dropout(0.08)(x)\n\n    outputs = layers.Dense(units=1, activation=\"sigmoid\")(x)\n\n    model = keras.Model(inputs, outputs, name=\"3dcnn\")\n\n    return model\n\n\nmodel = get_3d_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:48.863267Z","iopub.execute_input":"2021-12-02T09:44:48.863611Z","iopub.status.idle":"2021-12-02T09:44:49.037808Z","shell.execute_reply.started":"2021-12-02T09:44:48.863575Z","shell.execute_reply":"2021-12-02T09:44:49.036978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save Model here","metadata":{}},{"cell_type":"code","source":"# Save the entire model as a SavedModel.\n!mkdir -p saved_model\nmodel.save('saved_model/my_model')","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:49.039520Z","iopub.execute_input":"2021-12-02T09:44:49.039882Z","iopub.status.idle":"2021-12-02T09:44:53.021347Z","shell.execute_reply.started":"2021-12-02T09:44:49.039846Z","shell.execute_reply":"2021-12-02T09:44:53.020279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BrainTumorModel3D(tf.keras.Model):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)        \n        self.cnn = get_3d_model()\n        \n    @tf.function\n    def call(self, input_tensor, training=False, **kwargs):\n        x = self.cnn(input_tensor)\n        return x\n    \n    def build_graph(self, raw_shape):\n        x = tf.keras.layers.Input(shape=raw_shape)\n        return tf.keras.Model(inputs=[x], outputs=self.call(x))\n\n\nif tf.test.is_gpu_available():\n    device_name = tf.test.gpu_device_name()\nelse:\n    device_name = 'cpu:0'\n\nwith tf.device(device_name):\n    model = BrainTumorModel3D()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:53.029942Z","iopub.execute_input":"2021-12-02T09:44:53.030225Z","iopub.status.idle":"2021-12-02T09:44:53.190226Z","shell.execute_reply.started":"2021-12-02T09:44:53.030195Z","shell.execute_reply":"2021-12-02T09:44:53.189432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.Adam(learning_rate=config['learning_rate'])\n\nloss_fn = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n\ntrain_acc_metric = tf.keras.metrics.BinaryAccuracy()\nval_acc_metric = tf.keras.metrics.BinaryAccuracy()\n\nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)\nmodel_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n    filepath=config['output_path'],\n    save_weights_only=True,\n    monitor='val_loss',\n    mode='min',\n    save_best_only=True)\n\n@tf.function\ndef train_step(x, y):\n    \n    with tf.GradientTape() as tape:\n        pred_y = model(x, training=True)\n        train_loss = loss_fn(y, pred_y)\n        \n    grads = tape.gradient(train_loss, model.trainable_weights)\n    \n    optimizer.apply_gradients(zip(grads, model.trainable_weights))\n    \n    train_acc_metric.update_state(y_true=y, y_pred=pred_y)\n    \n    return train_loss\n\n\n@tf.function\ndef valid_step(x, y):\n    pred_y = model(x, training=False)\n    val_loss = loss_fn(y, pred_y)\n    \n    val_acc_metric.update_state(y_true=y, y_pred=pred_y)\n    \n    return val_loss","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:53.191480Z","iopub.execute_input":"2021-12-02T09:44:53.192023Z","iopub.status.idle":"2021-12-02T09:44:53.231475Z","shell.execute_reply.started":"2021-12-02T09:44:53.191981Z","shell.execute_reply":"2021-12-02T09:44:53.230759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_history = []\nvalid_history = []\n\nfor epoch in range(config['num_epochs']):\n    t = time.time()\n    \n    train_loss_list = []\n    val_loss_list = []\n    \n    for x, y in train_ds:\n        train_batch_loss = train_step(x, y)\n        train_loss_list.append(train_batch_loss)\n        \n    for x, y in valid_ds:\n        val_batch_loss = valid_step(x, y)\n        val_loss_list.append(val_batch_loss)\n        \n    train_loss = sum(train_loss_list) / len(train_loss_list)\n    val_loss = sum(val_loss_list) / len(val_loss_list)\n    \n    train_acc = train_acc_metric.result()\n    val_acc = val_acc_metric.result()\n    \n    train_history.append(train_loss)\n    valid_history.append(val_loss)\n    \n    template = 'ETA: {} -- epoch: {}, loss: {}  acc: {}  val_loss: {}  val_acc: {}\\n'\n    print(template.format(\n                   round((time.time() -  t) / 60, 2), epoch+1,\n                   (train_loss, '.3f'), (train_acc, '.3f'),\n                   (val_loss, '.3f'), (val_acc, '.3f'))\n         )\n    \n    train_acc_metric.reset_states()\n    val_acc_metric.reset_states()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:44:53.233144Z","iopub.execute_input":"2021-12-02T09:44:53.233445Z","iopub.status.idle":"2021-12-02T09:45:04.075322Z","shell.execute_reply.started":"2021-12-02T09:44:53.233421Z","shell.execute_reply":"2021-12-02T09:45:04.074401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Prediction ","metadata":{}},{"cell_type":"code","source":"proba = model.predict(test_ds, batch_size=config['batch_size'], verbose=1)\nproba","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:45:04.076651Z","iopub.execute_input":"2021-12-02T09:45:04.077016Z","iopub.status.idle":"2021-12-02T09:45:04.408778Z","shell.execute_reply.started":"2021-12-02T09:45:04.076987Z","shell.execute_reply":"2021-12-02T09:45:04.407952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['prediction'] = proba\nsample_df['MGMT_value'] = test_df['prediction']\nsample_df","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:45:04.410086Z","iopub.execute_input":"2021-12-02T09:45:04.410447Z","iopub.status.idle":"2021-12-02T09:45:04.426643Z","shell.execute_reply.started":"2021-12-02T09:45:04.410411Z","shell.execute_reply":"2021-12-02T09:45:04.425652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:45:04.428102Z","iopub.execute_input":"2021-12-02T09:45:04.428588Z","iopub.status.idle":"2021-12-02T09:45:04.434920Z","shell.execute_reply.started":"2021-12-02T09:45:04.428550Z","shell.execute_reply":"2021-12-02T09:45:04.433782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# cf. Gradient Accumulation Model","metadata":{}},{"cell_type":"code","source":"class GradAcumModel(tf.keras.Model):\n    def __init__(self, model, n_gradients=config['n_gradients'], *args, **kwargs):\n        super(GradAcumModel, self).__init__(*args, **kwargs)\n        self.model = model\n        self.n_gradients = tf.constant(n_gradients, dtype=tf.int32)\n        self.n_acum_step = tf.Variable(0, dtype=tf.int32, trainable=False)\n        self.gradient_accumulation = [tf.Variable(tf.zeros_like(v, dtype=tf.float32),\n                                                  trainable=False)\n                                       for v in self.model.trainable_variables]\n\n    @tf.function\n    def train_step(self, data):\n        self.n_acum_step.assign_add(1)\n        images, labels = data\n\n        with tf.GradientTape() as tape:\n            predictions = self.model(images, training=True)\n            loss = self.compiled_loss(labels, predictions)\n\n        gradients = tape.gradient(loss, self.model.trainable_variables)\n\n        for i in range(len(self.gradient_accumulation)):\n            self.gradient_accumulation[i].assign_add(gradients[i])\n\n        # If n_acum_step reach the n_gradients then we apply accumulated gradients -\n        # - to update the variables otherwise do nothing\n        tf.cond(tf.equal(self.n_acum_step, self.n_gradients),\n                self.apply_accu_gradients, lambda: None)\n        \n        self.compiled_metrics.update_state(labels, predictions)\n        return {m.name: m.result() for m in self.metrics}\n\n    def apply_accu_gradients(self):\n        self.optimizer.apply_gradients(zip(self.gradient_accumulation,\n                                           self.model.trainable_variables))\n        \n        # Reset\n        self.n_acum_step.assign(0)\n        for i in range(len(self.gradient_accumulation)):\n            self.gradient_accumulation[i].assign(\n                tf.zeros_like(self.model.trainable_variables[i], dtype=tf.float32)\n            )\n\n    @tf.function\n    def test_step(self, data):\n        images, labels = data\n\n        predictions = self.model(images, training=False)\n        loss = self.compiled_loss(labels, predictions)\n        self.compiled_metrics.update_state(labels, predictions)\n        return {m.name: m.result() for m in self.metrics}\n\n    def call(self, inputs, *args, **kwargs):\n        return self.model(inputs)\n\nwith tf.device(device_name):\n    grad_acum_model = GradAcumModel(model, n_gradients=4)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:45:04.436510Z","iopub.execute_input":"2021-12-02T09:45:04.437097Z","iopub.status.idle":"2021-12-02T09:45:04.467281Z","shell.execute_reply.started":"2021-12-02T09:45:04.436911Z","shell.execute_reply":"2021-12-02T09:45:04.466505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grad_acum_model.compile(\n    loss=tf.keras.losses.BinaryCrossentropy(from_logits=False),\n    optimizer='adam',\n    metrics=[tf.keras.metrics.BinaryAccuracy(name='acc'), ],\n)\n\nmodel_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n    filepath=config['output_path'],\n    save_weights_only=True,\n    monitor='val_loss',\n    mode='min',\n    save_best_only=True)\n\n# Train the model\n#grad_acum_model.fit(\n#    train_ds,\n#    epochs=config['num_epochs'],\n#    validation_data=valid_ds,\n#    callbacks=[model_checkpoint],\n#    verbose=1\n#)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:45:04.468547Z","iopub.execute_input":"2021-12-02T09:45:04.468945Z","iopub.status.idle":"2021-12-02T09:45:04.489138Z","shell.execute_reply.started":"2021-12-02T09:45:04.468911Z","shell.execute_reply":"2021-12-02T09:45:04.488354Z"},"trusted":true},"execution_count":null,"outputs":[]}]}