{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9138256,"sourceType":"datasetVersion","datasetId":5517720},{"sourceId":92043,"sourceType":"modelInstanceVersion","modelInstanceId":76796,"modelId":101447}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport pydicom\nimport numpy as np\nimport os\nimport glob\nfrom tqdm import tqdm\nimport warnings\nimport tensorflow as tf\nimport tensorflow.keras as keras\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications import EfficientNetV2B0\nfrom tensorflow.keras.initializers import GlorotUniform, HeNormal\nfrom sklearn.preprocessing import LabelEncoder\nfrom tensorflow.keras.losses import BinaryCrossentropy\n\nimport SimpleITK as sitk\n\n%matplotlib inline\nnp.random.seed(42)\ntf.random.set_seed(42)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-15T01:47:10.188096Z","iopub.execute_input":"2024-08-15T01:47:10.188639Z","iopub.status.idle":"2024-08-15T01:47:10.203601Z","shell.execute_reply.started":"2024-08-15T01:47:10.188604Z","shell.execute_reply":"2024-08-15T01:47:10.200680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use this cell to empty the output folder\n# !rm -rf /kaggle/working/*","metadata":{"execution":{"iopub.status.busy":"2024-08-15T01:47:10.207183Z","iopub.execute_input":"2024-08-15T01:47:10.207555Z","iopub.status.idle":"2024-08-15T01:47:10.233650Z","shell.execute_reply.started":"2024-08-15T01:47:10.207526Z","shell.execute_reply":"2024-08-15T01:47:10.231448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification'\ntrain_csv_path = os.path.join(root_path, 'train.csv')\ntrain_csv = pd.read_csv(train_csv_path)\ncoordinates_path = os.path.join(root_path, 'train_label_coordinates.csv')\ndf_coor = pd.read_csv(coordinates_path)","metadata":{"execution":{"iopub.status.busy":"2024-08-15T01:47:10.235656Z","iopub.execute_input":"2024-08-15T01:47:10.236104Z","iopub.status.idle":"2024-08-15T01:47:10.408239Z","shell.execute_reply.started":"2024-08-15T01:47:10.236067Z","shell.execute_reply":"2024-08-15T01:47:10.406877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_description_path = os.path.join(root_path, 'train_series_descriptions.csv')\ntrain_meta_df = pd.read_csv(train_description_path)\ntrain_meta_df = train_meta_df.drop_duplicates(subset=['study_id', 'series_description'])\ntrain_images_path = os.path.join(root_path, 'train_images')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read MRI volume\n# INPUT: A folder consisting of MRI slices\n# for example: /kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084\"\n# OUTPUT: Image \ndef readMRIVolume(mri_volume_path):\n    reader = sitk.ImageSeriesReader()\n    dicom_files = reader.GetGDCMSeriesFileNames(mri_volume_path)\n    reader.SetFileNames(dicom_files)\n    retrieved_mri_volume = reader.Execute()\n    return retrieved_mri_volume","metadata":{"execution":{"iopub.status.busy":"2024-08-15T01:47:10.409480Z","iopub.execute_input":"2024-08-15T01:47:10.410749Z","iopub.status.idle":"2024-08-15T01:47:10.418229Z","shell.execute_reply.started":"2024-08-15T01:47:10.410707Z","shell.execute_reply":"2024-08-15T01:47:10.416775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resample images to uniform voxel spacing\n# INPUT: Image\n# OUTPUT: Image\ndef resample_image(input_volume, out_spacing=[1, 1, 1]):\n  \n    original_spacing = input_volume.GetSpacing()\n    original_size = input_volume.GetSize()\n\n    out_size = [\n        int(np.round(original_size[0] * (original_spacing[0] / out_spacing[0]))),\n        int(np.round(original_size[1] * (original_spacing[1] / out_spacing[1]))),\n        int(np.round(original_size[2] * (original_spacing[2] / out_spacing[2])))]\n\n    resample = sitk.ResampleImageFilter()\n    resample.SetOutputSpacing(out_spacing)\n    resample.SetSize(out_size)\n    resample.SetOutputDirection(input_volume.GetDirection())\n    resample.SetOutputOrigin(input_volume.GetOrigin())\n    resample.SetTransform(sitk.Transform())\n    resample.SetDefaultPixelValue(input_volume.GetPixelIDValue())\n\n    return resample.Execute(input_volume)","metadata":{"execution":{"iopub.status.busy":"2024-08-15T01:47:10.422166Z","iopub.execute_input":"2024-08-15T01:47:10.422579Z","iopub.status.idle":"2024-08-15T01:47:10.439189Z","shell.execute_reply.started":"2024-08-15T01:47:10.422539Z","shell.execute_reply":"2024-08-15T01:47:10.437053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resize images to fixed spatial resolution in pixels\n# INPUT: Image\n# OUTPUT: Image\ndef resize_image(input_volume, output_size=320):\n    num_axial_slices = int(input_volume.GetSize()[-1])\n    output_size = [output_size, output_size, num_axial_slices]\n    scale = np.divide(input_volume.GetSize(), output_size)\n    spacing = np.multiply(input_volume.GetSpacing(), scale)\n    transform = sitk.AffineTransform(3)\n    resized_volume = sitk.Resample(input_volume, output_size, transform, sitk.sitkLinear, input_volume.GetOrigin(),\n                                  spacing, \n    input_volume.GetDirection())\n    return resized_volume","metadata":{"execution":{"iopub.status.busy":"2024-08-15T01:47:10.441107Z","iopub.execute_input":"2024-08-15T01:47:10.441686Z","iopub.status.idle":"2024-08-15T01:47:10.469068Z","shell.execute_reply.started":"2024-08-15T01:47:10.441619Z","shell.execute_reply":"2024-08-15T01:47:10.467708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INPUT: Image\n# OUTPUT: Numpy array\ndef extract_slices(image_volume):\n    image_array = sitk.GetArrayFromImage(image_volume)\n    return image_array","metadata":{"execution":{"iopub.status.busy":"2024-08-15T01:47:10.470444Z","iopub.execute_input":"2024-08-15T01:47:10.470852Z","iopub.status.idle":"2024-08-15T01:47:10.492471Z","shell.execute_reply.started":"2024-08-15T01:47:10.470819Z","shell.execute_reply":"2024-08-15T01:47:10.491043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom_stack(dicom_folder_path, index):\n    try:\n        mri_image_vol = readMRIVolume(dicom_folder_path) # this is excpected to throw exception\n        resampled_img = resample_image(mri_image_vol)\n        resized_img = resize_image(resampled_img) \n        img_numpy_arr =  extract_slices(resized_img) # someNumber * 320 * 320\n        img_slice = img_numpy_arr[index] # 320 * 320\n        return img_slice\n    except Exception as e:\n        print(f\"An exception occured occurred: {e}\")\n        return np.zeros((320, 320))","metadata":{"execution":{"iopub.status.busy":"2024-08-15T01:47:10.493995Z","iopub.execute_input":"2024-08-15T01:47:10.494885Z","iopub.status.idle":"2024-08-15T01:47:10.509505Z","shell.execute_reply.started":"2024-08-15T01:47:10.494840Z","shell.execute_reply":"2024-08-15T01:47:10.507934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_files(meta_df, image_path, data_points_input=None):\n    \n    count_df = meta_df.groupby('study_id').size().reset_index(name='count')\n    if data_points_input is not None:\n        count_df = count_df.head(data_points_input)\n    data_points = count_df.shape[0]\n    \n    num_files_df = pd.DataFrame(columns=['study_id', 'min_slices'])\n    rows = []\n    \n    for i in range(data_points):\n        study_ID = count_df.study_id.iloc[i]\n        study = meta_df.loc[meta_df.study_id == study_ID]\n        \n        min_slices = 1000 # a large number\n        for row in study.itertuples():\n            path = os.path.join(image_path, str(row.study_id), str(row.series_id))\n            num_files = count_files_in_directory(path)\n            min_slices = min(min_slices, num_files)\n    \n        # Add (study_ID, min_slices) to the DataFrame\n        rows.append({'study_id': study_ID, 'min_slices': min_slices})\n            \n    num_files_df = pd.concat([num_files_df, pd.DataFrame(rows)], ignore_index=True)\n    return num_files_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files_count = count_files(train_meta_df, train_images_path)\nprint(train_files_count)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INPUTS: \n#    meta_df (Pandas dataframe): metadata of Images\n#.   image_path (String): The path to Images\n#    OUTPUT: A Numpy array of size (data_points * 3 * 320 * 320)\n\n# TRAINING: \n#   meta_df = train_meta_df\n#   image_path = train_images_path\n\n# NOTE: not using tqdm because print() and tqdm.write() is causing problems\ndef create_dataset(meta_df, image_path, files_count_df, data_points_input=None):\n    count_df = meta_df.groupby('study_id').size().reset_index(name='count')\n    \n    if data_points_input is not None:\n        count_df = count_df.head(data_points_input)\n        \n    data_points = count_df.shape[0]\n    data_set = np.empty((data_points, 3, 320, 320)) \n    \n    # iterates through each of the 1975 study\n    for i in range(data_points):\n        print(f\"{i+1}th iteration out of {data_points}\")\n        study_ID = count_df.study_id.iloc[i]\n        study = meta_df.loc[meta_df.study_id == study_ID]\n        \n        # Find the min_slices value for the given study_ID\n        min_slices = files_count_df.loc[files_count_df['study_id'] == study_ID, 'min_slices'].values[0]\n        \n        # iterates through each of the 3 series in a single study \n        for row in study.itertuples():\n            path = os.path.join(image_path, str(row.study_id), str(row.series_id))\n\n            if row.series_description == \"Sagittal T2/STIR\":\n                sag_t2_slices = []\n                for slice_index in range(min_slices):\n                    curr_slice = load_dicom_stack(path, slice_index) # 320 * 320\n                    sag_t2_slices.append(curr_slice)\n                    \n            elif row.series_description == \"Sagittal T1\":\n                sag_t1_slices = []\n                for slice_index in range(min_slices):\n                    curr_slice = load_dicom_stack(path, slice_index) # 320 * 320\n                    sag_t1_slices.append(curr_slice)\n                    \n            elif row.series_description == \"Axial T2\":\n                ax_t2_slices = []\n                for slice_index in range(min_slices):\n                    curr_slice = load_dicom_stack(path, slice_index) # 320 * 320\n                    ax_t2_slices.append(curr_slice)\n                \n        for \n        curr_set = np.stack((sag_t2_mid_slice, sag_t1_mid_slice, ax_t2_mid_slice), axis=0) # (3*320*320)\n        data_set[i] = curr_set \n            \n    return data_set","metadata":{"execution":{"iopub.status.busy":"2024-08-15T01:47:10.510969Z","iopub.execute_input":"2024-08-15T01:47:10.511342Z","iopub.status.idle":"2024-08-15T01:47:10.531784Z","shell.execute_reply.started":"2024-08-15T01:47:10.511313Z","shell.execute_reply":"2024-08-15T01:47:10.530586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-08-15T02:12:01.284386Z","iopub.execute_input":"2024-08-15T02:12:01.284840Z","iopub.status.idle":"2024-08-15T02:12:01.290552Z","shell.execute_reply.started":"2024-08-15T02:12:01.284809Z","shell.execute_reply":"2024-08-15T02:12:01.289243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load(path):\n    shape = (1975, 320, 320, 3)\n    data_set = np.memmap(path, mode='r', shape=shape)\n    return data_set","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:03.874213Z","iopub.execute_input":"2024-08-13T05:48:03.874638Z","iopub.status.idle":"2024-08-13T05:48:03.881019Z","shell.execute_reply.started":"2024-08-13T05:48:03.874540Z","shell.execute_reply":"2024-08-13T05:48:03.879862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"[[3 3 4 ... 0 0 0]\n [3 5 7 ... 0 0 0]\n [3 5 9 ... 0 0 0]\n ...\n [1 3 4 ... 0 0 0]\n [1 3 4 ... 0 0 0]\n [1 1 1 ... 0 0 0]]\n \n ","metadata":{}},{"cell_type":"code","source":"def create_labels():\n    train_csv_copy = train_csv.copy()\n\n    new_columns = []\n    severities = ['Normal/Mild', 'Moderate', 'Severe']\n\n    for disease in train_csv_copy.columns[1:]:\n        for severity in severities:\n            new_col_name = f\"{disease}_{severity}\"\n            new_columns.append(new_col_name)\n            train_csv_copy[new_col_name] = 0\n            train_csv_copy.loc[train_csv_copy[disease] == severity, new_col_name] = 1\n\n    train_csv_copy = train_csv_copy.drop(columns=train_csv_copy.columns[0:26])\n    # print(train_csv_copy[0])\n    train_csv_copy = train_csv_copy.to_numpy()\n    # print(train_csv_copy)\n    \n    return train_csv_copy\n","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:03.882786Z","iopub.execute_input":"2024-08-13T05:48:03.883203Z","iopub.status.idle":"2024-08-13T05:48:03.895936Z","shell.execute_reply.started":"2024-08-13T05:48:03.883172Z","shell.execute_reply":"2024-08-13T05:48:03.894648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INPUT: numpy datasets \n# OUTPUT: tf datasets train_set, val_set\n# !!! num_samples=1975\ndef create_tf_dataset(data_set, labels, num_samples=1975, train_ratio=0.8, batch_size=32, seed=42):\n    data_set_tf = tf.data.Dataset.from_tensor_slices(data_set)\n    labels_tf = tf.data.Dataset.from_tensor_slices(labels)\n\n    # Combine the inputs and labels\n    combined_set = tf.data.Dataset.zip((data_set_tf, labels_tf))\n    \n    # Shuffle and split the dataset\n    # Lets skip shuffling for now\n    # combined_set = combined_set.shuffle(buffer_size=num_samples, seed=seed)\n    train_size = int(num_samples * train_ratio)\n    train_set = combined_set.take(train_size).batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    val_set = combined_set.skip(train_size).batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    \n    return train_set, val_set\n","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:03.897567Z","iopub.execute_input":"2024-08-13T05:48:03.897927Z","iopub.status.idle":"2024-08-13T05:48:03.915424Z","shell.execute_reply.started":"2024-08-13T05:48:03.897897Z","shell.execute_reply":"2024-08-13T05:48:03.914408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n","metadata":{"execution":{"iopub.status.busy":"2024-07-16T04:10:29.492316Z","iopub.execute_input":"2024-07-16T04:10:29.492672Z","iopub.status.idle":"2024-07-16T04:10:29.498052Z","shell.execute_reply.started":"2024-07-16T04:10:29.492647Z","shell.execute_reply":"2024-07-16T04:10:29.497074Z"}}},{"cell_type":"code","source":"def create_model(input_shape):\n    print(\"Creating the model....\")\n    model_input = tf.keras.Input(shape=input_shape, name='input')\n\n    base_model = EfficientNetV2B0(\n        include_top = False,\n        weights = '/kaggle/input/rsna-dataset-numpy-complete/efficientnetv2-b0_notop.h5', \n        input_shape = input_shape\n    )\n    base_model.trainable = False\n    \n    base_model_features = base_model(model_input, training=False)\n    base_model_features = layers.GlobalAveragePooling2D()(base_model_features)\n    # base_model_features = layers.Dropout(0.2)(base_model_features) \n    \n    output = keras.layers.Dense(75, activation='softmax', kernel_initializer=GlorotUniform(), name='output')(base_model_features)\n    \n    model = Model(inputs=[model_input], outputs=[output])\n    \n    num_layers = len(base_model.layers)\n    # print(f\"Number of layers in the base model: {num_layers}\")\n\n    return model, base_model","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:03.920971Z","iopub.execute_input":"2024-08-13T05:48:03.921566Z","iopub.status.idle":"2024-08-13T05:48:03.932795Z","shell.execute_reply.started":"2024-08-13T05:48:03.921506Z","shell.execute_reply":"2024-08-13T05:48:03.931534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\ndef train_test_save_model(train_dataset, validation_data, epochs_to_train):\n    \n    input_shape = (320, 320, 3)\n    model, base_model = create_model(input_shape)\n    print(model.summary(show_trainable=True))\n    \n    print(\"Compiling the model....\")\n    model.compile(\n        optimizer=keras.optimizers.Adam(), \n        loss=keras.losses.BinaryCrossentropy(), \n        metrics=['accuracy']\n    )\n    \n    print(\"Training the model....\")\n    history = model.fit(\n        train_dataset,\n        validation_data=validation_data,\n        epochs=epochs_to_train,  # Number of epochs to train\n        verbose=1  # Verbosity mode\n    )\n\n    print(\"Evaluating the model....\")\n    results = model.evaluate(validation_data)\n    print(f\"Validation results - {results}\")\n\n    print(\"Saving the model....\")\n    model.save('non_fine_tuned_model.h5')\n    \n    return model, base_model;","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:03.934113Z","iopub.execute_input":"2024-08-13T05:48:03.934484Z","iopub.status.idle":"2024-08-13T05:48:03.953943Z","shell.execute_reply.started":"2024-08-13T05:48:03.934456Z","shell.execute_reply":"2024-08-13T05:48:03.952339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fine tune the model\n# there are 270 layers in efficientNet_v2_b0\ndef train_test_save_fine_tune_model(train_dataset, validation_data, epochs_to_train, epochs_to_fine_tune,layers_to_unfreeze):\n    model, base_model = train_test_save_model(train_dataset, validation_data, epochs_to_train)\n    \n    print(\"Starting the fine tuning process....\")\n    \n    for layer in base_model.layers[-layers_to_unfreeze:]:\n        layer.trainable = True\n    # base_model.trainable = True \n    \n    model.summary(show_trainable=True)\n    \n    model.compile(\n        optimizer=keras.optimizers.Adam(1e-5),  # Low learning rate\n        loss=keras.losses.BinaryCrossentropy(),\n        metrics=['accuracy'],\n    )\n\n    print(\"Fitting the end-to-end model\")\n    model.fit(train_dataset, epochs=epochs_to_fine_tune, validation_data=validation_data)\n    \n    print(\"Evaluating the model....\")\n    results = model.evaluate(validation_data)\n    print(f\"Validation results - {results}\")\n    \n    print(\"Saving the model....\")\n    model.save('fine_tuned_model_one_layer_unfrozen.h5')","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:03.955574Z","iopub.execute_input":"2024-08-13T05:48:03.955943Z","iopub.status.idle":"2024-08-13T05:48:03.969537Z","shell.execute_reply.started":"2024-08-13T05:48:03.955910Z","shell.execute_reply":"2024-08-13T05:48:03.968148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing","metadata":{}},{"cell_type":"code","source":"# Path to all the files\ntest_images_path = os.path.join(root_path, 'test_images')\ntest_description_path = os.path.join(root_path, 'test_series_descriptions.csv')\ntest_meta_df = pd.read_csv(test_description_path)\ntest_meta_df = test_meta_df.drop_duplicates(subset=['study_id', 'series_description'])","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:03.972029Z","iopub.execute_input":"2024-08-13T05:48:03.972635Z","iopub.status.idle":"2024-08-13T05:48:03.992125Z","shell.execute_reply.started":"2024-08-13T05:48:03.972481Z","shell.execute_reply":"2024-08-13T05:48:03.990755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model):\n    test_data_set = create_dataset(test_meta_df, test_images_path)\n    test_data_set = np.transpose(test_data_set, (0, 2, 3, 1))\n    test_tf = tf.data.Dataset.from_tensor_slices(test_data_set)\n    test_tf = test_tf.batch(32)\n    predictions = model.predict(test_tf)\n    \n    return predictions","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:03.993918Z","iopub.execute_input":"2024-08-13T05:48:03.995143Z","iopub.status.idle":"2024-08-13T05:48:04.004141Z","shell.execute_reply.started":"2024-08-13T05:48:03.995095Z","shell.execute_reply":"2024-08-13T05:48:04.002637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creation of the submission file","metadata":{}},{"cell_type":"code","source":"def create_submission(predictions):\n    # Create the severity level columns\n    normal_mild_col = []\n    moderate_col = []\n    severe_col = []\n\n    for curr_75_element_array in predictions:\n        arrays = [normal_mild_col, moderate_col, severe_col]\n        for i, element in enumerate(curr_75_element_array):\n            arrays[i % 3].append(element) \n\n    normal_mild_col = np.array(normal_mild_col).flatten()\n    moderate_col = np.array(moderate_col).flatten()\n    severe_col = np.array(severe_col).flatten()\n    \n    # Create the row id column\n    row_id_col = []\n    test_meta_df_copy = test_meta_df.copy()\n    unique_study_ids = test_meta_df_copy[['study_id']].drop_duplicates().reset_index(drop=True)\n    diseases_array = train_csv.copy().columns[1:].to_numpy()\n    for study_id in unique_study_ids['study_id']:\n        for disease in diseases_array:\n            # row_id_col.append({'row_id': f'{study_id}_{disease}'})\n            row_id_col.append(f'{study_id}_{disease}')\n\n    data = {\n        'row_id': row_id_col,\n        'normal_mild': normal_mild_col,\n        'moderate': moderate_col,\n        'severe': severe_col\n    }\n    submission_df = pd.DataFrame(data)\n    submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:04.006203Z","iopub.execute_input":"2024-08-13T05:48:04.006674Z","iopub.status.idle":"2024-08-13T05:48:04.023120Z","shell.execute_reply.started":"2024-08-13T05:48:04.006631Z","shell.execute_reply":"2024-08-13T05:48:04.021018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The following cell contains the data creation code, which runs only once ","metadata":{}},{"cell_type":"code","source":"def np_dataset_creation():\n    data_set = create_dataset(train_meta_df, train_images_path) # 5922 * 3 * 320 * 320\n    data_set = np.transpose(data_set, (0, 2, 3, 1))  # 5922 * 320 * 320 * 3\n    print(f\"shape: {data_set.shape}\")\n    np.save('numpy_4d_npy_array', data_set)\n    \n# np_dataset_creation()","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:04.024318Z","iopub.execute_input":"2024-08-13T05:48:04.024742Z","iopub.status.idle":"2024-08-13T05:48:04.041614Z","shell.execute_reply.started":"2024-08-13T05:48:04.024710Z","shell.execute_reply":"2024-08-13T05:48:04.040289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data_set():\n    data_set_path = '/kaggle/input/rsna-dataset-numpy-complete/numpy_4d_npy_array.npy'\n    data_set = load(data_set_path)\n    print(data_set.shape)\n    return data_set\n","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:04.043319Z","iopub.execute_input":"2024-08-13T05:48:04.043781Z","iopub.status.idle":"2024-08-13T05:48:04.053235Z","shell.execute_reply.started":"2024-08-13T05:48:04.043747Z","shell.execute_reply":"2024-08-13T05:48:04.052131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def complete_training_workflow(data_set, epochs_to_train, epochs_to_fine_tune, layers_to_unfreeze):\n    labels = create_labels()\n    train_dataset, validation_data = create_tf_dataset(data_set, labels)\n    train_test_save_model(train_dataset, validation_data, epochs_to_train)\n\ndata_set = load_data_set()\ncomplete_training_workflow(data_set, epochs_to_train=10, epochs_to_fine_tune=5,layers_to_unfreeze=0)","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:04.054875Z","iopub.execute_input":"2024-08-13T05:48:04.055671Z","iopub.status.idle":"2024-08-13T05:48:08.439729Z","shell.execute_reply.started":"2024-08-13T05:48:04.055626Z","shell.execute_reply":"2024-08-13T05:48:08.437560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !! load function accesses from the output, not the input. Submitting doesn't have access to output\ndef complete_testing_workflow():\n    model_path = '/kaggle/input/eff_v2b0_frozen_10_epochs/keras/default/2/eff_v2b0_frozen_layer_10_epochs.h5'\n    model = tf.keras.models.load_model(model_path)\n    print(\"predicting...\")\n    predictions = predict(model)\n    \n    print(\"creating submission...\")\n    create_submission(predictions)\n    print(\"submission created\")\n\ncomplete_testing_workflow()","metadata":{"execution":{"iopub.status.busy":"2024-08-13T05:48:08.440961Z","iopub.status.idle":"2024-08-13T05:48:08.441876Z","shell.execute_reply.started":"2024-08-13T05:48:08.441414Z","shell.execute_reply":"2024-08-13T05:48:08.441442Z"},"trusted":true},"execution_count":null,"outputs":[]}]}