{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## First of All, credit goes to [Yaroslav Isaienkov](https://www.kaggle.com/ihelon/brain-tumor-eda-with-animations-and-modeling) for the great notebook to study and inspiration \n\n> ### His Original NB @ https://www.kaggle.com/ihelon/brain-tumor-eda-with-animations-and-modeling/notebook\n> 1st run gave an 0.60 accuracy on LB w/ no changes\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-17T09:08:45.39006Z","iopub.execute_input":"2021-08-17T09:08:45.390708Z","iopub.status.idle":"2021-08-17T09:08:45.405275Z","shell.execute_reply.started":"2021-08-17T09:08:45.390583Z","shell.execute_reply":"2021-08-17T09:08:45.403763Z"}}},{"cell_type":"markdown","source":"### The Format of This Notebook Will be as Follows:\n* Markup Code-Block Label\n1. The Original Code Block\n1. The Annotated Code Block\n1. **[Bonus] And the occasional demonstration of what functions are doing in the main code block in a separate code block**","metadata":{}},{"cell_type":"markdown","source":"### The Goal/Motivation\n> I dived head first into this notebook for several long hours trying to annotate this notebook so that I could improve my own skills and understanding\n\n> By posting the culmination of my annotations of this notebook so far the idea is that other will be able to glean more than they would otherwise(especially beginners)\n\n> And lastly, I left a few question within the Annotations that I couldn't find the answers to. Feedback by those who know what I am missing to connect the dots is SUPER appreciated","metadata":{}},{"cell_type":"markdown","source":"
\n\n

\n Annotations Begin Here: \n

\n
\n","metadata":{}},{"cell_type":"markdown","source":"## imports","metadata":{}},{"cell_type":"code","source":"import os\n#for file management\nimport json\n#for standardized data storage\nimport glob\n#for precise file selection(low verbosity)\nimport random\n#random amount generator\nimport collections\n#has premade datastructure objects that can be implemented\nimport numpy as np\n#linear algebra\nimport pandas as pd\n#succinct array and data handling\nimport pydicom\n#c based dicom modulation tool\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n#pixel handler that uses voi_lut to grab pixel data from within the frame of a window\nimport cv2\n#image data handler\nimport matplotlib.pyplot as plt\n#data visualization library\nimport seaborn as sns\n#data visualization library","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:12:24.301187Z","iopub.execute_input":"2021-08-17T10:12:24.301514Z","iopub.status.idle":"2021-08-17T10:12:25.444327Z","shell.execute_reply.started":"2021-08-17T10:12:24.301483Z","shell.execute_reply":"2021-08-17T10:12:25.443429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## dataloading","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\n#load data into train_df\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:13:34.350042Z","iopub.execute_input":"2021-08-17T10:13:34.350401Z","iopub.status.idle":"2021-08-17T10:13:34.366744Z","shell.execute_reply.started":"2021-08-17T10:13:34.350369Z","shell.execute_reply":"2021-08-17T10:13:34.365827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"def load_dicom(path):\n dicom = pydicom.read_file(path)\n data = dicom.pixel_array\n data = data - np.min(data)\n if np.max(data) != 0:\n data = data / np.max(data)\n data = (data * 255).astype(np.uint8)\n return data\n\n\ndef visualize_sample(\n brats21id, \n slice_i,\n mgmt_value,\n types=(\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\")\n):\n plt.figure(figsize=(16, 5))\n patient_path = os.path.join(\n \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/\", \n str(brats21id).zfill(5),\n )\n for i, t in enumerate(types, 1):\n t_paths = sorted(\n glob.glob(os.path.join(patient_path, t, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n data = load_dicom(t_paths[int(len(t_paths) * slice_i)])\n plt.subplot(1, 4, i)\n plt.imshow(data, cmap=\"gray\")\n plt.title(f\"{t}\", fontsize=16)\n plt.axis(\"off\")\n\n plt.suptitle(f\"MGMT_value: {mgmt_value}\", fontsize=16)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:13:36.709829Z","iopub.execute_input":"2021-08-17T10:13:36.710186Z","iopub.status.idle":"2021-08-17T10:13:36.719697Z","shell.execute_reply.started":"2021-08-17T10:13:36.710137Z","shell.execute_reply":"2021-08-17T10:13:36.718709Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#load dicom file from a given path\ndef load_dicom(path):\n #variable dicom = data returned from pydicom object(see metadata for more)\n dicom = pydicom.read_file(path)\n #variable data = pixel data within the frame(returned in a np array)\n data = dicom.pixel_array\n #variable data = elements of variable data - minima of the array\n data = data - np.min(data)\n #conditional to check if the dicom isn't empty(all elements of pixel data == 0 would indicate an empty dicom)\n if np.max(data) != 0:\n #normalize data(part 1) by dividing by the maxima of the data array(largest pixel quantity), this will produce a fractional quantity that we will scale in the next step\n data = data / np.max(data)\n #normalize data(part 2) we can get our pixel data on the 0-255 scale and the store it as uint8 \n '''\n why uint8 instead of merely leaving it as np data?\n as it turns out none really, the only reason I could find is, \"you can pass it directly to \n functions requiring pointers to the bytes like C functions, whereas for Data you have to do\n a bunch more gymnastics.\"-Jacob King from stackoverflow post https://stackoverflow.com/questions/39873282/data-vs-uint8\n \n i reckon it may mesh better with the pydicom library but don't know if thats necessarily true as they would have likely\n accounted for merely 8 bit data and not necessarily what you called it whether it be byte data, uint8, np data, etc etc\n \n \"Uint8 is specially used to store various images (including RGB, grayscale images, etc.), ranging from 0 to 255.\" \n '''\n data = (data * 255).astype(np.uint8)\n #worth noting at this stage that you technically might have been okay without handling the pixel data so thoroughly(speaking from experience) but it is a great way to just avoid\n #possible issues in the future and allows for higher liklihood of reusability in future projects\n return data\n\n\ndef visualize_sample(\n brats21id, #id between 1 and 1006\n slice_i, #varying amount of slices in given file e.g. \"Image-1.dcm\"\n mgmt_value, #0.0-1.0 rating \n types=(\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\") #4 subfiles that contain slices of each different type of image\n):\n #generates a matplot figure with given width and height\n plt.figure(figsize=(16, 5)) \n #variable patient_path is set to the give file location and str(brats21id).zfill(5)\n patient_path = os.path.join(str(brats21id).zfill(5), \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/\",\n )\n '''\n brats21id == value at given Brats21ID column location changed to string from int for functionality within the os.path.join()\n *note that the brats21id itself started out as a numeric value of say 1, thus necessitating the following zfill*\n .zfill(#) is a builtin python function that will return the operated value(e.g. 1) with the as many indices filled with 0 leftover and then the operated value(e.g. 00001)\n '''\n \n \n for i, t in enumerate(types, 1):\n '''\n for the indice/key i and the corresponding value t within the dict 'types' starting from 1 do the following(stringent on the next few lines of logic)\n the types dict is made up of FLAIR, T1w, T1wCE, T2w\n '''\n #t-paths variable set to be the sorted values(ascending order numerically) t is abbreviating the types \n '''\n #variable t_paths is set to be the path of the patient path + the current type str value + the rest of the files within that given type path\n the key= statement is a parameter of sorted, that dictates the order in which it reads the tuple itself\n as the .split() function is used here we then know we must have created a list that is comprised of the file paths of all of the contents of the folder T2w\n we have a comma after the key= statement to signify that we are not setting reverse= to True or False at all\n !!!!!if anyone has a better interpretation PLEASE leave a comment and I will upvote to the moon!!!!!!\n '''\n t_paths = sorted(\n glob.glob(os.path.join(patient_path, t, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n #set data variable to the associated RIA_image object data that is a 3D array of pixel data of the file pointed to within the t_paths\n #array index. the length of the t_paths array which would be the total amount of slices so we probably dont want 258 images, so it is\n #then multiplied by a fractional slice_i component to have less to look at\n #*note that we have 2d pixel array data but it still will work with this library's backend work\n data = load_dicom(t_paths[int(len(t_paths) * slice_i)]) #slice_i was later set to 0.5 for anyone immediately curious\n \n # subplot function takes (m,n,p) where its an mxn grid and p is where the axis forms as specified by p(position)\n #as such, we get a 1 by 4 set of converted dcm images with an i 'dimension' positionally which is slappin(==good news)\n #because i goes from 1 upward so the depth of our visualization is handled effortlessly\n plt.subplot(1, 4, i)\n #plt.imshow is also from matplotlib and will return an image from the given 2d array and make it the color specified in the cmap parameter\n plt.imshow(data, cmap=\"gray\")\n #style decisions for the title using matplotlib, uses a string literal(3.7>) to use the t value we set up so long ago to iterate with and sets a fontsize\n plt.title(f\"{t}\", fontsize=16)\n #removes the lining that is usually what we would see as the x and y axis intersecting\n plt.axis(\"off\")\n\n #another title for each respective slice groupthat indicates whether the MGMT value is 1(present) or 0(not present)\n plt.suptitle(f\"MGMT_value: {mgmt_value}\", fontsize=16)\n #shows the plot\n plt.show()\n \n'''\nfor those interested, load_dicom takes the following parameters as well as more, see the docs here: https://rdrr.io/cran/RIA/src/R/load_dicom.R \n\nload_dicom <- function(filename, mask_filename = NULL, keep_mask_values = 1, switch_z = TRUE, \n crop_in = TRUE, replace_in = TRUE, center_in = TRUE, zero_value = NULL, min_to = -1024,\n header_add = NULL, header_exclude = NULL, verbose_in = TRUE,\n recursive_in = TRUE, exclude_in = \"sql\",\n mode_in = \"integer\", transpose_in = TRUE, pixelData_in = TRUE,\n mosaic_in = FALSE, mosaicXY_in = NULL, sequence_in = FALSE, ...\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Quick Demonstration] of the sorted comprehension in the 'visualize_sample' function for those confused:","metadata":{}},{"cell_type":"markdown","source":"> consider an array of the following contents and length","metadata":{}},{"cell_type":"code","source":"new =['00001','00002','00003','00004','00005']","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:13:52.643913Z","iopub.execute_input":"2021-08-17T10:13:52.644234Z","iopub.status.idle":"2021-08-17T10:13:52.649425Z","shell.execute_reply.started":"2021-08-17T10:13:52.644204Z","shell.execute_reply":"2021-08-17T10:13:52.648449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> if we use the :-4 array manuever we will get the following","metadata":{}},{"cell_type":"code","source":"new[:-4]","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:13:53.702137Z","iopub.execute_input":"2021-08-17T10:13:53.702477Z","iopub.status.idle":"2021-08-17T10:13:53.708442Z","shell.execute_reply.started":"2021-08-17T10:13:53.702445Z","shell.execute_reply":"2021-08-17T10:13:53.707384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Therefore (x[:-4]) indicates 4 from the end inclusive of the final value you land one, in this case '00001'\n\n> We can then conclude that int(x[:-4].split(\"-\")[-1] is then grabbing the -4 value, which is T2w from the types (probably the last col Original author was evaluating)\n\n>list and then using the split function to create an array of the paths of the files of the slices, which is thereby \n\n>a list that has all of the paths to the slices within the given type, T2w\n\nsee the next code cell for optional confirmation of the sorted comprehension analysis","metadata":{}},{"cell_type":"code","source":"''' Run this cell for confirmation of what values are being pulled out with our sorted comprehension'''\ntypes=(\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\")\n# brats21id, #id between 1 and 1006\n# slice_i, #varying amount of slices in given file e.g. \"Image-1.dcm\"\n# mgmt_value, #0.0-1.0 rating \nfor i, t in enumerate(types, 1):\n '''\n for the indice/key i and the corresponding value t within the dict 'types' starting from 1 do the following(stringent on the next few lines of logic)\n the types dict is made up of FLAIR, T1w, T1wCE, T2w\n '''\n #t-paths variable set to be the sorted values(ascending order numerically) t is abbreviating the types \n '''\n #variable t_paths is set to be the path of the patient path + the current type str value + the rest of the files within that given type path\n the key= statement is a parameter of sorted, that dictates the order in which it reads the tuple itself\n as the .split() function is used here we then know we must have created a list that is comprised of the file paths of all of the contents of the folder T2w\n we have a comma after the key= statement to signify that we are not setting reverse= to True or False at all\n '''\n t_paths = sorted(\n glob.glob(os.path.join(patient_path, t, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n\nt_paths","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"def load_dicom(path):\n dicom = pydicom.read_file(path)\n data = dicom.pixel_array\n data = data - np.min(data)\n if np.max(data) != 0:\n data = data / np.max(data)\n data = (data * 255).astype(np.uint8)\n return data\n\n\ndef visualize_sample(\n brats21id, \n slice_i,\n mgmt_value,\n types=(\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\")\n):\n plt.figure(figsize=(16, 5))\n patient_path = os.path.join(\n \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/\", \n str(brats21id).zfill(5),\n )\n for i, t in enumerate(types, 1):\n t_paths = sorted(\n glob.glob(os.path.join(patient_path, t, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n data = load_dicom(t_paths[int(len(t_paths) * slice_i)])\n plt.subplot(1, 4, i)\n plt.imshow(data, cmap=\"gray\")\n plt.title(f\"{t}\", fontsize=16)\n plt.axis(\"off\")\n\n plt.suptitle(f\"MGMT_value: {mgmt_value}\", fontsize=16)\n plt.show()\n\n#sample visualizations\nfor i in random.sample(range(train_df.shape[0]), 10):\n _brats21id = train_df.iloc[i][\"BraTS21ID\"]\n _mgmt_value = train_df.iloc[i][\"MGMT_value\"]\n visualize_sample(brats21id=_brats21id, mgmt_value=_mgmt_value, slice_i=0.5)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:02.475349Z","iopub.execute_input":"2021-08-17T10:14:02.475685Z","iopub.status.idle":"2021-08-17T10:14:06.70663Z","shell.execute_reply.started":"2021-08-17T10:14:02.475656Z","shell.execute_reply":"2021-08-17T10:14:06.705807Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"def create_animation(ims):\n #generates a figure with given width and height\n fig = plt.figure(figsize=(6, 6))\n #removes the x and y axis\n '''\n To get rid of whitespace around the border, we can set bbox_inches='tight' \n in the savefig() method. Similarly, to remove the white border around the \n image while we set pad_inches = 0 in the savefig() method.\n '''\n plt.axis('off')\n #sets im varibale to the current img in the ims array @ indice 0 with a grey color map\n im = plt.imshow(ims[0], cmap=\"gray\")\n\n def animate_func(i):\n #set_array() function sets the color array of the given image(im assuming that it just set RGB, i have not used this functionality)\n im.set_array(ims[i])\n return [im]\n '''\n class matplotlib.animation.FuncAnimation(fig, func, frames=None, init_func=None, fargs=None, save_count=None, *, cache_frame_data=True, **kwargs)[source]\n the animation.FuncAnimation() is a matplot function that takes your fig parameters, the function you defined in animate_func() ->this is quite clever actually,\n the way it works is that for every time it is called it will load a new frame so you can effectively scroll through the entire slide of slices of a file in one condensed\n animation.\n \n the frames parameter is the data \"Source of data to pass func and each frame of the animation\" so what I am conceptualizing here is that the len(ims) indicates the \n total number of frames that will exist in this animation and we are iterating through the ims array and providing a frame for each indice so that works well logically\n \n the interval is just the delay between each animation in seconds, as such, 1000//24 = 41 and the units are ms(milleseconds). not sure why exactly he chose to do 41 or why\n he set up the eqn to do so. the default would have been 200 ms for those curious.\n \n documentation for this function: https://matplotlib.org/stable/api/_as_gen/matplotlib.animation.FuncAnimation.html#matplotlib.animation.FuncAnimation\n '''\n return animation.FuncAnimation(fig, animate_func, frames = len(ims), interval = 1000//24)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:56:58.111497Z","iopub.execute_input":"2021-08-17T09:56:58.112073Z","iopub.status.idle":"2021-08-17T09:56:58.119724Z","shell.execute_reply.started":"2021-08-17T09:56:58.112025Z","shell.execute_reply":"2021-08-17T09:56:58.118763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"def load_dicom_line(path):\n t_paths = sorted(\n glob.glob(os.path.join(path, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n images = []\n for filename in t_paths:\n data = load_dicom(filename)\n if data.max() == 0:\n continue\n images.append(data)\n \n return images","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:06.708031Z","iopub.execute_input":"2021-08-17T10:14:06.708523Z","iopub.status.idle":"2021-08-17T10:14:06.71503Z","shell.execute_reply.started":"2021-08-17T10:14:06.708484Z","shell.execute_reply":"2021-08-17T10:14:06.71421Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#create an images array that can be used as ims in the create_animation funnction above. \ndef load_dicom_line(path):\n #uses the same logic as def visualize_sample , except it grabs from the inserted path. so it will grab all contents of a given patient subfolder,\n #return the imgs in an array of file names\n t_paths = sorted(\n glob.glob(os.path.join(path, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n #create an images array to store the data values within each file path from t_paths\n images = []\n #a loop to convert all file paths to their respective data values and send to the images array\n for filename in t_paths:\n data = load_dicom(filename)\n #contingency for an empty data value(blank dcm file)from a filepath such that it won't save that dcm data to the img array and will move to the next value\n if data.max() == 0:\n continue\n images.append(data)\n \n return images","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Quick Demonstration] What the \" if data.max() == 0: \" did in the background","metadata":{}},{"cell_type":"code","source":"import numpy as np\n#quick note that the array.max() function works for numpy arrays, not regular python arrays\narray = np.array([0,0,0,0])\nnew = []\nfor _ in array:\n if array.max() == 0:\n continue\n new.append(_)\nprint(new)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:08.402094Z","iopub.execute_input":"2021-08-17T10:14:08.402423Z","iopub.status.idle":"2021-08-17T10:14:08.407976Z","shell.execute_reply.started":"2021-08-17T10:14:08.402393Z","shell.execute_reply":"2021-08-17T10:14:08.407193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

modeling imports

","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:33:10.46942Z","iopub.execute_input":"2021-08-17T09:33:10.469787Z","iopub.status.idle":"2021-08-17T09:33:10.475782Z","shell.execute_reply.started":"2021-08-17T09:33:10.469759Z","shell.execute_reply":"2021-08-17T09:33:10.474476Z"}}},{"cell_type":"code","source":"#he imports a reimplementation of the EfficientNet library that has pretrained models and saves the file path to the variable package_path\npackage_path = \"../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master/\"\n#he imports the sys module. the sys module allows for the manipulation of versioning of python and the sys.path environment variable allows for the \n#searching of all of the python variables within the file\nimport sys \n#this effectively acts as an import statement except it is grabbing from a file within kaggle that I am guessing kaggle doesn't usually provide(thus\n#necessitating this line to use the library of choice)\n#another way to phrase it: you are placing the python module of choice into the same path that you are currently working with so that can access said functions universally\nsys.path.append(package_path)\n\n#the time library is a c-based library for python. The author uses time() frmom this library frequently in order to \n#return the time in seconds since the epoch as a floating point number. A good way to watch your training progress quantitatively \nimport time\n\n#torch is the pytorch library that acts as gpu level linear algebra\nimport torch\n\n# the nn module is very extensive. it contains functions for initializing container, conv layer, pooling layers, padding layers, non-linear activations, and more\n# i feel the need to highlight the importance of this module a bit more: this module eliminates any vectorization by the kaggler themself, but it wouldn't be a \n# bad idea to learn how the underlying principles work both from a backend code and statistics perspective. for this I recommend the nn from scratch by \n# sentdex(https://www.youtube.com/watch?v=Wo5dMEP_BbI&list=PLQVvvaa0QuDcjD5BAw2DxE6OF2tius3V3&ab_channel=sentdex) and \n# statquest(https://www.youtube.com/user/joshstarmer) respectively\nfrom torch import nn \n\n#the module data from pytorch provides easily accessible functions that allow for batch creation(especially useful for large dataset handling) and other handy\n#dataloading utility.\n#documentation:https://pytorch.org/docs/stable/data.html\nfrom torch.utils import data as torch_data\n\n#sklearn model selection module that provides us with functions to easily split data into training data and testing data\n#documentation: https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html\nfrom sklearn import model_selection as sk_model_selection\n\n#imports the module with the torch nn library that is responsible for conv nets, pooling functions, loss functions, and more as also mentioned in torch nn import above\nfrom torch.nn import functional as torch_functional\n\n#efficientnet pytorch library import for faster conv net training speeds(i suppose that in order to get that edge you should aim to pay attention to groups like\n#these's progress on their model creation and updates)\n#documentation/website: https://pypi.org/project/efficientnet-pytorch/\n!pip install --upgrade efficientnet-pytorch\nimport efficientnet_pytorch\n\n#imports the stratified k fold functionality from the sklearn library(allows for easier paramaterization of data organization within splits)\n#documentation: https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.StratifiedKFold.html\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:09.938311Z","iopub.execute_input":"2021-08-17T10:14:09.93884Z","iopub.status.idle":"2021-08-17T10:14:16.164766Z","shell.execute_reply.started":"2021-08-17T10:14:09.938801Z","shell.execute_reply":"2021-08-17T10:14:16.163698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:34:27.790745Z","iopub.execute_input":"2021-08-17T09:34:27.791075Z","iopub.status.idle":"2021-08-17T09:34:27.794532Z","shell.execute_reply.started":"2021-08-17T09:34:27.791048Z","shell.execute_reply":"2021-08-17T09:34:27.79382Z"}}},{"cell_type":"code","source":"def set_seed(seed):\n random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n if torch.cuda.is_available():\n torch.cuda.manual_seed_all(seed)\n torch.backends.cudnn.deterministic = True\n\n\nset_seed(42)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:17.370221Z","iopub.execute_input":"2021-08-17T10:14:17.370586Z","iopub.status.idle":"2021-08-17T10:14:17.427698Z","shell.execute_reply.started":"2021-08-17T10:14:17.37053Z","shell.execute_reply":"2021-08-17T10:14:17.426869Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#this function appears toset every available possible seed generator to whatever the input seed value is in order to guarantee reproducibility\n#*this could be an incorrect interpretation of the intention, but if it is right its kinda humorous, like a very thorough wack-a-mole\ndef set_seed(seed):\n #pseudo-random number generator function from python standard library that is deterministic on the input of the 'seed' value in this case\n '''\n \"Use a random.seed() function with other random module functions to reproduce their output again and again.\"\n source-> https://pynative.com/python-random-seed/\n '''\n random.seed(seed)\n \n '''\n \"PYTHONHASHSEED. If this variable is not set or set to random, \n a random value is used to seed the hashes of str, bytes and datetime objects.\n If PYTHONHASHSEED is set to an integer value, it is used as a fixed seed for \n generating the hash() of the types covered by the hash randomization.\n \" source-> https://askinglot.com/what-is-pythonhashseed\n '''\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n \n #makes the random number predictable, the generated random numbers will be the same for the same seed value for np functions(see next code block for example)\n #src: https://stackoverflow.com/questions/21494489/what-does-numpy-random-seed0-do\n np.random.seed(seed)\n #makes the random number predictable, the generated random numbers will be the same for the same seed value for torch functions\n torch.manual_seed(seed)\n #if we have a gpu(s) available to us\n if torch.cuda.is_available():\n #Sets the seed for generating random numbers on all GPUs. It’s safe to call this function if CUDA is not available\n torch.cuda.manual_seed_all(seed)\n #CuDNN picks the same algorithms from the set they have available.\n #source: (user:tom) https://discuss.pytorch.org/t/what-is-the-differenc-between-cudnn-deterministic-and-cudnn-benchmark/38054/2\n torch.backends.cudnn.deterministic = True\n\n#set every seed available for psuedo random number generation to the integer value 42\nset_seed(42)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Quick Demonstration] of numpy.random.seed()","metadata":{}},{"cell_type":"code","source":"import numpy as numpy\nnumpy.random.seed(0) ; numpy.random.rand(4)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:20.754123Z","iopub.execute_input":"2021-08-17T10:14:20.754534Z","iopub.status.idle":"2021-08-17T10:14:20.761838Z","shell.execute_reply.started":"2021-08-17T10:14:20.754484Z","shell.execute_reply":"2021-08-17T10:14:20.760826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numpy.random.seed(0) ; numpy.random.rand(4)#outputs the same random values due to the same seed and same input array len","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:21.317805Z","iopub.execute_input":"2021-08-17T10:14:21.3181Z","iopub.status.idle":"2021-08-17T10:14:21.324427Z","shell.execute_reply.started":"2021-08-17T10:14:21.318071Z","shell.execute_reply":"2021-08-17T10:14:21.323476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\ndf_train, df_valid = sk_model_selection.train_test_split(\n df, \n test_size=0.2, \n random_state=42, \n stratify=train_df[\"MGMT_value\"],\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:22.79017Z","iopub.execute_input":"2021-08-17T10:14:22.790513Z","iopub.status.idle":"2021-08-17T10:14:22.805528Z","shell.execute_reply.started":"2021-08-17T10:14:22.790481Z","shell.execute_reply":"2021-08-17T10:14:22.804595Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#sets the df variable to the cotents of the train_labels.csv(we create a dataframe that we will be training with)\ndf = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\n#using the previously imported sklearn model selection library to split our dataframe and preprocess our data to be trained with\ndf_train, df_valid = sk_model_selection.train_test_split(\n df, #chosen dataframe\n test_size=0.2, #chosen fraction of test data to train with\n random_state=42, #randomization of the splitting of the data and putting it together into a random distribution of \n #test data(we don't want to just read our data in order for risk of the data being collected in a bias way beforehand)\n stratify=train_df[\"MGMT_value\"], #we split our train data proportionally to the amount of a given \"MGMT_value\". so like we have equal amounts of 0 and 1 values so that we don't create a skewed model\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Context For the Next Few Code Blocks] \nIn the following code block we will be creating a pytorch dataloader. it should be able to generate a dataset that takes the features we want and sets\nup a pytorch compatible dataset.\n\nthe main concept is that we are creating ids and corresponding data and then making that into a dictionary (it wil also return the 'tensor' channel ID(our data path) and the label)\n\nThe reason we are making the dataset afresh from what we were initially given has to do with the fact that we want:\n* modularity\n* batching\n* multiprocessing\n* pytorch dataset utility functions","metadata":{}},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"class DataRetriever(torch_data.Dataset):\n def __init__(self, paths, targets):\n self.paths = paths\n self.targets = targets\n \n def __len__(self):\n return len(self.paths)\n \n def __getitem__(self, index):\n _id = self.paths[index]\n patient_path = f\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/{str(_id).zfill(5)}/\"\n channels = []\n for t in (\"FLAIR\", \"T1w\", \"T1wCE\"): # \"T2w\"\n t_paths = sorted(\n glob.glob(os.path.join(patient_path, t, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n # start, end = int(len(t_paths) * 0.475), int(len(t_paths) * 0.525)\n x = len(t_paths)\n if x < 10:\n r = range(x)\n else:\n d = x // 10\n r = range(d, x - d, d)\n \n channel = []\n # for i in range(start, end + 1):\n for i in r:\n channel.append(cv2.resize(load_dicom(t_paths[i]), (256, 256)) / 255)\n channel = np.mean(channel, axis=0)\n channels.append(channel)\n \n y = torch.tensor(self.targets[index], dtype=torch.float)\n \n return {\"X\": torch.tensor(channels).float(), \"y\": y}","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:25.694939Z","iopub.execute_input":"2021-08-17T10:14:25.695307Z","iopub.status.idle":"2021-08-17T10:14:25.704928Z","shell.execute_reply.started":"2021-08-17T10:14:25.695272Z","shell.execute_reply":"2021-08-17T10:14:25.704039Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#characterizes a pytorch dataset such that it can be handled like one(src: https://stanford.edu/~shervine/blog/pytorch-how-to-generate-data-parallel)\nclass DataRetriever(torch_data.Dataset):#parameter is a map-style dataset that implements \n #\"__getitem__() and __len__() protocols, and represents a map from (possibly non-integral) indices/keys to data samples\"\n def __init__(self, paths, targets):\n #initialization of the choice path and target for the data we will generate\n self.paths = paths\n self.targets = targets\n \n def __len__(self):\n #the total number of samples we will be generating\n return len(self.paths)\n \n def __getitem__(self, index):\n #generating a sample of the data\n _id = self.paths[index]\n #set the _id variable to the element of the index of the paths array\n patient_path = f\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/{str(_id).zfill(5)}/\"\n #patient_path variable is set using an f string in order to dynamically adjust the _id value and subsequently prepend zeros to create a str that represents a path from\n #our data set, i.e. \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/00066/\"\n channels = []\n for t in (\"FLAIR\", \"T1w\", \"T1wCE\"): # author comment [ \"T2w\" ]-> he said but it really output the contents of T1wCE as it is the last indice\n t_paths = sorted(\n glob.glob(os.path.join(patient_path, t, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n # author comment [ start, end = int(len(t_paths) * 0.475), int(len(t_paths) * 0.525) ] \n # x variable holds the length of the array that contains the list that contains every slice from the T1wCE folder within patient #00000\n \n '''\n okay as kaggle user @Aman Arora so eloquently put it here: https://www.kaggle.com/ihelon/brain-tumor-eda-with-animations-and-modeling/comments#1388761\n the following data loading sequence grabs an effectively random small subsection of the data we have to work with, averages the chosen slice \n and does so for 3 of the 4 image type and uses those 'channels' array in order to train with. this code block's dataloading class is also one of two. This one\n is used to train the data(thus why we take paths and targets) and the next one will exclude the targets and use the paths of the test data and predict off of that\n absoluetly no disrespect to main author, however. this notebook has been fantastic to annotate thus far providing a ridiculous amount of insights\n '''\n \n x = len(t_paths)\n #if the length of t_paths array is less than 10\n if x < 10:\n r = range(x) #r = the range of between 0 and whatever the sub 10 value length of the t_paths array\n else:\n #if x is in the double digits, d = 1 for 10-19, 2 for 20-29, etc...\n d = x // 10\n #the python range function takes parameters (start, stop(stop point not included), step)\n r = range(d, x - d, d) # so range( some arbitrary amount, length of t_path-arbirary amount, with the step being the size of the arbitrary amount)\n #current thought process is that the author might have made it such that the len of the r array will always be consistent but I haven't been able to check that yet, feel free to comment if it is clear to you\n channel = []\n # for i in range(start, end + 1):\n for i in r:\n channel.append(cv2.resize(load_dicom(t_paths[i]), (256, 256)) / 255)\n #add the data from the corresponding dicom data(that is converted to image data) at the indice i within the t_paths array. The cv2.resize function effectively \n #changes the size of the data within the loaded data according to the input parameters, in this case:\n \n channel = np.mean(channel, axis=0) #takes the column-wise mean of the channel array\n channels.append(channel) # oh my lanta, I think this mad lad made a 2d representation of 3 dimensional data by considering each slice in one image as 'channels' with this process(correct me if misinterpreted that)\n #array made up of multiple 'channel' mean arrays\n \n y = torch.tensor(self.targets[index], dtype=torch.float)\n #sets the y value to the label of the chosen index that is passed with the precision of a float data type(some on the pytorch forumns state double is sufficient)\n \n return {\"X\": torch.tensor(channels).float(), \"y\": y}\n #returns the channels array in torch.tensor formatting and the y value","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Quick Demonstration] \n#### Breaking down the following logic;\n\nchannel.append(cv2.resize(load_dicom(t_paths[i]), (256, 256)) / 255)","metadata":{}},{"cell_type":"code","source":"#raw dcm to image data\nimage_data = load_dicom('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/00046/T2w/Image-300.dcm')#this is an indice that the t_path would be contain\nsize = image_data.size\ndimensions = image_data.ndim\nshape = image_data.shape\nprint(\"size: \"+ str(size), \"dimensions: \"+ str(dimensions), \"shape: \"+ str(shape))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:29.078206Z","iopub.execute_input":"2021-08-17T10:14:29.078569Z","iopub.status.idle":"2021-08-17T10:14:29.093774Z","shell.execute_reply.started":"2021-08-17T10:14:29.07852Z","shell.execute_reply":"2021-08-17T10:14:29.092651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#raw image data resized to 256,256\nimage_data = cv2.resize(image_data, (256,256))\ndimensions = image_data.ndim\nshape = image_data.shape\nprint(\"size: \"+ str(size), \"dimensions: \"+ str(dimensions), \"shape: \"+ str(shape))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:30.07826Z","iopub.execute_input":"2021-08-17T10:14:30.078602Z","iopub.status.idle":"2021-08-17T10:14:30.086826Z","shell.execute_reply.started":"2021-08-17T10:14:30.078566Z","shell.execute_reply":"2021-08-17T10:14:30.085806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#resized image data normalized with 255\nimage_data = (image_data / 255 )\ndimensions = image_data.ndim\nshape = image_data.shape\nprint(\"size: \"+ str(size), \"dimensions: \"+ str(dimensions), \"shape: \"+ str(shape))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:30.646266Z","iopub.execute_input":"2021-08-17T10:14:30.646602Z","iopub.status.idle":"2021-08-17T10:14:30.652434Z","shell.execute_reply.started":"2021-08-17T10:14:30.646567Z","shell.execute_reply":"2021-08-17T10:14:30.651619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(image_data, image_data[0])#2d array of empty image data(we are expecting values .003 and 1.003 [256/1 and 256/255])","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:00:00.922798Z","iopub.execute_input":"2021-08-17T10:00:00.92318Z","iopub.status.idle":"2021-08-17T10:00:00.932153Z","shell.execute_reply.started":"2021-08-17T10:00:00.923151Z","shell.execute_reply":"2021-08-17T10:00:00.931063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Quick Demonstration] what is in t_paths?","metadata":{}},{"cell_type":"code","source":"#quick side-note: what is in t_paths?\npatient_path = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/00000' #set this to a temp variable for analysis\nfor t in (\"FLAIR\", \"T1w\", \"T1wCE\"): # \"T2w\"\n t_paths = sorted(\n glob.glob(os.path.join(patient_path, t, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\nprint(t_paths)#outputs a list that contains every slice from the T1wCE folder within patient #00000","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:33.122314Z","iopub.execute_input":"2021-08-17T10:14:33.122759Z","iopub.status.idle":"2021-08-17T10:14:33.139137Z","shell.execute_reply.started":"2021-08-17T10:14:33.122709Z","shell.execute_reply":"2021-08-17T10:14:33.138311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Quick Context] for next annotation code block;","metadata":{"execution":{"iopub.status.busy":"2021-08-17T09:41:32.777182Z","iopub.execute_input":"2021-08-17T09:41:32.777524Z","iopub.status.idle":"2021-08-17T09:41:32.781745Z","shell.execute_reply.started":"2021-08-17T09:41:32.777494Z","shell.execute_reply":"2021-08-17T09:41:32.780597Z"}}},{"cell_type":"code","source":"df_train[\"BraTS21ID\"].values","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:36.862191Z","iopub.execute_input":"2021-08-17T10:14:36.862516Z","iopub.status.idle":"2021-08-17T10:14:36.87373Z","shell.execute_reply.started":"2021-08-17T10:14:36.862483Z","shell.execute_reply":"2021-08-17T10:14:36.872831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"MGMT_value\"].values","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:39.118397Z","iopub.execute_input":"2021-08-17T10:14:39.118779Z","iopub.status.idle":"2021-08-17T10:14:39.126423Z","shell.execute_reply.started":"2021-08-17T10:14:39.118746Z","shell.execute_reply":"2021-08-17T10:14:39.125251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_retriever = DataRetriever(\n df_train[\"BraTS21ID\"].values, \n df_train[\"MGMT_value\"].values, \n)\n\nvalid_data_retriever = DataRetriever(\n df_valid[\"BraTS21ID\"].values, \n df_valid[\"MGMT_value\"].values,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:41.130161Z","iopub.execute_input":"2021-08-17T10:14:41.130507Z","iopub.status.idle":"2021-08-17T10:14:41.135483Z","shell.execute_reply.started":"2021-08-17T10:14:41.130476Z","shell.execute_reply":"2021-08-17T10:14:41.134539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### A better look at the pytorch DataRetriever object;\n> more information: https://stanford.edu/~shervine/blog/pytorch-how-to-generate-data-parallel","metadata":{}},{"cell_type":"markdown","source":"## [Quick Context] for next annotation code block;","metadata":{}},{"cell_type":"code","source":"print(train_data_retriever)#the pytorch dataset object memory location we created with the the DataRetriever Class","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:43.006041Z","iopub.execute_input":"2021-08-17T10:14:43.006481Z","iopub.status.idle":"2021-08-17T10:14:43.011365Z","shell.execute_reply.started":"2021-08-17T10:14:43.006444Z","shell.execute_reply":"2021-08-17T10:14:43.010145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data_retriever[100])#access the train_data_retriever pytorch object and see the dictionary that it generates with the given BraTs21ID and MGMT_value values","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:43.60584Z","iopub.execute_input":"2021-08-17T10:14:43.606156Z","iopub.status.idle":"2021-08-17T10:14:43.944297Z","shell.execute_reply.started":"2021-08-17T10:14:43.606126Z","shell.execute_reply":"2021-08-17T10:14:43.943493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data_retriever[100][\"X\"])#access the contents of the keys of the pytorch dataretriever object dictionary","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:45.094099Z","iopub.execute_input":"2021-08-17T10:14:45.094426Z","iopub.status.idle":"2021-08-17T10:14:45.212093Z","shell.execute_reply.started":"2021-08-17T10:14:45.094396Z","shell.execute_reply":"2021-08-17T10:14:45.211291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_retriever[100][\"X\"].numpy()[0]#access the layer of the object that is contains a numpy array of the image data at the index 0","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:47.398093Z","iopub.execute_input":"2021-08-17T10:14:47.398458Z","iopub.status.idle":"2021-08-17T10:14:47.512626Z","shell.execute_reply.started":"2021-08-17T10:14:47.398425Z","shell.execute_reply":"2021-08-17T10:14:47.511643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\nfor i in range(3):\n plt.subplot(1, 3, i + 1)\n plt.imshow(train_data_retriever[100][\"X\"].numpy()[i], cmap=\"gray\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:49.114083Z","iopub.execute_input":"2021-08-17T10:14:49.114407Z","iopub.status.idle":"2021-08-17T10:14:49.856282Z","shell.execute_reply.started":"2021-08-17T10:14:49.114377Z","shell.execute_reply":"2021-08-17T10:14:49.855326Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#visualizing the data we have stored in the DataRetriever pytorch object\nplt.figure(figsize=(16, 6))\n#use matlab to create a figure that has the dim width 16 by height 6 (by the way, figsize by default is 6by4 with units in inches, src: https://www.pythonpool.com/matplotlib-figsize/)\nfor i in range(3):\n plt.subplot(1, 3, i+1)#create a subplot w/ following params(row=1,columns=3, index=i+1 or simply 4[bc 0,1,2,3])\n #subplot uses 1 based indexing and includes the last mention so i+1 equates to iterating through the contents of the indices 1,2,3 in the train_data_retriever \n plt.imshow(train_data_retriever[100][\"X\"].numpy()[i], cmap=\"gray\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n def __init__(self):\n super().__init__()\n self.net = efficientnet_pytorch.EfficientNet.from_name(\"efficientnet-b0\")\n checkpoint = torch.load(\"../input/efficientnet-pytorch/efficientnet-b0-08094119.pth\")\n self.net.load_state_dict(checkpoint)\n n_features = self.net._fc.in_features\n self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n \n def forward(self, x):\n out = self.net(x)\n return out","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:14:52.298258Z","iopub.execute_input":"2021-08-17T10:14:52.298613Z","iopub.status.idle":"2021-08-17T10:14:52.304355Z","shell.execute_reply.started":"2021-08-17T10:14:52.29858Z","shell.execute_reply":"2021-08-17T10:14:52.303408Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#\"base class for all neural network modules\"-> source: https://pytorch.org/docs/stable/generated/torch.nn.Module.html\n# same source: \"all models used should also be a subclass to this class\"\nclass Model(nn.Module):\n def __init__(self):\n #initialization \n super().__init__()\n #the super() method allows us to access methods from a parent class from within the child class \n self.net = efficientnet_pytorch.EfficientNet.from_name(\"efficientnet-b0\")\n #here the author has chosen the 'efficientnet-b0' from the EfficientNet library which, based on the graphing on \n #this blog(https://ai.googleblog.com/2019/05/efficientnet-improving-accuracy-and.html)is inferior to the efficientnet-b7\n checkpoint = torch.load(\"../input/efficientnet-pytorch/efficientnet-b0-08094119.pth\")\n #here we are using torches unpickling capabilities and loading an existing, deserialized model from the efficient net library\n self.net.load_state_dict(checkpoint)\n #here we load the state dict of the checkpoint variable which gives us access to the learnable params such as the conv layers used as well as the hyperparameters\n n_features = self.net._fc.in_features\n #here we set the variable n_features to the in_features contained in the state_dict of the loaded model(the size of each input sample)\n #more insight by user:hktxt @ https://discuss.pytorch.org/t/how-to-modify-the-final-fc-layer-based-on-the-torch-model/766/23\n #also see the docs for fc.in_features here: https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/linear.py\n self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n #applies a linear transformation using the features we extracted from the state_dict earlier and set the bias to true\n #bias refers to the additive b in the y = mX^n + b linear function \n \n def forward(self, x):\n out = self.net(x) # an example of the forward feeding net we could use is F.relu(self.conv1(x)) , where F is what we imported torch.nn.functional as\n return out","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### More info on the comments from the previous code block can be found here:\n\n> self.net.load_state_dict(): [one](https://pytorch.org/tutorials/beginner/saving_loading_models.html) and [two](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict)\n\n#### From the source: \" a state_dict is a python dictionary object that maps each layer to its parameter tensor(where the parameters within layers are learnable, i.e. conv layers, linear layers, and running_mean for bathnorm). optimizer objects(torch.optim) also have information about the optimizers state and the hyperparameters used \"\n","metadata":{}},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"class LossMeter:\n def __init__(self):\n self.avg = 0\n self.n = 0\n\n def update(self, val):\n self.n += 1\n # incremental update\n self.avg = val / self.n + (self.n - 1) / self.n * self.avg\n\n \nclass AccMeter:\n def __init__(self):\n self.avg = 0\n self.n = 0\n \n def update(self, y_true, y_pred):\n y_true = y_true.cpu().numpy().astype(int)\n y_pred = y_pred.cpu().numpy() >= 0\n last_n = self.n\n self.n += len(y_true)\n true_count = np.sum(y_true == y_pred)\n # incremental update\n self.avg = true_count / self.n + last_n / self.n * self.avg","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:15:00.019704Z","iopub.execute_input":"2021-08-17T10:15:00.020036Z","iopub.status.idle":"2021-08-17T10:15:00.027104Z","shell.execute_reply.started":"2021-08-17T10:15:00.020006Z","shell.execute_reply":"2021-08-17T10:15:00.026241Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#create a class responsible for accounting for the loss while training\nclass LossMeter:\n def __init__(self):\n #initialize values\n self.avg = 0 #set avg to 0 to begin with\n self.n = 0 #set n(count) to 0 to begin with\n\n def update(self, val):\n self.n += 1\n #author comment: [ incremental update ] \n self.avg = val / self.n + (self.n - 1) / self.n * self.avg\n #sets the average loss to the value passed divided by what is effectively;\n '''\n val / self.n + [ (self.n - 1)/self.n * self.avg ]\n !!!!presently not sure why we set self.avg like this, please leave a comment if it is something clear that I am missing here!!!!\n '''\n\n#create a class responsible for updating the change in accuracy of the model\nclass AccMeter:\n def __init__(self):\n #initialize values\n self.avg = 0 #set avg to 0 to begin with\n self.n = 0 #set n(count) to 0 to begin with\n \n \n def update(self, y_true, y_pred):\n y_true = y_true.cpu().numpy().astype(int)#.cpu() moves the value to the cpu for the operation of changing the value of y_true to a numpy array as type int\n y_pred = y_pred.cpu().numpy() >= 0 #.cpu() moves the value to the cpu for the operation setting array values in pred to True or False for if they are greater\n # than or equal to 0\n last_n = self.n #sets value of last_n to be the most recent n value\n self.n += len(y_true)\n #sets the n value to itself + the length of the array y_true\n true_count = np.sum(y_true == y_pred)\n #true_count is set to the following statement that compares the y_true with the y_pred and equals 1 or 0 for true or false respectively\n # collects the matching amount between the y_true and y_pred array (as per the sum aspect) and returns the total matching amount\n # incremental update\n self.avg = true_count / self.n + last_n / self.n * self.avg\n #average set to the sum of the number of matching predictions divided by the count and the value of the latest count divided by the count multiplied by the current average\n # !!!once again this math has a basis in the docs but I need to look deeper, if you happen to know please leave a comment !!!","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Quick Demonstration] Understanding the true_count variable from the AccMeter Class:\n","metadata":{}},{"cell_type":"code","source":"array1 = np.array([1,2,3,4])\narray2 = np.array([1,2,3,3])\nyo = np.sum(array1 == array2)\nprint(yo)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:15:03.230189Z","iopub.execute_input":"2021-08-17T10:15:03.230539Z","iopub.status.idle":"2021-08-17T10:15:03.23587Z","shell.execute_reply.started":"2021-08-17T10:15:03.230509Z","shell.execute_reply":"2021-08-17T10:15:03.235026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"class Trainer:\n def __init__(\n self, \n model, \n device, \n optimizer, \n criterion, \n loss_meter, \n score_meter\n ):\n self.model = model\n self.device = device\n self.optimizer = optimizer\n self.criterion = criterion\n self.loss_meter = loss_meter\n self.score_meter = score_meter\n \n self.best_valid_score = -np.inf\n self.n_patience = 0\n \n self.messages = {\n \"epoch\": \"[Epoch {}: {}] loss: {:.5f}, score: {:.5f}, time: {} s\",\n \"checkpoint\": \"The score improved from {:.5f} to {:.5f}. Save model to '{}'\",\n \"patience\": \"\\nValid score didn't improve last {} epochs.\"\n }\n \n def fit(self, epochs, train_loader, valid_loader, save_path, patience): \n for n_epoch in range(1, epochs + 1):\n self.info_message(\"EPOCH: {}\", n_epoch)\n \n train_loss, train_score, train_time = self.train_epoch(train_loader)\n valid_loss, valid_score, valid_time = self.valid_epoch(valid_loader)\n \n self.info_message(\n self.messages[\"epoch\"], \"Train\", n_epoch, train_loss, train_score, train_time\n )\n \n self.info_message(\n self.messages[\"epoch\"], \"Valid\", n_epoch, valid_loss, valid_score, valid_time\n )\n\n if True:\n# if self.best_valid_score < valid_score:\n self.info_message(\n self.messages[\"checkpoint\"], self.best_valid_score, valid_score, save_path\n )\n self.best_valid_score = valid_score\n self.save_model(n_epoch, save_path)\n self.n_patience = 0\n else:\n self.n_patience += 1\n \n if self.n_patience >= patience:\n self.info_message(self.messages[\"patience\"], patience)\n break\n \n def train_epoch(self, train_loader):\n self.model.train()\n t = time.time()\n train_loss = self.loss_meter()\n train_score = self.score_meter()\n \n for step, batch in enumerate(train_loader, 1):\n X = batch[\"X\"].to(self.device)\n targets = batch[\"y\"].to(self.device)\n self.optimizer.zero_grad()\n outputs = self.model(X).squeeze(1)\n \n loss = self.criterion(outputs, targets)\n loss.backward()\n\n train_loss.update(loss.detach().item())\n train_score.update(targets, outputs.detach())\n\n self.optimizer.step()\n \n _loss, _score = train_loss.avg, train_score.avg\n message = 'Train Step {}/{}, train_loss: {:.5f}, train_score: {:.5f}'\n self.info_message(message, step, len(train_loader), _loss, _score, end=\"\\r\")\n \n return train_loss.avg, train_score.avg, int(time.time() - t)\n \n def valid_epoch(self, valid_loader):\n self.model.eval()\n t = time.time()\n valid_loss = self.loss_meter()\n valid_score = self.score_meter()\n\n for step, batch in enumerate(valid_loader, 1):\n with torch.no_grad():\n X = batch[\"X\"].to(self.device)\n targets = batch[\"y\"].to(self.device)\n\n outputs = self.model(X).squeeze(1)\n loss = self.criterion(outputs, targets)\n\n valid_loss.update(loss.detach().item())\n valid_score.update(targets, outputs)\n \n _loss, _score = valid_loss.avg, valid_score.avg\n message = 'Valid Step {}/{}, valid_loss: {:.5f}, valid_score: {:.5f}'\n self.info_message(message, step, len(valid_loader), _loss, _score, end=\"\\r\")\n \n return valid_loss.avg, valid_score.avg, int(time.time() - t)\n \n def save_model(self, n_epoch, save_path):\n torch.save(\n {\n \"model_state_dict\": self.model.state_dict(),\n \"optimizer_state_dict\": self.optimizer.state_dict(),\n \"best_valid_score\": self.best_valid_score,\n \"n_epoch\": n_epoch,\n },\n save_path,\n )\n \n @staticmethod\n def info_message(message, *args, end=\"\\n\"):\n print(message.format(*args), end=end)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:15:06.278923Z","iopub.execute_input":"2021-08-17T10:15:06.279425Z","iopub.status.idle":"2021-08-17T10:15:06.312769Z","shell.execute_reply.started":"2021-08-17T10:15:06.279375Z","shell.execute_reply":"2021-08-17T10:15:06.311785Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#creating a trainer class for modularity\nclass Trainer:\n def __init__(\n #initializes variables \n self, \n model, #model of choice\n device, #cpu vs gpu\n optimizer, #optimizer, i.e. Adam\n criterion, #defines what determines if a y_pred is a y_true using cross_entropy w/ logits for example\n '''\n later defined as;\n optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n criterion = torch_functional.binary_cross_entropy_with_logits\n \n source for understanding how to use this module:https://sebastianraschka.com/faq/docs/pytorch-crossentropy.html\n what is a logit? src: https://www.sciencedirect.com/topics/mathematics/logit-link-function\n '''\n loss_meter, #loss tracker\n score_meter #accuracy grading\n ):\n self.model = model\n self.device = device\n self.optimizer = optimizer\n self.criterion = criterion\n self.loss_meter = loss_meter\n self.score_meter = score_meter\n \n self.best_valid_score = -np.inf #RUN IN A CODE BLOCK BY ITSELF!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n self.n_patience = 0 #number of epochs with no improvement after which learning rate will be reduced\n #initially set to zero, will eventually be used in a loop in order to facilitate the saving of the best model while training\n \n #setting up messages for corresponding model analytics while training. We can leverage a staticmethod decorator function populate our messages with relevant data\n #as our model trains\n self.messages = {\n \"epoch\": \"[Epoch {}: {}] loss: {:.5f}, score: {:.5f}, time: {} s\", # sets up epoch fstring\n \"checkpoint\": \"The score improved from {:.5f} to {:.5f}. Save model to '{}'\", #sets up epoch fstring\n \"patience\": \"\\nValid score didn't improve last {} epochs.\" #sets up pateince fstring\n }\n \n #training our data with our validation set and then saving the path of the model\n def fit(self, epochs, train_loader, valid_loader, save_path, patience): \n for n_epoch in range(1, epochs + 1):\n #for each epoch within the inclusive range of epochs were working with initialize info_message variable as such\n self.info_message(\"EPOCH: {}\", n_epoch)\n \n train_loss, train_score, train_time = self.train_epoch(train_loader)\n #another source that does a similar train_epoch process: https://github.com/Kulbear/pytorch-project-template/blob/master/trainers/mnist_trainer.py\n valid_loss, valid_score, valid_time = self.valid_epoch(valid_loader)\n \n self.info_message(\n self.messages[\"epoch\"], \"Train\", n_epoch, train_loss, train_score, train_time\n )\n #it is not immediately clear to me why we initialize info_message twice here\n self.info_message(\n self.messages[\"epoch\"], \"Valid\", n_epoch, valid_loss, valid_score, valid_time\n )\n\n if True: #will be true so long as the for loop continues\n# author's comment: [ if self.best_valid_score < valid_score: ]\n self.info_message(\n self.messages[\"checkpoint\"], self.best_valid_score, valid_score, save_path\n )\n self.best_valid_score = valid_score\n self.save_model(n_epoch, save_path)\n self.n_patience = 0\n else: #will occur when range of epochs is met and will increase n_patience value to 1\n self.n_patience += 1\n \n if self.n_patience >= patience:#if n_patience exceeds the input parameter patience, leave a message about it in the log and break the loop(end the function)\n self.info_message(self.messages[\"patience\"], patience)\n break\n \n #training epoch function using the dataset loaded by train_loader\n def train_epoch(self, train_loader):\n self.model.train()#trains model with dropout layers and BatchNorm actively doing their functions in the background\n t = time.time()#t variable set to the displacement of time since last epoch completion\n train_loss = self.loss_meter() #sets train loss variable\n train_score = self.score_meter() #sets train accuracy score variable\n \n for step, batch in enumerate(train_loader, 1): #enumerate step and batch of the trainloader object starting at 1\n X = batch[\"X\"].to(self.device) #set X to the corresponding numpy array of batch[\"X\"] and do it on the available accelerator\n targets = batch[\"y\"].to(self.device) #set y to the corresponding numpy array of batch[\"y\"] and do it on the available accelerator\n self.optimizer.zero_grad()# \"Sets the gradients of all optimized torch.Tensor s to zero\" source: https://pytorch.org/docs/stable/generated/torch.optim.Optimizer.zero_grad.html\n outputs = self.model(X).squeeze(1) #flatten the row-wise contents of the model results when X(training data) is run through it and save it to outputs\n \n loss = self.criterion(outputs, targets) #use criterion to determine your loss between the outputs and the targets(true)\n loss.backward() #computes the gradient based on the calculated loss\n\n train_loss.update(loss.detach().item())#update our weight with the detached singular python value that is the loss we\n #calculated in the previous line based on the train_loss which was our loss_meter we set earlier\n train_score.update(targets, outputs.detach())#update our score based on the targets and outputs(the detach() excludes the calculated gradient that ordinarily gets added on)\n\n self.optimizer.step() #update the current model by the predefined step with the gradient taken into account\n #great discussion of this her: https://discuss.pytorch.org/t/how-are-optimizer-step-and-loss-backward-related/7350\n \n _loss, _score = train_loss.avg, train_score.avg #set _loss and _score to the average train loss and train score respectively\n message = 'Train Step {}/{}, train_loss: {:.5f}, train_score: {:.5f}' #create a message variable to be read out later by our static method\n self.info_message(message, step, len(train_loader), _loss, _score, end=\"\\r\") #set message information to what we have collected thus far\n \n return train_loss.avg, train_score.avg, int(time.time() - t)#return the loss average, score average, and the time that has passed during this epoch\n \n #here we are just using the loaded model from earlier(transfer learning guy), nothing new is learned here(or at least that is what I am interpreting from this\n #like just in case just runnning inference was more effective as a model for our current data set than actually using the data to train with)\n def valid_epoch(self, valid_loader):\n self.model.eval()#eval() therefore no dropout layers or BatchNorm\n t = time.time()\n valid_loss = self.loss_meter()\n valid_score = self.score_meter()\n\n for step, batch in enumerate(valid_loader, 1):\n with torch.no_grad():#no_grad() therefore no \n X = batch[\"X\"].to(self.device)\n targets = batch[\"y\"].to(self.device)\n\n outputs = self.model(X).squeeze(1)\n loss = self.criterion(outputs, targets)\n\n valid_loss.update(loss.detach().item())\n valid_score.update(targets, outputs)\n \n _loss, _score = valid_loss.avg, valid_score.avg\n message = 'Valid Step {}/{}, valid_loss: {:.5f}, valid_score: {:.5f}'\n self.info_message(message, step, len(valid_loader), _loss, _score, end=\"\\r\")\n \n return valid_loss.avg, valid_score.avg, int(time.time() - t)\n \n #function for saving model as zip-file based file format \n def save_model(self, n_epoch, save_path):\n torch.save(\n { #object values; see docs: https://pytorch.org/docs/stable/generated/torch.save.html\n \"model_state_dict\": self.model.state_dict(),\n \"optimizer_state_dict\": self.optimizer.state_dict(),\n \"best_valid_score\": self.best_valid_score,\n \"n_epoch\": n_epoch,\n },\n save_path,#file name / location to save to\n )\n \n #staticmethod decorator(i.e. no ability to access or modify class state)\n @staticmethod\n def info_message(message, *args, end=\"\\n\"):#indiscriminately executes any message variables with their constituent fstring comprhensions for display in logs\n print(message.format(*args), end=end)#effectively prints out the message in the code logs when the notebook is run(?)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## [Quick Demonstration]\n#### What happens when we detach() a tensor? Let's see:\n> source: user: ptrblck @ https://discuss.pytorch.org/t/how-to-detach-specific-components-in-the-loss/13983/7","metadata":{}},{"cell_type":"code","source":"modelA = nn.Linear(10, 10)\nmodelB = nn.Linear(10, 10)\nmodelC = nn.Linear(10, 10)\n\nx = torch.randn(1, 10)\na = modelA(x)\nb = modelB(a.detach())\nb.mean().backward()\nprint(modelA.weight.grad)\nprint(modelB.weight.grad)\nprint(modelC.weight.grad)\n\nc = modelC(a)\nc.mean().backward()\nprint(modelA.weight.grad)\nprint(modelB.weight.grad)\nprint(modelC.weight.grad)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:15:10.491187Z","iopub.execute_input":"2021-08-17T10:15:10.491521Z","iopub.status.idle":"2021-08-17T10:15:10.570176Z","shell.execute_reply.started":"2021-08-17T10:15:10.49149Z","shell.execute_reply":"2021-08-17T10:15:10.5692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"import efficientnet_pytorch\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ntrain_data_retriever = DataRetriever(\n df_train[\"BraTS21ID\"].values, \n df_train[\"MGMT_value\"].values, \n)\n\nvalid_data_retriever = DataRetriever(\n df_valid[\"BraTS21ID\"].values, \n df_valid[\"MGMT_value\"].values,\n)\n\ntrain_loader = torch_data.DataLoader(\n train_data_retriever,\n batch_size=8,\n shuffle=True,\n num_workers=8,\n)\n\nvalid_loader = torch_data.DataLoader(\n valid_data_retriever, \n batch_size=8,\n shuffle=False,\n num_workers=8,\n)\n\nmodel = Model()\nmodel.to(device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\ncriterion = torch_functional.binary_cross_entropy_with_logits\n\ntrainer = Trainer(\n model, \n device, \n optimizer, \n criterion, \n LossMeter, \n AccMeter\n)\n\nhistory = trainer.fit(\n 10, \n train_loader, \n valid_loader, \n f\"best-model-0.pth\", \n 100,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:15:13.087454Z","iopub.execute_input":"2021-08-17T10:15:13.087814Z","iopub.status.idle":"2021-08-17T10:25:29.697292Z","shell.execute_reply.started":"2021-08-17T10:15:13.087782Z","shell.execute_reply":"2021-08-17T10:25:29.696362Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n#set accelerator to gpu if possible\n\n#Retrieve training data\ntrain_data_retriever = DataRetriever(\n df_train[\"BraTS21ID\"].values, \n df_train[\"MGMT_value\"].values, \n)\n\n#Retrieve validation data\nvalid_data_retriever = DataRetriever(\n df_valid[\"BraTS21ID\"].values, \n df_valid[\"MGMT_value\"].values,\n)\n\n#load train dataset with desired batch size, data order(shuffle), and workers\ntrain_loader = torch_data.DataLoader(\n train_data_retriever,\n batch_size=8,\n shuffle=True,\n num_workers=8,\n)\n\n#load validation dataset with desired batch size, data order(shuffle), and workers\nvalid_loader = torch_data.DataLoader(\n valid_data_retriever, \n batch_size=8,\n shuffle=False,\n num_workers=8,\n)\n\n#define model\nmodel = Model()\n#coordinate device for model to use\nmodel.to(device)\n\n#set optimizer for learning rate of model\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n#set criterion for determining true or false classification\ncriterion = torch_functional.binary_cross_entropy_with_logits\n\n#Train with chosen specifications and save it to the trainer variable\ntrainer = Trainer(\n model, \n device, \n optimizer, \n criterion, \n LossMeter, \n AccMeter\n)\n\n#for logging purposes?\nhistory = trainer.fit( #uses fit function from the Train class on the trainer object; should effectively fit the trainer object with the best model's state_dict() (i think? please correct me if I am mis reading this)\n 2, #specifying the loader amount(?)\n train_loader, \n valid_loader, \n f\"best-model-0.pth\", #stores the string that states that will alert you in the logs where the best model is stored \n 100, #flush_logs_every_n_steps where n = 100(?)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"models = []\nfor i in range(1):\n model = Model()\n model.to(device)\n \n checkpoint = torch.load(f\"best-model-{i}.pth\")\n model.load_state_dict(checkpoint[\"model_state_dict\"])\n model.eval()\n \n models.append(model)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:25:29.699241Z","iopub.execute_input":"2021-08-17T10:25:29.699651Z","iopub.status.idle":"2021-08-17T10:25:29.970645Z","shell.execute_reply.started":"2021-08-17T10:25:29.699604Z","shell.execute_reply":"2021-08-17T10:25:29.96972Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"models = []\n#set variable models to an empty array\nfor i in range(1):\n #one for loop for on evalue to generate a single model\n model = Model()\n #model variable = the chosen model function\n model.to(device)\n #load model into our choice accelerator\n \n checkpoint = torch.load(f\"best-model-{i}.pth\")\n #load the unserialized model from pkl to the variable checkpoint using torch library\n model.load_state_dict(checkpoint[\"model_state_dict\"])\n #load the state_dict information that is correspondent to this model(the layers, the hyperparameters, the optim, etc.)\n model.eval()\n #eval() will effectively remove the state_dict info that is responsible for quality training but not inference, for example it removes the dropout layers and\n #the BatchNorm layers(we pair this with no_grad() later on as well as the source also suggests)\n #source: https://stackoverflow.com/questions/60018578/what-does-model-eval-do-in-pytorch\n models.append(model)\n #add our freshly loaded and coordinated model to the models array","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Un/Annotated","metadata":{}},{"cell_type":"code","source":"models = []\n#set variable models to an empty array\nfor i in range(1):\n #one for loop for on evalue to generate a single model\n model = Model()\n #model variable = the chosen model function\n model.to(device)\n #load model into our choice accelerator\n \n checkpoint = torch.load(f\"best-model-{i}.pth\")\n #load the unserialized model from pkl to the variable checkpoint using torch library\n model.load_state_dict(checkpoint[\"model_state_dict\"])\n #load the state_dict information that is correspondent to this model(the layers, the hyperparameters, the optim, etc.)\n model.eval()\n #eval() will effectively remove the state_dict info that is responsible for quality training but not inference, for example it removes the dropout layers and\n #the BatchNorm layers(we pair this with no_grad() later on as well as the source also suggests)\n #source: https://stackoverflow.com/questions/60018578/what-does-model-eval-do-in-pytorch\n models.append(model)\n #add our freshly loaded and coordinated model to the models array","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:25:29.972403Z","iopub.execute_input":"2021-08-17T10:25:29.972773Z","iopub.status.idle":"2021-08-17T10:25:30.233253Z","shell.execute_reply.started":"2021-08-17T10:25:29.972726Z","shell.execute_reply":"2021-08-17T10:25:30.232415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### As far as I understand it, the next code block will be used as a reference for our own submission file generation so that our submission file has the same amount of rows\n> open to correction if I am misunderstanding main author's intentions here","metadata":{}},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"#same DataRetriever logic as before except we exclude the self.targets from the initialization phase as we aim to use this dataretriever for predictions\nclass DataRetriever(torch_data.Dataset):\n def __init__(self, paths):\n self.paths = paths\n \n def __len__(self):\n return len(self.paths)\n \n def __getitem__(self, index):\n _id = self.paths[index]\n patient_path = f\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/test/{str(_id).zfill(5)}/\"\n channels = []\n for t in (\"FLAIR\", \"T1w\", \"T1wCE\"): # \"T2w\"\n t_paths = sorted(\n glob.glob(os.path.join(patient_path, t, \"*\")), \n key=lambda x: int(x[:-4].split(\"-\")[-1]),\n )\n # start, end = int(len(t_paths) * 0.475), int(len(t_paths) * 0.525)\n x = len(t_paths)\n if x < 10:\n r = range(x)\n else:\n d = x // 10\n r = range(d, x - d, d)\n \n channel = []\n # for i in range(start, end + 1):\n for i in r:\n channel.append(cv2.resize(load_dicom(t_paths[i]), (256, 256)) / 255)\n channel = np.mean(channel, axis=0)\n channels.append(channel)\n \n return {\"X\": torch.tensor(channels).float(), \"id\": _id}","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:21.916053Z","iopub.execute_input":"2021-08-17T10:31:21.916411Z","iopub.status.idle":"2021-08-17T10:31:21.926266Z","shell.execute_reply.started":"2021-08-17T10:31:21.916378Z","shell.execute_reply":"2021-08-17T10:31:21.925291Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\n\ntest_data_retriever = DataRetriever(\n submission[\"BraTS21ID\"].values, \n)\n\ntest_loader = torch_data.DataLoader(\n test_data_retriever,\n batch_size=4,\n shuffle=False,\n num_workers=8,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:35.821034Z","iopub.execute_input":"2021-08-17T10:31:35.821358Z","iopub.status.idle":"2021-08-17T10:31:35.832718Z","shell.execute_reply.started":"2021-08-17T10:31:35.821328Z","shell.execute_reply":"2021-08-17T10:31:35.831879Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#sets submission to sample_submission.csv\nsubmission = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")\n\n#retrieves sample_submission BraTS21ID values\ntest_data_retriever = DataRetriever(\n submission[\"BraTS21ID\"].values, \n)\n\n#uses DataLoader to load the sample_submission data as we would with our submission data\ntest_loader = torch_data.DataLoader(\n test_data_retriever,\n batch_size=4,\n shuffle=False,\n num_workers=8,\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"y_pred = []\nids = []\n\nfor e, batch in enumerate(test_loader):\n print(f\"{e}/{len(test_loader)}\", end=\"\\r\")\n with torch.no_grad():\n tmp_pred = np.zeros((batch[\"X\"].shape[0], ))\n for model in models:\n tmp_res = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n tmp_pred += tmp_res\n y_pred.extend(tmp_pred)\n ids.extend(batch[\"id\"].numpy().tolist())","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:47.36453Z","iopub.execute_input":"2021-08-17T10:31:47.364887Z","iopub.status.idle":"2021-08-17T10:31:57.335695Z","shell.execute_reply.started":"2021-08-17T10:31:47.364857Z","shell.execute_reply":"2021-08-17T10:31:57.334789Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"y_pred = [] #set an array for prediction values\nids = [] #set an array for the corresponding ids\n\nfor e, batch in enumerate(test_loader):\n #for loop that can reference the element indice(e) and the corresponding key(batch) from the object test_loader\n print(f\"{e}/{len(test_loader)}\", end=\"\\r\") # the end=\"\\r\" is just a new line statement\n #prints an f-string that gives the current batch the model is predicting on\n with torch.no_grad():\n #no_grad() indicates that we are disabling gradient descent which is good for inference where we don't want to consume extra memory by avoiding the Tensor.backward() call\n #docs: https://pytorch.org/docs/stable/generated/torch.no_grad.html\n tmp_pred = np.zeros((batch[\"X\"].shape[0], ))\n #initializing a temporary predictions numpy array that has the shape of the batch array\n for model in models:\n tmp_res = torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze()\n #runs inference using the chosen model on the current batch, gets the predictions, and uses the squeeze function to put the preds into a 1d array(flattens the tensor)\n tmp_pred += tmp_res\n #appends to the tmp_pred array\n y_pred.extend(tmp_pred)\n #extends the y_pred by adding the contents of the tmp_pred array contents to it\n ids.extend(batch[\"id\"].numpy().tolist())\n #extends the numpy id array with the current batch of id's and changes the numpy array to a python list with pytorch tolist() function","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:19.903853Z","iopub.execute_input":"2021-08-17T10:32:19.904179Z","iopub.status.idle":"2021-08-17T10:32:19.911165Z","shell.execute_reply.started":"2021-08-17T10:32:19.904147Z","shell.execute_reply":"2021-08-17T10:32:19.910138Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"#creating a dataframe out of our selected ids and their constituent predictions\nsubmission = pd.DataFrame({\"BraTS21ID\": ids, \"MGMT_value\": y_pred})\n#converting the dataframe to a csv with the proper indexing for submission\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unannotated","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(5, 5))\nplt.hist(submission[\"MGMT_value\"]);","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:31:57.754309Z","iopub.execute_input":"2021-08-17T10:31:57.754652Z","iopub.status.idle":"2021-08-17T10:31:57.897919Z","shell.execute_reply.started":"2021-08-17T10:31:57.754617Z","shell.execute_reply":"2021-08-17T10:31:57.896915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Annotated","metadata":{}},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-08-17T10:32:10.85137Z","iopub.execute_input":"2021-08-17T10:32:10.851758Z","iopub.status.idle":"2021-08-17T10:32:10.865506Z","shell.execute_reply.started":"2021-08-17T10:32:10.851722Z","shell.execute_reply":"2021-08-17T10:32:10.864525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}