{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport tensorflow as tf\nimport pandas as pd\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nimport datetime\nimport math\n\nfrom scipy import ndimage\n\nusing_data_from_kaggle = True\n\ndef PrintTime(msg=\"\"):\n    if len(msg) > 0:\n        print(\"\\n\",msg,\" at Time: \",datetime.datetime.now(),\"\\n\")\n    else:\n        print(\"\\n\",datetime.datetime.now(),\"\\n\")\n\nPrintTime(\"Start Notebook\")","metadata":{"execution":{"iopub.status.busy":"2021-08-31T18:57:12.944595Z","iopub.execute_input":"2021-08-31T18:57:12.94513Z","iopub.status.idle":"2021-08-31T18:57:19.003196Z","shell.execute_reply.started":"2021-08-31T18:57:12.945023Z","shell.execute_reply":"2021-08-31T18:57:19.002048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def GetSizeOfImage(x_low,x_mid,x_high):\n    x = x_high * 65536 + x_mid * 256 + x_low\n    x_half = x / 2\n    x_half_sr = math.sqrt(x_half)\n    num_rows = 0\n    if math.fabs(512-x_half_sr) < math.fabs(256-x_half_sr):\n        num_rows = 512\n    else:\n        num_rows = 256\n    num_cols = int(x / num_rows)\n    return num_rows,num_cols\n\ndef FindIndexOfStartOfImage(f):\n    # Input: f is a .dcm file\n    # Return: the position in the file where the actual data resides (typically 1004)\n    \n    f.seek(0,0)\n    buf = bytearray(2048)\n    subbytes = bytearray(4)\n    subbytes[0] = 224 # x'E0'\n    subbytes[1] = 127 # x'7F'\n    subbytes[2] = 16  # x'10'\n    subbytes[3] = 0   # x'00'\n    \n    subbytes_rowcol = bytearray(8)\n    subbytes_rowcol[0] = 40\n    subbytes_rowcol[1] = 0\n    subbytes_rowcol[2] = 16\n    subbytes_rowcol[3] = 0\n    subbytes_rowcol[4] = 2\n    subbytes_rowcol[5] = 0\n    subbytes_rowcol[6] = 0\n    subbytes_rowcol[7] = 0\n    \n    \n    \n    buf = f.read(2048)\n    idx = buf.find(subbytes)\n    idx_row_col = buf.find(subbytes_rowcol)\n    num_row = 0\n    num_col = 0\n    if idx_row_col > 0 and idx > 0:\n        num_row = 256 * buf[idx_row_col + 8 + 1] + buf[idx_row_col + 8 + 0]\n        num_col = 256 * buf[idx_row_col + 10 + 8 + 1] + buf[idx_row_col + 10 + 8 + 0]\n        num_col *= 2\n        size = 65536 * buf[idx+6] + 256 * buf[idx+5] + buf[idx+4]\n        return idx + 8,num_row,num_col\n    elif idx > 0:\n        num_row,num_col = GetSizeOfImage(buf[idx+4],buf[idx+5],buf[idx+6])\n        return idx + 8,num_row,num_col\n    else:\n        return -1,-1,-1\n    \n    \ndef resize_volume(img,desired_width,desired_height,desired_depth):\n\n    # Get current depth\n    current_depth = img.shape[-1]\n    current_width = img.shape[0]\n    current_height = img.shape[1]\n\n    # Compute depth factor\n    depth = current_depth / desired_depth\n    width = current_width / desired_width\n    height = current_height / desired_height\n    depth_factor = 1 / depth\n    width_factor = 1 / width\n    height_factor = 1 / height\n\n    # Rotate\n    img = ndimage.rotate(img, 90, reshape=False)\n    \n    # Resize across z-axis\n    img = ndimage.zoom(img, (width_factor, height_factor, depth_factor), order=1)\n    \n    return img\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-31T18:57:36.124828Z","iopub.execute_input":"2021-08-31T18:57:36.125181Z","iopub.status.idle":"2021-08-31T18:57:36.142007Z","shell.execute_reply.started":"2021-08-31T18:57:36.125151Z","shell.execute_reply":"2021-08-31T18:57:36.140838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The number of rows X columns X slices for each type\ntarget_num_rows = 512\ntarget_num_cols = 512\ntarget_num_slices_per_filetype = 32\n\n# The number of rows X columns X slices that actually get fed into the 3D CNN for each type\ncnn_num_rows = 256\ncnn_num_cols = 256\ncnn_num_slices_per_filetype = 16\n\ndef Create_128by128by16(directory):\n    # Input: directory is a folder like train/00000/FLAIR or test/00091/T1wCE\n    #\n    # Return: a ndarray of shape (cnn_num_rows,cnn_num_cols,cnn_num_slices_per_filetype)\n    # IMPORTANT: It is assumed the image cols will only take the low-order byte!\n    # so num_col = 512 means the original image has 1024 bytes\n    \n    big_buf = np.zeros( (target_num_rows,target_num_cols,512),dtype = np.float32)  \n        # we will first resize z-axis to target_num_slices_per_filetype later\n        # then we will resize all 3 axis to cnn_num_rows X cnn_num_cols X cnn_num_slices_per_filetype\n\n    count = 0\n    for filename in os.listdir(directory):\n        file_path = os.path.join(directory, filename)\n        f = open(file_path,\"rb\")\n        pos_of_image,size_of_image_rows,size_of_image_cols = FindIndexOfStartOfImage(f)\n        if pos_of_image > 0:\n            \n            \"\"\"\n            # Is this a known format?\n            # argg... line continuation doesn't work!!\n            if (size_of_image_rows == 512 and size_of_image_cols == 1024):\n                pass\n            elif (size_of_image_rows == 512 and size_of_image_cols == 800):\n                pass\n            elif (size_of_image_rows == 256 and size_of_image_cols == 448):\n                pass\n            elif (size_of_image_rows == 256 and size_of_image_cols == 512):\n                pass\n            elif (size_of_image_rows == 256 and size_of_image_cols == 384):\n                pass\n            else:\n                #print(f\"{file_path} has abnormal size: \",size_of_image_rows,size_of_image_cols)\n                pass\n            \"\"\"\n            \n            f.seek(pos_of_image,0)\n            size_of_image = size_of_image_rows *  size_of_image_cols\n            buf = np.fromfile(f, dtype=np.int8, count=size_of_image)\n            buf = buf.astype(np.float32)\n            buf = buf.reshape((size_of_image_rows,size_of_image_cols))\n            actual_cols = int(size_of_image_cols / 2)\n            buf = buf[:,0:size_of_image_cols:2]  # get rid of high byte\n            if size_of_image_rows <= target_num_rows and actual_cols <= target_num_cols:\n                big_buf[0:size_of_image_rows,0:actual_cols,count] = buf\n            else:\n                # resize image to num_rows X num_cols\n                upd = ndimage.zoom(buf,(target_num_rows/size_of_image_rows,target_num_cols/actual_cols))\n                \n                big_buf[0:target_num_rows,0:target_num_cols,count] = upd\n                \n        else:\n            print(f\"WARNING! file_path {file_path} has pos_of_image = 0 \")\n                \n        f.close()\n        count += 1\n        if count >= 512:  # GDD TODO: CHANGE THIS back to 512\n            break\n        \n    # Now compress the slices of images so it is exactly target_num_rows X target_num_cols X target_num_slices_per_filetype\n    compressed_img1 = resize_volume(big_buf[:,:,0:count],target_num_rows,target_num_cols,target_num_slices_per_filetype)\n\n    # Now compress the slices of images so it is exactly cnn_num_rows X cnn_num_cols X cnn_num_slices_per_filetype\n    compressed_img = resize_volume(compressed_img1,cnn_num_rows,cnn_num_cols,cnn_num_slices_per_filetype)\n    \n    return compressed_img\n\ndef CreateCaseData_128by128by64(directory):\n    # Input: directory is something like \"train/00000\" or \"test/00019\"\n    arr1 = Create_128by128by16(f\"{directory}/FLAIR\")\n    arr2 = Create_128by128by16(f\"{directory}/T1w\")\n    arr3 = Create_128by128by16(f\"{directory}/T1wCE\")\n    arr4 = Create_128by128by16(f\"{directory}/T2w\")\n    arr = np.concatenate([arr1,arr2,arr3,arr4],axis=2)\n    return arr\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-31T18:57:46.762584Z","iopub.execute_input":"2021-08-31T18:57:46.762968Z","iopub.status.idle":"2021-08-31T18:57:46.778739Z","shell.execute_reply.started":"2021-08-31T18:57:46.762938Z","shell.execute_reply":"2021-08-31T18:57:46.777448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import threading\nimport time\nimport pickle\n\nclass myTestThread (threading.Thread):\n    \n    def __init__(self, caseID,directory):\n        threading.Thread.__init__(self)\n        self.caseID = str(caseID).zfill(5)\n        self.directory = directory\n        self.arr = np.zeros((cnn_num_rows,cnn_num_cols,4*cnn_num_slices_per_filetype))\n\n    def run(self):\n        folder = f\"{self.directory}/{self.caseID}\"\n        #print(\"In thread for {self.caseID}, working on folder {folder}\")\n        arr = CreateCaseData_128by128by64(folder)\n        self.arr = arr\n","metadata":{"execution":{"iopub.status.busy":"2021-08-31T18:57:52.424591Z","iopub.execute_input":"2021-08-31T18:57:52.425019Z","iopub.status.idle":"2021-08-31T18:57:52.432552Z","shell.execute_reply.started":"2021-08-31T18:57:52.424983Z","shell.execute_reply":"2021-08-31T18:57:52.431531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Test using 'saved off files'\nmodel_path = \"../input/save-tlgp120gp145/Save_TransferLearning-210830-WED-Gp120Gp145\"\nprint(\"Loaded Saved Model:\",model_path)\nmodel_load = keras.models.load_model(model_path)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-31T18:57:56.469479Z","iopub.execute_input":"2021-08-31T18:57:56.469861Z","iopub.status.idle":"2021-08-31T18:57:58.641092Z","shell.execute_reply.started":"2021-08-31T18:57:56.469827Z","shell.execute_reply":"2021-08-31T18:57:58.640274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def WaitTillProcessingTestDataFinishes(testThreads,numThreads):  \n    num_times_thru_loop = 0\n    while True:\n        num_times_thru_loop += 1\n        \n        atleastOneActive = False\n        count_active_threads = 0\n\n        # check status once per second\n        time.sleep(5)\n\n        for i in range(numThreads):\n            status = testThreads[i].is_alive()\n            if status == True:\n                atleastOneActive = True\n                count_active_threads += 1\n                \n        if num_times_thru_loop % 12 == 0:\n            print(f\"{count_active_threads} out of {numThreads} are still active at \",datetime.datetime.now())\n\n        if atleastOneActive == False:\n            break\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-31T18:58:03.675419Z","iopub.execute_input":"2021-08-31T18:58:03.675975Z","iopub.status.idle":"2021-08-31T18:58:03.682078Z","shell.execute_reply.started":"2021-08-31T18:58:03.675938Z","shell.execute_reply":"2021-08-31T18:58:03.681292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now test the test-set\nPrintTime(\"Start making predictions on test cases \")\n\n# open file to store predictions - file must be named \"submissions.txt\"\npred_file = open(\"./submission.csv\",\"w+\")\npred_file.write(\"BraTS21ID,MGMT_value\\n\")\n\nbase_dir_for_data = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"\nbase_dir_test = f\"{base_dir_for_data}/test\"\nif using_data_from_kaggle == False:\n    base_dir_test = f\"test\"\n\nthreads = []\nnumber_cases_per_loop = 4\n\ncount = 0\nall_cases = []\nfor case in os.listdir(base_dir_test):\n    all_cases.append(case)\n    \nnum_test_groups = int(len(all_cases) / 4)\nnum_cases_last_group = 4\n\nrem_test_cases = len(all_cases) - 4 * num_test_groups\nif rem_test_cases > 0:\n    num_test_groups += 1\n    num_cases_last_group = rem_test_cases\n\nprint(f\"Number of test groups: {num_test_groups}, Number remaining test cases: {rem_test_cases}\")\n\nfor gp_no in range(num_test_groups):\n    threads = []\n    num_cases_this_group = 4\n    if gp_no == (num_test_groups - 1):\n        num_cases_this_group = num_cases_last_group\n        \n    for i in range(num_cases_this_group):\n        case = all_cases[number_cases_per_loop * gp_no + i]\n        print(f\"GN:{gp_no},working on {case} at time {datetime.datetime.now()}\")\n        aThread = myTestThread(case,base_dir_test)\n        aThread.start()\n        threads.append(aThread)\n        \n    WaitTillProcessingTestDataFinishes(threads,num_cases_this_group)\n    \n    # Get the data\n    for i in range(num_cases_this_group):\n        case = all_cases[number_cases_per_loop * gp_no + i]\n        prediction = model_load.predict(np.expand_dims(threads[i].arr, axis=0))[0]  \n    \n        print(f\"case={case}, prediction={prediction[0]}\")\n        pred_file.write(f\"{case},{prediction[0]}\\n\")\n        \n    #if gp_no == 2:  # GDD TODO: REMOVE THIS  \n    #    break\n\n\npred_file.close()\n\nPrintTime(\"END making predictions on test cases\")","metadata":{"execution":{"iopub.status.busy":"2021-08-31T18:58:17.110971Z","iopub.execute_input":"2021-08-31T18:58:17.111474Z","iopub.status.idle":"2021-08-31T19:03:42.726954Z","shell.execute_reply.started":"2021-08-31T18:58:17.11144Z","shell.execute_reply":"2021-08-31T19:03:42.726076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}