{"cells":[{"metadata":{"_kg_hide-output":false,"_kg_hide-input":false,"trusted":true,"_uuid":"141fc8090e16a14132c0a90ef2b776c92500c98b"},"cell_type":"code","source":"import os\nimport pandas as pd\nimport shutil\n\n\nfileList = os.listdir(\"./\") # get file list in the path directory\n\n# list files\nfor f in fileList: \n    print(f)\nfileList2 = os.listdir(\"./model\") # get file list in the path directory\n# list files\nfor f2 in fileList2: \n    print(f2)\n\n#shutil.os.mkdir(\"./graph\")\n#shutil.os.mkdir(\"./model\")\n#shutil.os.mkdir(\"./model/model.h5\")\n#shutil.os.mkdir(\"./model/model.json\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebeae0f194703e4c3b2871bac376267c54ce994a"},"cell_type":"code","source":"# So until here we have input : (m,1024,1024,1), output : (m,1)\nfrom keras import layers\nfrom keras.layers import Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, \\\n    AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D\nfrom keras.models import Model, load_model\nfrom keras.preprocessing import image\nfrom keras.utils import layer_utils\nfrom keras.utils.data_utils import get_file\nfrom keras.applications.imagenet_utils import preprocess_input\nfrom keras.utils.vis_utils import model_to_dot\nfrom keras.utils import plot_model\nfrom keras.initializers import glorot_uniform \nimport scipy.misc\nimport keras.backend as K\n\nK.set_image_data_format('channels_last')\nK.set_learning_phase(1)\n\n\ndef identity_block(X, f, filters, stage, block):\n    \"\"\"\n    Implementation of the identity block as defined in Figure 3\n\n    Arguments:\n    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)\n    f -- integer, specifying the shape of the middle CONV's window for the main path\n    filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n    stage -- integer, used to name the layers, depending on their position in the network\n    block -- string/character, used to name the layers, depending on their position in the network\n\n    Returns:\n    X -- output of the identity block, tensor of shape (n_H, n_W, n_C)\n    \"\"\"\n\n    # defining name basis\n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n\n    # Retrieve Filters: number of filters\n    F1, F2, F3 = filters\n\n    # Save the input value. You'll need this later to add back to the main path.\n    X_shortcut = X\n\n    # First component of main path\n    X = Conv2D(filters=F1, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2a',\n               kernel_initializer=glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n\n    # Second component of main path (≈3 lines)\n    X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b',\n               kernel_initializer=glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n\n    # Third component of main path (≈2 lines)\n    X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c',\n               kernel_initializer=glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X)\n\n    # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)\n    X = Add()([X, X_shortcut])\n    X = Activation('relu')(X)\n\n    return X\n\n\ndef convolutional_block(X, f, filters, stage, block, s=2):\n    \"\"\"\n    Implementation of the convolutional block as defined in Figure 4\n\n    Arguments:\n    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)\n    f -- integer, specifying the shape of the middle CONV's window for the main path\n    filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n    stage -- integer, used to name the layers, depending on their position in the network\n    block -- string/character, used to name the layers, depending on their position in the network\n    s -- Integer, specifying the stride to be used\n\n    Returns:\n    X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)\n    \"\"\"\n\n    # defining name basis\n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n\n    # Retrieve Filters\n    F1, F2, F3 = filters\n\n    # Save the input value\n    X_shortcut = X\n\n    ##### MAIN PATH #####\n    # First component of main path\n    X = Conv2D(F1, (1, 1), strides=(s, s), padding='valid', name=conv_name_base + '2a',\n               kernel_initializer=glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n\n    # Second component of main path (≈3 lines)\n    X = Conv2D(F2, (f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b',\n               kernel_initializer=glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n\n    # Third component of main path (≈2 lines)\n    X = Conv2D(F3, (1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c',\n               kernel_initializer=glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X)\n\n    ##### SHORTCUT PATH #### (≈2 lines)\n    X_shortcut = Conv2D(F3, (1, 1), strides=(s, s), padding='valid', name=conv_name_base + '1',\n                        kernel_initializer=glorot_uniform(seed=0))(X_shortcut)\n    X_shortcut = BatchNormalization(axis=3, name=bn_name_base + '1')(X_shortcut)\n\n    # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)\n    X = Add()([X, X_shortcut])\n    X = Activation('relu')(X)\n\n    return X\n\n\n#ResNet50\n\ndef ResNet50(input_shape=(1024, 1024, 1), classes=1):\n    \"\"\"\n    Implementation of the popular ResNet50 the following architecture:\n    CONV2D -> BATCHNORM -> RELU -> MAXPOOL -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3\n    -> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> TOPLAYER\n\n    Arguments:\n    input_shape -- shape of the images of the dataset\n    classes -- integer, number of classes\n\n    Returns:\n    model -- a Model() instance in Keras\n    \"\"\"\n\n    # Define the input as a tensor with shape input_shape\n    X_input = Input(input_shape)\n\n    # Zero-Padding\n    X = ZeroPadding2D((3, 3))(X_input)\n\n    # Stage 1\n    X = Conv2D(64, (7, 7), strides=(2, 2), name='conv1', kernel_initializer=glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name='bn_conv1')(X)\n    X = Activation('relu')(X)\n    X = MaxPooling2D((3, 3), strides=(2, 2))(X)\n\n    # Stage 2\n    X = convolutional_block(X, f=3, filters=[64, 64, 256], stage=2, block='a', s=1)\n    X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')\n    X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')\n\n    # Stage 3 (≈4 lines)\n    X = convolutional_block(X, f=3, filters=[128, 128, 512], stage=3, block='a', s=2)\n    X = identity_block(X, 3, [128, 128, 512], stage=3, block='b')\n    X = identity_block(X, 3, [128, 128, 512], stage=3, block='c')\n    X = identity_block(X, 3, [128, 128, 512], stage=3, block='d')\n\n    # Stage 4 (≈6 lines)\n    X = convolutional_block(X, f=3, filters=[256, 256, 1024], stage=4, block='a', s=2)\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='b')\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='c')\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='d')\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='e')\n    X = identity_block(X, 3, [256, 256, 1024], stage=4, block='f')\n\n    # Stage 5 (≈3 lines)\n    X = convolutional_block(X, f=3, filters=[512, 512, 2048], stage=5, block='a', s=2)\n    X = identity_block(X, 3, [512, 512, 2048], stage=5, block='b')\n    X = identity_block(X, 3, [512, 512, 2048], stage=5, block='c')\n\n    print('X.shape : ', X.shape)\n    # AVGPOOL (≈1 line). Use \"X = AveragePooling2D(...)(X)\"\n    X = AveragePooling2D((2, 2), name='avg_pool')(X)\n\n    print('X.shape : ', X.shape)\n    # output layer\n    X = Flatten()(X)\n    #X_FOR_BOUNDINGBOX = X\n    print('X.shape : ', X.shape)\n    X = Dense(classes, activation='sigmoid', name='fc' + str(classes), kernel_initializer=glorot_uniform(seed=0))(X)\n    #X_FOR_BOUNDINGBOX = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer=glorot_uniform(seed=0))(X_FOR_BOUNDINGBOX)\n    print('X.shape after Dense : ', X.shape)\n\n    # Create model\n    model = Model(inputs=X_input, outputs=X, name='ResNet50')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c09d94ca98b92636e743f66e23dde8c7f63c674"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport seaborn as sns\nimport math\nimport pydicom\nfrom skimage.transform import rescale, resize, downscale_local_mean\nfrom keras.optimizers import Adam\nfrom keras.callbacks import TensorBoard\nimport os\nimport time\n\n#get all image file\ntrain_images_file_list = os.listdir(\"../input/stage_1_train_images\")\ntest_images_file_list = os.listdir(\"../input/stage_1_test_images\")\n#25684 images\n\n#-----stage_1_train_labels.csv-----\ndata_stage_1_train_labels = pd.read_csv((\"../input/stage_1_train_labels.csv\"))\n'''\n['patientId', 'x', 'y', 'width', 'height', 'Target']\n'''\n\n#-----stage_1_detailed_class_info.csv-----\ndata_stage_1_detailed_class_info = pd.read_csv((\"../input/stage_1_detailed_class_info.csv\"))\n'''\nProvides detailed information about the type of positive or negative class for each image:\n['patientId', 'class']\n'''\n\n\n#-----stage_1_sample_submission.csv-----\ndata_stage_1_sample_submission = pd.read_csv((\"../input/stage_1_sample_submission.csv\"))\n'''\nIt is just one sample datasets :\n['patientId', 'PredictionString']\npatientId PredictionString\n0  000924cf-0f8d-42bd-9158-1af53881a557  0.5 0 0 100 100\n'''\n\n'''\nm_stage_1_train_images :  16226\nm_stage_1_test_images :  1000\ntrain image shape : (1024, 1024,1)\n\nNeed to create the data_train_input_image : (16626,1024,1024,1)\n'''\ndata_train_input_patientid = []\ndata_train_input_image=[]\ndata_test_input_image_file = []\ndata_test_input_image=[]\n#use small train data\nsmall_m=5000#16225\nsmall_m_test=10\nimagesize = 64\nfor i in range(small_m):\n    dcmFile = \"../input/stage_1_train_images/\"+train_images_file_list[i]\n    readDcmFile = pydicom.dcmread(dcmFile)\n    data_train_input_patientid.append(readDcmFile.PatientID)\n    dcmPixelArray = readDcmFile.pixel_array\n    #change the image from (1024,1024) to (64,64)\n    image_resized = resize(dcmPixelArray, (dcmPixelArray.shape[0] / 16, dcmPixelArray.shape[1] / 16),anti_aliasing=True)\n    data_train_input_image.append(image_resized)\n\ndata_train_input_image_arr = np.asarray(data_train_input_image)\ndata_train_input_image_arr = data_train_input_image_arr.reshape(small_m,imagesize,imagesize)\n\ndata_train_input_image_arr = data_train_input_image_arr/255\n\n#transform to (m,1024,1024,1)\ndata_train_input_image_arr = data_train_input_image_arr.reshape(small_m,imagesize,imagesize,1)\ndata_train_input_image_arr[:,[2]] = 1\n\n'''\nI will use classfication + regression model to train the 'target' and bounding box\nI begin with classfication :\nclassfication :input : (m,1024,1024) ['patientId','Target']\n               output : (m , 1)\n'''\nsmall_data_stage_1_train_labels_patientId = []\nsmall_data_stage_1_train_labels_target = []\n#{patientId:target}\npatientid_target = {}\ndata_stage_1_train_labels_patientId = data_stage_1_train_labels['patientId']\ndata_stage_1_train_labels_target = data_stage_1_train_labels['Target']\n#transform into dictionary\nfor i in range(len(data_stage_1_train_labels_patientId)):\n    small_data_stage_1_train_labels_patientId.append(data_stage_1_train_labels_patientId[i])\n    patientid_target[data_stage_1_train_labels_patientId[i]] = data_stage_1_train_labels_target[i]\nfor i in range(len(data_train_input_patientid)):\n    patientId = data_train_input_patientid[i]\n    try:\n        small_data_stage_1_train_labels_target.append(patientid_target[patientId])\n    except:\n        small_data_stage_1_train_labels_target.append(np.nan)\nsmall_data_stage_1_train_labels_target_arr = np.asarray(small_data_stage_1_train_labels_target).reshape(small_m,1)\n\n\n#delete NaN from small_data_stage_1_train_labels_target_arr and not use image corresponded\nidx_delete=[]\nfor i in range(len(small_data_stage_1_train_labels_target_arr)):\n    if (math.isnan(small_data_stage_1_train_labels_target_arr[i][0])):\n        idx_delete.append(i)\n\nsmall_data_stage_1_train_labels_target_arr = np.delete(small_data_stage_1_train_labels_target_arr, idx_delete, 0)\ndata_train_input_image_arr = np.delete(data_train_input_image_arr, idx_delete, 0)\n\n\nmodel = ResNet50(input_shape = (imagesize, imagesize,1), classes = 1)\n\ntbCallBack = TensorBoard(log_dir='./graph', histogram_freq=0, write_graph=True, write_images=True)\n\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\nmodel.fit(data_train_input_image_arr, small_data_stage_1_train_labels_target_arr, epochs = 9, batch_size = 64,\n          callbacks=[tbCallBack])\n\n#save model\n'''\nprint(\"Save model ongoing.....\")\n\nif not os.path.exists(\"./model/model.json\"):\n    with open('./model/model.json', 'w'):\n        pass\n    \nif not os.path.exists(\"./model/model.h5\"):\n    with open('./model/model.h5', 'w'):\n        pass\nmodel_json = model.to_json()\nwith open(\"./model/model.json\",\"w\") as json_file:\n    json_file.write(model_json)\nmodel.save_weights(\"./model/model.h5\")\nprint(\"Saved model to disk\")\n'''\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96408bc5f95be24189ab53cc6f8628c223005981"},"cell_type":"code","source":"\n\n5000/5000 [==============================] - 573s 115ms/step - loss: 0.4589 - acc: 0.7794\nEpoch 9/9\n64/5000 [..............................] - ETA: 9:23 - loss: 0.3616 - acc: 0.8594\n\nI have  acc : 0.85 here(first 64 batch of the 9th epoch)  \n                \n4928/5000 [============================>.] - ETA: 8s - loss: 0.4620 - acc: 0.7845\n\n4992/5000 [============================>.] - ETA: 0s - loss: 0.4623 - acc: 0.7851\n\n5000/5000 [==============================] - 560s 112ms/step - loss: 0.4624 - acc: 0.7850","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}