{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from numpy.random import seed\nseed(101)\nfrom tensorflow import set_random_seed\nset_random_seed(101)\n\nimport pandas as pd\nimport numpy as np\n\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Activation\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\n\nimport os\nimport cv2\n\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport itertools\nimport shutil\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7bbdd52c81188b8e9c528b88d9fd0da176bf4bc"},"cell_type":"code","source":"\nIMAGE_SIZE = 96\nIMAGE_CHANNELS = 3\n\nSAMPLE_SIZE = 80000 # the number of images we use from each of the two classes\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"699bb899bde433ba20fb0d086fa0f33a0f61a250"},"cell_type":"code","source":"os.listdir('../input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54461212efed65ac377369a468c80e7d708010f4"},"cell_type":"code","source":"\nprint(len(os.listdir('../input/train')))\nprint(len(os.listdir('../input/test')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9c9f40ffab35044641b0dc7d9b18609af1aa25e"},"cell_type":"code","source":"df_data = pd.read_csv('../input/train_labels.csv')\ndf_data[df_data['id'] != 'dd6dfed324f9fcb6f93f46f32fc800f2ec196be2']\ndf_data[df_data['id'] != '9369c7278ec8bcc6c880d99194de09fc2bd4efbe']\nprint(df_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e18560bf69d3dfc0c4772e7c79bb119fd2eb634b"},"cell_type":"code","source":"df_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"547f9f571ee82e20b7647e16ed36de7550046032"},"cell_type":"code","source":"df_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"270fc18640b552ecc3cb0e1dd3036441db7a4a2b"},"cell_type":"code","source":"df_0 = df_data[df_data['label'] == 0].sample(SAMPLE_SIZE, random_state = 101)\ndf_1 = df_data[df_data['label'] == 1].sample(SAMPLE_SIZE, random_state = 101)\n# concat the dataframes\ndf_data = pd.concat([df_0, df_1], axis=0).reset_index(drop=True)\ndf_data = shuffle(df_data)\ndf_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a166dec3ef84c66ad9cd815b63fc1a753df2eb76"},"cell_type":"code","source":"df_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15ba9792e6a370b7560330af15b3cfe21185c1cb"},"cell_type":"code","source":"y = df_data['label']\ndf_train, df_val = train_test_split(df_data, test_size=0.10, random_state=101, stratify=y)\nprint(df_train.shape)\nprint(df_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7de70d915a5f1d2599725e00bdb3b9103d947883"},"cell_type":"code","source":"df_train['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"392c0eea00be8e43a6e55438d1458650e842030b"},"cell_type":"code","source":"df_val['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff8acc2e92a1b1b5002d6e1bf9a1180c3256f19d"},"cell_type":"code","source":"# Create a new directory\nbase_dir = 'base_dir'\nos.mkdir(base_dir)\ntrain_dir = os.path.join(base_dir, 'train_dir')\nos.mkdir(train_dir)\n# val_dir\nval_dir = os.path.join(base_dir, 'val_dir')\nos.mkdir(val_dir)\n# create new folders inside train_dir\nno_tumor_tissue = os.path.join(train_dir, 'a_no_tumor_tissue')\nos.mkdir(no_tumor_tissue)\nhas_tumor_tissue = os.path.join(train_dir, 'b_has_tumor_tissue')\nos.mkdir(has_tumor_tissue)\n# create new folders inside val_dir\nno_tumor_tissue = os.path.join(val_dir, 'a_no_tumor_tissue')\nos.mkdir(no_tumor_tissue)\nhas_tumor_tissue = os.path.join(val_dir, 'b_has_tumor_tissue')\nos.mkdir(has_tumor_tissue)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e84c8a9642b030094b1888af3299063f883112a6"},"cell_type":"code","source":"df_data.set_index('id', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afb8969a9ee75c13bddc808a4bcc326611baaaaf"},"cell_type":"code","source":"\n\ntrain_list = list(df_train['id'])\nval_list = list(df_val['id'])\n\n# Transfer the train images\n\nfor image in train_list:\n    \n    # adding extension .tif to image name\n    fname = image + '.tif'\n    # label for a certain image\n    target = df_data.loc[image,'label']\n    \n    if target == 0:\n        label = 'a_no_tumor_tissue'\n    if target == 1:\n        label = 'b_has_tumor_tissue'\n    \n    # source path to image\n    src = os.path.join('../input/train', fname)\n    # destination path to image\n    dst = os.path.join(train_dir, label, fname)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dst)\n\n\n# Transfer the val images\n\nfor image in val_list:\n    \n    # the id in the csv file does not have the .tif extension therefore we add it here\n    fname = image + '.tif'\n    # get the label for a certain image\n    target = df_data.loc[image,'label']\n    \n    # these must match the folder names\n    if target == 0:\n        label = 'a_no_tumor_tissue'\n    if target == 1:\n        label = 'b_has_tumor_tissue'\n    \n\n    # source path to image\n    src = os.path.join('../input/train', fname)\n    # destination path to image\n    dst = os.path.join(val_dir, label, fname)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dst)\n    \n\n\n   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71532bfc32608289b1f773ffdbc8a7cea1bfb94c"},"cell_type":"code","source":"# check how many train images we have in each folder\n\nprint(len(os.listdir('base_dir/train_dir/a_no_tumor_tissue')))\nprint(len(os.listdir('base_dir/train_dir/b_has_tumor_tissue')))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"897e9df543bb65b47bb00019dc681125ca08ee5d"},"cell_type":"code","source":"# check how many val images we have in each folder\n\nprint(len(os.listdir('base_dir/val_dir/a_no_tumor_tissue')))\nprint(len(os.listdir('base_dir/val_dir/b_has_tumor_tissue')))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef4fe7be09f11ff4badfd22d5fd5e03f8521ed58"},"cell_type":"code","source":"train_path = 'base_dir/train_dir'\nvalid_path = 'base_dir/val_dir'\ntest_path = '../input/test'\n\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 10\nval_batch_size = 10\n\n\ntrain_steps = np.ceil(num_train_samples / train_batch_size)\nval_steps = np.ceil(num_val_samples / val_batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68fbd9d5fbb80859a82f94a12e335ce05a93bd51"},"cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1.0/255)\n\ntrain_gen = datagen.flow_from_directory(train_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=train_batch_size,\n                                        class_mode='categorical')\n\nval_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=val_batch_size,\n                                        class_mode='categorical')\n\n# Note: shuffle=False causes the test dataset to not be shuffled\ntest_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='categorical',\n                                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9835ea0fd0bca54138904895c39d38227a70c22"},"cell_type":"code","source":"\n\nkernel_size = (3,3)\npool_size= (2,2)\nfirst_filters = 32\nsecond_filters = 64\nthird_filters = 128\n\ndropout_conv = 0.3\ndropout_dense = 0.3\n\n\nmodel = Sequential()\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu', input_shape = (96, 96, 3)))\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu'))\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu'))\nmodel.add(MaxPooling2D(pool_size = pool_size)) \nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(second_filters, kernel_size, activation ='relu'))\nmodel.add(Conv2D(second_filters, kernel_size, activation ='relu'))\nmodel.add(Conv2D(second_filters, kernel_size, activation ='relu'))\nmodel.add(MaxPooling2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu'))\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu'))\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu'))\nmodel.add(MaxPooling2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(dropout_dense))\nmodel.add(Dense(2, activation = \"softmax\"))\n\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9de9715f49a63b55775b10abd2f461b395e23b5d"},"cell_type":"code","source":"model.compile(Adam(lr=0.0001), loss='binary_crossentropy', \n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"a746769db61563f226288eba9aa8a6584b9e8e0b"},"cell_type":"code","source":"filepath = \"model.h5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, \n                             save_best_only=True, mode='max')\n\nreduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=2, \n                                   verbose=1, mode='max', min_lr=0.00001)\n                              \n                              \ncallbacks_list = [checkpoint, reduce_lr]\n\nhistory = model.fit_generator(train_gen, steps_per_epoch=train_steps, \n                    validation_data=val_gen,\n                    validation_steps=val_steps,\n                    epochs=5, verbose=1,\n                   callbacks=callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree('base_dir')\nshutil.rmtree('test_dir')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}