{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport shutil\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport math\n\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.preprocessing.image import ImageDataGenerator\n\nimport keras\nfrom keras.models import Sequential, load_model\nfrom keras.models import *\nfrom keras.layers import *\n\nimport utilities","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/histopathologic-cancer-detection/sample_submission.csv\", dtype=str)\nlabeled_df = pd.read_csv(\"../input/histopathologic-cancer-detection/train_labels.csv\", dtype=str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labeled_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labeled_df.id = labeled_df.id + '.tif'\ntest_id = test_df.id\ntest_df.id = test_df.id + '.tif'\nprint(labeled_df.head())\nprint(test_df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = \"../input/histopathologic-cancer-detection/train/\"\ntest_path = \"../input/histopathologic-cancer-detection/test/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(os.listdir(train_path)))\nprint(len(os.listdir(test_path)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labeled_df.label.value_counts() / len(labeled_df.label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labeled_df.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 80000\nnegative_sample = labeled_df.loc[labeled_df.label == '0', :].sample(n, random_state=1)\npositive_sample = labeled_df.loc[labeled_df.label == '1', :].sample(n, random_state=1)\nlabeled_sample = pd.concat([negative_sample, positive_sample], axis=0).reset_index(drop=True)\n\ntrain_df, valid_df = train_test_split(labeled_sample, test_size=0.2, random_state=1, stratify=labeled_sample.label)\n\nprint(train_df.shape)\nprint(valid_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_dir = 'images/'\ntrain_dir = 'images/train/'\nvalid_dir = 'images/valid/'\ntest_dir = 'images/test/'\n\nos.mkdir(base_dir)\nos.mkdir(train_dir)\nos.mkdir(valid_dir)\nos.mkdir(test_dir)\n\nos.mkdir(train_dir + 'negative')\nos.mkdir(train_dir + 'positive')\nos.mkdir(valid_dir + 'negative')\nos.mkdir(valid_dir + 'positive')\nos.mkdir(test_dir + 'unlabeled')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n\nfor i in range(len(train_df.id)):\n    \n    src = train_path + train_df.id.iloc[i]\n        \n    if train_df.label.iloc[i] == '0':    \n        dest = train_dir + 'negative/' + train_df.id.iloc[i]\n    else: \n        dest = train_dir + 'positive/' + train_df.id.iloc[i]\n        \n    shutil.copyfile(src, dest)\n\nprint(len(os.listdir(train_dir + 'negative')))\nprint(len(os.listdir(train_dir + 'positive')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n\nfor i in range(len(valid_df.id)):\n    \n    src = train_path + valid_df.id.iloc[i]\n        \n    if valid_df.label.iloc[i] == '0':    \n        dest = valid_dir + 'negative/' + valid_df.id.iloc[i]\n    else: \n        dest = valid_dir + 'positive/' + valid_df.id.iloc[i]\n        \n    shutil.copyfile(src, dest)\n\nprint(len(os.listdir(valid_dir + 'negative')))\nprint(len(os.listdir(valid_dir + 'positive')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bs = 64\n\ntrain_datagen = ImageDataGenerator(rescale=1/255)\nvalid_datagen = ImageDataGenerator(rescale=1/255)\ntest_datagen = ImageDataGenerator(rescale=1/255)\n\ntrain_generator = train_datagen.flow_from_directory(\n    directory = train_dir,\n    batch_size = bs,\n    shuffle = True,\n    class_mode = \"binary\",\n    target_size = (96,96))\n\nvalid_generator = train_datagen.flow_from_directory(\n    directory = valid_dir,\n    batch_size = bs,\n    shuffle = True,\n    class_mode = \"binary\",\n    target_size = (96,96))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_size = 128000 \nva_size = 32000\n\ntr_steps = math.ceil(tr_size / bs)\nva_steps = math.ceil(va_size / bs)\n\n\nprint('Number of training batches:  ', tr_steps)\nprint('Number of validation batches:', va_steps)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def training_images(seed):\n    np.random.seed(seed)\n    train_generator.reset()\n    imgs, labels = next(train_generator)\n        \n    plt.figure(figsize=(12,12))\n    for i in range(16):\n        plt.subplot(4,4,i+1)\n        plt.imshow(imgs[i,:,:,:])\n        if(labels[i] == 1):\n            plt.text(0, -5, 'Positive', color='r')\n        else:\n            plt.text(0, -5, 'Negative', color='b')\n        plt.axis('off')\n    plt.show()\n\ntraining_images(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(1)\n\ncnn = Sequential()\ncnn.add(Cropping2D(cropping=((32,32), (32,32)), input_shape=(96,96,3)))\ncnn.add(Conv2D(64, (3,3), activation = 'relu', padding = 'same'))\ncnn.add(Conv2D(64, (3,3), activation = 'relu', padding = 'same'))\ncnn.add(MaxPooling2D(2,2))\n#cnn.add(Dropout(0.25))\ncnn.add(BatchNormalization())\n\ncnn.add(Conv2D(128, (3,3), activation = 'relu', padding = 'same'))\ncnn.add(Conv2D(128, (3,3), activation = 'relu', padding = 'same'))\ncnn.add(MaxPooling2D(2,2))\n#cnn.add(Dropout(0.25))\ncnn.add(BatchNormalization())\n\ncnn.add(Flatten())\ncnn.add(Dense(512, activation='relu'))\n#cnn.add(Dropout(0.25))\ncnn.add(BatchNormalization())\ncnn.add(Dense(1, activation='sigmoid'))\n\ncnn.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n\nopt = keras.optimizers.Adam(0.002)\ncnn.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])\n\nh1 = cnn.fit_generator(train_generator, steps_per_epoch=tr_steps, epochs=5,\n                       validation_data=valid_generator, validation_steps=va_steps, \n                       verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"utilities.vis_training([h1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = keras.optimizers.Adam(lr=0.0002)\n\nh2 = cnn.fit_generator(train_generator, steps_per_epoch=tr_steps, epochs=20,\n                       validation_data=valid_generator, validation_steps=va_steps, \n                       verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"utilities.vis_training([h1,h2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = keras.optimizers.Adam(lr=0.00001)\n\nh3 = cnn.fit_generator(train_generator, steps_per_epoch=tr_steps, epochs=15,\n                       validation_data=valid_generator, validation_steps=va_steps, \n                       verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"utilities.vis_training([h1,h2, h3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn.save('cnn_v01.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree('/kaggle/working/images/train')\nshutil.rmtree('/kaggle/working/images/valid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfor i in range(len(test_df.id)):\n    \n    src = test_path + test_df.id.iloc[i]\n    dest = test_dir + 'unlabeled/' + test_df.id.iloc[i]\n    shutil.copyfile(src, dest)\n    \nprint(len(os.listdir(test_dir + 'unlabeled')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bs = 64\n\ntest_datagen = ImageDataGenerator(rescale=1/255)\n\ntest_generator = test_datagen.flow_from_directory(\n    directory = test_dir,\n    batch_size = bs,\n    shuffle = False,\n    class_mode = None,\n    target_size = (96,96))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"te_size = 57458\nte_steps = math.ceil(te_size / bs)\nprint('Number of test batches:    ', te_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred = cnn.predict_generator(test_generator, steps = te_steps, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_fnames = test_generator.filenames\ntest_fnames[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_fnames = [x.split('.')[0] for x in test_fnames]\ntest_fnames = [x.split('/')[1] for x in test_fnames]\ntest_fnames[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test_pred.shape)\n#pred_classes = np.argmax(test_pred, axis=1)\npred_classes= np.where(test_pred > 0.5, 1, 0)\n\nprint(pred_classes[:5])\n\nprint(np.sum(pred_classes == 0))\nprint(np.sum(pred_classes == 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(test_fnames))\nprint(pred_classes.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\n    'id':test_fnames,\n    'label':pred_classes.reshape(-1,)\n})\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree('/kaggle/working/images')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}