{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-12T18:38:15.63393Z","iopub.execute_input":"2021-07-12T18:38:15.634235Z","iopub.status.idle":"2021-07-12T18:38:15.640308Z","shell.execute_reply.started":"2021-07-12T18:38:15.634184Z","shell.execute_reply":"2021-07-12T18:38:15.639312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.listdir(\"../input/train\"))","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2021-07-12T18:38:15.646Z","iopub.execute_input":"2021-07-12T18:38:15.646575Z","iopub.status.idle":"2021-07-12T18:38:15.654175Z","shell.execute_reply.started":"2021-07-12T18:38:15.646527Z","shell.execute_reply":"2021-07-12T18:38:15.653437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport tensorflow as tf\n\nfrom sklearn.model_selection import train_test_split\nfrom collections import Counter\nfrom sklearn.model_selection import train_test_split\nfrom collections import Counter\n\nimport cv2\nfrom concurrent import futures\nimport threading\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom keras.utils import to_categorical\n\nimport datetime","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:38:15.656535Z","iopub.execute_input":"2021-07-12T18:38:15.657099Z","iopub.status.idle":"2021-07-12T18:38:15.666557Z","shell.execute_reply.started":"2021-07-12T18:38:15.657052Z","shell.execute_reply":"2021-07-12T18:38:15.665969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#getting the total number of images in the training set\n\nbase_dir = os.path.join('../input/train/train')\ntype1_dir = os.path.join(base_dir,'Type_1')\ntype2_dir = os.path.join(base_dir,'Type_2')\ntype3_dir = os.path.join(base_dir,'Type_3')\n\ntype1_files = glob.glob(type1_dir+'/*.jpg')\ntype2_files = glob.glob(type2_dir+'/*.jpg')\ntype3_files = glob.glob(type3_dir+'/*.jpg')\n\nlen(type1_files),len(type2_files),len(type3_files)","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:38:15.668152Z","iopub.execute_input":"2021-07-12T18:38:15.668866Z","iopub.status.idle":"2021-07-12T18:38:15.685536Z","shell.execute_reply.started":"2021-07-12T18:38:15.668724Z","shell.execute_reply":"2021-07-12T18:38:15.684691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Building a dataframe mapping images and Cancer type\nnp.random.seed(42)\n\nfiles_df = pd.DataFrame({\n    'filename': type1_files + type2_files + type3_files,\n    'label': ['Type_1'] * len(type1_files) + ['Type_2'] * len(type2_files) + ['Type_3'] * len(type3_files)\n}).sample(frac=1, random_state=42).reset_index(drop=True)\n\nfiles_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:38:15.688588Z","iopub.execute_input":"2021-07-12T18:38:15.688852Z","iopub.status.idle":"2021-07-12T18:38:15.705019Z","shell.execute_reply.started":"2021-07-12T18:38:15.688798Z","shell.execute_reply":"2021-07-12T18:38:15.703995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split training,dev and test set : 60:10:30\n\ntrain_files, test_files, train_labels, test_labels = train_test_split(files_df['filename'].values,\n                                                                      files_df['label'].values, \n                                                                      test_size=0.3, random_state=42)\ntrain_files, val_files, train_labels, val_labels = train_test_split(train_files,\n                                                                    train_labels, \n                                                                    test_size=0.1, random_state=42)\n\nprint(train_files.shape, val_files.shape, test_files.shape)\nprint('Train:', Counter(train_labels), '\\nVal:', Counter(val_labels), '\\nTest:', Counter(test_labels))","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:38:15.708245Z","iopub.execute_input":"2021-07-12T18:38:15.708734Z","iopub.status.idle":"2021-07-12T18:38:15.719208Z","shell.execute_reply.started":"2021-07-12T18:38:15.708538Z","shell.execute_reply":"2021-07-12T18:38:15.718195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#getting the summary of the dataset dimensions\ndef get_img_shape_parallel(idx, img, total_imgs):\n    if idx % 5000 == 0 or idx == (total_imgs - 1):\n        print('{}: working on img num: {}'.format(threading.current_thread().name,\n                                                  idx))\n    return cv2.imread(img).shape\n  \nex = futures.ThreadPoolExecutor(max_workers=None)\ndata_inp = [(idx, img, len(train_files)) for idx, img in enumerate(train_files)]\nprint('Starting Img shape computation:')\ntrain_img_dims_map = ex.map(get_img_shape_parallel, \n                            [record[0] for record in data_inp],\n                            [record[1] for record in data_inp],\n                            [record[2] for record in data_inp])\ntrain_img_dims = list(train_img_dims_map)\nprint('Min Dimensions:', np.min(train_img_dims, axis=0)) \nprint('Avg Dimensions:', np.mean(train_img_dims, axis=0))\nprint('Median Dimensions:', np.median(train_img_dims, axis=0))\nprint('Max Dimensions:', np.max(train_img_dims, axis=0))\n","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:38:15.720856Z","iopub.execute_input":"2021-07-12T18:38:15.721141Z","iopub.status.idle":"2021-07-12T18:40:00.463352Z","shell.execute_reply.started":"2021-07-12T18:38:15.721096Z","shell.execute_reply":"2021-07-12T18:40:00.462679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nIMG_DIMS = (224, 224)\n\ndef get_img_data_parallel(idx, img, total_imgs):\n    if idx % 5000 == 0 or idx == (total_imgs - 1):\n        print('{}: working on img num: {}'.format(threading.current_thread().name,\n                                                  idx))\n    img = cv2.imread(img)\n    img = cv2.resize(img, dsize=IMG_DIMS, \n                     interpolation=cv2.INTER_CUBIC)\n    img = np.array(img, dtype=np.float32)\n    return img\n\nex = futures.ThreadPoolExecutor(max_workers=None)\ntrain_data_inp = [(idx, img, len(train_files)) for idx, img in enumerate(train_files)]\nval_data_inp = [(idx, img, len(val_files)) for idx, img in enumerate(val_files)]\ntest_data_inp = [(idx, img, len(test_files)) for idx, img in enumerate(test_files)]\n\nprint('Loading Train Images:')\ntrain_data_map = ex.map(get_img_data_parallel, \n                        [record[0] for record in train_data_inp],\n                        [record[1] for record in train_data_inp],\n                        [record[2] for record in train_data_inp])\ntrain_data = np.array(list(train_data_map))\n\nprint('\\nLoading Validation Images:')\nval_data_map = ex.map(get_img_data_parallel, \n                        [record[0] for record in val_data_inp],\n                        [record[1] for record in val_data_inp],\n                        [record[2] for record in val_data_inp])\nval_data = np.array(list(val_data_map))\n\nprint('\\nLoading Test Images:')\ntest_data_map = ex.map(get_img_data_parallel, \n                        [record[0] for record in test_data_inp],\n                        [record[1] for record in test_data_inp],\n                        [record[2] for record in test_data_inp])\ntest_data = np.array(list(test_data_map))\n\ntrain_data.shape, val_data.shape, test_data.shape  ","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:40:00.466841Z","iopub.execute_input":"2021-07-12T18:40:00.467085Z","iopub.status.idle":"2021-07-12T18:42:51.6101Z","shell.execute_reply.started":"2021-07-12T18:40:00.46704Z","shell.execute_reply":"2021-07-12T18:42:51.609427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#viewing some sample images\nplt.figure(1 , figsize = (8 , 8))\nn = 0 \nfor i in range(16):\n    n += 1 \n    r = np.random.randint(0 , train_data.shape[0] , 1)\n    plt.subplot(4 , 4 , n)\n    plt.subplots_adjust(hspace = 0.5 , wspace = 0.5)\n    plt.imshow(train_data[r[0]]/255.)\n    plt.title('{}'.format(train_labels[r[0]]))\n    plt.xticks([]) , plt.yticks([])","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:42:51.612298Z","iopub.execute_input":"2021-07-12T18:42:51.612625Z","iopub.status.idle":"2021-07-12T18:42:52.715883Z","shell.execute_reply.started":"2021-07-12T18:42:51.612573Z","shell.execute_reply":"2021-07-12T18:42:52.715133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#encode text categories with labels\nBATCH_SIZE = 64\nNUM_CLASSES = 2\nEPOCHS = 20\nINPUT_SHAPE = (224, 224, 3)\n\ntrain_imgs_scaled = train_data / 255.\nval_imgs_scaled = val_data / 255.\n\nle = LabelEncoder()\nle.fit(train_labels)\ntrain_labels_enc = le.transform(train_labels)\nval_labels_enc = le.transform(val_labels)\n\ntrain_labels_1hotenc = to_categorical(train_labels_enc, num_classes=3)\nval_labels_1hotenc = to_categorical(val_labels_enc, num_classes=3)\n\nprint(train_labels[:6], train_labels_enc[:6])\nprint(train_labels[:6], train_labels_1hotenc[:6])\n","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:42:52.717007Z","iopub.execute_input":"2021-07-12T18:42:52.717299Z","iopub.status.idle":"2021-07-12T18:42:53.088833Z","shell.execute_reply.started":"2021-07-12T18:42:52.717248Z","shell.execute_reply":"2021-07-12T18:42:53.088161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using VGG19 pre-trained model\nvgg = tf.keras.applications.vgg19.VGG19(include_top=False, weights='imagenet', \n                                        input_shape=INPUT_SHAPE)\nvgg.trainable = False\n# Freeze the layers\nfor layer in vgg.layers:\n    layer.trainable = False\n    \nbase_vgg = vgg\nbase_out = base_vgg.output\npool_out = tf.keras.layers.Flatten()(base_out)\nhidden1 = tf.keras.layers.Dense(512, activation='relu')(pool_out)\ndrop1 = tf.keras.layers.Dropout(rate=0.3)(hidden1)\nhidden2 = tf.keras.layers.Dense(512, activation='relu')(drop1)\ndrop2 = tf.keras.layers.Dropout(rate=0.3)(hidden2)\n\nout = tf.keras.layers.Dense(3, activation='softmax')(drop2)\n\nmodel = tf.keras.Model(inputs=base_vgg.input, outputs=out)\nmodel.compile(optimizer=tf.keras.optimizers.RMSprop(lr=1e-4),\n                loss='categorical_crossentropy',\n                metrics=['accuracy'])\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:42:53.090227Z","iopub.execute_input":"2021-07-12T18:42:53.090541Z","iopub.status.idle":"2021-07-12T18:42:58.658243Z","shell.execute_reply.started":"2021-07-12T18:42:53.090493Z","shell.execute_reply":"2021-07-12T18:42:58.657389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x=train_imgs_scaled, y=train_labels_1hotenc, \n                    batch_size=BATCH_SIZE,\n                    epochs=EPOCHS, \n                    validation_data=(val_imgs_scaled, val_labels_1hotenc),\n                    verbose = 1)\n                   ","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:42:58.659513Z","iopub.execute_input":"2021-07-12T18:42:58.659789Z","iopub.status.idle":"2021-07-12T18:44:04.75701Z","shell.execute_reply.started":"2021-07-12T18:42:58.659743Z","shell.execute_reply":"2021-07-12T18:44:04.756319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\nt = f.suptitle('VGG19 Transfer Learning Perfomance', fontsize=12)\nf.subplots_adjust(top=0.85, wspace=0.3)\n\nmax_epoch = len(history.history['acc'])+1\nepoch_list = list(range(1,max_epoch))\nax1.plot(epoch_list, history.history['acc'], label='Train Accuracy')\nax1.plot(epoch_list, history.history['val_acc'], label='Validation Accuracy')\nax1.set_xticks(np.arange(1, max_epoch, 5))\nax1.set_ylabel('Accuracy Value')\nax1.set_xlabel('Epoch')\nax1.set_title('Accuracy')\nl1 = ax1.legend(loc=\"best\")\n\nax2.plot(epoch_list, history.history['loss'], label='Train Loss')\nax2.plot(epoch_list, history.history['val_loss'], label='Validation Loss')\nax2.set_xticks(np.arange(1, max_epoch, 5))\nax2.set_ylabel('Loss Value')\nax2.set_xlabel('Epoch')\nax2.set_title('Loss')\nl2 = ax2.legend(loc=\"best\")\n","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:44:04.758983Z","iopub.execute_input":"2021-07-12T18:44:04.75922Z","iopub.status.idle":"2021-07-12T18:44:05.203212Z","shell.execute_reply.started":"2021-07-12T18:44:04.759176Z","shell.execute_reply":"2021-07-12T18:44:05.202183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(history.history.keys())","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:44:05.204783Z","iopub.execute_input":"2021-07-12T18:44:05.205058Z","iopub.status.idle":"2021-07-12T18:44:05.209234Z","shell.execute_reply.started":"2021-07-12T18:44:05.205013Z","shell.execute_reply":"2021-07-12T18:44:05.20848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#scaling the test set and one-hot encoding the test labels\ntest_imgs_scaled = test_data / 255.\ntest_imgs_scaled.shape, test_labels.shape\n\nle = LabelEncoder()\nle.fit(test_labels)\ntest_labels_enc = le.transform(test_labels)\n\ntest_labels_1hotenc = to_categorical(test_labels_enc, num_classes=3)\n\n\nprint(test_labels[:6], test_labels_enc[:6])\nprint(test_labels[:6], test_labels_1hotenc[:6])","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:44:05.210707Z","iopub.execute_input":"2021-07-12T18:44:05.211255Z","iopub.status.idle":"2021-07-12T18:44:05.389618Z","shell.execute_reply.started":"2021-07-12T18:44:05.211178Z","shell.execute_reply":"2021-07-12T18:44:05.387674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_imgs_scaled,test_labels_1hotenc)","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:44:05.391001Z","iopub.execute_input":"2021-07-12T18:44:05.391278Z","iopub.status.idle":"2021-07-12T18:44:07.978268Z","shell.execute_reply.started":"2021-07-12T18:44:05.39123Z","shell.execute_reply":"2021-07-12T18:44:07.977651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('my_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-07-12T18:46:23.77555Z","iopub.execute_input":"2021-07-12T18:46:23.77585Z","iopub.status.idle":"2021-07-12T18:46:24.004327Z","shell.execute_reply.started":"2021-07-12T18:46:23.775802Z","shell.execute_reply":"2021-07-12T18:46:24.003542Z"},"trusted":true},"execution_count":null,"outputs":[]}]}