{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0,"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4d77e4db-bddd-0f60-4bbc-3980d022c802","_active":false},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom skimage.io import imread, imshow\nimport cv2\n\n%matplotlib inline\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input/train\"]).decode(\"utf8\"))","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"abb0b87a-655b-37f5-f757-ebb2d14cedab","_active":false},"outputs":[],"source":"from glob import glob\nbasepath = '../input/train/'\n\nall_cervix_images = []\n\nfor path in sorted(glob(basepath + \"*\")):\n    cervix_type = path.split(\"/\")[-1]\n    cervix_images = sorted(glob(basepath + cervix_type + \"/*\"))\n    all_cervix_images = all_cervix_images + cervix_images\n\nall_cervix_images = pd.DataFrame({'imagepath': all_cervix_images})\nall_cervix_images['filetype'] = all_cervix_images.apply(lambda row: row.imagepath.split(\".\")[-1], axis=1)\nall_cervix_images['type'] = all_cervix_images.apply(lambda row: row.imagepath.split(\"/\")[-2], axis=1)\nall_cervix_images.head()","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"8f70f603-2e6a-683e-635f-ab9643b38c46","_active":false},"outputs":[],"source":"print('We have a total of {} images in the whole dataset'.format(all_cervix_images.shape[0]))\ntype_aggregation = all_cervix_images.groupby(['type', 'filetype']).agg('count')\ntype_aggregation_p = type_aggregation.apply(lambda row: 1.0*row['imagepath']/all_cervix_images.shape[0], axis=1)\n\nfig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10, 8))\n\ntype_aggregation.plot.barh(ax=axes[0])\naxes[0].set_xlabel(\"image count\")\ntype_aggregation_p.plot.barh(ax=axes[1])\naxes[1].set_xlabel(\"training size fraction\")","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b42092b9-5d56-ec2d-ff32-f9a7d593acec","_active":false},"outputs":[],"source":"fig = plt.figure(figsize=(12,8))\n\ni = 1\nfor t in all_cervix_images['type'].unique():\n    ax = fig.add_subplot(1,3,i)\n    i+=1\n    f = all_cervix_images[all_cervix_images['type'] == t]['imagepath'].values[0]\n    plt.imshow(plt.imread(f))\n    plt.title('sample for cervix {}'.format(t))\n    ","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f1747239-5192-2831-4bd5-8ec92c616dd9","_active":false},"outputs":[],"source":"from collections import defaultdict\n\nimages = defaultdict(list)\n\nfor t in all_cervix_images['type'].unique():\n    sample_counter = 0\n    for _, row in all_cervix_images[all_cervix_images['type'] == t].iterrows():\n        #print('reading image {}'.format(row.imagepath))\n        try:\n            img = imread(row.imagepath)\n            sample_counter +=1\n            images[t].append(img)\n        except:\n            print('image read failed for {}'.format(row.imagepath))\n        if sample_counter > 35:\n            break","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"64bcbfce-4952-d8b2-d65f-bd2dd1caaa5e","_active":false},"outputs":[],"source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom skimage.io import imread, imshow\nimport cv2\nfrom random import shuffle\nfrom tqdm import tqdm\n\nTRAIN_DIR = \"../input/train\"\nTEST_DIR = \"../input/test\"\nIMG_SIZE = 50\nLR = 1e-3\n\nMODEL_NAME = 'cervicalcancer-{}-{}.model'.format(LR, '2conv-basic')","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"58932386-d558-5172-73fe-ca668d4874e0","_active":false},"outputs":[],"source":"def label_img(img):\n    word_label = img.split('.')[-3]\n    if word_label == 'Type_1': return [1,0,0]\n    elif word_label == 'Type_2': return [0,1,0]\n    elif word_label == 'Type_3': return [0,0,1]","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a969872a-eb4f-94cf-2b64-66e4e53bfa10","_active":false},"outputs":[],"source":"null","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2436d6ae-986f-afdf-6b2a-8da6e26de6b7","_active":false},"outputs":[],"source":"from glob import glob\nbasepath = '../input/train/'\n\nall_cervix_images = []\n\nfor path in sorted(glob(basepath + \"*\")):\n    cervix_type = path.split(\"/\")[-1]\n    cervix_images = sorted(glob(basepath + cervix_type + \"/*\"))\n    all_cervix_images = all_cervix_images + cervix_images\n\nall_cervix_images = pd.DataFrame({'imagepath': all_cervix_images})\nall_cervix_images['filetype'] = all_cervix_images.apply(lambda row: row.imagepath.split(\".\")[-1], axis=1)\nall_cervix_images['type'] = all_cervix_images.apply(lambda row: row.imagepath.split(\"/\")[-2], axis=1)\nall_cervix_images.head()","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e6fe6e1d-6885-84ab-f722-d412b9523fe7","_active":false},"outputs":[],"source":"print(cervix_type)","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4a041059-67af-549b-2e6f-cbcef1ca7d8f","_active":false},"outputs":[],"source":"def create_train_data():\n    training_data = []\n    for img in tqdm(os.listdir(TRAIN_DIR)):\n        path = os.path.join(TRAIN_DIR,img)\n        img = cv2.imread(path,cv2.IMREAD_GRAYSCALE)\n        img = cv2.resize(img, (IMG_SIZE,IMG_SIZE))\n        training_data.append([np.array(img),np.array(label)])\n    shuffle(training_data)\n    np.save('train_data.npy', training_data)\n    return training_data","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f0a947c1-4250-6681-0d55-0d4ef7edb046","_active":false},"outputs":[],"source":"def process_test_data():\n    testing_data = []\n    for img in tqdm(os.listdir(TEST_DIR)):\n        path = os.path.join(TEST_DIR,img)\n        img_num = img.split('.')[0]\n        img = cv2.imread(path,cv2.IMREAD_GRAYSCALE)\n        img = cv2.resize(img, (IMG_SIZE,IMG_SIZE))\n        testing_data.append([np.array(img), img_num])\n        \n    shuffle(testing_data)\n    np.save('test_data.npy', testing_data)\n    return testing_data","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"8638bcdd-bff5-871f-0cc3-1440bb74c82a","_active":false},"outputs":[],"source":"train_data = create_train_data()\n# If you have already created the dataset:\n#train_data = np.load('train_data.npy')","execution_state":"idle"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"dc6e77a5-1e50-e326-54f5-3affef73201a","_active":false,"collapsed":false},"outputs":[],"source":"import tflearn\nfrom tflearn.layers.conv import conv_2d, max_pool_2d\nfrom tflearn.layers.core import input_data, dropout, fully_connected\nfrom tflearn.layers.estimator import regression\n\nIMG_HEIGHT = 80\n\nIMG_WIDTH = 80\n\nconvnet = input_data(shape=[None, IMG_WIDTH, IMG_HEIGHT, 1], name='input')\n\nconvnet = conv_2d(convnet, 32, 5, activation='relu')\nconvnet = max_pool_2d(convnet, 5)\n\nconvnet = conv_2d(convnet, 64, 5, activation='relu')\nconvnet = max_pool_2d(convnet, 5)\n\nconvnet = conv_2d(convnet, 128, 5, activation='relu')\nconvnet = max_pool_2d(convnet, 5)\n\nconvnet = conv_2d(convnet, 64, 5, activation='relu')\nconvnet = max_pool_2d(convnet, 5)\n\nconvnet = conv_2d(convnet, 32, 5, activation='relu')\nconvnet = max_pool_2d(convnet, 5)\n\nconvnet = fully_connected(convnet, 1024, activation='relu')\nconvnet = dropout(convnet, 0.8)\n\nconvnet = fully_connected(convnet, 2, activation='softmax')\nconvnet = regression(convnet, optimizer='adam', learning_rate=LR, loss='categorical_crossentropy', name='targets')\n\nmodel = tflearn.DNN(convnet, tensorboard_dir='log')","execution_state":"idle"},{"metadata":{"_cell_guid":"169debc9-a43f-d128-7e30-556950304556","_active":false,"collapsed":false},"source":null,"execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"3fa6c718-4e78-7c58-fc92-fb5de00ccca0","_active":false,"collapsed":false},"source":"train = train_data[:-500]\ntest = train_data[-500:]","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"cb67980d-fd32-d20c-bf57-5830638d3789","_active":false,"collapsed":false},"source":"X = np.array([i[0] for i in train])\nY = [i[1] for i in train]\n\ntest_x = np.array([i[0] for i in test])\ntest_y = [i[1] for i in test]","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"0ab13f2b-3509-8aae-f682-1430e6a67bf3","_active":false,"collapsed":false},"source":"model.fit({'input': X}, {'targets': Y}, n_epoch=3, validation_set=({'input': test_x}, {'targets': test_y}), \n    snapshot_step=500, show_metric=True, run_id=MODEL_NAME)","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"02a9f6ba-f651-e2ee-3057-c67dfce1cadf","_active":true,"collapsed":false},"source":null,"execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"}]}