{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Load the modules\nfrom glob import glob \nimport os\nimport numpy as np\nimport pandas as pd\nimport itertools\nimport shutil\nfrom sklearn.utils import shuffle\nimport keras,math\n!pip install opencv-python==4.0.0.21\nimport cv2 as cv\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation\nfrom keras.layers import Conv2D, MaxPool2D\n\nimport gc #garbage collection, we need to save all the RAM we can\n\nfrom tqdm import tqdm_notebook,trange\nimport matplotlib.pyplot as plt\n\nimport gc #garbage collection, we need to save all the RAM we can","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(os.listdir('../input/train')))\nprint(len(os.listdir('../input/test')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#set paths to training and test data\npath = \"../input/\" #adapt this path, when running locally\ntrain_path = path + 'train/'\ntest_path = path + 'test/'\n\ndf = pd.DataFrame({'path': glob(os.path.join(train_path,'*.tif'))}) # load the filenames\ndf['id'] = df.path.map(lambda x: x.split('/')[3].split(\".\")[0]) # keep only the file names in 'id'\nlabels = pd.read_csv(path+\"train_labels.csv\") # read the provided labels\ndf = df.merge(labels, on = \"id\") # merge labels and filepaths\ndf.head(3) # print the first three entrys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(N,df):\n    \"\"\" This functions loads N images using the data df\n    \"\"\"\n    # allocate a numpy array for the images (N, 96x96px, 3 channels, values 0 - 255)\n    X = np.zeros([N,96,96,3],dtype=np.uint8) \n    #convert the labels to a numpy array too\n    y = np.squeeze(df.as_matrix(columns=['label']))[0:N]\n    #read images one by one, tdqm notebook displays a progress bar\n    for i, row in tqdm_notebook(df.iterrows(), total=N):\n        if i == N:\n            break\n        X[i] = cv.imread(row['path'])\n          \n    return X,y\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N=10000\nX,y = load_data(N=N,df=df) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10, 4), dpi=150)\nnp.random.seed(100) #we can use the seed to get a different set of random images\nfor plotNr,idx in enumerate(np.random.randint(0,N,8)):\n    ax = fig.add_subplot(2, 8//2, plotNr+1, xticks=[], yticks=[]) #add subplots\n    plt.imshow(X[idx]) #plot image\n    ax.set_title('Label: ' + str(y[idx])) #show the label corresponding to the image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(4, 2),dpi=150)\nplt.bar([1,0], [(y==0).sum(), (y==1).sum()]); #plot a bar chart of the label frequency\nplt.xticks([1,0],[\"Negative (N={})\".format((y==0).sum()),\"Positive (N={})\".format((y==1).sum())]);\nplt.ylabel(\"# of samples\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = cv.imread('../input/train/019ce31cc317087ca287f66ad757776952826594.tif')\nr, g, b = cv.split(img)\nr_avg = cv.mean(r)[0]\ng_avg = cv.mean(g)[0]\nb_avg = cv.mean(b)[0]\n \nk = (r_avg + g_avg + b_avg) / 3\nkr = k / r_avg\nkg = k / g_avg\nkb = k / b_avg\n \nr = cv.addWeighted(src1=r, alpha=kr, src2=0, beta=0, gamma=0)\ng = cv.addWeighted(src1=g, alpha=kg, src2=0, beta=0, gamma=0)\nb = cv.addWeighted(src1=b, alpha=kb, src2=0, beta=0, gamma=0)\n \nbalance_img = cv.merge([b, g, r])\n\nplt.figure(figsize=(25,12))\nplt.subplot(121)\nplt.imshow(img)\nplt.subplot(122)\nplt.imshow(balance_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = '../input/train'\ntrain_imgs = ['../input/train/{}'.format(i) for i in os.listdir(train_dir)]\n\nplt.figure(figsize=(25,12))\nfor idx, train_img in enumerate(train_imgs):\n    if idx >= 15:\n        break\n    \n    temp_img = cv.imread(train_img, cv.IMREAD_COLOR)        \n    \n    \n    plt.subplot(3,5, idx + 1)\n    plt.imshow(temp_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(25,12))\nfor idx, train_img in enumerate(train_imgs):\n    if idx >= 15:\n        break\n    temp_img = cv.imread(train_img, cv.IMREAD_COLOR)\n    r, g, b = cv.split(temp_img)\n    r_avg = cv.mean(r)[0]\n    g_avg = cv.mean(g)[0]\n    b_avg = cv.mean(b)[0]\n    k = (r_avg + g_avg + b_avg) / 3\n    kr = k / r_avg\n    kg = k / g_avg\n    kb = k / b_avg\n    r = cv.addWeighted(src1=r, alpha=kr, src2=0, beta=0, gamma=0)\n    g = cv.addWeighted(src1=g, alpha=kg, src2=0, beta=0, gamma=0)\n    b = cv.addWeighted(src1=b, alpha=kb, src2=0, beta=0, gamma=0)\n    balance_img = cv.merge([b, g, r])\n\n    plt.subplot(3 ,5, idx + 1)\n    plt.imshow(balance_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = 130000\nX,y = load_data(N=N,df=df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(X[632])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in tqdm_notebook(range(len(X))):\n    r ,g, b = cv.split(X[i])\n    r_avg = cv.mean(r)[0]\n    g_avg = cv.mean(g)[0]\n    b_avg = cv.mean(b)[0]\n \n    k = (r_avg + g_avg + b_avg) / 3\n    kr = k / r_avg\n    kg = k / g_avg\n    kb = k / b_avg\n \n    r = cv.addWeighted(src1=r, alpha=kr, src2=0, beta=0, gamma=0)\n    g = cv.addWeighted(src1=g, alpha=kg, src2=0, beta=0, gamma=0)\n    b = cv.addWeighted(src1=b, alpha=kb, src2=0, beta=0, gamma=0)\n \n    X[i] = cv.merge([b, g, r])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(X[632])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Collect garbage\npositives_samples = None\nnegative_samples = None\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_portion = 0.8 # Specify training/validation ratio\nsplit_idx = int(np.round(training_portion * y.shape[0])) #Compute split idx\n\nnp.random.seed(42) #set the seed to ensure reproducibility\n\n#shuffle\nidx = np.arange(y.shape[0])\nnp.random.shuffle(idx)\nX = X[idx]\ny = y[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#just some network parameters, see above link regarding the layers for details\nkernel_size = (3,3)\npool_size= (2,2)\nfirst_filters = 32\nsecond_filters = 64\nthird_filters = 128\n\n#dropout is used for regularization here with a probability of 0.3 for conv layers, 0.5 for the dense layer at the end\ndropout_conv = 0.3\ndropout_dense = 0.5\n\n#initialize the model\nmodel = Sequential()\n\n#now add layers to it\n#conv block 1\nmodel.add(Conv2D(first_filters, kernel_size, input_shape = (96, 96, 3)))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(first_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size)) \nmodel.add(Dropout(dropout_conv))\n\n#conv block 2\nmodel.add(Conv2D(second_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(second_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\n#conv block 3\nmodel.add(Conv2D(third_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(third_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\n#a fully connected (also called dense) layer at the end\nmodel.add(Flatten())\nmodel.add(Dense(256, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Dropout(dropout_dense))\n\n#finally convert to values of 0 to 1 using the sigmoid activation function\nmodel.add(Dense(1, activation = \"sigmoid\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 50\n\nmodel.compile(loss=keras.losses.binary_crossentropy,\n              optimizer=keras.optimizers.Adam(0.001), \n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 12\nfor epoch in range(epochs):\n    iterations = np.floor(split_idx / batch_size).astype(int)\n    loss,acc = 0,0 \n    with trange(iterations) as t: \n        for i in t:\n            start_idx = i * batch_size \n            x_batch = X[start_idx:start_idx+batch_size] \n            y_batch = y[start_idx:start_idx+batch_size] \n\n            metrics = model.train_on_batch(x_batch, y_batch) #train the model on a batch\n\n            loss = loss + metrics[0] \n            acc = acc + metrics[1] \n            t.set_description('Running training epoch ' + str(epoch)) \n            t.set_postfix(loss=\"%.2f\" % round(loss / (i+1),2),acc=\"%.2f\" % round(acc / (i+1),2)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iterations = np.floor((y.shape[0]-split_idx) / batch_size).astype(int) \nloss,acc = 0,0 \nwith trange(iterations) as t: \n    for i in t:\n        start_idx = i * batch_size \n        x_batch = X[start_idx:start_idx+batch_size] \n        y_batch = y[start_idx:start_idx+batch_size] \n        \n        metrics = model.test_on_batch(x_batch, y_batch) \n        \n        loss = loss + metrics[0] \n        acc = acc + metrics[1] \n        t.set_description('Running training') \n        t.set_description('Running validation')\n        t.set_postfix(loss=\"%.2f\" % round(loss / (i+1),2),acc=\"%.2f\" % round(acc / (i+1),2))\n        \nprint(\"Validation loss:\",loss / iterations)\nprint(\"Validation accuracy:\",acc / iterations)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = None\ny = None\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_test_dir = path + 'test/' #specify test data folder\ntest_files = glob(os.path.join(base_test_dir,'*.tif')) #find the test file names\nsubmission = pd.DataFrame() #create a dataframe to hold results\nfile_batch = 5000 #we will predict 5000 images at a time\nmax_idx = len(test_files) #last index to use\nfor idx in range(0, max_idx, file_batch): #iterate over test image batches\n    print(\"Indexes: %i - %i\"%(idx, idx+file_batch))\n    test_df = pd.DataFrame({'path': test_files[idx:idx+file_batch]}) #add the filenames to the dataframe\n    test_df['id'] = test_df.path.map(lambda x: x.split('/')[3].split(\".\")[0]) #add the ids to the dataframe\n    test_df['image'] = test_df['path'].map(cv.imread) #read the batch\n    K_test = np.stack(test_df[\"image\"].values) #convert to numpy array\n    for i in tqdm_notebook(range(len(K_test))):\n        r ,g, b = cv.split(K_test[i])\n        r_avg = cv.mean(r)[0]\n        g_avg = cv.mean(g)[0]\n        b_avg = cv.mean(b)[0]\n \n        k = (r_avg + g_avg + b_avg) / 3\n        kr = k / r_avg\n        kg = k / g_avg\n        kb = k / b_avg\n \n        r = cv.addWeighted(src1=r, alpha=kr, src2=0, beta=0, gamma=0)\n        g = cv.addWeighted(src1=g, alpha=kg, src2=0, beta=0, gamma=0)\n        b = cv.addWeighted(src1=b, alpha=kb, src2=0, beta=0, gamma=0)\n \n        K_test[i] = cv.merge([b, g, r])\n    predictions = model.predict(K_test,verbose = 1) #predict the labels for the test data\n    test_df['label'] = predictions #store them in the dataframe\n    submission = pd.concat([submission, test_df[[\"id\", \"label\"]]])\nsubmission.head() #display first lines","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index = False, header = True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}