{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob, pylab, pandas as pd\nimport pydicom, numpy as np\nfrom os import listdir\nfrom os.path import isfile, join\nimport matplotlib.pylab as plt\nimport matplotlib.pyplot as plt2\nfrom plotly.offline import init_notebook_mode\nimport plotly.graph_objs as go\nfrom plotly import tools\nimport os\nimport seaborn as sns\nfrom keras import layers\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.layers import *\nfrom keras.applications import DenseNet121\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nimport tensorflow as tf\nfrom tqdm import tqdm\nimport cv2\nfrom PIL import Image\nfrom plotly.offline import iplot\nimport cufflinks\n#from tpu_helper import *\nimport cv2 as cv\ncufflinks.go_offline()\ncufflinks.set_config_file(world_readable=True, theme='pearl')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SIIM-ISIC Melanonma Classification\n\nThis competition is to identify melanoma (a type of skin cancer) using lesion images.\n\nThe given data set includes 4 folders jpeg, tfrecords, test, train, and three .csv files: test.csv, train.csv and sample_submission.csv.\n\nHere, the train.csv and test.csv are metadata which contains information about images provided in train and test folder where images stored in DICOM format.\n\nImages are also provided in JPEG and TFRecord format (in the jpeg and tfrecords directories, respectively). Images in TFRecord format have been resized to a uniform 1024x1024.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images_dir = '../input/siim-isic-melanoma-classification/train/'\ntrain_images = [f for f in listdir(train_images_dir) if isfile(join(train_images_dir, f))]\ntest_images_dir = '../input/siim-isic-melanoma-classification/test/'\ntest_images = [f for f in listdir(test_images_dir) if isfile(join(test_images_dir, f))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=plt.figure(figsize=(15, 10))\ncolumns = 5; rows = 4\nfor i in range(1, columns*rows +1):\n    ds = pydicom.dcmread(train_images_dir + train_images[i])\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(ds.pixel_array, cmap=plt.cm.bone)\n    fig.add_subplot","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explanatory Data Analysis","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**File train.csv have 8 columns, 33126 rows**\n* image_name - unique identifier, points to filename of related DICOM image\n* patient_id - unique patient identifier\n* sex - the sex of the patient (when unknown, will be blank)\n* age_approx - approximate patient age at time of imaging\n* anatom_site_general_challenge - location of imaged site\n* diagnosis - detailed diagnosis information (train only): unknown, nevus, melanoma,...\n* benign_malignant - indicator of malignancy of imaged lesion: benign = harmless, malignant = harmful.\n* target - binarized version of the target variable (1:melanoma; 0: non)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* **File train.csv have only 5 columns, 10982 rows** :image_name, patient_id, sex, age_approx, anatom_site_general_challenge","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Submission**: assign each image in the test set (10982 rows) to a target variable: from 0 to 1 (to indicate the percentage of having melanoma or non)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing\n\nAs we can see, the size of these lension images are not the same. \n\nTherefore, we do need to resize the images and also do the augmentation. cv2 module can be used to complete these two tasks.\n\nThis part is from https://www.kaggle.com/nxrprime/siim-eda-augmentations-model-seresnet-unet","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"****Augmentation and Visualization****","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"We will first try to visualize in grayscale (only gray colors) so that it is possible for us to clearly visualize the varied differences in color, region, and shape.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\ndef view_images_aug1(images, title = '', aug = None):\n    width = 6\n    height = 5\n    fig, axs = plt.subplots(height, width, figsize=(15,15))\n    for im in range(0, height * width):\n        data = pydicom.read_file(os.path.join(train_images_dir, list(images)[im]+ '.dcm'))\n        image = data.pixel_array\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n        image = cv2.resize(image, (256, 256))\n        i = im // width\n        j = im % width\n        axs[i,j].imshow(image, cmap=plt.cm.bone) \n        axs[i,j].axis('off')\n        \n    plt.suptitle(title)\nview_images_aug1(train[train['diagnosis']=='lentigo NOS']['image_name'], title=\"Lentigo NOS's growth\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The second method we will use is Ben Graham's method from APTOS.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_images_aug2(images, title = '', aug = None):\n    width = 6\n    height = 5\n    fig, axs = plt.subplots(height, width, figsize=(15,15))\n    for im in range(0, height * width):  \n        data = pydicom.read_file(os.path.join(train_images_dir, list(images)[im]+ '.dcm'))\n        image = data.pixel_array\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n        image = cv2.resize(image, (256, 256))\n        image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , 256/10) ,-4 ,128)\n        i = im // width\n        j = im % width\n        axs[i,j].imshow(image, cmap=plt.cm.bone) \n        axs[i,j].axis('off')\n        \n    plt.suptitle(title)\nview_images_aug2(train[train['diagnosis']=='lentigo NOS']['image_name'], title=\"Lentigo NOS's growth\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can further observe the clear color distinctions by using Neuron Engineer's method (an improved version of Ben Graham's).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_images_aug3(images, title = '', aug = None):\n    width = 6\n    height = 5\n    fig, axs = plt.subplots(height, width, figsize=(15,15))\n    for im in range(0, height * width):  \n        data = pydicom.read_file(os.path.join(train_images_dir, list(images)[im]+ '.dcm'))\n        image = data.pixel_array\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (256, 256))\n        image=cv2.addWeighted ( image,4, cv2.GaussianBlur( image , (0,0) , 10) ,-4 ,128)\n        i = im // width\n        j = im % width\n        axs[i,j].imshow(image, cmap=plt.cm.bone) \n        axs[i,j].axis('off')\n        \n    plt.suptitle(title)\nview_images_aug3(train[train['diagnosis']=='lentigo NOS']['image_name'], title=\"Lentigo NOS's growth\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we can finally visualize the clear distinctions in our data. The clear regions, the clear color differences, the clear everything! \n\nCircular crop may not be feasible for images where the tumor is on the edge of the image. It does not seem so feasible, so I would recommend you try to be smarter in your methods for preprocessing. Remember, you can build upon Ben Graham's work as a starting point, then try Neuron Engineer's or circle crop or even build you own method.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\n    \ndef circle_crop(img, sigmaX=10):   \n    \"\"\"\n    Create circular crop around image centre    \n    \"\"\"    \n    \n    img = crop_image_from_gray(img)    \n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    height, width, depth = img.shape    \n    \n    x = int(width/2)\n    y = int(height/2)\n    r = np.amin((x,y))\n    \n    circle_img = np.zeros((height, width), np.uint8)\n    cv2.circle(circle_img, (x,y), int(r), 1, thickness=-1)\n    img = cv2.bitwise_and(img, img, mask=circle_img)\n    img = crop_image_from_gray(img)\n    img=cv2.addWeighted ( img,4, cv2.GaussianBlur( img , (0,0) , sigmaX) ,-4 ,128)\n    return img \n\ndef view_images_aug4(images, title = '', aug = None):\n    width = 6\n    height = 5\n    fig, axs = plt.subplots(height, width, figsize=(15,15))\n    for im in range(0, height * width):  \n        data = pydicom.read_file(os.path.join(train_images_dir, list(images)[im]+ '.dcm'))\n        image = data.pixel_array\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (256, 256))\n        image= circle_crop(image)\n        i = im // width\n        j = im % width\n        axs[i,j].imshow(image, cmap=plt.cm.bone) \n        axs[i,j].axis('off')\n        \n    plt.suptitle(title)\nview_images_aug4(train[train['diagnosis']=='lentigo NOS']['image_name'], title=\"Lentigo NOS's growth\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we are using auto-cropping as a method of preprocessing, which is a more \"refined\" circle crop if you will. Think of circle crop as C, and think of auto-cropping as C++. Auto-cropping indeed is powerful, but the risk is that you will lose valuable data in the image.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img\n    \ndef view_images_aug(images, title = '', aug = None):\n    width = 6\n    height = 5\n    fig, axs = plt.subplots(height, width, figsize=(15,15))\n    for im in range(0, height * width):  \n        data = pydicom.read_file(os.path.join(train_images_dir, list(images)[im]+ '.dcm'))\n        image = data.pixel_array\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (256, 256))\n        image= crop_image_from_gray(image)\n        i = im // width\n        j = im % width\n        axs[i,j].imshow(image, cmap=plt.cm.bone) \n        axs[i,j].axis('off')\n        \n    plt.suptitle(title)\nview_images_aug(train[train['diagnosis']=='lentigo NOS']['image_name'], title=\"Lentigo NOS's growth\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Another thing you can do is background subtraction.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fgbg = cv.createBackgroundSubtractorMOG2()\n    \ndef view_images_aug(images, title = '', aug = None):\n    width = 6\n    height = 5\n    fig, axs = plt.subplots(height, width, figsize=(15,15))\n    for im in range(0, height * width):  \n        data = pydicom.read_file(os.path.join(train_images_dir, list(images)[im]+ '.dcm'))\n        image = data.pixel_array\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (256, 256))\n        image= fgbg.apply(image)\n        i = im // width\n        j = im % width\n        axs[i,j].imshow(image, cmap=plt.cm.bone) \n        axs[i,j].axis('off')\n        \n    plt.suptitle(title)\nview_images_aug(train[train['diagnosis']=='lentigo NOS']['image_name'], title=\"Lentigo NOS's growth\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Also, you can use the albumentations library to create a lot of simulated images for your model. Remember, your model MUST BE ABLE TO GENERALIZE!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as A\nimage_folder_path = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/\"\nchosen_image = cv2.imread(os.path.join(image_folder_path, \"ISIC_0079038.jpg\"))\nalbumentation_list = [A.RandomSunFlare(p=1), A.RandomFog(p=1), A.RandomBrightness(p=1),\n                      A.RandomCrop(p=1,height = 512, width = 512), A.Rotate(p=1, limit=90),\n                      A.RGBShift(p=1), A.RandomSnow(p=1),\n                      A.HorizontalFlip(p=1), A.VerticalFlip(p=1), A.RandomContrast(limit = 0.5,p = 1),\n                      A.HueSaturationValue(p=1,hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=50)]\n\nimg_matrix_list = []\nbboxes_list = []\nfor aug_type in albumentation_list:\n    img = aug_type(image = chosen_image)['image']\n    img_matrix_list.append(img)\n\nimg_matrix_list.insert(0,chosen_image)    \n\ntitles_list = [\"Original\",\"RandomSunFlare\",\"RandomFog\",\"RandomBrightness\",\n               \"RandomCrop\",\"Rotate\", \"RGBShift\", \"RandomSnow\",\"HorizontalFlip\", \"VerticalFlip\", \"RandomContrast\",\"HSV\"]\n\n##reminder of helper function\ndef plot_multiple_img(img_matrix_list, title_list, ncols, main_title=\"\"):\n    fig, myaxes = plt.subplots(figsize=(20, 15), nrows=3, ncols=ncols, squeeze=False)\n    fig.suptitle(main_title, fontsize = 30)\n    fig.subplots_adjust(wspace=0.3)\n    fig.subplots_adjust(hspace=0.3)\n    for i, (img, title) in enumerate(zip(img_matrix_list, title_list)):\n        myaxes[i // ncols][i % ncols].imshow(img)\n        myaxes[i // ncols][i % ncols].set_title(title, fontsize=15)\n    plt.show()\n    \nplot_multiple_img(img_matrix_list, titles_list, ncols = 4,main_title=\"Different Types of Augmentations\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have much more augmentations we can try like:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image_folder_path = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/\"\nchosen_image = cv2.imread(os.path.join(image_folder_path, \"ISIC_0079038.jpg\"))\nalbumentation_list = [A.RandomSunFlare(p=1), A.GaussNoise(p=1), A.CLAHE(p=1),\n                      A.RandomRain(p=1), A.Rotate(p=1, limit=90),\n                      A.RGBShift(p=1), A.RandomSnow(p=1),\n                      A.HorizontalFlip(p=1), A.VerticalFlip(p=1), A.RandomContrast(limit = 0.5,p = 1),\n                      A.HueSaturationValue(p=1,hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=50)]\n\nimg_matrix_list = []\nbboxes_list = []\nfor aug_type in albumentation_list:\n    img = aug_type(image = chosen_image)['image']\n    img_matrix_list.append(img)\n\nimg_matrix_list.insert(0,chosen_image)    \n\ntitles_list = [\"Original\",\"RandomSunFlare\",\"GaussNoise\",\"CLAHE\",\n               \"RandomRain\",\"Rotate\", \"RGBShift\", \"RandomSnow\",\"HorizontalFlip\", \"VerticalFlip\", \"RandomContrast\",\"HSV\"]\n\n##reminder of helper function\ndef plot_multiple_img(img_matrix_list, title_list, ncols, main_title=\"\"):\n    fig, myaxes = plt.subplots(figsize=(20, 15), nrows=3, ncols=ncols, squeeze=False)\n    fig.suptitle(main_title, fontsize = 30)\n    fig.subplots_adjust(wspace=0.3)\n    fig.subplots_adjust(hspace=0.3)\n    for i, (img, title) in enumerate(zip(img_matrix_list, title_list)):\n        myaxes[i // ncols][i % ncols].imshow(img)\n        myaxes[i // ncols][i % ncols].set_title(title, fontsize=15)\n    plt.show()\n    \nplot_multiple_img(img_matrix_list, titles_list, ncols = 4,main_title=\"Different Types of Augmentations\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Resized Images**\n\nThe images should be resized to speed up the testing progress.\nThe original kernel is here https://www.kaggle.com/tunguz/image-resizing-32x32-train\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport json\nimport math\nimport cv2\nimport PIL\nfrom PIL import Image\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\nimport scipy\nfrom tqdm import tqdm\n%matplotlib inline\nfrom keras.preprocessing import image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#These code will run for very long time. Then, we can use the alread-done files created in \n#here https://www.kaggle.com/tunguz/siimisic-melanoma-resized-images\n\"\"\"\nimage_Size = 32\ndef preprocess_image(image_path, desired_size=image_Size):\n    im = Image.open(image_path)\n    im = im.resize((desired_size, )*2, resample=Image.LANCZOS)\n    \n    return im\n\n# get the number of training images from the target\\id dataset\nN = train.shape[0]\n# create an empty matrix for storing the images\nx_train = np.empty((N, image_Size, image_Size, 3), dtype=np.uint8)\n\n# loop through the images from the images ids from the target\\id dataset\n# then grab the cooresponding image from disk, pre-process, and store in matrix in memory\nfor i, image_id in enumerate(tqdm(train['image_name'])):\n    x_train[i, :, :, :] = preprocess_image(\n        f'../input/siim-isic-melanoma-classification/jpeg/train/{image_id}.jpg'\n    )\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nNtest = test.shape[0]\n# create an empty matrix for storing the images\nx_test = np.empty((Ntest, image_Size, image_Size, 3), dtype=np.uint8)\n\n# loop through the images from the images ids from the target\\id dataset\n# then grab the cooresponding image from disk, pre-process, and store in matrix in memory\nfor i, image_id in enumerate(tqdm(test['image_name'])):\n    x_test[i, :, :, :] = preprocess_image(\n        f'../input/siim-isic-melanoma-classification/jpeg/test/{image_id}.jpg'\n    )\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nnp.save('x_train_32', x_train)\nnp.save('x_test_32', x_test)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# The Baseline Model\n\nThe most simple baseline here is to assign every case 0.5. That means for every cases we have 50% of melanoma. The score for this kind of submission is 0.5 (Ofcourse!).\n\nNow, we can try to implement our first baseline model with simple CNN.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"1. Before using machine learning tools, let try with the basic statistic based off of mean and count of the target variable to get final prediction.\n\nThis part is from this notebook https://www.kaggle.com/titericz/simple-baseline","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Filling NA\ntrain['sex'] = train['sex'].fillna('na')\ntrain['age_approx'] = train['age_approx'].fillna(0)\ntrain['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].fillna('na')\n\ntest['sex'] = test['sex'].fillna('na')\ntest['age_approx'] = test['age_approx'].fillna(0)\ntest['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].fillna('na')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['sex','age_approx','anatom_site_general_challenge']\nMean_train = train.target.mean() #mean of target columns in train set (0.01762965646320111)\n#Grouping the train set using features, then adding the mean and count of each group\ngroupped_train = train.groupby(features)['target'].agg(['mean','count']).reset_index()\ngroupped_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Writing the prediction with mean of each group\n#A paremeter L is added for the bias of the whole train set.\nL=15\ngroupped_train['prediction'] = ((groupped_train['mean']*groupped_train['count'])+(Mean_train*L))/(groupped_train['count']+L)\ndel groupped_train['mean'], groupped_train['count']\n\ntest = test.merge( groupped_train, on=features, how='left' )\ntest['prediction'] = test['prediction'].fillna(Mean_train)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.target = test.prediction.values\nsample_submission.head(5)\nsample_submission.to_csv( 'submission_GroupedMean.csv', index=False )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}