{"cells":[{"metadata":{},"cell_type":"markdown","source":"hello evryone,\nThis notebook goes through the whole process of model training from EDA , data wrngling to feature engineering to processingg the ct data to final model training \n\nplease upvote if you like my work","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings \n%matplotlib inline\n\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest_df = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsub_df = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Exploration and Feature Engineering\n----\n----","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'number of unique patients = {len(train_df.patient_id.unique())}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(4, 6))\nsns.countplot(x = 'benign_malignant' , data=train_df)\nplt.title('class distribution')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"okay so we have a class imbalance problem","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df.shape)\ntrain_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure()\nsns.distplot(train_df.age_approx)\nplt.title('Distribution of Age')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see that the Age has discrete values and a normal distribtuon. we can use countplot to better visualize the age feature and how our outcome is dependent on AGE","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.age_approx.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10, 16))\nfig.add_subplot(2,1,1)\nsns.countplot(x='age_approx' , data = train_df)\nplt.title('freq count of AGE')\n\nfig.add_subplot(2,1,2)\nsns.countplot(x='benign_malignant' , hue='age_approx' , data=train_df)\nplt.title('distribution of age with respect to outcome')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"disttribution by sex","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x='sex' , data= train_df)\nplt.title('distribution by sex')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets explore anatom_site_general_challenge feature","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.anatom_site_general_challenge.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,6))\nsns.countplot(x='anatom_site_general_challenge' , data=train_df)\nplt.title('dustribution of anatom_site_general_challenge')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets handle the missing values\n\n* ## 1. AGE - we can fill the missing values with mean","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['age_approx'] = train_df.age_approx.fillna(np.mean(train_df['age_approx']))\ntest_df['age_approx'] = test_df.age_approx.fillna(np.mean(test_df['age_approx']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ## 2. SEX -- \nif the unknown values are also present in the test set we will replace the nan with an unknown class U otherwise we will look to find its value in train set first if not found we will drop the unknown","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"unknown_sex= train_df.loc[train_df.sex.isnull()]\nun_sex_list = unknown_sex.patient_id.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor p in un_sex_list:\n    a = train_df.loc[(train_df.patient_id == p) & (train_df.sex.notnull()) , 'sex']\n    print(a)\n    #train_df.loc[(train_df.patient_id == p)].fillna(a , inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"so there are only two pateints whose sex we don't know so now lest look if there are any null values for sex in test set or not","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now since there are no null values for sex in the test set we can drop these two pateints out of our training set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.dropna(subset=['sex'] , inplace=True)\nprint(train_df.shape)\ntrain_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"converting sex to a numerical feature","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['sex'] = train_df['sex'].replace('male' , 0)\ntrain_df['sex'] = train_df['sex'].replace('female' , 1)\n\ntest_df['sex'] = test_df['sex'].replace('male' ,0)\ntest_df['sex'] = test_df['sex'].replace('female' , 1)\n\ntrain_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ## 3. anatom_site_general_challenge --\nnow we numerically encode the labels and use unknown class U for unknown labels","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# filling nan with U\ntrain_df['anatom_site_general_challenge'] = train_df['anatom_site_general_challenge'].fillna('U')\ntest_df['anatom_site_general_challenge'] = test_df['anatom_site_general_challenge'].fillna('U')\nprint(len(train_df['anatom_site_general_challenge'].unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_enc_dict = { p: i for i, p in enumerate(train_df['anatom_site_general_challenge'].unique())}\nnum_enc_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['anatom_site_general_challenge'] = train_df['anatom_site_general_challenge'].map(num_enc_dict)\ntest_df['anatom_site_general_challenge'] = test_df['anatom_site_general_challenge'].map(num_enc_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* at last fixing the diagnosis columns","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_enc_dict = { p: i for i, p in enumerate(train_df['diagnosis'].unique())}\nnum_enc_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['diagnosis'] = train_df['diagnosis'].map(num_enc_dict)\n#test_df['diagnosis'] = test_df['diagnosis'].map(num_enc_dict)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Hurrah! we replaced all the unknown values with a proper value and now both our test and train set are clean and contain no null values\n\nnow for once final time lets look at the count of all the features in the dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,10))\ni =1\nfor p in train_df.columns[2:]:\n    ax = fig.add_subplot(3,3,i)\n    sns.countplot(x=p, data=train_df)\n    ax.set_xticks([])\n    i +=1\n\nfig.tight_layout(pad=3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image data processing\nnow lets look at image data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\nfrom PIL import Image\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport cv2\nimport io\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = os.path.join('../input/siim-isic-melanoma-classification/jpeg/train/')\ntest_path = os.path.join('../input/siim-isic-melanoma-classification/jpeg/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TFRpath = os.path.join('../input/siim-isic-melanoma-classification/tfrecords/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for f in os.listdir(train_path)[:30]:\n    a = cv2.imread(train_path + f)\n    print(a.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"so we can see that the images are pretty big and we can use fed these directly to the the convnets so we will need to reduce the size of of these images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# image dimension\nIMG_DIM = 64\nIMG_DEPTH = 3\nBATCH_SIZE = 32\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"raw_data = tf.data.TFRecordDataset('../input/siim-isic-melanoma-classification/tfrecords/train00-2071.tfrec')\nfor r in raw_data.take(1):\n    ex = tf.train.Example()\n    ex.ParseFromString(r.numpy())\n    print(ex)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_description = {\n    'image': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'image_name': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'target': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n}\ndef parse_data_func(ex_proto):\n    return tf.io.parse_single_example(ex_proto , feature_description)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dic = {}\nparsed_data = raw_data.map(parse_data_func)\nfor p in parsed_data.take(10):\n    img = p['image'].numpy()\n    name = p['image_name'].numpy()\n    name = name.decode('utf-8')\n    img = np.array(Image.open(io.BytesIO(img)))\n    img = cv2.resize(img , (IMG_DIM , IMG_DIM))\n    print(img.shape)\n    di = {name : img}\n    dic.update(di)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now we can use the same approcah to write our function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"parsed_data = raw_data.map(parse_data_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function to load TFR data into a dictionary\ndef load_TFR_data(sample, img_dim):\n    img = sample['image'].numpy()\n    name = sample['image_name'].numpy()\n    name = name.decode('utf-8')\n    img = np.array(Image.open(io.BytesIO(img)))\n    img = cv2.resize(img , (img_dim , img_dim))\n    img = img/255\n    img = img.astype(np.uint8)\n    return {name : img}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ct_data = {}\nfor p in parsed_data:\n    train_ct_data.update(load_TFR_data(p , IMG_DIM))\n\nlen(train_ct_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_ct_data_to_dict(directory , _set='train'):\n    data_dict = {}\n    for p in os.listdir(directory):\n        if p.startswith(_set):\n            path = directory+p\n            raw_data = tf.data.TFRecordDataset(path)\n            parsed_data = raw_data.map(parse_data_func)\n            for p in parsed_data:\n                data_dict.update(load_TFR_data(p , IMG_DIM))\n    return data_dict\n            \n                ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ct_data_dict = load_ct_data_to_dict(TFRpath , _set='train')\ntest_ct_data_dict = load_ct_data_to_dict(TFRpath , _set='test')\n\nprint(f'number of samples of ct in train set = {len(train_ct_data_dict)}')\nprint(f'number of samples of ct in test set = {len(test_ct_data_dict)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_images = []\nfor p in train_df['image_name']:\n    x_train_images.append(train_ct_data_dict[p])\n    del(train_ct_data_dict[p])\n    \nx_train_images = np.array(x_train_images)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_images.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test_images = []\nfor p in test_df['image_name']:\n    x_test_images.append(test_ct_data_dict[p])\n    del(test_ct_data_dict[p])\n    \nx_test_images = np.array(x_test_images)\nx_test_images.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('x_train_images.npy' , x_train_images)\nnp.save('x_test_images.npy' , x_test_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_images.nbytes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now we have  training and testing images preparesd\nlets prepare our numerical data\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_numerical = train_df[['sex' ,'age_approx' ,'anatom_site_general_challenge']]\nx_train_numerical = np.array(x_train_numerical)\ny_train = train_df[['target']]\ny_train = np.array(y_train)\nx_test_numerical = test_df[['sex' ,'age_approx' ,'anatom_site_general_challenge']]\nx_test_numerical = np.array(x_test_numerical)\n\nprint(f'shape of train images : {x_train_images.shape}')\nprint(f'shape of train numerical : {x_train_numerical.shape}')\nprint(f'shape of train target (y) : {y_train.shape}')\nprint(f'shape of test images : {x_test_images.shape}')\nprint(f'shape of test numerical : {x_test_numerical.shape}')\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now our data is ready for training and we can go on to create our model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# model creation and training","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"we will be using a two branch model which will process the different tupes inputs\n* branch 1 -- processes the image data\n* branch 2 -- processes the tabular data\n* final model will predict on [branch 1+ branch 2]\nas shown in the figure \n\n![](https://www.pyimagesearch.com/wp-content/uploads/2019/02/keras_multi_input_design.png)\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Conv2D , Activation , MaxPool2D , concatenate , GlobalAvgPool2D\nfrom tensorflow.keras.layers import Flatten , Dense , Input , BatchNormalization , Dropout \nfrom tensorflow.keras.models import Model , Sequential","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"lets create the multilayer perceptron first","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def perceptron_model(dim):\n    model = Sequential()\n    model.add(Dense(12 , input_dim=dim , activation='relu'))\n    model.add(Dense(8 , activation = 'relu'))\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now the CNN","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def CNN_model(h,w,d):\n    input_shape = (h,w,d)\n    _input = Input(shape=input_shape)\n    x = Conv2D(16 ,(3,3) , activation='relu' , padding='same')(_input)\n    x = Conv2D(32 ,(3,3) , activation ='relu' , padding='same')(x)\n    x = MaxPool2D((2,2))(x)\n    x = BatchNormalization()(x)\n    x = Conv2D(64 , (3,3) , activation='relu' , padding='same')(x)\n    x = MaxPool2D((2,2))(x)\n    x = BatchNormalization()(x)\n    x = Flatten()(x)\n    x = Dense(1024 , activation='relu')(x)\n    x = Dropout(0.5)(x)\n    x = Dense(128 , activation='relu')(x)\n    model = Model(inputs=_input , outputs=x)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets create a checkpoint so that we can store the weights of our model pause and continue the training the way we want. first lets create a directory to hold our checkpoint","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"! mkdir model_checkpoints","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath = 'model_checkpoints/model_weights.hdf5' , save_best_only = True , verbose =1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets define our final model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"linear_model = perceptron_model(3)\nCnn_model = CNN_model(IMG_DIM , IMG_DIM, 3)\n\nf = concatenate([linear_model.output , Cnn_model.output])\nf = Dense(128 , activation = 'relu')(f)\nf = Dropout(0.3)(f)\nf = Dense(64 , activation = 'relu')(f)\nf = Dropout(0.5)(f)\nf = Dense(1 , activation = 'sigmoid')(f)\n\nfinal_model = Model(inputs=[linear_model.input , Cnn_model.input] , outputs=f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now our final model is ready and we can now compile it. remember the metric we need is roc_auc_score","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"final_model.compile(optimizer='adam' , loss='binary_crossentropy' , metrics=['accuracy'])\nfinal_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now our model is ready lets satrt the training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"history = final_model.fit(x=[x_train_numerical , x_train_images] , y = y_train , epochs=30 , batch_size=32 , callbacks=[cp_callback])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = final_model.predict(x=[x_test_numerical , x_test_images])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['target'] = preds\nsub_df.set_index('image_name' , inplace=True)\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now this was a baseline model which didn't use any validation set. the score of this model was 0.5757 so now we create a validation and use early stopping to save our model from overfitting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating validation set\n# number of samples in validation set = k\nk = 5000\nx_val_numerical = x_train_numerical[len(x_train_numerical) - k :]\nx_val_images = x_train_images[len(x_train_images) -k :]\ny_val = y_train[len(y_train)-k:]\n\nx_train_numerical = x_train_numerical[:len(x_train_numerical) - k]\nx_train_images = x_train_images[:len(x_train_images) - k]\ny_train = y_train[:len(y_train)-k]\n\nprint(f'shape of training numerical : {x_train_numerical.shape}')\nprint(f'shape of training images : {x_train_images.shape}')\nprint(f'shape of training target : {y_train.shape}')\nprint(f'shape of validation set (numerical) : {x_val_numerical.shape}')\nprint(f'shape of validation set (images) : {x_val_images.shape}')\nprint(f'shape of validation set (target) : {y_val.shape}')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"and lets recompile our model which use auc score as a metrics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"METRICS = [\n      tf.keras.metrics.TruePositives(name='tp'),\n      tf.keras.metrics.FalsePositives(name='fp'),\n      tf.keras.metrics.TrueNegatives(name='tn'),\n      tf.keras.metrics.FalseNegatives(name='fn'), \n      tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n      tf.keras.metrics.Precision(name='precision'),\n      tf.keras.metrics.Recall(name='recall'),\n      tf.keras.metrics.AUC(name='auc'),\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_model.compile(optimizer='adam' , loss='binary_crossentropy' , metrics=METRICS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# early_stopping\nes_callback = tf.keras.callbacks.EarlyStopping(monitor='val_auc' , verbose=1, patience=10 , mode='max' ,restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history2 = final_model.fit(x= [x_train_numerical , x_train_images] , \n                           y = y_train , \n                           epochs = 50 , \n                           validation_data=([x_val_numerical , x_val_images] , y_val), \n                           callbacks = [cp_callback , es_callback])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = plt.figure()\nax1 = plt.plot(history2.history['loss'])\nax2 = plt.plot(history2.history['val_loss'])\nplt.title('LOSS')\nplt.legend(labels =['loss' ,'val_loss'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history2.history['auc'])\nplt.plot(history2.history['val_auc'])\nplt.title('AUC')\nplt.legend(labels = ['auc' ,'val_auc'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets make a prediction with this model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = final_model.predict([x_test_numerical , x_test_images])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['target'] = preds\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"so this time our score improved to 0.6770\n\nnow one thing we can also do is to get rid of class imabalnce as we saw in the beginning there is a huge imbalnce between classes so we need to handle this imbalnce\n\nthe techninque we are going to use is class weighting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# first lets look at the count of each class in our training sample\npos = np.count_nonzero(y_train==0)\nneg = np.count_nonzero(y_train==1)\ntotal = len(y_train)\nprint('class 0 : ' + str(pos))\nprint('class 1 : ' + str(neg))\nprint('total : ' +str(total))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"this is a huge imbalnce. so lets how weighting of classes is done","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"w_0 = (1/pos)*(total)/2\nw_1 = (1/neg)*(total)/2\n\nclass_weights = {0:w_0 ,1:w_1}\nprint(f'weight of class 0 : {w_0}')\nprint(f'weight of class 1 : {w_1}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now we will feed this class weight to the model while training this will penalise the model weights for the imbalance between classes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"history3 = final_model.fit(x= [x_train_numerical , x_train_images] , \n                           y = y_train , \n                           epochs = 50 , \n                           validation_data=([x_val_numerical , x_val_images] , y_val), \n                           callbacks = [cp_callback , es_callback],\n                           class_weight=class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = plt.figure()\nax1 = plt.plot(history3.history['loss'])\nax2 = plt.plot(history3.history['val_loss'])\nplt.title('LOSS')\nplt.legend(labels =['loss' ,'val_loss'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history3.history['auc'])\nplt.plot(history3.history['val_auc'])\nplt.title('AUC')\nplt.legend(labels = ['auc' ,'val_auc'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = final_model.predict([x_test_numerical , x_test_images])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['target'] = preds\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we can see our val loss was going during the training so lets apply early stopping on val_loss and check how the model performs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"es_callback2 = tf.keras.callbacks.EarlyStopping(monitor='val_loss' , verbose=1, patience=10 , mode='auto' ,restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history4 = final_model.fit(x= [x_train_numerical , x_train_images] , \n                           y = y_train , \n                           epochs = 50 , \n                           validation_data=([x_val_numerical , x_val_images] , y_val), \n                           callbacks = [cp_callback , es_callback2],\n                           class_weight=class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"z = final_model.evaluate([x_val_numerical , x_val_images] , y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = final_model.predict([x_test_numerical , x_test_images])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['target'] = preds\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}