{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import glob\nimport numpy as np\nimport pandas as pd\nimport os\nimport PIL\nimport matplotlib.pyplot as plt\nimport math\n\nfrom sklearn.model_selection import train_test_split\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\nfrom keras.models import Sequential, Model\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Activation, Dropout, Flatten, Dense, Input\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.callbacks import Callback","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"cat_dtype = {'sex':'category', 'anatom_site_general_challenge':'category'}\n\ntrain_df = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/train.csv\", dtype=cat_dtype)\ntest_df = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/test.csv\", dtype=cat_dtype)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define the function to change the category column from string to int16\ndef chg_cat_int(df, cat_col):\n    \n    for col, col_dtype in cat_col.items():\n        if col_dtype == 'category':\n            df[col] = df[col].cat.codes.astype('int16')\n            df[col] -= df[col].min()\n            \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fill the NA value and change the category string to int\ntrain_df['sex'] = train_df['sex'].cat.add_categories('unknown').fillna('unknown')\ntrain_df['anatom_site_general_challenge'] = train_df['anatom_site_general_challenge'].cat.add_categories('unknown').fillna('unknown')\ntrain_df['age_approx'] = train_df['age_approx'].fillna(train_df['age_approx'].mean())\n\ntrain_df = chg_cat_int(train_df, cat_dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['sex'] = test_df['sex'].cat.add_categories('unknown').fillna('unknown')\ntest_df['anatom_site_general_challenge'] = test_df['anatom_site_general_challenge'].cat.add_categories('unknown').fillna('unknown')\ntest_df['age_approx'] = test_df['age_approx'].fillna(test_df['age_approx'].mean())\n\ntest_df = chg_cat_int(test_df, cat_dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Training dataset: Number of data with target=1: ', len(train_df[train_df['target'] == 1]))\nprint('Training dataset: Number of data with target=0: ', len(train_df[train_df['target'] == 0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\n\ntrain_df['file_path'] = train_path + train_df['image_name'] + '.jpg'\ntest_df['file_path'] = test_path + test_df['image_name'] + '.jpg'\n\ncols = ['image_name', 'patient_id', 'sex', 'age_approx', 'anatom_site_general_challenge', 'diagnosis', 'benign_malignant', 'file_path', 'target']\ntrain_df = train_df[cols]\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_center(img,cropx,cropy):\n    y,x = img.shape[1:3]\n    startx = x//2-(cropx//2)\n    starty = y//2-(cropy//2)    \n    return img[:,starty:starty+cropy,startx:startx+cropx:,]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_validate, y_train, y_validate = train_test_split(train_df, train_df['target'], test_size=0.2, random_state=9)\nX_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_cancer_len = len(X_train[X_train['target'] == 1])\nX_validate_cancer_len = len(X_validate[X_validate['target'] == 1])\nprint(X_train_cancer_len)\nprint(X_validate_cancer_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# separate the data in normal and cancer status\n#normal_data = train_df[train_df['target'] == 0]\n#cancer_data = train_df[train_df['target'] == 1]\n\n# shuffle the normal data and take 10000 records only to save training time\n#normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n#normal_data = normal_data[:500] \n\n# append the cancer data and product X and y for model training data\n#normal_data = normal_data.append(cancer_data, ignore_index=True)\n#normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n#X = normal_data.iloc[:, 0:-1]\n#y = normal_data['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Train_Generator(Sequence):\n\n    def __init__(self, train_df, num_samples=5000, batch_size=50, target_dim=(400, 400)):\n        \n        # separate the data in normal and cancer status\n        normal_data = train_df[train_df['target'] == 0]\n        cancer_data = train_df[train_df['target'] == 1]\n\n        # shuffle the normal data and take num_samples records only to save training time\n        normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n        normal_data = normal_data[:num_samples] \n        \n        # append the cancer data and product X and y for model training data\n        normal_data = normal_data.append(cancer_data, ignore_index=True)\n        normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n        x_set = normal_data['file_path']\n        y_set = normal_data['target']\n        \n        self.df = train_df\n        self.x, self.y = x_set, y_set\n        self.samples = num_samples\n        self.batch_size = batch_size\n        self.target_size = target_dim\n        print('Init function, self.x=', self.x[0], ', lengh=', len(self.x))\n\n    def __len__(self):\n        return math.ceil(len(self.x) / self.batch_size)\n\n    def __getitem__(self, idx):\n        #print('index:', idx)\n        batch_x = self.x[idx * self.batch_size:(idx + 1) * self.batch_size]\n        batch_y = self.y[idx * self.batch_size:(idx + 1) * self.batch_size]\n        image_x = np.array([img_to_array(load_img(img, target_size=self.target_size)) for img in batch_x])\n        image_x /= 255.0\n        \n        crop_image_x = crop_center(image_x, self.target_size[0]-100, self.target_size[1]-100)\n\n        return crop_image_x, batch_y\n    \n    def on_epoch_end(self):\n        normal_data = self.df[self.df['target'] == 0]\n        cancer_data = self.df[self.df['target'] == 1]\n        \n        normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n        normal_data = normal_data[:self.samples] \n        \n        normal_data = normal_data.append(cancer_data, ignore_index=True)\n        normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n        x_set = normal_data['file_path']\n        y_set = normal_data['target']\n        \n        self.x, self.y = x_set, y_set\n        print('on_epoch_end, self.x=', self.x[0], ', lengh=', len(self.x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#X_train_files = X_train['file_path']\n#X_validate_files = X_validate['file_path']\n\n#target_dim = (300, 300)\n#train_images = [img_to_array(load_img(img, target_size=target_dim)) for img in X_train_files]\n#train_images = np.array(train_images)\n\n#validate_images = [img_to_array(load_img(img, target_size=target_dim)) for img in X_validate_files]\n#validate_images = np.array(validate_images)\n\n#train_images.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32, (3, 3), activation='relu', input_shape=(300, 300, 3)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(128, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(128, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(2048, activation='relu', name='deep_1'))\nmodel.add(Dense(1024, activation='relu', name='deep_2'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(1, activation='sigmoid'))\n\n#model.compile(loss='binary_crossentropy',\n#              optimizer=optimizers.RMSprop(lr=1e-4),\n#              metrics=['accuracy'])\nmodel.compile(optimizer='adam', \n                  loss='binary_crossentropy',\n                  metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 50\ntarget_dim = (400, 400)\nX_train_files = X_train['file_path']\nX_validate_files = X_validate['file_path']\n\n#class printbatch(Callback):\n#    def on_batch_end(self, epoch, logs={}):\n#        print(logs)\n\n#pb = printbatch()\n\nhistory = model.fit_generator(\n    Train_Generator(X_train, 12000, batch_size, target_dim),\n    steps_per_epoch = (12000+X_train_cancer_len) // batch_size,\n    validation_data = Train_Generator(X_validate, 1500, batch_size, target_dim),\n    validation_steps = (1500+X_validate_cancer_len) // batch_size,\n    #callbacks=[pb],\n    epochs = 2,\n    workers=6,\n    use_multiprocessing=True,\n    verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"deep_feature_model = Model(inputs=model.input, outputs=model.get_layer('deep_2').output)\ndeep_feature_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Test_Generator(Sequence):\n\n    def __init__(self, x_set, batch_size=50, target_dim=(200, 200)):\n        self.x = x_set\n        self.batch_size = batch_size\n        self.target_size = target_dim\n\n    def __len__(self):\n        return math.ceil(len(self.x) / self.batch_size)\n\n    def __getitem__(self, idx):\n\n        batch_x = self.x[idx * self.batch_size:(idx + 1) * self.batch_size]\n        image_x = np.array([img_to_array(load_img(img, target_size=self.target_size)) for img in batch_x])\n        image_x /= 255.0\n        \n        crop_image_x = crop_center(image_x, self.target_size[0]-100, self.target_size[1]-100)\n\n        return crop_image_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# separate the data in normal and cancer status\nnormal_data = train_df[train_df['target'] == 0]\ncancer_data = train_df[train_df['target'] == 1]\n\n# shuffle the normal data and take 20000 records only to save training time\nnormal_data = normal_data.sample(frac=1).reset_index(drop=True)\nnormal_data = normal_data[:20000] \n\n# append the cancer data and product X and y for model training data\nnormal_data = normal_data.append(cancer_data, ignore_index=True)\nnormal_data = normal_data.sample(frac=1).reset_index(drop=True)\nX_files = normal_data['file_path']\ny_lables = normal_data['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_files, X_validate_files, y_train, y_validate = train_test_split(X_files, y_lables, test_size=0.2, random_state=9)\nX_train_files.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_feature_predict = deep_feature_model.predict(Test_Generator(X_train_files, batch_size, target_dim), \n                                                        steps=(len(X_train_files) // batch_size)+1,\n                                                        workers=6, use_multiprocessing=True)\nprint(len(X_train_files))\nprint(X_train_feature_predict.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_validate_feature_predict = deep_feature_model.predict(Test_Generator(X_validate_files, batch_size, target_dim), \n                                                        steps=(len(X_validate_files) // batch_size)+1,\n                                                        workers=6, use_multiprocessing=True)\nX_validate_feature_predict.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = test_df['file_path']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_feature_predict = deep_feature_model.predict(Test_Generator(test_files, batch_size, target_dim),\n                                                   steps=(len(test_files) // batch_size)+1,\n                                                        workers=6, use_multiprocessing=True)\nX_test_feature_predict.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = [f\"col_{num}\" for num in range(1024)]\ntrain_feature_df = pd.DataFrame(X_train_feature_predict, columns=cols)\nvalidate_feature_df = pd.DataFrame(X_validate_feature_predict, columns=cols)\ntest_feature_df = pd.DataFrame(X_test_feature_predict, columns=cols)\n\ntrain_img_name = [file.split('/')[-1].split('.')[0].strip() for file in X_train_files]\nvalidate_img_name = [file.split('/')[-1].split('.')[0].strip() for file in X_validate_files]\ntest_img_name = [file.split('/')[-1].split('.')[0].strip() for file in test_files]\n\ntrain_feature_df['image_name'] = train_img_name\nvalidate_feature_df['image_name'] = validate_img_name\ntest_feature_df['image_name'] = test_img_name\n\ntrain_feature_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feature_df = train_feature_df.merge(train_df, on='image_name')\nvalidate_feature_df = validate_feature_df.merge(train_df, on='image_name')\ntest_feature_df = test_feature_df.merge(test_df, on='image_name')\n\n\nprint('length of train features dataframe: ', len(train_feature_df))\ntrain_feature_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = [f\"col_{num}\" for num in range(1024)]\ncat_cols = ['sex', 'anatom_site_general_challenge']\nage_col = ['age_approx']\n\ncolumns = cat_cols + age_col + cols\n\nX_feature_train = train_feature_df[columns]\ny_feature_train = train_feature_df['target']\n\nX_feature_validate = validate_feature_df[columns]\ny_feature_validate = validate_feature_df['target']\n\nX_feature_test = test_feature_df[columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\n# use the lightGBM model, the category features is not required to performe the OneHotEnconder as the lightGBM have the parameter to indicate the category features. \ndef create_GBMmodel(X_train, y_train, X_validate, y_validate, cat_features):\n            \n    train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)\n    validate_data = lgb.Dataset(X_validate, label=y_validate, categorical_feature=cat_features)\n    \n    params = {\n        \"objective\" : \"xentropy\",\n        \"metric\" :\"binary_logloss\",\n        \"force_row_wise\" : True,\n        'verbosity': 1,\n    }\n    \n    num_round = 200\n    m_lgb = lgb.train(params, train_data, num_round, valid_sets = [validate_data], early_stopping_rounds=5, verbose_eval=25) \n        \n    return m_lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bst = create_GBMmodel(X_feature_train, y_feature_train, X_feature_validate, y_feature_validate, cat_cols)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test = bst.predict(X_feature_test, num_iteration=bst.best_iteration)\n\ntest_feature_df['target'] = y_test\n\ntest_feature_df[['image_name', 'target']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}