{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n'''\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n'''\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-28T22:19:32.607132Z","iopub.execute_input":"2022-02-28T22:19:32.607635Z","iopub.status.idle":"2022-02-28T22:19:32.640414Z","shell.execute_reply.started":"2022-02-28T22:19:32.607508Z","shell.execute_reply":"2022-02-28T22:19:32.639376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nimport cv2\nimport random\n\nfrom sklearn.utils import shuffle\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, accuracy_score, precision_score, recall_score\nfrom imblearn.over_sampling import RandomOverSampler\n#from sklearn.metrics import accuracy_score\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPool2D, Dropout, Dense, Flatten, BatchNormalization\nfrom tensorflow.keras.optimizers import SGD, Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2022-03-01T02:02:35.452034Z","iopub.execute_input":"2022-03-01T02:02:35.452494Z","iopub.status.idle":"2022-03-01T02:02:35.461169Z","shell.execute_reply.started":"2022-03-01T02:02:35.452436Z","shell.execute_reply":"2022-03-01T02:02:35.460392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input')","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.622574Z","iopub.execute_input":"2022-02-28T22:19:41.622904Z","iopub.status.idle":"2022-02-28T22:19:41.629529Z","shell.execute_reply.started":"2022-02-28T22:19:41.622864Z","shell.execute_reply":"2022-02-28T22:19:41.628875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = os.path.join(\"../input\", 'siim-isic-melanoma-classification')\nprint(path)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.631445Z","iopub.execute_input":"2022-02-28T22:19:41.631938Z","iopub.status.idle":"2022-02-28T22:19:41.655538Z","shell.execute_reply.started":"2022-02-28T22:19:41.631885Z","shell.execute_reply":"2022-02-28T22:19:41.654817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(path)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.657031Z","iopub.execute_input":"2022-02-28T22:19:41.657481Z","iopub.status.idle":"2022-02-28T22:19:41.671502Z","shell.execute_reply.started":"2022-02-28T22:19:41.657420Z","shell.execute_reply":"2022-02-28T22:19:41.670700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.673070Z","iopub.execute_input":"2022-02-28T22:19:41.673363Z","iopub.status.idle":"2022-02-28T22:19:41.796057Z","shell.execute_reply.started":"2022-02-28T22:19:41.673330Z","shell.execute_reply":"2022-02-28T22:19:41.794890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.797709Z","iopub.execute_input":"2022-02-28T22:19:41.798286Z","iopub.status.idle":"2022-02-28T22:19:41.839950Z","shell.execute_reply.started":"2022-02-28T22:19:41.798235Z","shell.execute_reply":"2022-02-28T22:19:41.838882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.index","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.841622Z","iopub.execute_input":"2022-02-28T22:19:41.841948Z","iopub.status.idle":"2022-02-28T22:19:41.847999Z","shell.execute_reply.started":"2022-02-28T22:19:41.841911Z","shell.execute_reply":"2022-02-28T22:19:41.847309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.loc[train['image_name'] == 'ISIC_0068279'].index)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.849266Z","iopub.execute_input":"2022-02-28T22:19:41.849862Z","iopub.status.idle":"2022-02-28T22:19:41.873751Z","shell.execute_reply.started":"2022-02-28T22:19:41.849827Z","shell.execute_reply":"2022-02-28T22:19:41.872638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr =[] \narr.append(train['target'][train.loc[train['image_name'] == 'ISIC_0068279'].index])\nprint(arr[0] +1)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.877238Z","iopub.execute_input":"2022-02-28T22:19:41.877558Z","iopub.status.idle":"2022-02-28T22:19:41.893275Z","shell.execute_reply.started":"2022-02-28T22:19:41.877521Z","shell.execute_reply":"2022-02-28T22:19:41.892101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['image_name'][0]","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.895256Z","iopub.execute_input":"2022-02-28T22:19:41.895858Z","iopub.status.idle":"2022-02-28T22:19:41.907095Z","shell.execute_reply.started":"2022-02-28T22:19:41.895805Z","shell.execute_reply":"2022-02-28T22:19:41.906032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train['image_name'])","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.908434Z","iopub.execute_input":"2022-02-28T22:19:41.909408Z","iopub.status.idle":"2022-02-28T22:19:41.921337Z","shell.execute_reply.started":"2022-02-28T22:19:41.909350Z","shell.execute_reply":"2022-02-28T22:19:41.920329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/siim-isic-melanoma-classification/\"\nos.listdir(path)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.922848Z","iopub.execute_input":"2022-02-28T22:19:41.923728Z","iopub.status.idle":"2022-02-28T22:19:41.937203Z","shell.execute_reply.started":"2022-02-28T22:19:41.923677Z","shell.execute_reply":"2022-02-28T22:19:41.935599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = plt.imread(os.path.join(path, 'jpeg', 'train', train['image_name'][0] + '.jpg'))\nimg = cv2.resize(img, (128,128))\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:41.938933Z","iopub.execute_input":"2022-02-28T22:19:41.939399Z","iopub.status.idle":"2022-02-28T22:19:42.881595Z","shell.execute_reply.started":"2022-02-28T22:19:41.939348Z","shell.execute_reply":"2022-02-28T22:19:42.880477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:42.883235Z","iopub.execute_input":"2022-02-28T22:19:42.884190Z","iopub.status.idle":"2022-02-28T22:19:42.892438Z","shell.execute_reply.started":"2022-02-28T22:19:42.884142Z","shell.execute_reply":"2022-02-28T22:19:42.891657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_arrays (path) :\n    images_data = []\n    labels_data = []\n    train = pd.read_csv(os.path.join(path, 'train.csv'))\n    for k in tqdm(range(len(train['image_name']))) :\n        try :\n            img = plt.imread(os.path.join(path, 'jpeg', 'train', train['image_name'][k] + '.jpg'))\n            img = cv2.resize(img, (128,128))\n            images_data.append(img)\n            labels_data.append(train['target'][k])\n        except :\n            continue\n    print(\"finished : \", len(images_data))\n    return images_data, labels_data","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:42.893621Z","iopub.execute_input":"2022-02-28T22:19:42.894973Z","iopub.status.idle":"2022-02-28T22:19:42.903760Z","shell.execute_reply.started":"2022-02-28T22:19:42.894923Z","shell.execute_reply":"2022-02-28T22:19:42.902774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_data, labels_data = prepare_arrays(path)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T22:19:42.905330Z","iopub.execute_input":"2022-02-28T22:19:42.906182Z","iopub.status.idle":"2022-03-01T00:58:25.282065Z","shell.execute_reply.started":"2022-02-28T22:19:42.906131Z","shell.execute_reply":"2022-03-01T00:58:25.278899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(images_data[30000])\nprint(labels_data[30000])","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:25.286472Z","iopub.execute_input":"2022-03-01T00:58:25.286988Z","iopub.status.idle":"2022-03-01T00:58:25.584732Z","shell.execute_reply.started":"2022-03-01T00:58:25.286944Z","shell.execute_reply":"2022-03-01T00:58:25.583782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(images_data)\n#print(labels_data[0])","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:25.586400Z","iopub.execute_input":"2022-03-01T00:58:25.586629Z","iopub.status.idle":"2022-03-01T00:58:25.592679Z","shell.execute_reply.started":"2022-03-01T00:58:25.586602Z","shell.execute_reply":"2022-03-01T00:58:25.591857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = images_data.copy()\nlabels = labels_data.copy()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:25.594591Z","iopub.execute_input":"2022-03-01T00:58:25.594924Z","iopub.status.idle":"2022-03-01T00:58:25.607986Z","shell.execute_reply.started":"2022-03-01T00:58:25.594889Z","shell.execute_reply":"2022-03-01T00:58:25.606872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_samples (images, labels) :\n    real_labels = [\"benign\", \"malignant\"]\n    fig = plt.figure(figsize = (20,25))\n    rand = random.randint(0, len(images))\n\n    for i in range(20): \n        fig.add_subplot(5,4, i+1)\n        plt.title(real_labels[int(labels[rand])])\n        plt.imshow(images[rand])\n        rand = random.randint(0, len(images))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:25.610034Z","iopub.execute_input":"2022-03-01T00:58:25.610533Z","iopub.status.idle":"2022-03-01T00:58:25.625345Z","shell.execute_reply.started":"2022-03-01T00:58:25.610489Z","shell.execute_reply":"2022-03-01T00:58:25.624064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_samples(images, labels)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:25.627288Z","iopub.execute_input":"2022-03-01T00:58:25.628131Z","iopub.status.idle":"2022-03-01T00:58:30.382694Z","shell.execute_reply.started":"2022-03-01T00:58:25.628080Z","shell.execute_reply":"2022-03-01T00:58:30.381812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# processing the data for the model","metadata":{}},{"cell_type":"code","source":"def resize_array(images, size) :\n    for i in range(len(images)) :\n        images[i] = cv2.resize(images[i], size)\n    return images","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:30.383969Z","iopub.execute_input":"2022-03-01T00:58:30.384353Z","iopub.status.idle":"2022-03-01T00:58:30.390324Z","shell.execute_reply.started":"2022-03-01T00:58:30.384321Z","shell.execute_reply":"2022-03-01T00:58:30.389180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = LabelEncoder()\ndef preprocess_data(images, labels) :\n    labels = le.fit_transform(labels)\n    labels = to_categorical(labels, 2)\n    images, labels = shuffle(images, labels, random_state = 32)\n    x_train, x_test, y_train, y_test = train_test_split(images, labels, test_size = 0.25, random_state = 32)\n    x_train = np.array(x_train) \n    x_test = np.array(x_test) \n    y_train = np.array(y_train) \n    y_test = np.array(y_test)\n    return x_train, x_test, y_train, y_test","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:30.391817Z","iopub.execute_input":"2022-03-01T00:58:30.392073Z","iopub.status.idle":"2022-03-01T00:58:30.405221Z","shell.execute_reply.started":"2022-03-01T00:58:30.392042Z","shell.execute_reply":"2022-03-01T00:58:30.404284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = resize_array(images, tuple((32,32)))\nx_train, x_test, y_train, y_test = preprocess_data(images, labels)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:30.406379Z","iopub.execute_input":"2022-03-01T00:58:30.406900Z","iopub.status.idle":"2022-03-01T00:58:32.652300Z","shell.execute_reply.started":"2022-03-01T00:58:30.406859Z","shell.execute_reply":"2022-03-01T00:58:32.651282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# modeling","metadata":{}},{"cell_type":"code","source":"def build_model () :\n    model = Sequential()\n    model.add(Conv2D(filters = 128, kernel_size = (4,4), input_shape = (32, 32, 3), activation = 'relu'))\n    model.add(MaxPool2D(pool_size = (4,4)))\n    model.add(Conv2D(filters = 64, kernel_size = (2,2), activation = 'relu'))\n    model.add(MaxPool2D(pool_size = (2,2)))\n    model.add(BatchNormalization())\n    #model.add(GlobalAveragePooling2D())\n\n    model.add(Flatten())\n    model.add(Dense(128, activation = 'relu'))\n    model.add(Dropout(0.1))\n    model.add(Dense(2, activation = 'sigmoid')) # sigmoid is better for binary classification\n\n    model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:32.653724Z","iopub.execute_input":"2022-03-01T00:58:32.654333Z","iopub.status.idle":"2022-03-01T00:58:32.663106Z","shell.execute_reply.started":"2022-03-01T00:58:32.654292Z","shell.execute_reply":"2022-03-01T00:58:32.662152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:32.664521Z","iopub.execute_input":"2022-03-01T00:58:32.664853Z","iopub.status.idle":"2022-03-01T00:58:33.035958Z","shell.execute_reply.started":"2022-03-01T00:58:32.664819Z","shell.execute_reply":"2022-03-01T00:58:33.035083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, epochs_num) :\n    model.compile(loss = 'binary_crossentropy', optimizer = 'Adam', metrics = ['accuracy'])\n    early_stop = EarlyStopping(monitor = 'val_loss', patience = 3)\n    hist = model.fit(x_train, y_train, epochs = epochs_num, validation_data = (x_test, y_test))#, callbacks = early_stop)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:35:14.460634Z","iopub.execute_input":"2022-03-01T04:35:14.460984Z","iopub.status.idle":"2022-03-01T04:35:14.469776Z","shell.execute_reply.started":"2022-03-01T04:35:14.460947Z","shell.execute_reply":"2022-03-01T04:35:14.468848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(model, 12)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T00:58:33.048330Z","iopub.execute_input":"2022-03-01T00:58:33.048650Z","iopub.status.idle":"2022-03-01T01:02:47.506437Z","shell.execute_reply.started":"2022-03-01T00:58:33.048609Z","shell.execute_reply":"2022-03-01T01:02:47.505450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_scores (model) :\n    scores = pd.DataFrame(model.history.history)\n    titles = [\"accuaracy vs val_accuracy\", \"loss vs val_loss\"]\n    fig, axes = plt.subplots(nrows = 1, ncols = 2, figsize = (24,6))\n    #fig, ax = plt.figure(figsize = (20,8))\n\n    for i in range(2): \n        fig.add_subplot(1,2, i+1)\n        axes[i].set_title(titles[i])\n        if i==0 : plt.plot(scores[['accuracy','val_accuracy']])#, color = 'green')\n        if i==1 : plt.plot(scores[['loss','val_loss']])#, color = 'green')\n        #scores[['accuracy', 'val_accuracy']].plot(ax = axes[i])","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:02:47.508392Z","iopub.execute_input":"2022-03-01T01:02:47.509154Z","iopub.status.idle":"2022-03-01T01:02:47.518238Z","shell.execute_reply.started":"2022-03-01T01:02:47.509113Z","shell.execute_reply":"2022-03-01T01:02:47.517220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_scores(model) ","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:02:47.519909Z","iopub.execute_input":"2022-03-01T01:02:47.520926Z","iopub.status.idle":"2022-03-01T01:02:48.171534Z","shell.execute_reply.started":"2022-03-01T01:02:47.520883Z","shell.execute_reply":"2022-03-01T01:02:48.170580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('Normal_skin_cancer_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:02:48.172977Z","iopub.execute_input":"2022-03-01T01:02:48.173253Z","iopub.status.idle":"2022-03-01T01:02:48.249783Z","shell.execute_reply.started":"2022-03-01T01:02:48.173219Z","shell.execute_reply":"2022-03-01T01:02:48.248793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input')","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:02:48.251284Z","iopub.execute_input":"2022-03-01T01:02:48.251514Z","iopub.status.idle":"2022-03-01T01:02:48.256770Z","shell.execute_reply.started":"2022-03-01T01:02:48.251486Z","shell.execute_reply":"2022-03-01T01:02:48.256179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"final results for loss and accuracy : \")\nprint(model.evaluate(x_train, y_train))\nprint(model.evaluate(x_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:25:51.856569Z","iopub.execute_input":"2022-03-01T01:25:51.856962Z","iopub.status.idle":"2022-03-01T01:26:00.366875Z","shell.execute_reply.started":"2022-03-01T01:25:51.856925Z","shell.execute_reply":"2022-03-01T01:26:00.365823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# classification report ","metadata":{}},{"cell_type":"code","source":"pred = np.argmax(model.predict(x_test), axis = -1)\ny_testc = np.argmax(y_test, axis = -1)\n\nprint(classification_report(y_testc, pred))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:39:53.111422Z","iopub.execute_input":"2022-03-01T01:39:53.111771Z","iopub.status.idle":"2022-03-01T01:39:55.636225Z","shell.execute_reply.started":"2022-03-01T01:39:53.111721Z","shell.execute_reply":"2022-03-01T01:39:55.634013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"very bad recall and f1 score class 1 (maignant)","metadata":{}},{"cell_type":"code","source":"print(pred)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:18:23.623630Z","iopub.execute_input":"2022-03-01T01:18:23.624283Z","iopub.status.idle":"2022-03-01T01:18:23.630407Z","shell.execute_reply.started":"2022-03-01T01:18:23.624215Z","shell.execute_reply":"2022-03-01T01:18:23.629582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(y_test)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:18:34.044416Z","iopub.execute_input":"2022-03-01T01:18:34.044899Z","iopub.status.idle":"2022-03-01T01:18:34.050242Z","shell.execute_reply.started":"2022-03-01T01:18:34.044868Z","shell.execute_reply":"2022-03-01T01:18:34.049624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# modeling image augmentation","metadata":{}},{"cell_type":"code","source":"#be calm and watch wa3i\ndef image_augmentation(x_train, x_test, y_train, y_test) :\n    image_generator = ImageDataGenerator(\n        samplewise_center = True,\n        samplewise_std_normalization = True,\n        rotation_range = 25, \n        width_shift_range = 0.1, \n        height_shift_range = 0.1,\n        shear_range = 0.2,\n        zoom_range = 0.2,\n        horizontal_flip = True#, fill_mode = \"nearst\"\n    )\n    train_generator = image_generator.flow(x_train, y_train, batch_size = 32)\n    test_generator = image_generator.flow(x_test, y_test, batch_size = 32)\n    return train_generator, test_generator","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:02:48.257959Z","iopub.execute_input":"2022-03-01T01:02:48.258311Z","iopub.status.idle":"2022-03-01T01:02:48.271422Z","shell.execute_reply.started":"2022-03-01T01:02:48.258282Z","shell.execute_reply":"2022-03-01T01:02:48.270307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator, test_generator = image_augmentation(x_train, x_test, y_train, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:02:48.272628Z","iopub.execute_input":"2022-03-01T01:02:48.273214Z","iopub.status.idle":"2022-03-01T01:02:48.760101Z","shell.execute_reply.started":"2022-03-01T01:02:48.273178Z","shell.execute_reply":"2022-03-01T01:02:48.758814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelA =  build_model()","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:02:48.761764Z","iopub.execute_input":"2022-03-01T01:02:48.762441Z","iopub.status.idle":"2022-03-01T01:02:48.840797Z","shell.execute_reply.started":"2022-03-01T01:02:48.762384Z","shell.execute_reply":"2022-03-01T01:02:48.839496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trainA(model, epochs_num) :\n    model.compile(loss = 'binary_crossentropy', optimizer = 'Adam', metrics = ['accuracy'])\n    early_stop = EarlyStopping(monitor = 'val_loss', patience = 3)\n    hist = model.fit(train_generator, epochs = epochs_num, validation_data = (test_generator))#, callbacks = early_stop)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:34:51.770893Z","iopub.execute_input":"2022-03-01T04:34:51.771213Z","iopub.status.idle":"2022-03-01T04:34:51.778717Z","shell.execute_reply.started":"2022-03-01T04:34:51.771180Z","shell.execute_reply":"2022-03-01T04:34:51.777809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainA(modelA, 12)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:02:48.851342Z","iopub.execute_input":"2022-03-01T01:02:48.851690Z","iopub.status.idle":"2022-03-01T01:11:35.746145Z","shell.execute_reply.started":"2022-03-01T01:02:48.851645Z","shell.execute_reply":"2022-03-01T01:11:35.745099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_scores(modelA) ","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:11:35.748036Z","iopub.execute_input":"2022-03-01T01:11:35.748321Z","iopub.status.idle":"2022-03-01T01:11:36.311715Z","shell.execute_reply.started":"2022-03-01T01:11:35.748288Z","shell.execute_reply":"2022-03-01T01:11:36.310810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelA.save('Augmented_skin_cancer_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:11:36.313024Z","iopub.execute_input":"2022-03-01T01:11:36.313289Z","iopub.status.idle":"2022-03-01T01:11:36.352638Z","shell.execute_reply.started":"2022-03-01T01:11:36.313258Z","shell.execute_reply":"2022-03-01T01:11:36.351806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# classification report","metadata":{}},{"cell_type":"code","source":"predA = np.argmax(modelA.predict(x_test), axis = -1)\ny_testc = np.argmax(y_test, axis = -1)\n\nprint(classification_report(y_testc, predA))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T01:42:43.392442Z","iopub.execute_input":"2022-03-01T01:42:43.393479Z","iopub.status.idle":"2022-03-01T01:42:45.228443Z","shell.execute_reply.started":"2022-03-01T01:42:43.393421Z","shell.execute_reply":"2022-03-01T01:42:45.227461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"very bad precision and recall","metadata":{}},{"cell_type":"markdown","source":"# oversamping our data","metadata":{}},{"cell_type":"code","source":"images_arr = np.array(images)\nprint(images_arr.shape)\nreshaped_images = images_arr.reshape(images_arr.shape[0], -1)\nprint(reshaped_images.shape)\n\noversample = RandomOverSampler()\noversample_images_non_normal, oversample_labels = oversample.fit_resample(reshaped_images, labels)\n\noversample_images = oversample_images_non_normal.reshape(-1, 32,32,3)\nprint(oversample_images.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T02:10:29.400725Z","iopub.execute_input":"2022-03-01T02:10:29.401060Z","iopub.status.idle":"2022-03-01T02:10:29.677707Z","shell.execute_reply.started":"2022-03-01T02:10:29.401024Z","shell.execute_reply":"2022-03-01T02:10:29.676539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = 0\nfor i in oversample_labels :\n    if i == 1:\n        c += 1\n        \nprint(c)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T02:14:35.044252Z","iopub.execute_input":"2022-03-01T02:14:35.044553Z","iopub.status.idle":"2022-03-01T02:14:35.058914Z","shell.execute_reply.started":"2022-03-01T02:14:35.044523Z","shell.execute_reply":"2022-03-01T02:14:35.057903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(oversample_images[0])","metadata":{"execution":{"iopub.status.busy":"2022-03-01T02:16:16.417962Z","iopub.execute_input":"2022-03-01T02:16:16.418486Z","iopub.status.idle":"2022-03-01T02:16:16.588506Z","shell.execute_reply.started":"2022-03-01T02:16:16.418427Z","shell.execute_reply":"2022-03-01T02:16:16.587502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# New models","metadata":{}},{"cell_type":"code","source":"x_train, x_test, y_train, y_test = preprocess_data(oversample_images, oversample_labels)\nmodel = build_model()\ntrain(model, 12)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T02:18:37.838941Z","iopub.execute_input":"2022-03-01T02:18:37.839306Z","iopub.status.idle":"2022-03-01T02:26:56.674909Z","shell.execute_reply.started":"2022-03-01T02:18:37.839272Z","shell.execute_reply":"2022-03-01T02:26:56.673913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_scores(model) ","metadata":{"execution":{"iopub.status.busy":"2022-03-01T02:27:23.326011Z","iopub.execute_input":"2022-03-01T02:27:23.326427Z","iopub.status.idle":"2022-03-01T02:27:23.889512Z","shell.execute_reply.started":"2022-03-01T02:27:23.326385Z","shell.execute_reply":"2022-03-01T02:27:23.888182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = np.argmax(model.predict(x_test), axis = -1)\ny_testc = np.argmax(y_test, axis = -1)\n\nprint(classification_report(y_testc, pred))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T02:27:31.228208Z","iopub.execute_input":"2022-03-01T02:27:31.228762Z","iopub.status.idle":"2022-03-01T02:27:35.030763Z","shell.execute_reply.started":"2022-03-01T02:27:31.228705Z","shell.execute_reply":"2022-03-01T02:27:35.029714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('Normal_skin_cancer_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-03-01T02:28:01.196036Z","iopub.execute_input":"2022-03-01T02:28:01.196337Z","iopub.status.idle":"2022-03-01T02:28:01.241478Z","shell.execute_reply.started":"2022-03-01T02:28:01.196307Z","shell.execute_reply":"2022-03-01T02:28:01.240766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# oversampling with augmentation","metadata":{}},{"cell_type":"code","source":"train_generator, test_generator = image_augmentation(x_train, x_test, y_train, y_test)\nmodelA = build_model()\ntrainA(modelA, 24)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:34:37.859912Z","iopub.execute_input":"2022-03-01T04:34:37.860279Z","iopub.status.idle":"2022-03-01T04:34:38.961358Z","shell.execute_reply.started":"2022-03-01T04:34:37.860243Z","shell.execute_reply":"2022-03-01T04:34:38.959971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_scores(modelA) ","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:14:09.765485Z","iopub.execute_input":"2022-03-01T04:14:09.766027Z","iopub.status.idle":"2022-03-01T04:14:10.286426Z","shell.execute_reply.started":"2022-03-01T04:14:09.765984Z","shell.execute_reply":"2022-03-01T04:14:10.285600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelA.evaluate(train_generator)\nmodelA.evaluate(test_generator)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:15:29.022488Z","iopub.execute_input":"2022-03-01T04:15:29.022840Z","iopub.status.idle":"2022-03-01T04:16:30.111156Z","shell.execute_reply.started":"2022-03-01T04:15:29.022805Z","shell.execute_reply":"2022-03-01T04:16:30.110414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predA = np.argmax(modelA.predict(x_test), axis = -1)\ny_testc = np.argmax(y_test, axis = -1)\n\n\nprint(len(x_test))\nprint(len(y_test))\n#print(accuracy_score(y_testc, predA))\n#print(classification_report(y_testc, predA))","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:18:15.095550Z","iopub.execute_input":"2022-03-01T04:18:15.096642Z","iopub.status.idle":"2022-03-01T04:18:18.869596Z","shell.execute_reply.started":"2022-03-01T04:18:15.096577Z","shell.execute_reply":"2022-03-01T04:18:18.868597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelA.save('Augmented_skin_cancer_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:18:49.015381Z","iopub.execute_input":"2022-03-01T04:18:49.016249Z","iopub.status.idle":"2022-03-01T04:18:49.060403Z","shell.execute_reply.started":"2022-03-01T04:18:49.016191Z","shell.execute_reply":"2022-03-01T04:18:49.059614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# evaluation","metadata":{}},{"cell_type":"code","source":"def prepare_eval_array (path) :\n    eval_array = []\n    train = pd.read_csv(os.path.join(path, 'train.csv'))\n    for k in tqdm(range(10)) :\n        rand = random.randint(0, len(test['image_name']))\n        try :\n            img = plt.imread(os.path.join(path, 'jpeg', 'test', test['image_name'][rand] + '.jpg'))\n            img = cv2.resize(img, (128,128))\n            eval_array.append(img)\n        except :\n            continue\n    print(\"finished : \", len(eval_array))\n    return eval_array","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:18:52.048944Z","iopub.execute_input":"2022-03-01T04:18:52.049581Z","iopub.status.idle":"2022-03-01T04:18:52.057727Z","shell.execute_reply.started":"2022-03-01T04:18:52.049541Z","shell.execute_reply":"2022-03-01T04:18:52.056923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_array = prepare_eval_array(path)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:18:56.048952Z","iopub.execute_input":"2022-03-01T04:18:56.049275Z","iopub.status.idle":"2022-03-01T04:18:59.119510Z","shell.execute_reply.started":"2022-03-01T04:18:56.049240Z","shell.execute_reply":"2022-03-01T04:18:59.118543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_predictions (model) :\n    fig = plt.figure(figsize = (20,8))\n    real_labels = [\"benign\", \"malignant\"]\n\n    for i in range(10) :\n        img = eval_array[i]\n        img = cv2.resize(img, (32,32))\n        img = img.reshape(1,32,32,3)\n        val = np.argmax(model.predict(img))\n        fig.add_subplot(2,5, i+1)\n        plt.title(real_labels[val])\n        plt.imshow(eval_array[i])","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:19:01.388251Z","iopub.execute_input":"2022-03-01T04:19:01.388550Z","iopub.status.idle":"2022-03-01T04:19:01.397468Z","shell.execute_reply.started":"2022-03-01T04:19:01.388520Z","shell.execute_reply":"2022-03-01T04:19:01.396502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_predictions(model)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:19:05.065174Z","iopub.execute_input":"2022-03-01T04:19:05.065501Z","iopub.status.idle":"2022-03-01T04:19:07.389290Z","shell.execute_reply.started":"2022-03-01T04:19:05.065466Z","shell.execute_reply":"2022-03-01T04:19:07.388092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_predictions(modelA)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:19:14.482627Z","iopub.execute_input":"2022-03-01T04:19:14.482969Z","iopub.status.idle":"2022-03-01T04:19:16.861890Z","shell.execute_reply.started":"2022-03-01T04:19:14.482935Z","shell.execute_reply":"2022-03-01T04:19:16.860717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# images from website","metadata":{}},{"cell_type":"code","source":"os.listdir(\"../input\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:31:55.535602Z","iopub.execute_input":"2022-03-01T04:31:55.536002Z","iopub.status.idle":"2022-03-01T04:31:55.543967Z","shell.execute_reply.started":"2022-03-01T04:31:55.535967Z","shell.execute_reply":"2022-03-01T04:31:55.543006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_labels = [\"benign\", \"malignant\"]\nimg = plt.imread(\"../input/evaluation-images/binign_skin1.jpg\")\nimg1 = img.copy()\nimg1 = cv2.resize(img1, (32,32))\nimg1 = img1.reshape(1,32,32,3)\nval = np.argmax(model.predict(img1))\nplt.title(real_labels[val])\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:29:34.625337Z","iopub.execute_input":"2022-03-01T04:29:34.625927Z","iopub.status.idle":"2022-03-01T04:29:34.929396Z","shell.execute_reply.started":"2022-03-01T04:29:34.625881Z","shell.execute_reply":"2022-03-01T04:29:34.928455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(\"../input/melanoma-image\")","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:28:37.131915Z","iopub.execute_input":"2022-03-01T04:28:37.132205Z","iopub.status.idle":"2022-03-01T04:28:37.144073Z","shell.execute_reply.started":"2022-03-01T04:28:37.132174Z","shell.execute_reply":"2022-03-01T04:28:37.143310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_labels = [\"benign\", \"malignant\"]\nimg = plt.imread(\"../input/melanoma-image/melanoma-14.jpg\")\nimg1 = img.copy()\nimg1 = cv2.resize(img1, (32,32))\nimg1 = img1.reshape(1,32,32,3)\nval = np.argmax(model.predict(img1))\nplt.title(real_labels[val])\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-03-01T04:29:09.719134Z","iopub.execute_input":"2022-03-01T04:29:09.719455Z","iopub.status.idle":"2022-03-01T04:29:10.147068Z","shell.execute_reply.started":"2022-03-01T04:29:09.719422Z","shell.execute_reply":"2022-03-01T04:29:10.146111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_labels = [\"benign\", \"malignant\"]\nimg = plt.imread(\"../input/melanoma-image/melanoma-14.jpg\")\nimg1 = img.copy()\nimg1 = cv2.resize(img1, (32,32))\nimg1 = img1.reshape(1,32,32,3)\nval = np.argmax(modelA.predict(img1))\nplt.title(real_labels[val])\nplt.imshow(img)","metadata":{},"execution_count":null,"outputs":[]}]}