{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import das bibliotecas\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom glob import glob \nimport os\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom tensorflow.python.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nimport xgboost as xgb\n\nnp.random.seed(42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"/kaggle/input/histopathologic-cancer-detection/\" \nlabels = pd.read_csv(path + 'train_labels.csv')\ntrain_path = path + 'train/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'path': glob(os.path.join(train_path,'*.tif'))})\ndf['id'] = df.path.map(lambda x: ((x.split(\"n\")[-1].split('.')[0])[1:]))\ndf = df.merge(labels, on = \"id\")\ndf.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 96\nBATCH_SIZE = 128","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"Distribuição das classes\");\n\nplt.pie(df['label'].value_counts(), labels=['Sem cancer',\n          'Com Cancer'], startangle=180, autopct='%1.1f', \n           colors=['#00ff99','#FF96A7'], shadow=True);\nplt.figure(figsize=(16,16));\nplt.show();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #fig = plt.figure(figsize=(30, 6))\n# #np.random.choice(df['path'], 20)\n# X = []\n\n# #plt.title(\"Exemplos de imagens \\n\")\n# for idx, img in enumerate(np.random.choice(df['path'])):\n#     #ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])    \n#     im = cv2.imread(img)    \n#     X.append(im)\n#     if()\n#     #plt.imshow(im)\n#     #lab = df.loc[df['id'] == img.split('/')[-1].split('.')[0], 'label'].values[0]\n#     #ax.set_title('Label: %s'%lab) \n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = []\ny = []\n# plt.title(\"Exemplos de imagens \\n\")\nfor idx in tqdm.tqdm(range(df.shape[0])):\n    X.append(cv2.imread(df.iloc[idx]['path']))\n    y.append(df.iloc[idx]['label'])\n    if idx == 10000:\n        break\n\nX = np.array(X)\ny = np.array(y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"Primeira imagem do dataset\")\nimagem = cv2.imread(df['path'][0], cv2.IMREAD_GRAYSCALE)\nplt.imshow(imagem);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\nX_test, X_valid, y_test, y_valid = train_test_split(X_test, y_test, test_size=0.3, random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow\nfrom tensorflow import keras","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 96","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications.resnet50 import ResNet50\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation\nfrom keras.layers import Conv2D, MaxPool2D\n\ndropout_fc = 0.5\n\nconv_base = ResNet50(weights = 'imagenet', include_top = False, input_shape = (IMG_SIZE,IMG_SIZE,3))\n\nmy_model = Sequential()\n\nmy_model.add(conv_base)\nmy_model.add(Flatten())\nmy_model.add(Dense(256, use_bias=False))\nmy_model.add(BatchNormalization())\nmy_model.add(Activation(\"relu\"))\nmy_model.add(Dropout(dropout_fc))\nmy_model.add(Dense(1, activation = \"sigmoid\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conv_base.Trainable=True\n\nset_trainable=False\nfor layer in conv_base.layers:\n    if layer.name == 'res5a_branch2a':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import optimizers\nmy_model.compile(optimizers.Adam(0.001), loss = \"binary_crossentropy\", metrics = [\"accuracy\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = my_model.fit(X_train, y_train, epochs=10, validation_data=(X_test, y_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = [i for i in range(1, len(history.history['loss'])+1)]\n\nplt.plot(epochs, history.history['loss'], color='blue', label=\"training_loss\")\nplt.plot(epochs, history.history['val_loss'], color='red', label=\"validation_loss\")\nplt.legend(loc='best')\nplt.title('training')\nplt.xlabel('epoch')\nplt.savefig(\"training.png\", bbox_inches='tight')\nplt.show()\n\nplt.plot(epochs, history.history['accuracy'], color='blue', label=\"training_accuracy\")\nplt.plot(epochs, history.history['val_accuracy'], color='red',label=\"validation_accuracy\")\nplt.legend(loc='best')\nplt.title('validation')\nplt.xlabel('epoch')\nplt.savefig(\"validation.png\", bbox_inches='tight')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc, roc_auc_score\n\npredictions = my_model.predict(X_valid)\nfalse_positive_rate, true_positive_rate, threshold = roc_curve(y_valid, predictions)\narea_under_curve = auc(false_positive_rate, true_positive_rate)\n\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(false_positive_rate, true_positive_rate, label='AUC = {:.3f}'.format(area_under_curve))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.savefig('ROC_PLOT.png', bbox_inches='tight')\nplt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = my_model.predict(X_valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.array([0 if x < 0.5 else 1 for x in y_pred])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_valid, y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import das bibliotecas\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom glob import glob \nimport os\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom tensorflow.python.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nnp.random.seed(42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}