{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"%ls '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pydicom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport keras\nfrom matplotlib import pyplot as plt\nimport numpy as np\nimport gzip\nimport cv2\nimport pydicom\nimport random\nfrom tqdm import tqdm\n\n%matplotlib inline\nfrom keras.models import Model\nfrom keras.optimizers import RMSprop\nfrom keras.layers import Input,Dense,Flatten,Dropout,merge,Reshape,Conv2D,MaxPooling2D,UpSampling2D,Conv2DTranspose\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.models import Model,Sequential\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.optimizers import Adadelta, RMSprop,SGD,Adam\nfrom keras import regularizers\nfrom keras import backend as K\nfrom keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_files = len([f for f in os.listdir('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images')if os.path.isfile(os.path.join('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images', f))])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(num_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"desired_size = 28","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_image(image_path, desired_size=28):\n    im = pydicom.dcmread(image_path)\n    im = cv2.resize(np.array(im.pixel_array),(desired_size,desired_size))\n    #im = crop_image_from_gray(im)\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    #im = np.expand_dims(im, axis=-1)\n    return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#N = train_df.shape[0]\nx_train = np.empty((3000,desired_size, desired_size, 3), dtype=np.uint8)\n\n#num_break = 3001\n\n#random.shuffle(num_break)\n#x_train = np.zeros((num_files, desired_size, desired_size, 1)).astype('float')\n\n#for i,image_id in enumerate(os.listdir('./stage_2_train_images')):\nfor i,image_id in enumerate(tqdm(os.listdir('/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images')[:3000])):\n    x_train[i, :, :, :] = preprocess_image(\n        f'/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images/{image_id}'\n    )\n    #cv2.imwrite('/content/gdrive/My Drive/Desafio5_autoencoder_pulmao/rsna-pneumonia_files/' , {image_id})\n    #if i >= num_break:\n    #    break\n    #i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainX = x_train.astype(\"float32\") / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.max(trainX)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n(train_X, val_X) = train_test_split(trainX, test_size=0.2,\n\trandom_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nepochs = 3\ninChannel = 3\nx, y = desired_size, desired_size\ninput_img = Input(shape = (x, y, inChannel))\nnum_classes = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def encoder(input_img):\n    #encoder\n    #input = 28 x 28 x 1 (wide and thin)\n    conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(input_img) #28 x 28 x 32\n    conv1 = BatchNormalization()(conv1)\n    conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)\n    conv1 = BatchNormalization()(conv1)\n    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1) #14 x 14 x 32\n    conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1) #14 x 14 x 64\n    conv2 = BatchNormalization()(conv2)\n    conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)\n    conv2 = BatchNormalization()(conv2)\n    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) #7 x 7 x 64\n    conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2) #7 x 7 x 128 (small and thick)\n    conv3 = BatchNormalization()(conv3)\n    conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv3)\n    conv3 = BatchNormalization()(conv3)\n    conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv3) #7 x 7 x 256 (small and thick)\n    conv4 = BatchNormalization()(conv4)\n    conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv4)\n    conv4 = BatchNormalization()(conv4)\n    return conv4\n\ndef decoder(conv4):    \n    #decoder\n    conv5 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv4) #7 x 7 x 128\n    conv5 = BatchNormalization()(conv5)\n    conv5 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv5)\n    conv5 = BatchNormalization()(conv5)\n    conv6 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv5) #7 x 7 x 64\n    conv6 = BatchNormalization()(conv6)\n    conv6 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv6)\n    conv6 = BatchNormalization()(conv6)\n    up1 = UpSampling2D((2,2))(conv6) #14 x 14 x 64\n    conv7 = Conv2D(32, (3, 3), activation='relu', padding='same')(up1) # 14 x 14 x 32\n    conv7 = BatchNormalization()(conv7)\n    conv7 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv7)\n    conv7 = BatchNormalization()(conv7)\n    up2 = UpSampling2D((2,2))(conv7) # 28 x 28 x 32\n    decoded = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(up2) # 28 x 28 x 3 se for 1 channel(gray) alterar para Conv2D(1,(3,3))\n    return decoded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"autoencoder = Model(input_img, decoder(encoder(input_img)))\nautoencoder.compile(loss='mean_squared_error', optimizer = RMSprop())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"autoencoder_train = autoencoder.fit(train_X, train_X, batch_size=batch_size,epochs=epochs,verbose=1,validation_data=(val_X,val_X))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_files_predict = len([f for f in os.listdir('/kaggle/input/chest-xray-anomaly-detection/images/')if os.path.isfile(os.path.join('/kaggle/input/chest-xray-anomaly-detection/images/', f))])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(num_files_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_image_predict(image_path, desired_size=28):\n    im = cv2.imread(image_path)\n    im = cv2.resize(im, (desired_size, desired_size))\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pre processando dados teste\ndf_test = pd.read_csv(\"../input/chest-xray-anomaly-detection/sample_submission.csv\")\n\ndepth = 3\n\nN = df_test.shape[0]\nX_test = np.empty((N, desired_size, desired_size, depth), dtype=np.uint8)\n\ny_teste = []\ny_mse = []\nthresh = 0.11931779579818275\n\nfor i, image_id in enumerate(tqdm(df_test['fileName'])):\n    X_test[i, :, :, :] = preprocess_image_predict(\n        f'../input/chest-xray-anomaly-detection/images/{image_id}'\n    )\n    result = autoencoder.predict(X_test)\n    #X_test = X_test.astype(\"float32\") / 255.0\n    mse = np.mean((X_test - result) ** 2)\n    #result = np.argmax(result, axis = 1)\n    #y_teste.append(result)\n    \n    if np.array(mse) >= thresh:\n       df_test[\"anomaly\"][i] = 1\n    else:\n       df_test[\"anomaly\"][i] = 0\n    #Save images preprocessadas\n    #cv2.imwrite('../test/preprocessada/' + image_id ,X_test[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.to_csv('./submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}