{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pydicom\n\ndef img_to_np(path, size, resize=True):\n    img_array = []\n#     view_pos = []\n    for fname in os.listdir(path):\n        dicom = pydicom.dcmread(os.path.join(path, fname))\n        img = dicom.pixel_array\n        if resize:\n            img = cv2.resize(img, size)\n        img = img.astype('float32') / 255.\n#         img = np.stack((img,) * 3, axis=-1)\n        img_array.append(np.asarray(img))\n#         view_pos.append(dicom.ViewPosition)\n    img_array = np.array(img_array)\n#     view_pos = np.array(view_pos)\n    return img_array#, view_pos\n\n        \npath_train = \"../input/rsna-pneumonia-detection-challenge/stage_2_train_images\"\npath_test = \"../input/rsna-pneumonia-detection-challenge/stage_2_test_images\"\n\n# train_df = pd.read_csv(\"../input/chest-xray-anomaly-detection/train.csv\")\npath_train_sub = \"../input/chest-xray-anomaly-detection/images\"\n\n# sub_df = pd.read_csv(\"../input/chest-xray-anomaly-detection/sample_submission.csv\")\npath_test_sub = \"../input/chest-xray-anomaly-detection/images\"\n\noutput = \"./output\"\n\nsize = (128, 128)\nshape = (128, 128, 1)\nnrows = 128\n# batch = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow.keras,os\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, UpSampling2D\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport numpy as np\n\ndef create_model():\n    model = Sequential()\n    model.add(Conv2D(input_shape=shape,filters=64,kernel_size=(3,3),padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=64,kernel_size=(3,3),padding=\"same\", activation=\"relu\"))\n    model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))\n    model.add(Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))\n    model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))\n    \n    model.add(Dense(2, activation='sigmoid'))\n    \n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(UpSampling2D(size=(2,2)))#,strides=(2,2)))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(UpSampling2D(size=(2,2)))#,strides=(2,2)))\n    model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(UpSampling2D(size=(2,2)))#,strides=(2,2)))\n    model.add(Conv2D(filters=64,kernel_size=(3,3),padding=\"same\", activation=\"relu\"))\n    model.add(UpSampling2D(size=(2,2)))#,strides=(2,2)))\n    model.add(Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=\"relu\"))\n    model.add(UpSampling2D(size=(2,2)))#,strides=(2,2)))\n    model.add(Conv2D(filters=64,kernel_size=(3,3),padding=\"same\", activation=\"relu\"))\n    model.add(Conv2D(filters=1,kernel_size=(3,3),padding=\"same\", activation=\"relu\"))\n    \n#     autoencoder = keras.Model(input_img, decoded)\n    opt = tensorflow.keras.optimizers.Adam(learning_rate=0.00001)\n    model.compile(optimizer=opt, loss='mean_squared_error')\n\n    return model\n\nautoencoder = create_model()\nautoencoder.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = img_to_np(path_train, size, resize=True)\nprint(x_train.shape)\nx_test = img_to_np(path_test, size, resize=True)\nprint(x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train = np.reshape(x_train, (len(x_train), nrows, nrows, 1))\nx_train, x_train_test = train_test_split(x_train, test_size=0.20, random_state=42)\nprint(x_train.shape)\nprint(x_train_test.shape)\nx_test = np.reshape(x_test, (len(x_test), nrows, nrows, 1))\nprint(x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import *\n\ncallbacks = [EarlyStopping(monitor='val_loss', patience=15)]\nautoencoder.fit(x_train, x_train,\n                epochs=100,\n                batch_size=16,\n                shuffle=True,\n                validation_data=(x_train_test, x_train_test),\n                callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\ndecoded_imgs = autoencoder.predict(x_test)\n\nn = 10\nplt.figure(figsize=(20, 4))\nfor i in range(1, n + 1):\n    # Display original\n    ax = plt.subplot(2, n, i)\n    plt.imshow(x_test[i].reshape(nrows, nrows))\n    plt.gray()\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\n\n    # Display reconstruction\n    ax = plt.subplot(2, n, i + n)\n    plt.imshow(decoded_imgs[i].reshape(nrows, nrows))\n    plt.gray()\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd \n\ndef test_to_np(path, size, resize=True, lista=None):\n    img_array = []\n#     view_pos = []\n    if lista is None:\n        lista = os.listdir(path)\n    for fname in lista:\n        color_img = cv2.imread(os.path.join(path, fname))\n        img = cv2.cvtColor(color_img, cv2.COLOR_BGR2GRAY)\n        if resize:\n            img = cv2.resize(img, size)\n        img = img.astype('float32') / 255.\n#         img = np.stack((img,) * 3, axis=-1)\n        img_array.append(np.asarray(img))\n#         view_pos.append(dicom.ViewPosition)\n    img_array = np.array(img_array)\n#     view_pos = np.array(view_pos)\n    return img_array#, view_pos\n\n\ndf_train = pd.read_csv(\"../input/chest-xray-anomaly-detection/train.csv\")\nfiles_train = df_train[\"fileName\"].tolist()\ntarget_train =  df_train[\"anomaly\"].tolist()\ntest = test_to_np(path_test_sub, size, resize=True, lista=files_train)\nprint(test.shape)\n\ntest = np.reshape(test, (len(test), nrows, nrows, 1))\nprint(test.shape)\n\ndecoded_test = autoencoder.predict(test)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 10\nplt.figure(figsize=(20, 4))\nfor i in range(1, n + 1):\n    #error \n    error = np.sqrt(np.sum(np.power((test[i] * 255) - (decoded_test[i] * 255),2)))\n    # Display original\n    ax = plt.subplot(3, n, i)\n    ax.set_title(error)\n    plt.imshow(test[i].reshape(nrows, nrows))\n    plt.gray()\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\n\n    # Display reconstruction\n    ax = plt.subplot(3, n, i + n)\n    plt.imshow(decoded_test[i].reshape(nrows, nrows))\n    plt.gray()\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\n    \n    # Display reconstruction\n    ax = plt.subplot(3, n, (i + n*2))\n    plt.imshow((test[i]- decoded_test[i]).reshape(nrows, nrows))\n    plt.gray()\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(type(test[0]))\nprint(np.count_nonzero(test[0] < 10))\nprint(np.max(test[0]))\nprint(np.max(decoded_test[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = []\nwhites = []\nfor real, decoded in zip(test, decoded_test):\n    real = real * 255\n    decoded = decoded * 255\n    white_count = np.count_nonzero(real > 50)\n    error = np.sqrt(np.sum(np.power(real - decoded,2)))\n    whites.append(white_count)\n    result.append(error)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = np.array(result)\ntarget_train = np.array(target_train)\nresult_0 = result[target_train == 0]\nresult_1 = result[target_train == 1]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"whites = np.array(whites)\nwhites_0 = whites[target_train == 0]\nwhites_1 = whites[target_train == 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(result_0, whites_0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(result_1, whites_1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(whites_0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(whites_1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize =(10, 7)) \nax = fig.add_axes([0, 0, 1, 1]) \nbp = ax.boxplot([result_0, result_1]) \nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize =(10, 7)) \nax = fig.add_axes([0, 0, 1, 1]) \nbp = ax.boxplot([result_0 / whites_0, result_1 / whites_1]) \nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(\"../input/chest-xray-anomaly-detection/sample_submission.csv\")\nfiles_test = df_test[\"fileName\"].tolist()\nsub = test_to_np(path_test_sub, size, resize=True, lista=files_test)\nprint(sub.shape)\n\nsub = np.reshape(sub, (len(sub), nrows, nrows, 1))\nprint(sub.shape)\n\ndecoded_sub = autoencoder.predict(sub)\n\n\nresult_sub = []\nos.mkdir(\"./result\")\nfor real, decoded, file in zip(sub, decoded_sub, files_test):\n    real = real * 255\n    decoded = decoded * 255\n    error = np.sqrt(np.sum(np.power(real - decoded,2)))\n    white_count = np.count_nonzero(real > 50)\n    error = 0 if error <= 2800 and white_count > 8000 else 1\n    result_sub.append(error)\n#     cv2.imwrite(os.path.join(\"./result\",f\"{int(error)}__{file}\"), decoded)\n    \n\nresults=pd.DataFrame({\"fileName\":files_test,\n                      \"anomaly\": result_sub})\nresults.to_csv(\"results.csv\",index=False)\n\nfig = plt.figure(figsize =(10, 7)) \nplt.boxplot(result_sub) \nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import shutil\n# shutil.make_archive(\"./res.zip\", 'zip', \"./result\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}