{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport pydicom\n\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntrain.head(70)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.Patient[train.SmokingStatus == 'Ex-smoker']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image visualization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"d = pydicom.dcmread('../input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/11.dcm')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10,10))\n\nplt.imshow(d.pixel_array)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = {}\nfor p in train.Patient.values:\n    train_data[p] = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## AutoEncoder","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"![](https://upload.wikimedia.org/wikipedia/commons/2/28/Autoencoder_structure.png)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Pipeline released in this kernel is very simple:\n\n* train encoder and used one for translate image in vector\n* visulize results vectors used PCA decomposition algorithm","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.utils import Sequence\nimport cv2\n\nclass IGenerator(Sequence):\n    BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']\n    def __init__(self, keys=list(train_data.keys()), train_data=train_data, batch_size=32):\n        self.keys = [k for k in keys if k not in self.BAD_ID]\n        self.train_data = train_data\n        self.batch_size = batch_size\n        \n    def __len__(self):\n        return 1000\n    \n    def __getitem__(self, idx):\n        x = []\n        keys = np.random.choice(self.keys, size = self.batch_size)\n        for k in keys:\n            try:\n                i = np.random.choice(self.train_data[k], size=1)[0]\n                d = pydicom.dcmread(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')\n                x.append(cv2.resize(d.pixel_array / 2**10, (512, 512)))\n            except:\n                print(k, i)\n        x = np.array(x)\n        x = np.expand_dims(x, axis=-1)\n        return x, x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import (\n    Dense, Dropout, Activation, Flatten, Input, BatchNormalization, UpSampling2D, Add, Conv2D, MaxPooling2D, LeakyReLU\n)\n\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.optimizers import Nadam\n\ndef get_encoder(shape=(512, 512, 1)):\n    def res_block(x, n_features):\n        _x = x\n        x = BatchNormalization()(x)\n        x = LeakyReLU()(x)\n    \n        x = Conv2D(n_features, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n        x = Add()([_x, x])\n        return x\n    \n    inp = Input(shape=shape)\n    \n    # 512\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(inp)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 256\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 32)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 128\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 32)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 64\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 64)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 32\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 64)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)    \n    \n    # 16\n    x = Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 128)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x) \n    \n    # 8\n    x = Conv2D(1, kernel_size=(1, 1), strides=(1, 1), padding='same')(x)\n    return Model(inp, x)\n\n\n\ndef get_decoder(shape=(8, 8, 1)):\n    inp = Input(shape=shape)\n\n    # 8\n    x = UpSampling2D((2, 2))(inp)\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    # 16\n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    # 32\n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    # 64\n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    # 128\n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    # 256\n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(8, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU()(x)\n    \n    x = Conv2D(1, kernel_size=(1, 1), strides=(1, 1), padding='same')(x)\n    return Model(inp, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder = get_encoder((512, 512, 1))\ndecoder = get_decoder((8, 8, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = Input((512, 512, 1))\ne = encoder(inp)\nd = decoder(e)\nmodel = Model(inp, d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=Nadam(lr=2*1e-3, schedule_decay=1e-5), loss='mse')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit_generator(IGenerator(), steps_per_epoch=500, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keys = [k for k in list(train_data.keys()) if k not in ['ID00011637202177653955184', 'ID00052637202186188008618']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"emb = {}\nfor k in tqdm(keys, total=len(keys)):\n    x = []\n    for i in train_data[k]:\n        d = pydicom.dcmread(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')\n        x.append(cv2.resize(d.pixel_array / 2**10, (512, 512)))\n    x = np.expand_dims(x, axis=-1)\n    emb[k] = encoder.predict(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder.save('encoder.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del encoder, decoder, model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del IGenerator","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del x\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization PCA components","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.decomposition import PCA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vec = []\nfor k in emb:\n    vec.extend(emb[k][..., 0].reshape((len(emb[k]), 64)).tolist())\n    \npca = PCA(n_components=3)\npca.fit(vec)\n\ndel vec\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca.explained_variance_ratio_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"emb3 = {}\n\nfor k in tqdm(emb):\n    emb3[k] = pca.transform(emb[k][..., 0].reshape((len(emb[k]), 64)).tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\n\nvec = []\nfor k in train.Patient.values:\n    if k in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n        continue\n    vec.extend(emb3[k].tolist())\nvec = np.array(vec)\n\nplt.plot(vec[:, 0], vec[:, 1], '.', alpha=0.05, label='Male')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\n\nvec = []\nfor k in train.Patient[train.Sex == 'Male'].values[:100]:\n    if k in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n        continue\n    vec.extend(emb3[k].tolist())\n\nvec = np.array(vec)\n\nplt.plot(vec[:, 0], vec[:, 1], '.', alpha=0.05, label='Male')\n\n\nvec = []\nfor k in train.Patient[train.Sex == 'Female'].values[:100]:\n    if k in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n        continue\n    vec.extend(emb3[k].tolist())\n\nvec = np.array(vec)\n\nplt.plot(vec[:, 0], vec[:, 1], '.', alpha=0.05, label='Female')\n\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\n\nvec = []\nfor k in train.Patient[train.SmokingStatus == 'Ex-smoker'].values[:100]:\n    if k in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n        continue\n    vec.extend(emb3[k].tolist())\n\nvec = np.array(vec)\n\nplt.plot(vec[:, 0], vec[:, 1], '.', alpha=0.05, label='Ex-smoker')\n\n\nvec = []\nfor k in train.Patient[train.SmokingStatus == 'Never smoked'].values[:100]:\n    if k in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n        continue\n    vec.extend(emb3[k].tolist())\n\nvec = np.array(vec)\n\nplt.plot(vec[:, 0], vec[:, 1], '.', alpha=0.05, label='Never smoked')\n\n\nvec = []\nfor k in train.Patient[train.SmokingStatus == 'Currently smokes'].values[:100]:\n    if k in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n        continue\n    vec.extend(emb3[k].tolist())\n\nvec = np.array(vec)\n\nplt.plot(vec[:, 0], vec[:, 1], '.', alpha=0.05, label='Currently smokes')\n\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['mc1'] = 0\ntrain['sc1'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k in tqdm(train.Patient.values):\n    if k in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n        continue\n    train.loc[train.Patient == k,'mc1'] = emb3[k][:, 0].max()\n    train.loc[train.Patient == k,'sc1'] = emb3[k][:, 0].std()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Depend 1 component stats with targets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\n\nplt.plot(train.mc1, train.FVC, '.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\n\nplt.plot(train.mc1, train.Percent, '.')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}