{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Basic Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\nimport tensorflow.keras as keras\nimport tensorflow.keras.layers as layers\n\nimport os\nimport random","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Seed all"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_all(seed = 2016):\n    os.environ['PYTHONHASHSEED']=str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_all(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Environment variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_dir = '/kaggle/input/osic-pulmonary-fibrosis-progression'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Get data"},{"metadata":{},"cell_type":"markdown","source":"Train data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data_input = pd.read_csv(os.path.join(base_dir, 'train.csv'))\n\ndf_data_input.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Submission data"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_baseline = pd.read_csv(os.path.join(base_dir, 'test.csv'))\n\nsubmission_baseline.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Submission sample"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_sample = pd.read_csv(os.path.join(base_dir, 'sample_submission.csv'))\nsubmission_sample.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Get images and join with data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport pydicom\n\nimg_data_raw = []\n\nerror_count = 0\n\ni = 1\n\ntotal = df_data_input['Patient'].count()\n\nfor index, row in df_data_input.iterrows():\n    if i % 200 == 0:\n        print('Iter: ', str(i), '/', str(total))\n    i = i+1\n    if(row['Weeks'] == 0):\n        for file in os.listdir(os.path.join(base_dir, 'train', row['Patient'])):\n            path = os.path.join(base_dir, 'train', row['Patient'], file)\n            try:\n                img = pydicom.dcmread(path).pixel_array\n                img = cv2.resize(img, (256,256))\n                img_data_raw.append([\n                    row['Patient'], \n                    row['Weeks'], \n                    row['FVC'], \n                    row['Percent'], \n                    row['Age'],\n                    row['Sex'],\n                    row['SmokingStatus'],\n                    img,\n                ])\n            except:\n                print('ERROR----------------------------------')\n                error_count = error_count + 1\n    else:\n        img_data_raw.append([\n            row['Patient'], \n            row['Weeks'], \n            row['FVC'], \n            row['Percent'], \n            row['Age'],\n            row['Sex'],\n            row['SmokingStatus'],\n            None,\n        ])\n        \nprint('FINISHED!!!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create inputs and outputs"},{"metadata":{},"cell_type":"markdown","source":"Convert list to numpy array"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"img_data_raw_np = np.array(img_data_raw)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate the output"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_data = img_data_raw_np[:,2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate input data"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"X_data = np.delete(img_data_raw_np, 0, 1)\nX_data = np.delete(X_data, 2, 1)\nX_data = np.delete(X_data, 5, 1)\nX_data = np.delete(X_data, 1, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Encoder"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\n\nlblEncoder = LabelEncoder()\nX_data[:, 2] = lblEncoder.fit_transform(X_data[:, 2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columnTransformer = ColumnTransformer([('encoder', OneHotEncoder(), [3])], remainder='passthrough')\nX_data = np.array(columnTransformer.fit_transform(X_data), dtype = np.str)\nX_data = X_data[:, 1:]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate input image"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_img = np.delete(img_data_raw_np, 0, 1)\nX_img = np.delete(X_img, 0, 1)\nX_img = np.delete(X_img, 0, 1)\nX_img = np.delete(X_img, 0, 1)\nX_img = np.delete(X_img, 0, 1)\nX_img = np.delete(X_img, 0, 1)\nX_img = np.delete(X_img, 0, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make 0 for images that we don't have information (Weeks <> 0)"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_img_len = X_img.shape[0]\n\nX_img_zer = []\ny_img = []\n\nsmp = 0\n\nfor i in range(0, X_img_len):\n    if(X_img[i] == None):\n        continue\n    else:\n        smp = X_img[i]\n        break\n        \nfor i in range(0, X_img_len):\n    if(X_img[i] == None):\n        X_img_zer.append(np.zeros((256,256)))\n        y_img.append(0)\n    else:\n        X_img_zer.append(X_img[i][0])\n        y_img.append(y_data[i])\n        \nX_img_zer = np.array(X_img_zer)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Split to get some test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_img_zer_train, X_img_zer_test, y_img_zer_train, y_img_zer_test = train_test_split(X_img_zer, y_data, test_size=0.1, random_state=2016)\n\nX_data_train, X_data_test, y_data_train, y_data_test = train_test_split(X_data, y_data, test_size=0.1, random_state=2016)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Convert all to float and prepare shapes"},{"metadata":{},"cell_type":"markdown","source":"Image input"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_img_zer_train = np.asarray(X_img_zer_train, dtype=np.float)\nX_img_zer_train = np.expand_dims(X_img_zer_train, -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_img_zer_test = np.asarray(X_img_zer_test, dtype=np.float)\nX_img_zer_test = np.expand_dims(X_img_zer_test, -1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data input"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_data_train = np.asarray(X_data_train, dtype=np.float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_data_test = np.asarray(X_data_test, dtype=np.float)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Output"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_data_train = np.asarray(y_data_train, dtype=np.float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_data_test = np.asarray(y_data_test, dtype=np.float)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Neural Network - This is deep learning!!! yeahh"},{"metadata":{},"cell_type":"markdown","source":"## Validate GPU's"},{"metadata":{"trusted":true},"cell_type":"code","source":"physical_devices = tf.config.experimental.list_physical_devices('GPU')\nprint(\"Num GPUs Available: \", len(physical_devices))\ntf.config.experimental.set_memory_growth(physical_devices[0], True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build a functional model"},{"metadata":{"trusted":true},"cell_type":"code","source":"img_input = keras.Input(shape=(256, 256, 1), name='img_input')\n\nx = layers.Conv2D(64, (3, 3), activation = 'relu')(img_input)\nx = layers.MaxPooling2D(pool_size = (2, 2))(x)\nx = layers.Conv2D(32, (3, 3), activation = 'relu')(x)\nx = layers.MaxPooling2D(pool_size = (2, 2))(x)\nx = layers.Flatten()(x)\n\ndata_input = keras.Input(shape=(5,), name='data_input')\nx = layers.concatenate([x, data_input])\nx = layers.Dense(128, activation=\"relu\")(x)\nx = layers.Dense(64, activation=\"relu\")(x)\n\noutput = layers.Dense(1, name='all_output')(x)\n\nmodel = keras.Model(inputs=[img_input, data_input], outputs=output, name=\"JB_MODEL_V0\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(\n    optimizer=keras.optimizers.Adam(learning_rate=0.001),\n    loss='mean_absolute_error')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    {\"img_input\": X_img_zer_train, \"data_input\": X_data_train},\n    {\"all_output\": y_data_train}, \n    batch_size=32, \n    epochs=15,\n    validation_split=0.2\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot results of training"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(history.history['loss'], label='loss')\nplt.plot(history.history['val_loss'], label='val_loss')\nplt.ylim([0, 1500])\nplt.xlabel('Epoch')\nplt.ylabel('Error [FVC]')\nplt.legend()\nplt.grid(True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analyze results on test set"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_results = {}\n\ntest_results['FVC'] = model.evaluate(\n    {\"img_input\": X_img_zer_test, \"data_input\": X_data_test},\n    {\"all_output\": y_data_test})","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"print('TEST MEAN ABSOLUTE ERROR: ', test_results['FVC'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict({\"img_input\": X_img_zer_test, \"data_input\": X_data_test})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = 100-((test_results['FVC']/y_data_test.mean())*100)\n\nprint('MODEL ACCURACY: ', str(round(acc, 2)), '%')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('JB_MODEL_V0.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_img_data_raw = []\n\nerror_count = 0\n\ni = 1\n\ntotal = submission_baseline['Patient'].count()\n\nfor index, row in submission_baseline.iterrows():\n    for i in range(-12, 134):\n        sub_img_data_raw.append([\n                row['Patient'], \n                i, \n                row['Age'],\n                row['Sex'],\n                row['SmokingStatus'],\n                None,\n            ])\n    \nprint('FINISHED!!!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_img_data_raw_np = np.array(sub_img_data_raw)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare input data"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_data_sub = np.delete(sub_img_data_raw_np, 0, 1)\nX_data_sub = np.delete(X_data_sub, 4, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_data_sub[:, 2] = lblEncoder.transform(X_data_sub[:, 2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_data_sub = np.array(columnTransformer.transform(X_data_sub), dtype = np.str)\nX_data_sub = X_data_sub[:, 1:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_img_sub = np.delete(sub_img_data_raw_np, 0, 1)\nX_img_sub = np.delete(X_img_sub, 0, 1)\nX_img_sub = np.delete(X_img_sub, 0, 1)\nX_img_sub = np.delete(X_img_sub, 0, 1)\nX_img_sub = np.delete(X_img_sub, 0, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_img_sub_len = X_img_sub.shape[0]\n\nX_img_sub_zer = []\ny_img_sub = []\n\nsmp_sub = 0\n\nfor i in range(0, X_img_sub_len):\n    if(X_img_sub[i] == None):\n        continue\n    else:\n        smp = X_img_sub[i]\n        break\n        \nfor i in range(0, X_img_sub_len):\n    if(X_img_sub[i] == None):\n        X_img_sub_zer.append(np.zeros((256,256)))\n    else:\n        X_img_sub_zer.append(X_img_sub[i][0])\n        \nX_img_sub_zer = np.array(X_img_sub_zer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_img_sub_zer = np.asarray(X_img_sub_zer, dtype=np.float)\nX_img_sub_zer = np.expand_dims(X_img_sub_zer, -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_data_sub = np.asarray(X_data_sub, dtype=np.float)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_sub_pred = model.predict({\"img_input\": X_img_sub_zer, \"data_input\": X_data_sub})\ny_sub_pred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction for submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"final = []\nfor i in range(0, sub_img_data_raw_np.shape[0]):\n    final.append([str(sub_img_data_raw_np[i][0]) + '_' + str(sub_img_data_raw_np[i][1]), y_sub_pred[i][0], round(acc, 2)])\n\ndf_final = pd.DataFrame(final, columns=['Patient_Week','FVC','Confidence'])\ndf_final","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_final.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}