{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom pydicom import dcmread\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.preprocessing import StandardScaler\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils import data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing labels","metadata":{}},{"cell_type":"code","source":"label_data = pd.read_csv('../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv')\ncolumns = ['patientId', 'Target']\n\nlabel_data = label_data.filter(columns)\nlabel_data.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dividing labels for train and validation set","metadata":{}},{"cell_type":"code","source":"train_labels, val_labels = train_test_split(label_data.values, test_size=0.2)\nprint(train_labels.shape)\nprint(val_labels.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'patientId: {train_labels[0][0]}, Target: {train_labels[0][1]}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing train and validation image paths","metadata":{}},{"cell_type":"code","source":"train_f = '../input/rsna-pneumonia-detection-challenge/stage_2_train_images'\ntest_f = '../input/rsna-pneumonia-detection-challenge/stage_2_test_images'\n\ntrain_paths = [os.path.join(train_f, image[0]) for image in train_labels]\nval_paths = [os.path.join(train_f, image[0]) for image in val_labels]\n\nprint(len(train_paths))\nprint(len(val_paths))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transforming and Compiling Data","metadata":{}},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.Resize(128)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_paths)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_np = dcmread(f'{train_paths[0]}.dcm').pixel_array\narr = np.array(transform(img_np)).flatten()\narr","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_pixels = np.zeros([len(train_paths), 16384])\ntrain_data_labels = np.zeros([len(train_paths), 1])\nfor i in range(0, len(train_paths)):\n    img_np = dcmread(f'{train_paths[i]}.dcm').pixel_array\n    train_data_pixels[i] = np.array(transform(img_np)).flatten()\n    train_data_labels[i] = train_labels[i][1]\n    print(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_data = pd.DataFrame(train_data_pixels)\ndf_train_data['label'] = pd.Series(train_data_labels.flatten(), index=df_train_data.index)\ndf_train_data.to_csv('train_data_and_labels2.csv')\n\nfrom IPython.display import FileLink\nFileLink(r'train_data_and_labels2.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_data_pixels = np.zeros([len(val_paths), 16384])\nval_data_labels = np.zeros([len(val_paths), 1])\nfor i in range(0, len(val_paths)):\n    img_np = dcmread(f'{val_paths[i]}.dcm').pixel_array\n    val_data_pixels[i] = np.array(transform(img_np)).flatten()\n    val_data_labels[i] = val_labels[i][1]\n    print(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val_data = pd.DataFrame(val_data_pixels)\ndf_val_data['label'] = pd.Series(val_data_labels.flatten(), index=df_val_data.index)\ndf_val_data.to_csv('val_data_and_labels2.csv')\n\nfrom IPython.display import FileLink\nFileLink(r'val_data_and_labels2.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}