{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm \nimport gc\nimport glob, os\nimport pydicom\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\nfrom sklearn.decomposition import PCA\nfrom bayes_opt import BayesianOptimization\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.decomposition import NMF\npd.set_option('display.max_columns', 500)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in tqdm(df.columns):\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_scans = pd.read_pickle('../input/ct-scans-to-dataframe/train_out.pkl')\n# tmp = train_scans[['Instance','Patient']]\n# train_scans = pd.get_dummies(train_scans[[x for x in train_scans.columns if x not in ['Instance','Patient']]], drop_first = True)\n\n# pca = NMF(n_components=2000)\n# train_scans = pca.fit_transform(train_scans)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_inf = pd.read_pickle('../input/osic-transform-dicom-into-dataframe/output_data.pkl')\ntrain_scans = pd.read_pickle('../input/ct-scans-to-dataframe/train_out.pkl')\n\ntest_inf = pd.read_pickle('../input/osic-transform-dicom-into-dataframe/output_data_test.pkl')\ntest_scans  = pd.read_pickle('../input/ct-scans-to-dataframe/test_out.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_scans.columns = train_scans.columns.astype(np.str)\npca = PCA(n_components=200)\nnew_pixels = pca.fit_transform(train_scans.loc[:, '0': '7395'])\n\ntrain_scans.drop(train_scans.loc[:, '0': '7395'].columns, axis = 1, inplace = True)\n\ntrain_scans = pd.merge(train_scans, pd.DataFrame(new_pixels), left_index=True, right_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_scans.columns = test_scans.columns.astype(np.str)\npca = PCA(n_components=200)\nnew_pixels = pca.fit_transform(test_scans.loc[:, '0': '7395'])\n\ntest_scans.drop(test_scans.loc[:, '0': '7395'].columns, axis = 1, inplace = True)\n\ntest_scans = pd.merge(test_scans, pd.DataFrame(new_pixels), left_index=True, right_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col_to_drop = ['Modality', 'ImageType', 'SOPInstanceUID', 'PatientName', 'PatientID', 'PatientSex', 'DeidentificationMethod', 'BodyPartExamined'\n              ,'GantryDetectorTilt', 'RotationDirection','StudyInstanceUID', 'SeriesInstanceUID', 'StudyID', 'ImagePositionPatient'\n              , 'ImageOrientationPatient', 'FrameOfReferenceUID', 'SamplesPerPixel', 'PhotometricInterpretation', 'PixelSpacing', 'BitsAllocated'\n              , 'RescaleSlope'\n#               , 'Patient'\n              ]\ntrain_inf = train_inf[[x for x in train_inf.columns if x not in col_to_drop]]\ntest_inf = test_inf[[x for x in test_inf.columns if x not in col_to_drop]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_inf.loc[train_inf.ManufacturerModelName == '','ManufacturerModelName'] = 'unk'\ntrain_inf.SliceThickness = train_inf.SliceThickness.astype('float')\ntrain_inf.KVP = train_inf.KVP.astype('float')\ntrain_inf.SpacingBetweenSlices = train_inf.SpacingBetweenSlices.astype('float')\ntrain_inf.TableHeight = train_inf.TableHeight.astype('float')\ntrain_inf.XRayTubeCurrent = train_inf.XRayTubeCurrent.astype('int')\ntrain_inf.InstanceNumber = train_inf.InstanceNumber.astype('int')\ntrain_inf.loc[train_inf.PositionReferenceIndicator == '','PositionReferenceIndicator'] = 'unk'\ntrain_inf.SliceLocation = train_inf.SliceLocation.astype('float')\ntrain_inf.Rows = train_inf.Rows.astype('int')\ntrain_inf.Columns = train_inf.Columns.astype('int')\ntrain_inf.BitsStored = train_inf.BitsStored.astype('int')\ntrain_inf.HighBit = train_inf.HighBit.astype('int')\ntrain_inf.loc[train_inf.WindowCenter == '[-500, 40]','WindowCenter'] = '-500'\ntrain_inf.loc[train_inf.WindowCenter == '-500.0','WindowCenter'] = '-500'\ntrain_inf.WindowCenter = train_inf.WindowCenter.astype('int')\ntrain_inf.loc[train_inf.WindowWidth == '[1500, 350]','WindowWidth'] = '1500'\ntrain_inf.loc[train_inf.WindowWidth == '-1500.0','WindowWidth'] = '-1500'\ntrain_inf.WindowWidth = train_inf.WindowWidth.astype('int')\ntrain_inf.loc[train_inf.RescaleIntercept == '0.','RescaleIntercept'] = '0'\ntrain_inf.loc[train_inf.RescaleIntercept == '-1024.','RescaleIntercept'] = '-1024'\ntrain_inf.loc[train_inf.RescaleIntercept == '-1024.0','RescaleIntercept'] = '-1024'\ntrain_inf.RescaleIntercept = train_inf.RescaleIntercept.astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_inf.loc[test_inf.ManufacturerModelName == '','ManufacturerModelName'] = 'unk'\ntest_inf.SliceThickness = test_inf.SliceThickness.astype('float')\ntest_inf.KVP = test_inf.KVP.astype('float')\ntest_inf.SpacingBetweenSlices = test_inf.SpacingBetweenSlices.astype('float')\ntest_inf.TableHeight = test_inf.TableHeight.astype('float')\ntest_inf.XRayTubeCurrent = test_inf.XRayTubeCurrent.astype('int')\ntest_inf.InstanceNumber = test_inf.InstanceNumber.astype('int')\ntest_inf.loc[test_inf.PositionReferenceIndicator == '','PositionReferenceIndicator'] = 'unk'\ntest_inf.SliceLocation = test_inf.SliceLocation.astype('float')\ntest_inf.Rows = test_inf.Rows.astype('int')\ntest_inf.Columns = test_inf.Columns.astype('int')\ntest_inf.BitsStored = test_inf.BitsStored.astype('int')\ntest_inf.HighBit = test_inf.HighBit.astype('int')\ntest_inf.loc[test_inf.WindowCenter == '[-500, 40]','WindowCenter'] = '-500'\ntest_inf.loc[test_inf.WindowCenter == '-500.0','WindowCenter'] = '-500'\ntest_inf.WindowCenter = test_inf.WindowCenter.astype('int')\ntest_inf.loc[test_inf.WindowWidth == '[1500, 350]','WindowWidth'] = '1500'\ntest_inf.loc[test_inf.WindowWidth == '-1500.0','WindowWidth'] = '-1500'\ntest_inf.WindowWidth = test_inf.WindowWidth.astype('int')\ntest_inf.loc[test_inf.RescaleIntercept == '0.','RescaleIntercept'] = '0'\ntest_inf.loc[test_inf.RescaleIntercept == '-1024.','RescaleIntercept'] = '-1024'\ntest_inf.loc[test_inf.RescaleIntercept == '-1024.0','RescaleIntercept'] = '-1024'\ntest_inf.RescaleIntercept = test_inf.RescaleIntercept.astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_inf = reduce_mem_usage(train_inf)\ntest_inf = reduce_mem_usage(test_inf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_final = train_inf.merge(train_scans, left_on=['Patient', 'InstanceNumber'], right_on=['Patient', 'Instance'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_final.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_final.drop('InstanceNumber', axis = 1, inplace = True)\ntrain_final.to_pickle(\"train_final.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_final = test_inf.merge(test_scans, left_on=['Patient', 'InstanceNumber'], right_on=['Patient', 'Instance'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_final[['FVC','Percent']] = -1\ntest_final.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_final.drop('InstanceNumber', axis = 1, inplace = True)\ntest_final.to_pickle(\"test_final.pkl\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}