{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\n# Printing the file names is not necessary for this notebook. I will comment them out.\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-12T19:04:00.345375Z","iopub.execute_input":"2022-07-12T19:04:00.345770Z","iopub.status.idle":"2022-07-12T19:04:00.352595Z","shell.execute_reply.started":"2022-07-12T19:04:00.345740Z","shell.execute_reply":"2022-07-12T19:04:00.351359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv', dtype=str)\n\nprint('Training Shape:',train_labels.shape)\n\nprint('Input example :')\ntrain_labels.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T19:05:57.571842Z","iopub.execute_input":"2022-07-12T19:05:57.572267Z","iopub.status.idle":"2022-07-12T19:05:57.841631Z","shell.execute_reply.started":"2022-07-12T19:05:57.572233Z","shell.execute_reply":"2022-07-12T19:05:57.840376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We will cast the input labels as float type instead of object\ntrain_labels['label'] = train_labels['label'].astype(float)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T19:06:30.956489Z","iopub.execute_input":"2022-07-12T19:06:30.956861Z","iopub.status.idle":"2022-07-12T19:06:31.005190Z","shell.execute_reply.started":"2022-07-12T19:06:30.956832Z","shell.execute_reply":"2022-07-12T19:06:31.004221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at Test/Train size difference\n\nprint(len(os.listdir('../input/histopathologic-cancer-detection/train/')))\nprint(len(os.listdir('../input/histopathologic-cancer-detection/test/')))\n\nTrainProportion = len(os.listdir('../input/histopathologic-cancer-detection/train/'))/ (len(os.listdir('../input/histopathologic-cancer-detection/train/')) + len(os.listdir('../input/histopathologic-cancer-detection/test/')))\n\nprint('Proportion of data in Train set: ',TrainProportion)\nprint('Proportion of data in Test set: ',1- TrainProportion)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T19:08:58.615866Z","iopub.execute_input":"2022-07-12T19:08:58.616292Z","iopub.status.idle":"2022-07-12T19:08:59.268062Z","shell.execute_reply.started":"2022-07-12T19:08:58.616256Z","shell.execute_reply":"2022-07-12T19:08:59.266889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Begin EDA and refine data for model","metadata":{}},{"cell_type":"code","source":"# Establish distribution of training labels\n\ntrain_labels['label'].value_counts()\ntrain_labels['label'].value_counts().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T19:13:48.700731Z","iopub.execute_input":"2022-07-12T19:13:48.701749Z","iopub.status.idle":"2022-07-12T19:13:48.893350Z","shell.execute_reply.started":"2022-07-12T19:13:48.701708Z","shell.execute_reply":"2022-07-12T19:13:48.892269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Imbalanced data will need to be dealt with","metadata":{}},{"cell_type":"code","source":"# Make occurence of 0s equivalent to 1s in the training data\n\ntrain_pos = train_labels[train_labels['label'] == 1]\ntrain_neg = train_labels[train_labels['label'] == 0]\ntrain_neg = train_neg.sample(n = len(train_pos))\n\n# Check new sizes\nprint(len(train_neg))\nprint(len(train_pos))","metadata":{"execution":{"iopub.status.busy":"2022-07-12T19:20:16.713100Z","iopub.execute_input":"2022-07-12T19:20:16.713561Z","iopub.status.idle":"2022-07-12T19:20:16.749570Z","shell.execute_reply.started":"2022-07-12T19:20:16.713529Z","shell.execute_reply":"2022-07-12T19:20:16.748190Z"},"trusted":true},"execution_count":null,"outputs":[]}]}