{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport math\nimport time\nimport cv2\nfrom sklearn import metrics","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-28T14:18:04.851527Z","iopub.execute_input":"2023-04-28T14:18:04.851951Z","iopub.status.idle":"2023-04-28T14:18:04.858459Z","shell.execute_reply.started":"2023-04-28T14:18:04.851912Z","shell.execute_reply":"2023-04-28T14:18:04.857149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_train = \"/kaggle/input/histopathologic-cancer-detection/train/\"\nfolder_test = \"/kaggle/input/histopathologic-cancer-detection/test/\"","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:18:06.945435Z","iopub.execute_input":"2023-04-28T14:18:06.946698Z","iopub.status.idle":"2023-04-28T14:18:06.951920Z","shell.execute_reply.started":"2023-04-28T14:18:06.946632Z","shell.execute_reply":"2023-04-28T14:18:06.950590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(\"/kaggle/input/histopathologic-cancer-detection/sample_submission.csv\")\nlabel_df = pd.read_csv(\"/kaggle/input/histopathologic-cancer-detection/train_labels.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:18:07.952224Z","iopub.execute_input":"2023-04-28T14:18:07.952902Z","iopub.status.idle":"2023-04-28T14:18:08.305150Z","shell.execute_reply.started":"2023-04-28T14:18:07.952852Z","shell.execute_reply":"2023-04-28T14:18:08.304090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_name = label_df[\"id\"].to_numpy()\ntest_name =  sample_df[\"id\"].to_numpy()\ntrain_label = label_df[\"label\"].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:18:08.307407Z","iopub.execute_input":"2023-04-28T14:18:08.308254Z","iopub.status.idle":"2023-04-28T14:18:08.318404Z","shell.execute_reply.started":"2023-04-28T14:18:08.308199Z","shell.execute_reply":"2023-04-28T14:18:08.317227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Samples","metadata":{}},{"cell_type":"code","source":"def plot_photo(N, nc = 4, pos = True):\n    \n    nr =  math.ceil(N/nc)\n    n_train = train_name.shape[0]\n    fig, ax = plt.subplots(nr, nc, figsize = (15, nr*3.2))\n    \n    np.random.seed(1)\n    \n    IDs = np.random.choice(n_train, N, replace = False)\n    \n    if pos:\n        filter1 = train_label == 1\n    else:\n        filter1 = train_label == 0\n        \n    names_1 = train_name[filter1]\n    \n    names = np.random.choice(names_1, N, replace = False)\n    \n    for k in range(N):\n        i =  int(k/nc)\n        j = k % nc\n        fname =  folder_train + names[k] + \".tif\"\n        img = cv2.imread(fname)\n        ax[i,j].imshow(img)\n        \n","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:50:02.863508Z","iopub.execute_input":"2023-04-28T04:50:02.864636Z","iopub.status.idle":"2023-04-28T04:50:02.874031Z","shell.execute_reply.started":"2023-04-28T04:50:02.864574Z","shell.execute_reply":"2023-04-28T04:50:02.873115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_photo(16)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:50:03.619962Z","iopub.execute_input":"2023-04-28T04:50:03.620734Z","iopub.status.idle":"2023-04-28T04:50:06.241079Z","shell.execute_reply.started":"2023-04-28T04:50:03.620665Z","shell.execute_reply":"2023-04-28T04:50:06.239929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_photo(16, pos = False)","metadata":{"execution":{"iopub.status.busy":"2023-04-26T04:47:50.864096Z","iopub.execute_input":"2023-04-26T04:47:50.864828Z","iopub.status.idle":"2023-04-26T04:47:53.403976Z","shell.execute_reply.started":"2023-04-26T04:47:50.864780Z","shell.execute_reply":"2023-04-26T04:47:53.402105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Balanced sampling and create X and y\n### functions","metadata":{}},{"cell_type":"code","source":"def balanced_sampling(n_sample):\n\n    indx_pos = np.where(train_label == 1)[0]\n    indx_neg = np.where(train_label == 0)[0]\n    \n    np.random.seed(1)\n    sample_pos = np.random.choice(indx_pos, n_sample, replace = False)\n    sample_neg = np.random.choice(indx_neg, n_sample, replace = False)\n    \n    sample_idx = np.concatenate((sample_pos, sample_neg))\n    \n    np.random.shuffle(sample_idx)\n    \n    return sample_idx\n    ","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:18:54.256701Z","iopub.execute_input":"2023-04-28T14:18:54.257141Z","iopub.status.idle":"2023-04-28T14:18:54.264368Z","shell.execute_reply.started":"2023-04-28T14:18:54.257097Z","shell.execute_reply":"2023-04-28T14:18:54.263163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_traindata(idxs, margin):\n    \n    time1 = time.time()\n    \n    n = idxs.shape[0]\n    \n    X_array = np.zeros((n, 32 + 2*margin, 32 + 2*margin, 3)).astype(\"uint8\")\n    \n    names = train_name[idxs]\n    \n    y_array = train_label[idxs]\n    \n    start = 32 - margin\n    end = 64 + margin\n    \n    for i in range(n):\n        \n        fname =  folder_train + names[i] + \".tif\"\n        img = cv2.imread(fname)\n        \n        X_array[i,:,:,:] = img[start:end,start:end,:]\n        \n        \n    time2 = time.time()\n    \n    time3 = np.round(time2 - time1)\n    print(time3, \"sec\")\n                     \n    return X_array, y_array.reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:19:02.241592Z","iopub.execute_input":"2023-04-28T14:19:02.242472Z","iopub.status.idle":"2023-04-28T14:19:02.251224Z","shell.execute_reply.started":"2023-04-28T14:19:02.242422Z","shell.execute_reply":"2023-04-28T14:19:02.249662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_testdata(margin):\n    \n    time1 = time.time()\n    \n    names = test_name\n    \n    n = names.shape[0]\n    print(\"n_test = \", n)\n    X_array = np.zeros((n, 32 + 2*margin, 32 + 2*margin, 3)).astype(\"uint8\")\n\n    start = 32 - margin\n    end = 64 + margin\n    \n    for i in range(n):\n        \n        fname =  folder_test + names[i] + \".tif\"\n        img = cv2.imread(fname)\n        \n        X_array[i,:,:,:] = img[start:end,start:end,:]\n        \n        \n    time2 = time.time()\n    \n    time3 = np.round(time2 - time1)\n    print(time3, \"sec\")\n                     \n    return X_array","metadata":{"execution":{"iopub.status.busy":"2023-04-28T04:53:24.432314Z","iopub.execute_input":"2023-04-28T04:53:24.432730Z","iopub.status.idle":"2023-04-28T04:53:24.441082Z","shell.execute_reply.started":"2023-04-28T04:53:24.432676Z","shell.execute_reply":"2023-04-28T04:53:24.439671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### create data","metadata":{}},{"cell_type":"code","source":"n_sample = int(170000/2) \nsample_indx = balanced_sampling(n_sample)\nn_val = 10000\nval_idxs = sample_indx[:n_val]\ntrain_idxs = sample_indx[n_val:]\nprint(val_idxs.shape, train_idxs.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:19:13.894867Z","iopub.execute_input":"2023-04-28T14:19:13.895305Z","iopub.status.idle":"2023-04-28T14:19:13.918104Z","shell.execute_reply.started":"2023-04-28T14:19:13.895266Z","shell.execute_reply":"2023-04-28T14:19:13.916747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label[sample_indx].mean()","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:19:20.635853Z","iopub.execute_input":"2023-04-28T14:19:20.636834Z","iopub.status.idle":"2023-04-28T14:19:20.645588Z","shell.execute_reply.started":"2023-04-28T14:19:20.636784Z","shell.execute_reply":"2023-04-28T14:19:20.644188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"margin = 32","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:19:44.135888Z","iopub.execute_input":"2023-04-28T14:19:44.137027Z","iopub.status.idle":"2023-04-28T14:19:44.142935Z","shell.execute_reply.started":"2023-04-28T14:19:44.136971Z","shell.execute_reply":"2023-04-28T14:19:44.141936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val, y_val = create_traindata(val_idxs[0:10], margin)\nX_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:19:46.088297Z","iopub.execute_input":"2023-04-28T14:19:46.089062Z","iopub.status.idle":"2023-04-28T14:19:46.210743Z","shell.execute_reply.started":"2023-04-28T14:19:46.089004Z","shell.execute_reply":"2023-04-28T14:19:46.209466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val, y_val = create_traindata(val_idxs, margin)\nX_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:20:05.308712Z","iopub.execute_input":"2023-04-28T14:20:05.309950Z","iopub.status.idle":"2023-04-28T14:21:53.841410Z","shell.execute_reply.started":"2023-04-28T14:20:05.309902Z","shell.execute_reply":"2023-04-28T14:21:53.839961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_val[2])","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:21:53.843491Z","iopub.execute_input":"2023-04-28T14:21:53.843822Z","iopub.status.idle":"2023-04-28T14:21:54.066192Z","shell.execute_reply.started":"2023-04-28T14:21:53.843791Z","shell.execute_reply":"2023-04-28T14:21:54.065144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, y_train = create_traindata(train_idxs, margin)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:21:54.067433Z","iopub.execute_input":"2023-04-28T14:21:54.067741Z","iopub.status.idle":"2023-04-28T14:47:48.440711Z","shell.execute_reply.started":"2023-04-28T14:21:54.067711Z","shell.execute_reply":"2023-04-28T14:47:48.439459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:47:48.443208Z","iopub.execute_input":"2023-04-28T14:47:48.443561Z","iopub.status.idle":"2023-04-28T14:47:48.451780Z","shell.execute_reply.started":"2023-04-28T14:47:48.443526Z","shell.execute_reply":"2023-04-28T14:47:48.450558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = create_testdata(margin)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-26T10:56:41.216697Z","iopub.execute_input":"2023-04-26T10:56:41.217090Z","iopub.status.idle":"2023-04-26T10:56:41.232276Z","shell.execute_reply.started":"2023-04-26T10:56:41.217055Z","shell.execute_reply":"2023-04-26T10:56:41.230797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### save files","metadata":{}},{"cell_type":"code","source":"#validation data  (balanced)\nnp.save(\"X_val\", X_val)\nnp.save(\"y_val\", y_val)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:47:48.453665Z","iopub.execute_input":"2023-04-28T14:47:48.454062Z","iopub.status.idle":"2023-04-28T14:47:48.716818Z","shell.execute_reply.started":"2023-04-28T14:47:48.454028Z","shell.execute_reply":"2023-04-28T14:47:48.715497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#50000 samples of train data (balanced)\nnp.save(\"X_train_s\", X_train[0:50000])\nnp.save(\"y_train_s\", y_train[0:50000])","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:47:48.718064Z","iopub.execute_input":"2023-04-28T14:47:48.719044Z","iopub.status.idle":"2023-04-28T14:47:49.990068Z","shell.execute_reply.started":"2023-04-28T14:47:48.719006Z","shell.execute_reply":"2023-04-28T14:47:49.988858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#full train data (balanced)\nnp.save(\"X_train\", X_train)\nnp.save(\"y_train\", y_train)","metadata":{"execution":{"iopub.status.busy":"2023-04-28T14:47:49.991659Z","iopub.execute_input":"2023-04-28T14:47:49.992011Z","iopub.status.idle":"2023-04-28T14:47:59.907894Z","shell.execute_reply.started":"2023-04-28T14:47:49.991975Z","shell.execute_reply":"2023-04-28T14:47:59.906764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test data\nnp.save(\"X_test\", X_test)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-26T04:26:59.118266Z","iopub.execute_input":"2023-04-26T04:26:59.119162Z","iopub.status.idle":"2023-04-26T04:26:59.513576Z","shell.execute_reply.started":"2023-04-26T04:26:59.119122Z","shell.execute_reply":"2023-04-26T04:26:59.512399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}