{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!apt install zip","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:04:27.922441Z","iopub.execute_input":"2021-08-26T03:04:27.923259Z","iopub.status.idle":"2021-08-26T03:04:30.899474Z","shell.execute_reply.started":"2021-08-26T03:04:27.923135Z","shell.execute_reply":"2021-08-26T03:04:30.898362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import glob\nimport joblib\nimport numpy as np\nimport PIL\nfrom PIL import Image\nimport pydicom\nimport tqdm\nimport os\nimport cv2\nimport zipfile\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-26T03:04:43.868340Z","iopub.execute_input":"2021-08-26T03:04:43.868746Z","iopub.status.idle":"2021-08-26T03:04:44.616713Z","shell.execute_reply.started":"2021-08-26T03:04:43.868708Z","shell.execute_reply":"2021-08-26T03:04:44.615340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get images paths","metadata":{}},{"cell_type":"code","source":"# Desired output size.\nRESIZED_WIDTH, RESIZED_HEIGHT = 224, 224\nEACH_WIDTH, EACH_HEIGHT = RESIZED_WIDTH // 2, RESIZED_HEIGHT // 2\n\nOUTPUT_FORMAT = \"jpg\"\n\nOUTPUT_DIR = \"data_argument_224_224\"","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:04:46.719138Z","iopub.execute_input":"2021-08-26T03:04:46.719492Z","iopub.status.idle":"2021-08-26T03:04:46.724679Z","shell.execute_reply.started":"2021-08-26T03:04:46.719461Z","shell.execute_reply":"2021-08-26T03:04:46.723471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_num2str = {0: 'powdery_mildew',\n                     1: 'scab',\n                     2: 'complex',\n                     3: 'frog_eye_leaf_spot',\n                     4: 'rust'}\n    \nlabel_str2num = {'powdery_mildew': 0,\n                 'scab': 1,\n                 'complex': 2,\n                 'frog_eye_leaf_spot': 3,\n                 'rust': 4}","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:04:49.663215Z","iopub.execute_input":"2021-08-26T03:04:49.663645Z","iopub.status.idle":"2021-08-26T03:04:49.669048Z","shell.execute_reply.started":"2021-08-26T03:04:49.663604Z","shell.execute_reply":"2021-08-26T03:04:49.668033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = \"../input/plant-pathology-2021-fgvc8\"\n!ls {data_dir}","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-08-26T03:04:51.842876Z","iopub.execute_input":"2021-08-26T03:04:51.843241Z","iopub.status.idle":"2021-08-26T03:04:52.586492Z","shell.execute_reply.started":"2021-08-26T03:04:51.843210Z","shell.execute_reply":"2021-08-26T03:04:52.585102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"train_images\"\ntrain_paths = glob.glob(f\"{data_dir}/{train_dir}/*.jpg\")\n# test_dir = \"test_images\"\n# test_paths = glob.glob(f\"{data_dir}/{test_dir}/*.jpg\")\n# len(train_paths), len(test_paths)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:04:54.133507Z","iopub.execute_input":"2021-08-26T03:04:54.133886Z","iopub.status.idle":"2021-08-26T03:04:54.506286Z","shell.execute_reply.started":"2021-08-26T03:04:54.133852Z","shell.execute_reply":"2021-08-26T03:04:54.505501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess all data\n\nFirst declare a bunch of useful functions.","metadata":{}},{"cell_type":"code","source":"!mkdir -p {OUTPUT_DIR}/{train_dir}","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:05:06.282040Z","iopub.execute_input":"2021-08-26T03:05:06.282705Z","iopub.status.idle":"2021-08-26T03:05:07.015463Z","shell.execute_reply.started":"2021-08-26T03:05:06.282665Z","shell.execute_reply":"2021-08-26T03:05:07.014023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load converted images\n\nLet's test that everything is ok!","metadata":{}},{"cell_type":"code","source":"TRAIN_DF = pd.read_csv('../input/pp-csv/clearned_train.csv')\nTRAIN_DF","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:18:22.129880Z","iopub.execute_input":"2021-08-26T03:18:22.130338Z","iopub.status.idle":"2021-08-26T03:18:22.179767Z","shell.execute_reply.started":"2021-08-26T03:18:22.130300Z","shell.execute_reply":"2021-08-26T03:18:22.178556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bad_df = pd.read_csv('../input/dup-csv/train_bad_images.csv')\n# dup_df = pd.read_csv('../input/dup-csv/train_duplicates_320.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:08:52.524638Z","iopub.execute_input":"2021-08-26T03:08:52.525102Z","iopub.status.idle":"2021-08-26T03:08:52.546966Z","shell.execute_reply.started":"2021-08-26T03:08:52.525066Z","shell.execute_reply":"2021-08-26T03:08:52.546025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for index, row in bad_df.iterrows():\n#     TRAIN_DF = TRAIN_DF.drop(TRAIN_DF[TRAIN_DF['image'] == row['image']].index)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:13:26.605900Z","iopub.execute_input":"2021-08-26T03:13:26.606315Z","iopub.status.idle":"2021-08-26T03:13:26.637827Z","shell.execute_reply.started":"2021-08-26T03:13:26.606278Z","shell.execute_reply":"2021-08-26T03:13:26.636711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for index, row in dup_df.iterrows():\n#     for image in row['image'][1:-1].split(','):\n#         image = image.replace(\"'\", '')\n#         image = image.replace(\" \", \"\")\n#         TRAIN_DF = TRAIN_DF.drop(TRAIN_DF[TRAIN_DF['image'] == image].index)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:16:09.501259Z","iopub.execute_input":"2021-08-26T03:16:09.501660Z","iopub.status.idle":"2021-08-26T03:16:13.612613Z","shell.execute_reply.started":"2021-08-26T03:16:09.501626Z","shell.execute_reply":"2021-08-26T03:16:13.611610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TRAIN_DF.to_csv('clearned_train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:16:42.239057Z","iopub.execute_input":"2021-08-26T03:16:42.239443Z","iopub.status.idle":"2021-08-26T03:16:42.299098Z","shell.execute_reply.started":"2021-08-26T03:16:42.239411Z","shell.execute_reply":"2021-08-26T03:16:42.298027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EXTRA_TRAIN = pd.DataFrame(columns=[\"image\", \"labels\"])","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:05:42.773879Z","iopub.execute_input":"2021-08-26T03:05:42.774405Z","iopub.status.idle":"2021-08-26T03:05:42.780219Z","shell.execute_reply.started":"2021-08-26T03:05:42.774372Z","shell.execute_reply":"2021-08-26T03:05:42.779406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def concat_img(imgs):\n    if len(imgs) < 4:\n        return\n    output_shape = (EACH_WIDTH, EACH_HEIGHT)\n    imgs = [cv2.resize(img, output_shape) for img in imgs]\n    upper_img = cv2.vconcat([imgs[0], imgs[1]])\n    lower_img = cv2.vconcat([imgs[2], imgs[3]])\n    img = cv2.hconcat([upper_img, lower_img])\n    return img","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:05:46.938991Z","iopub.execute_input":"2021-08-26T03:05:46.939516Z","iopub.status.idle":"2021-08-26T03:05:46.946986Z","shell.execute_reply.started":"2021-08-26T03:05:46.939482Z","shell.execute_reply":"2021-08-26T03:05:46.946096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"length = len(TRAIN_DF)\nindexs = np.random.randint(0, length, size=4)\n\nfor count in range(3000):\n    imgs = []\n    new_labels = []\n    indexs = np.random.randint(0, length, size=4)\n    for index in indexs:\n        filepath = train_paths[index]\n        img = cv2.imread(filepath)\n        imgs.append(img)\n        labels = TRAIN_DF.iloc[index, 1]\n        for disease in labels.split(' '):\n            if disease == 'healthy' or disease in new_labels:\n                continue\n            new_labels.append(disease)\n    if not new_labels:\n        new_label = 'healthy'\n    else:\n        new_label = ' '.join(new_labels)\n#     print(new_label)\n    img_name = f\"{count}.jpg\"\n    new_img = concat_img(imgs)\n    new_file_path = f\"{OUTPUT_DIR}/{train_dir}/{img_name}\"\n    cv2.imwrite(new_file_path, new_img)\n    EXTRA_TRAIN = EXTRA_TRAIN.append({\"image\":img_name, \"labels\":new_label}, ignore_index=True)\n#     print(f\"finished {count}th image!\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T03:06:05.228064Z","iopub.execute_input":"2021-08-26T03:06:05.228469Z","iopub.status.idle":"2021-08-26T03:07:08.478682Z","shell.execute_reply.started":"2021-08-26T03:06:05.228432Z","shell.execute_reply":"2021-08-26T03:07:08.477381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EXTRA_TRAIN","metadata":{"execution":{"iopub.status.busy":"2021-08-25T06:44:26.922265Z","iopub.execute_input":"2021-08-25T06:44:26.922709Z","iopub.status.idle":"2021-08-25T06:44:26.939103Z","shell.execute_reply.started":"2021-08-25T06:44:26.92267Z","shell.execute_reply":"2021-08-25T06:44:26.938373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EXTRA_TRAIN.to_csv(f\"{OUTPUT_DIR}/extra_train.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T22:04:33.034164Z","iopub.execute_input":"2021-08-22T22:04:33.034487Z","iopub.status.idle":"2021-08-22T22:04:34.846151Z","shell.execute_reply.started":"2021-08-22T22:04:33.034435Z","shell.execute_reply":"2021-08-22T22:04:34.845404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r {OUTPUT_DIR}_resized.zip ./{OUTPUT_DIR}/*","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r -f ./{OUTPUT_DIR}","metadata":{"execution":{"iopub.status.busy":"2021-07-27T02:25:03.260648Z","iopub.execute_input":"2021-07-27T02:25:03.260973Z","iopub.status.idle":"2021-07-27T02:25:03.898613Z","shell.execute_reply.started":"2021-07-27T02:25:03.26092Z","shell.execute_reply":"2021-07-27T02:25:03.897575Z"},"trusted":true},"execution_count":null,"outputs":[]}]}