{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\n\nimport matplotlib.pyplot as plt\n\nimport pathlib\nimport PIL\n\nfrom pathlib import Path","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-23T07:09:59.410503Z","iopub.execute_input":"2021-11-23T07:09:59.410845Z","iopub.status.idle":"2021-11-23T07:09:59.415863Z","shell.execute_reply.started":"2021-11-23T07:09:59.410809Z","shell.execute_reply":"2021-11-23T07:09:59.415248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/tensorflow-great-barrier-reef/train.csv\")\ntest = pd.read_csv(\"../input/tensorflow-great-barrier-reef/test.csv\")\nsub = pd.read_csv(\"../input/tensorflow-great-barrier-reef/example_sample_submission.csv\")\n\npath = Path('../input/tensorflow-great-barrier-reef/train_images')\nfilepaths = list(path.glob(r'**/*.jpg'))\norigianalFilePaths = filepaths.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:09:59.417109Z","iopub.execute_input":"2021-11-23T07:09:59.417509Z","iopub.status.idle":"2021-11-23T07:10:06.391708Z","shell.execute_reply.started":"2021-11-23T07:09:59.417474Z","shell.execute_reply":"2021-11-23T07:10:06.390563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.393617Z","iopub.execute_input":"2021-11-23T07:10:06.393896Z","iopub.status.idle":"2021-11-23T07:10:06.405777Z","shell.execute_reply.started":"2021-11-23T07:10:06.393861Z","shell.execute_reply":"2021-11-23T07:10:06.404860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# annotations != \"[]\"","metadata":{}},{"cell_type":"code","source":"# train = train.loc[train[\"annotations\"] != \"[]\"]\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.407971Z","iopub.execute_input":"2021-11-23T07:10:06.408741Z","iopub.status.idle":"2021-11-23T07:10:06.425466Z","shell.execute_reply.started":"2021-11-23T07:10:06.408696Z","shell.execute_reply":"2021-11-23T07:10:06.424512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.428435Z","iopub.execute_input":"2021-11-23T07:10:06.429091Z","iopub.status.idle":"2021-11-23T07:10:06.442731Z","shell.execute_reply.started":"2021-11-23T07:10:06.429041Z","shell.execute_reply":"2021-11-23T07:10:06.441412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.444388Z","iopub.execute_input":"2021-11-23T07:10:06.445434Z","iopub.status.idle":"2021-11-23T07:10:06.456748Z","shell.execute_reply.started":"2021-11-23T07:10:06.445382Z","shell.execute_reply":"2021-11-23T07:10:06.455848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape , test.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.458774Z","iopub.execute_input":"2021-11-23T07:10:06.459584Z","iopub.status.idle":"2021-11-23T07:10:06.470165Z","shell.execute_reply.started":"2021-11-23T07:10:06.459496Z","shell.execute_reply":"2021-11-23T07:10:06.469136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.473283Z","iopub.execute_input":"2021-11-23T07:10:06.474533Z","iopub.status.idle":"2021-11-23T07:10:06.491543Z","shell.execute_reply.started":"2021-11-23T07:10:06.474467Z","shell.execute_reply":"2021-11-23T07:10:06.490815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.492822Z","iopub.execute_input":"2021-11-23T07:10:06.493697Z","iopub.status.idle":"2021-11-23T07:10:06.505647Z","shell.execute_reply.started":"2021-11-23T07:10:06.493657Z","shell.execute_reply":"2021-11-23T07:10:06.504914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train.isnull().sum()/ len(train))*100 ","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.506951Z","iopub.execute_input":"2021-11-23T07:10:06.507653Z","iopub.status.idle":"2021-11-23T07:10:06.520981Z","shell.execute_reply.started":"2021-11-23T07:10:06.507604Z","shell.execute_reply":"2021-11-23T07:10:06.520362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"video_frame\"].unique().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:18:52.001109Z","iopub.execute_input":"2021-11-23T07:18:52.001637Z","iopub.status.idle":"2021-11-23T07:18:52.009977Z","shell.execute_reply.started":"2021-11-23T07:18:52.001597Z","shell.execute_reply":"2021-11-23T07:18:52.008937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"sequence\"].unique().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:19:05.086079Z","iopub.execute_input":"2021-11-23T07:19:05.086748Z","iopub.status.idle":"2021-11-23T07:19:05.094686Z","shell.execute_reply.started":"2021-11-23T07:19:05.086692Z","shell.execute_reply":"2021-11-23T07:19:05.093542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# get folder name => Target ","metadata":{}},{"cell_type":"code","source":"def getLastFolderName(x): \n    pathwithoutFilename = os.path.split(x)[0] \n    lastFolderName = pathwithoutFilename.split(\"/\")[-1] \n    lastFolderName = lastFolderName.replace(\"video_\",\"\")\n    return lastFolderName\n\ndef getFileName(x): \n    pathwithoutFilename = os.path.split(x)[1] \n    lastFolderName = pathwithoutFilename.split(\".\")[0] \n    return lastFolderName\n\n\ndef getImageId(x): \n    imageId = getLastFolderName(x) + \"-\" + getFileName(x)\n    return imageId\n\nfilePaths = filepaths.copy()\nfileNames = list(map(lambda x: getFileName(x),filepaths)) \nfileTypes = list(map(lambda x: getLastFolderName(x),filepaths)) \nimage_ids = list(map(lambda x: getImageId(x),filepaths)) \nprint(fileTypes[:5])\nprint(fileNames[:5])\nprint(filepaths[:5])\nprint(image_ids[:5])","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.523126Z","iopub.execute_input":"2021-11-23T07:10:06.523844Z","iopub.status.idle":"2021-11-23T07:10:06.757768Z","shell.execute_reply.started":"2021-11-23T07:10:06.523791Z","shell.execute_reply":"2021-11-23T07:10:06.756573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# make train dataframe","metadata":{}},{"cell_type":"code","source":"data = {'imagePaths': origianalFilePaths, 'ImageName': fileNames,'ImageType':fileTypes ,\"image-id\":image_ids}\n\nimage_df = pd.DataFrame(data)\n# train= train.sample(frac = 1).reset_index(drop=True) # all shuffle\nimage_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.759323Z","iopub.execute_input":"2021-11-23T07:10:06.759653Z","iopub.status.idle":"2021-11-23T07:10:06.787987Z","shell.execute_reply.started":"2021-11-23T07:10:06.759611Z","shell.execute_reply":"2021-11-23T07:10:06.786719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.790542Z","iopub.execute_input":"2021-11-23T07:10:06.790921Z","iopub.status.idle":"2021-11-23T07:10:06.812598Z","shell.execute_reply.started":"2021-11-23T07:10:06.790871Z","shell.execute_reply":"2021-11-23T07:10:06.811739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = image_df.astype({'ImageName': 'int32'})\nimage_df = image_df.astype({'ImageType': 'int32'})\nimage_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.815326Z","iopub.execute_input":"2021-11-23T07:10:06.815783Z","iopub.status.idle":"2021-11-23T07:10:06.839722Z","shell.execute_reply.started":"2021-11-23T07:10:06.815723Z","shell.execute_reply":"2021-11-23T07:10:06.838674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.841291Z","iopub.execute_input":"2021-11-23T07:10:06.841562Z","iopub.status.idle":"2021-11-23T07:10:06.859829Z","shell.execute_reply.started":"2021-11-23T07:10:06.841529Z","shell.execute_reply":"2021-11-23T07:10:06.859174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = image_df.sort_values([\"ImageType\",\"ImageName\"])\nimage_df = image_df.reset_index(drop=True)\nimage_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:14:49.230626Z","iopub.execute_input":"2021-11-23T07:14:49.231012Z","iopub.status.idle":"2021-11-23T07:14:49.257861Z","shell.execute_reply.started":"2021-11-23T07:14:49.230970Z","shell.execute_reply":"2021-11-23T07:14:49.256814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(500)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:17:27.585614Z","iopub.execute_input":"2021-11-23T07:17:27.586070Z","iopub.status.idle":"2021-11-23T07:17:27.599913Z","shell.execute_reply.started":"2021-11-23T07:17:27.586035Z","shell.execute_reply":"2021-11-23T07:17:27.599261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# draw images","metadata":{}},{"cell_type":"code","source":"f,a = plt.subplots(nrows=4, ncols=4,figsize=(15, 10),\n                        subplot_kw={'xticks': [], 'yticks': []})\n\nfor i, ax in enumerate(a.flat):\n    ax.imshow(plt.imread(image_df.imagePaths[i]))\n    ax.set_title(str(image_df.ImageType[i]))\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T07:10:06.891686Z","iopub.execute_input":"2021-11-23T07:10:06.891936Z","iopub.status.idle":"2021-11-23T07:10:10.332739Z","shell.execute_reply.started":"2021-11-23T07:10:06.891903Z","shell.execute_reply":"2021-11-23T07:10:10.331450Z"},"trusted":true},"execution_count":null,"outputs":[]}]}