{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This kernel use simple transfer learning to detect ship exist or not\n*** For biginner, can get about 85% accuracy train on 10 epoch (~20min one epoch no GPU, ~100s one epoch have GPU)**\n* Use ResNet50 to do transfer learning \n* Load 5000 picture to be training data \n* Split 4000 training set , 1000 validate set  (0.2%)\n* Image size 256 x 256, RGB data\n* Using ImageGenerator to do data augumatation","metadata":{"_uuid":"1634aad69553b73cfca7e59cf65518a934198a86"}},{"cell_type":"markdown","source":"## Load segmentation file","metadata":{"_uuid":"8dd5f64e5d8a9c8f05d60348dff63baa30b360fc"}},{"cell_type":"code","source":"import os\nimport gc\nprint(os.listdir(\"../input\"))\nimport numpy as np \nimport pandas as pd\nimport time","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-21T05:37:58.075013Z","iopub.execute_input":"2021-11-21T05:37:58.075425Z","iopub.status.idle":"2021-11-21T05:37:58.095952Z","shell.execute_reply.started":"2021-11-21T05:37:58.075345Z","shell.execute_reply":"2021-11-21T05:37:58.095041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/train_ship_segmentations_v2.csv')","metadata":{"_uuid":"6de6d2bae5755886b31604cfdd074e0768ce3360","execution":{"iopub.status.busy":"2021-11-21T05:37:58.097505Z","iopub.execute_input":"2021-11-21T05:37:58.097999Z","iopub.status.idle":"2021-11-21T05:37:59.42968Z","shell.execute_reply.started":"2021-11-21T05:37:58.097949Z","shell.execute_reply":"2021-11-21T05:37:59.428869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"_uuid":"3f17c5fca1c05816d5f5fef51c96c08471b578cb","execution":{"iopub.status.busy":"2021-11-21T05:37:59.431838Z","iopub.execute_input":"2021-11-21T05:37:59.432134Z","iopub.status.idle":"2021-11-21T05:37:59.458842Z","shell.execute_reply.started":"2021-11-21T05:37:59.432086Z","shell.execute_reply":"2021-11-21T05:37:59.458031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tranfer EncodedPixels to target \n* have ship ==> 1\n* No ship ==> 0","metadata":{"_uuid":"3c71c03554f932704ebf66c6811f3ee2ac21de15"}},{"cell_type":"code","source":"train['exist_ship'] = train['EncodedPixels'].fillna(0)\ntrain.loc[train['exist_ship']!=0,'exist_ship']=1\ndel train['EncodedPixels']","metadata":{"_uuid":"1e4661d18418f77cbe9e608a2916ad171694caf2","execution":{"iopub.status.busy":"2021-11-21T05:37:59.461167Z","iopub.execute_input":"2021-11-21T05:37:59.461659Z","iopub.status.idle":"2021-11-21T05:37:59.573995Z","shell.execute_reply.started":"2021-11-21T05:37:59.461606Z","shell.execute_reply":"2021-11-21T05:37:59.572877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We found there are some duplicate image in training data\n* groupby duplicate image ","metadata":{"_uuid":"7e6563b24a8c46962b5b9460c9da8faf703b20ab"}},{"cell_type":"code","source":"print(len(train['ImageId']))\nprint(train['ImageId'].value_counts().shape[0])\ntrain_gp = train.groupby('ImageId').sum().reset_index()\ntrain_gp.loc[train_gp['exist_ship']>0,'exist_ship']=1","metadata":{"_uuid":"042cefd7a9a3a26fb67b3067c0618df8edbda6e4","execution":{"iopub.status.busy":"2021-11-21T05:37:59.575021Z","iopub.execute_input":"2021-11-21T05:37:59.575263Z","iopub.status.idle":"2021-11-21T05:38:20.474476Z","shell.execute_reply.started":"2021-11-21T05:37:59.575219Z","shell.execute_reply":"2021-11-21T05:38:20.473787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Balance have chip and no chip data\n* Remove 100000 data of no chip","metadata":{"_uuid":"1f04a80c515d5295d1f65efcbb15c9299c1ae762"}},{"cell_type":"code","source":"print(train_gp['exist_ship'].value_counts())\ntrain_gp= train_gp.sort_values(by='exist_ship')\ntrain_gp = train_gp.drop(train_gp.index[0:100000])","metadata":{"_uuid":"4b106073db3ce8ee7d27a3cdeeebd2c6855b52e6","execution":{"iopub.status.busy":"2021-11-21T05:38:20.475308Z","iopub.execute_input":"2021-11-21T05:38:20.475542Z","iopub.status.idle":"2021-11-21T05:38:20.527688Z","shell.execute_reply.started":"2021-11-21T05:38:20.475499Z","shell.execute_reply":"2021-11-21T05:38:20.526703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set Training set count\n* prevent large data cause much time ","metadata":{"_uuid":"2b3818001efbd28bf6b2e3079eab115a81c54420"}},{"cell_type":"code","source":"print(train_gp['exist_ship'].value_counts())\ntrain_sample = train_gp.sample(5000)\nprint(train_sample['exist_ship'].value_counts())\nprint (train_sample.shape)","metadata":{"_uuid":"b3052c37f6b60b9ddc73fd52f99a1c5d56ca0ce0","execution":{"iopub.status.busy":"2021-11-21T05:38:20.528926Z","iopub.execute_input":"2021-11-21T05:38:20.530359Z","iopub.status.idle":"2021-11-21T05:38:20.546709Z","shell.execute_reply.started":"2021-11-21T05:38:20.53031Z","shell.execute_reply":"2021-11-21T05:38:20.545817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load training data function\n* load training data to numpy array for training ","metadata":{"_uuid":"f3df290c8e72ef71e02060ff7b8d76c74986e2eb"}},{"cell_type":"code","source":"Train_path = '../input/train_v2/'\nTest_path = '../input/test_v2/'","metadata":{"_uuid":"fb283b096d5244c228549d3ccf477dd1e996794f","execution":{"iopub.status.busy":"2021-11-21T05:38:20.549564Z","iopub.execute_input":"2021-11-21T05:38:20.550052Z","iopub.status.idle":"2021-11-21T05:38:20.553888Z","shell.execute_reply.started":"2021-11-21T05:38:20.549999Z","shell.execute_reply":"2021-11-21T05:38:20.553019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntraining_img_data = []\ntarget_data = []\nfrom PIL import Image\ndata = np.empty((len(train_sample['ImageId']),256, 256,3), dtype=np.uint8)\ndata_target = np.empty((len(train_sample['ImageId'])), dtype=np.uint8)\nimage_name_list = os.listdir(Train_path)\nindex = 0\nfor image_name in image_name_list:\n    if image_name in list(train_sample['ImageId']):\n        imageA = Image.open(Train_path+image_name).resize((256,256)).convert('RGB')\n        data[index]=imageA\n        data_target[index]=train_sample[train_gp['ImageId'].str.contains(image_name)]['exist_ship'].iloc[0]\n        index+=1\n        \nprint(data.shape)\nprint(data_target.shape)","metadata":{"_uuid":"77ade061a3070f84b8b16cf99c4a506ff2643953","execution":{"iopub.status.busy":"2021-11-21T05:38:20.555454Z","iopub.execute_input":"2021-11-21T05:38:20.556001Z","iopub.status.idle":"2021-11-21T05:46:28.720627Z","shell.execute_reply.started":"2021-11-21T05:38:20.555951Z","shell.execute_reply":"2021-11-21T05:46:28.719616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Doing One hot on target\n* Set target to one hot target for classification problem","metadata":{"_uuid":"04d0f83dbfe25e02e64c180339c9fef7170abc4b"}},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\ntargets =data_target.reshape(len(data_target),-1)\nenc = OneHotEncoder()\nenc.fit(targets)\ntargets = enc.transform(targets).toarray()\nprint(targets.shape)","metadata":{"_uuid":"f4adca93a229ee7e418b1a70bb23874476f09a7b","execution":{"iopub.status.busy":"2021-11-21T05:46:28.721624Z","iopub.execute_input":"2021-11-21T05:46:28.721865Z","iopub.status.idle":"2021-11-21T05:46:29.158518Z","shell.execute_reply.started":"2021-11-21T05:46:28.721824Z","shell.execute_reply":"2021-11-21T05:46:29.157735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split Training data to training data and validate data to detect overfit ","metadata":{"_uuid":"2072d6a685db2d6e92273b87a918f3d206992781"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_val, y_train, y_val = train_test_split(data,targets, test_size = 0.2)\nx_train.shape, x_val.shape, y_train.shape, y_val.shape","metadata":{"_uuid":"9ad67d24f997b669da87dc658aa7f63f854c7985","execution":{"iopub.status.busy":"2021-11-21T05:46:29.159601Z","iopub.execute_input":"2021-11-21T05:46:29.159885Z","iopub.status.idle":"2021-11-21T05:46:29.769352Z","shell.execute_reply.started":"2021-11-21T05:46:29.159838Z","shell.execute_reply":"2021-11-21T05:46:29.768418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install scikit-learn==0.23.1\n#!pip install scikit-learn","metadata":{"execution":{"iopub.status.busy":"2021-11-21T05:53:20.103256Z","iopub.execute_input":"2021-11-21T05:53:20.103585Z","iopub.status.idle":"2021-11-21T05:53:25.951959Z","shell.execute_reply.started":"2021-11-21T05:53:20.10353Z","shell.execute_reply":"2021-11-21T05:53:25.9511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install lazypredict","metadata":{"execution":{"iopub.status.busy":"2021-11-21T05:53:25.953919Z","iopub.execute_input":"2021-11-21T05:53:25.954224Z","iopub.status.idle":"2021-11-21T05:53:43.696888Z","shell.execute_reply.started":"2021-11-21T05:53:25.954174Z","shell.execute_reply":"2021-11-21T05:53:43.696083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install xgboost","metadata":{"execution":{"iopub.status.busy":"2021-11-21T05:50:26.478463Z","iopub.execute_input":"2021-11-21T05:50:26.478776Z","iopub.status.idle":"2021-11-21T05:50:31.960349Z","shell.execute_reply.started":"2021-11-21T05:50:26.478716Z","shell.execute_reply":"2021-11-21T05:50:31.95937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install imbalanced-learn==0.4","metadata":{"execution":{"iopub.status.busy":"2021-11-21T05:51:43.922823Z","iopub.execute_input":"2021-11-21T05:51:43.923153Z","iopub.status.idle":"2021-11-21T05:51:50.988552Z","shell.execute_reply.started":"2021-11-21T05:51:43.923092Z","shell.execute_reply":"2021-11-21T05:51:50.987723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lazypredict.Supervised import LazyClassifier, LazyRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import datasets","metadata":{"execution":{"iopub.status.busy":"2021-11-21T05:49:39.205411Z","iopub.execute_input":"2021-11-21T05:49:39.205937Z","iopub.status.idle":"2021-11-21T05:49:39.51963Z","shell.execute_reply.started":"2021-11-21T05:49:39.205724Z","shell.execute_reply":"2021-11-21T05:49:39.518484Z"},"trusted":true},"execution_count":null,"outputs":[]}]}