{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path\nimport os.path\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport seaborn as sns\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-25T23:55:00.734103Z","iopub.execute_input":"2022-03-25T23:55:00.734894Z","iopub.status.idle":"2022-03-26T00:00:38.306154Z","shell.execute_reply.started":"2022-03-25T23:55:00.734795Z","shell.execute_reply":"2022-03-26T00:00:38.305269Z"},"_kg_hide-input":true,"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](https://miro.medium.com/max/1400/0*aFWv6THbp58fCjBk)becominghuman.ai","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path\nimport os.path\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport seaborn as sns\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:40:23.720275Z","iopub.execute_input":"2022-03-26T02:40:23.720650Z","iopub.status.idle":"2022-03-26T02:40:29.658469Z","shell.execute_reply.started":"2022-03-26T02:40:23.720557Z","shell.execute_reply":"2022-03-26T02:40:29.657475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import requests\nfrom bs4 import BeautifulSoup\nimport lxml\nimport os\nimport urllib\nimport sys\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport cv2\nimport csv\nimport multiprocessing\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:40:34.545230Z","iopub.execute_input":"2022-03-26T02:40:34.545511Z","iopub.status.idle":"2022-03-26T02:40:35.087621Z","shell.execute_reply.started":"2022-03-26T02:40:34.545483Z","shell.execute_reply":"2022-03-26T02:40:35.086725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nimport json, codecs\n\nwith codecs.open(\"../input/iwildcam2022-fgvc9/metadata/metadata/iwildcam2022_train_annotations.json\", 'r',\n                 encoding='utf-8', errors='ignore') as f:\n    train_meta = json.load(f)\n    \nwith codecs.open(\"../input/iwildcam2022-fgvc9/metadata/metadata/iwildcam2022_test_information.json\", 'r',\n                 encoding='utf-8', errors='ignore') as f:\n    test_meta = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:40:40.863912Z","iopub.execute_input":"2022-03-26T02:40:40.864205Z","iopub.status.idle":"2022-03-26T02:40:43.277047Z","shell.execute_reply.started":"2022-03-26T02:40:40.864175Z","shell.execute_reply":"2022-03-26T02:40:43.276004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nwith codecs.open(\"../input/iwildcam2022-fgvc9/metadata/metadata/gps_locations.json\", 'r',\n                 encoding='utf-8', errors='ignore') as f:\n    gps_meta = json.load(f)\n    \nwith codecs.open(\"../input/iwildcam2022-fgvc9/metadata/metadata/iwildcam2022_mdv4_detections.json\", 'r',\n                 encoding='utf-8', errors='ignore') as f:\n    iwildc_meta = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:13:15.236843Z","iopub.execute_input":"2022-03-26T01:13:15.237110Z","iopub.status.idle":"2022-03-26T01:13:20.604448Z","shell.execute_reply.started":"2022-03-26T01:13:15.237081Z","shell.execute_reply":"2022-03-26T01:13:20.603077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\ndisplay(train_meta.keys())","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:10:00.207716Z","iopub.execute_input":"2022-03-26T01:10:00.208550Z","iopub.status.idle":"2022-03-26T01:10:00.216816Z","shell.execute_reply.started":"2022-03-26T01:10:00.208522Z","shell.execute_reply":"2022-03-26T01:10:00.215649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\ndisplay(test_meta.keys())","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:40:51.431582Z","iopub.execute_input":"2022-03-26T02:40:51.431903Z","iopub.status.idle":"2022-03-26T02:40:51.440317Z","shell.execute_reply.started":"2022-03-26T02:40:51.431868Z","shell.execute_reply":"2022-03-26T02:40:51.439142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nwcam4 = pd.DataFrame(test_meta['images'])\n#train_cat.columns = [ 'category_id', 'scientificName','family', 'genus']\ndisplay(wcam4)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:41:02.311648Z","iopub.execute_input":"2022-03-26T02:41:02.311980Z","iopub.status.idle":"2022-03-26T02:41:02.566203Z","shell.execute_reply.started":"2022-03-26T02:41:02.311946Z","shell.execute_reply":"2022-03-26T02:41:02.565094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(iwildc_meta.keys())","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:13:32.252510Z","iopub.execute_input":"2022-03-26T01:13:32.253494Z","iopub.status.idle":"2022-03-26T01:13:32.263374Z","shell.execute_reply.started":"2022-03-26T01:13:32.253425Z","shell.execute_reply":"2022-03-26T01:13:32.261752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nwcam = pd.DataFrame(train_meta['categories'])\n#train_cat.columns = [ 'category_id', 'scientificName','family', 'genus']\ndisplay(wcam)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T00:07:30.947523Z","iopub.execute_input":"2022-03-26T00:07:30.947876Z","iopub.status.idle":"2022-03-26T00:07:30.974459Z","shell.execute_reply.started":"2022-03-26T00:07:30.947843Z","shell.execute_reply":"2022-03-26T00:07:30.973794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nwcam1 = pd.DataFrame(train_meta['annotations'])\n#train_cat.columns = [ 'category_id', 'scientificName','family', 'genus']\ndisplay(wcam1)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:11:14.434603Z","iopub.execute_input":"2022-03-26T01:11:14.434860Z","iopub.status.idle":"2022-03-26T01:11:14.776602Z","shell.execute_reply.started":"2022-03-26T01:11:14.434837Z","shell.execute_reply":"2022-03-26T01:11:14.775276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nwcam2 = pd.DataFrame(train_meta['images'])\n#train_cat.columns = [ 'category_id', 'scientificName','family', 'genus']\ndisplay(wcam2)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:12:31.591629Z","iopub.execute_input":"2022-03-26T01:12:31.592276Z","iopub.status.idle":"2022-03-26T01:12:32.556428Z","shell.execute_reply.started":"2022-03-26T01:12:31.592229Z","shell.execute_reply":"2022-03-26T01:12:32.554932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nwcam3 = pd.DataFrame(iwildc_meta['images'])\n#train_cat.columns = [ 'category_id', 'scientificName','family', 'genus']\ndisplay(wcam3)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:13:50.514593Z","iopub.execute_input":"2022-03-26T01:13:50.514940Z","iopub.status.idle":"2022-03-26T01:13:50.823802Z","shell.execute_reply.started":"2022-03-26T01:13:50.514916Z","shell.execute_reply":"2022-03-26T01:13:50.822777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nlen(os.listdir('../input/iwildcam2022-fgvc9/train/train'))","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:14:25.778340Z","iopub.execute_input":"2022-03-26T01:14:25.778616Z","iopub.status.idle":"2022-03-26T01:14:30.152139Z","shell.execute_reply.started":"2022-03-26T01:14:25.778588Z","shell.execute_reply":"2022-03-26T01:14:30.151575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(wcam1.id)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T00:58:12.713180Z","iopub.execute_input":"2022-03-26T00:58:12.713584Z","iopub.status.idle":"2022-03-26T00:58:12.722572Z","shell.execute_reply.started":"2022-03-26T00:58:12.713552Z","shell.execute_reply":"2022-03-26T00:58:12.721647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Checking one image since I got errors with resize","metadata":{}},{"cell_type":"code","source":"import cv2 as cv\n\nimg = cv.imread('../input/iwildcam2022-fgvc9/train/train/87422d6c-21bc-11ea-a13a-137349068a90.jpg')\nprint( img.shape )","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:45:06.240954Z","iopub.execute_input":"2022-03-26T01:45:06.241298Z","iopub.status.idle":"2022-03-26T01:45:06.301199Z","shell.execute_reply.started":"2022-03-26T01:45:06.241263Z","shell.execute_reply":"2022-03-26T01:45:06.300023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nimg=[]\nfilename=wcam1.image_id[:10000]#original is id\nlabel=wcam1.category_id[:10000]\nfor file in filename:\n    image=cv2.imread(\"../input/iwildcam2022-fgvc9/train/train/\"+file+'.jpg')\n    res=cv2.resize(image,(32,32))\n    img.append(res)\nimg=np.array(img)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T01:58:41.950288Z","iopub.execute_input":"2022-03-26T01:58:41.950558Z","iopub.status.idle":"2022-03-26T02:05:47.601401Z","shell.execute_reply.started":"2022-03-26T01:58:41.950529Z","shell.execute_reply":"2022-03-26T02:05:47.599812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#It took more time than I expected. Though the resize returned error when I applied only id\n\n\"OpenCV(4.5.4) /tmp/pip-req-build-21t5esfk/opencv/modules/imgproc/src/resize.cpp:4051: error: (-215:Assertion failed) !ssize.empty() in function 'resize'","metadata":{}},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nplt.figure(figsize=(15,15))\nfor i in range(9):\n    plt.subplot(3,3,i+1)\n    plt.imshow(img[i])","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:06:13.132784Z","iopub.execute_input":"2022-03-26T02:06:13.133173Z","iopub.status.idle":"2022-03-26T02:06:14.304997Z","shell.execute_reply.started":"2022-03-26T02:06:13.133145Z","shell.execute_reply":"2022-03-26T02:06:14.304066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nnp.random.seed(921)\nfrom keras.utils import np_utils\nfrom keras.models import Sequential\nfrom keras.layers import Convolution2D,Dense,MaxPool2D,Activation,Dropout,Flatten\nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\n#from keras.layers.normalization import BatchNormalization\nfrom tensorflow.keras.layers import BatchNormalization\nX_train,X_test,y_train,y_test=train_test_split(img,label,test_size=0.2)\ndel img\ny_train=y_train.astype(int)\ny_test=y_test.astype(int)\ny_train=np.array(y_train).reshape(-1,1)\ny_test=np.array(y_test).reshape(-1,1)\nX_train=X_train.reshape(-1,32,32,3)/255 #Normalize\nX_test=X_test.reshape(-1,32,32,3)/255\ny_train=np_utils.to_categorical(y_train,num_classes=max(label)+1)\ny_test=np_utils.to_categorical(y_test,num_classes=max(label)+1)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:10:31.125188Z","iopub.execute_input":"2022-03-26T02:10:31.125687Z","iopub.status.idle":"2022-03-26T02:10:31.212333Z","shell.execute_reply.started":"2022-03-26T02:10:31.125663Z","shell.execute_reply":"2022-03-26T02:10:31.210842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nmodel=Sequential()\nmodel.add(Convolution2D(filters=32,kernel_size=(3,3),input_shape=(32,32,3),padding='same'))\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\nmodel.add(Dropout(rate=0.35))\n\nmodel.add(MaxPool2D(pool_size=(2,2),padding='same'))\n\nmodel.add(Convolution2D(filters=64,kernel_size=(3,3),padding='same'))\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\nmodel.add(Dropout(rate=0.45))\n\nmodel.add(MaxPool2D(pool_size=(2,2),padding='same'))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(1024,activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\nmodel.add(Dropout(rate=0.75))\n\nmodel.add(Dense(max(label)+1,activation='softmax'))\n\nmodel.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n\ntrain_history=model.fit(X_train,y_train,validation_split=0.2,epochs=20,batch_size=128,verbose=1)\naccuracy=model.evaluate(X_test,y_test,verbose=1)\nprint(\"test accuracy:\",accuracy[1])#accuracy for test set","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:10:45.054662Z","iopub.execute_input":"2022-03-26T02:10:45.054929Z","iopub.status.idle":"2022-03-26T02:13:29.379308Z","shell.execute_reply.started":"2022-03-26T02:10:45.054900Z","shell.execute_reply":"2022-03-26T02:13:29.378689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#That above was faster than I imagined.","metadata":{}},{"cell_type":"markdown","source":"#Don't abbreviate acc (accuracy) as the original code.","metadata":{}},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\ndef show_train_history(train_history,train,validation):\n        plt.plot(train_history.history[train])\n        plt.plot(train_history.history[validation])\n        plt.title('Train History')\n        plt.ylabel('train')\n        plt.xlabel('Epoch')\n        plt.legend(['train','validation'],loc='upper left')\n        plt.show()\n\nshow_train_history(train_history,'accuracy','val_accuracy') #acc:accuracy for training set. val_acc:accuracy for validation.","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:15:24.609882Z","iopub.execute_input":"2022-03-26T02:15:24.610644Z","iopub.status.idle":"2022-03-26T02:15:24.791853Z","shell.execute_reply.started":"2022-03-26T02:15:24.610602Z","shell.execute_reply":"2022-03-26T02:15:24.790909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nprediction=model.predict(X_test)#original update to predict\nprint(prediction[0:10])","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:17:23.318827Z","iopub.execute_input":"2022-03-26T02:17:23.319069Z","iopub.status.idle":"2022-03-26T02:17:25.087507Z","shell.execute_reply.started":"2022-03-26T02:17:23.319046Z","shell.execute_reply":"2022-03-26T02:17:25.085875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Checking the test image","metadata":{}},{"cell_type":"code","source":"img2 = cv.imread('../input/iwildcam2022-fgvc9/test/test/94cf8632-21bc-11ea-a13a-137349068a90.jpg')\nprint( img2.shape )","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:32:08.842455Z","iopub.execute_input":"2022-03-26T02:32:08.842976Z","iopub.status.idle":"2022-03-26T02:32:08.865466Z","shell.execute_reply.started":"2022-03-26T02:32:08.842942Z","shell.execute_reply":"2022-03-26T02:32:08.864189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nimg_test=[]\nfilename_test=wcam4.id[:10000]\nfor file in filename_test:\n    image=cv2.imread(\"../input/iwildcam2022-fgvc9/test/test/\"+file+'.jpg')\n    res=cv2.resize(image,(32,32))\n    img_test.append(res)\nimg_test=np.array(img_test)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T02:35:30.041114Z","iopub.execute_input":"2022-03-26T02:35:30.041340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#I gave up after the \"Corrupt JPEG data: premature end of data segment\" above.","metadata":{}},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nprediction=model.predict(img_test)\nprint(prediction[0:10])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Jensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nsubmit=pd.DataFrame({'Id':filename_test,'Predicted':prediction})\nsubmit.to_csv('submission.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Since I didn't make any image of the Wild. Just try a last one.","metadata":{}},{"cell_type":"code","source":"from PIL import Image\n\nimgs_dir = '../input/iwildcam2022-fgvc9/train/train/'\nImage.open(imgs_dir + '86760c00-21bc-11ea-a13a-137349068a90.jpg')","metadata":{"execution":{"iopub.status.busy":"2022-03-26T03:04:01.421893Z","iopub.execute_input":"2022-03-26T03:04:01.422233Z","iopub.status.idle":"2022-03-26T03:04:02.389440Z","shell.execute_reply.started":"2022-03-26T03:04:01.422184Z","shell.execute_reply":"2022-03-26T03:04:02.388349Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Acknowledgements:\n\nJensen https://www.kaggle.com/code/a03102030/simple-cnn-for-top-10000-data\n\nVentakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook","metadata":{}}]}