{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport os, sys, glob\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport keras.backend as K\nfrom IPython.display import Image\nimport random\nimport tqdm\nimport seaborn as sns\nfrom sklearn.model_selection import GridSearchCV\nfrom keras.wrappers.scikit_learn import KerasClassifier\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimgdir = \"../input/background-removed-happywhale-dataset/\"\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:19.588988Z","iopub.execute_input":"2022-04-28T22:11:19.589237Z","iopub.status.idle":"2022-04-28T22:11:19.600262Z","shell.execute_reply.started":"2022-04-28T22:11:19.589207Z","shell.execute_reply":"2022-04-28T22:11:19.599497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install visualkeras --upgrade","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:19.604625Z","iopub.execute_input":"2022-04-28T22:11:19.605398Z","iopub.status.idle":"2022-04-28T22:11:27.759192Z","shell.execute_reply.started":"2022-04-28T22:11:19.605357Z","shell.execute_reply":"2022-04-28T22:11:27.758315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style(\"white\")\nimport visualkeras as vk","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:27.762240Z","iopub.execute_input":"2022-04-28T22:11:27.762540Z","iopub.status.idle":"2022-04-28T22:11:27.769141Z","shell.execute_reply.started":"2022-04-28T22:11:27.762500Z","shell.execute_reply":"2022-04-28T22:11:27.768070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(imgdir)\nfor file in glob.glob(os.path.join(imgdir, \"*\")): \n    print(f\" \\_ {file}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:27.770692Z","iopub.execute_input":"2022-04-28T22:11:27.770958Z","iopub.status.idle":"2022-04-28T22:11:27.783892Z","shell.execute_reply.started":"2022-04-28T22:11:27.770917Z","shell.execute_reply":"2022-04-28T22:11:27.783195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.models import Sequential\nfrom keras import layers\n\nfrom keras.preprocessing import image\nfrom keras.applications.imagenet_utils import preprocess_input\nfrom keras.layers import Input, Dense, Activation, BatchNormalization, Flatten, Conv2D\nfrom keras.layers import AveragePooling2D, MaxPooling2D, Dropout\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom keras.models import Model\nfrom tensorflow.keras.optimizers import Adam","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:27.785691Z","iopub.execute_input":"2022-04-28T22:11:27.786767Z","iopub.status.idle":"2022-04-28T22:11:27.793971Z","shell.execute_reply.started":"2022-04-28T22:11:27.786732Z","shell.execute_reply":"2022-04-28T22:11:27.792783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(imgdir + \"seg_train.csv\")\ntrain = train[0:10000]\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:27.795347Z","iopub.execute_input":"2022-04-28T22:11:27.796684Z","iopub.status.idle":"2022-04-28T22:11:27.964994Z","shell.execute_reply.started":"2022-04-28T22:11:27.796647Z","shell.execute_reply":"2022-04-28T22:11:27.964254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ntrain['species'].replace('kiler_whale', 'killer_whale', inplace=True)\ntrain['species'].replace('bottlenose_dolpin', 'bottlenose_dolphin', inplace=True)\ntrain['species'].replace(('globis', 'pilot_whale'), 'short_finned_pilot_whale', \n                         inplace=True)\ntrain['species_id'] = encoder.fit_transform(train['species'])","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:27.966178Z","iopub.execute_input":"2022-04-28T22:11:27.966975Z","iopub.status.idle":"2022-04-28T22:11:27.982898Z","shell.execute_reply.started":"2022-04-28T22:11:27.966933Z","shell.execute_reply":"2022-04-28T22:11:27.981878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(dpi=200)\nsns.histplot(data=train, x='species', hue='species', stat='percent', \n             discrete=True, palette=\"rocket\")\nplt.xticks(rotation=90)\nplt.title(\"Distribution of different whale species\")\nplt.legend()\nsns.despine()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:27.984045Z","iopub.execute_input":"2022-04-28T22:11:27.984374Z","iopub.status.idle":"2022-04-28T22:11:30.212037Z","shell.execute_reply.started":"2022-04-28T22:11:27.984322Z","shell.execute_reply":"2022-04-28T22:11:30.211375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image(filename= imgdir + \"seg_img/\" + random.choice(train['image']))","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:30.213478Z","iopub.execute_input":"2022-04-28T22:11:30.214001Z","iopub.status.idle":"2022-04-28T22:11:30.228199Z","shell.execute_reply.started":"2022-04-28T22:11:30.213957Z","shell.execute_reply":"2022-04-28T22:11:30.227525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tqdm.tqdm(desc=\"processing\", total=train.shape[0]) as progress:\n    X = np.zeros((train.shape[0], 32, 32, 3))\n    #X = np.zeros((train.shape[0], 64, 64, 3))\n    for n, i in enumerate(train['image']):\n        img = image.load_img(imgdir + 'seg_img/' + i, target_size=(32, 32, 3))\n        #img = image.load_img(imgdir + 'seg_img/' + i, target_size=(64, 64, 3))\n        X[n] = preprocess_input(image.img_to_array(img))\n        progress.update(1)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:11:30.229625Z","iopub.execute_input":"2022-04-28T22:11:30.230072Z","iopub.status.idle":"2022-04-28T22:12:23.865597Z","shell.execute_reply.started":"2022-04-28T22:11:30.230037Z","shell.execute_reply":"2022-04-28T22:12:23.863614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.astype('float32') / 255.0\ny = train['species_id']","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:23.868601Z","iopub.execute_input":"2022-04-28T22:12:23.868839Z","iopub.status.idle":"2022-04-28T22:12:23.948806Z","shell.execute_reply.started":"2022-04-28T22:12:23.868806Z","shell.execute_reply":"2022-04-28T22:12:23.948003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_labels(y):\n    values = np.array(y)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n    y = onehot_encoded\n    return y, label_encoder","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:23.950094Z","iopub.execute_input":"2022-04-28T22:12:23.950463Z","iopub.status.idle":"2022-04-28T22:12:23.957366Z","shell.execute_reply.started":"2022-04-28T22:12:23.950422Z","shell.execute_reply":"2022-04-28T22:12:23.956465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=69)\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=69)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:23.958983Z","iopub.execute_input":"2022-04-28T22:12:23.959253Z","iopub.status.idle":"2022-04-28T22:12:24.040380Z","shell.execute_reply.started":"2022-04-28T22:12:23.959219Z","shell.execute_reply":"2022-04-28T22:12:24.039602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X[0])","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:24.041743Z","iopub.execute_input":"2022-04-28T22:12:24.042025Z","iopub.status.idle":"2022-04-28T22:12:24.279797Z","shell.execute_reply.started":"2022-04-28T22:12:24.041989Z","shell.execute_reply":"2022-04-28T22:12:24.279134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y, label_encoder = prepare_labels(train['individual_id'])","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:24.281009Z","iopub.execute_input":"2022-04-28T22:12:24.281402Z","iopub.status.idle":"2022-04-28T22:12:24.348662Z","shell.execute_reply.started":"2022-04-28T22:12:24.281365Z","shell.execute_reply":"2022-04-28T22:12:24.347959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import NASNetLarge\nbase_model = NASNetLarge(\n               input_shape=(32,32,3), \n               weights=None,\n               include_top=False)\n\nlayer = base_model.output\nlayer = Flatten()(layer)\npredictions = Dense(y.shape[1], activation='softmax')(layer)\nmodel = Model(inputs=base_model.input, outputs=predictions)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:24.349869Z","iopub.execute_input":"2022-04-28T22:12:24.350194Z","iopub.status.idle":"2022-04-28T22:12:30.494038Z","shell.execute_reply.started":"2022-04-28T22:12:24.350156Z","shell.execute_reply":"2022-04-28T22:12:30.493254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vk.layered_view(model)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:30.495281Z","iopub.execute_input":"2022-04-28T22:12:30.495630Z","iopub.status.idle":"2022-04-28T22:12:31.399989Z","shell.execute_reply.started":"2022-04-28T22:12:30.495592Z","shell.execute_reply":"2022-04-28T22:12:31.398573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#best = \"best.hdf5\"\n#callbacks = [\n        #ReduceLROnPlateau(monitor='val_accuracy', factor=0.2, patience=3, min_lr=1e-7),\n        #EarlyStopping(monitor='val_accuracy', patience=5, min_delta=1e-5), \n        #ModelCheckpoint(best, monitor='val_accuracy', verbose=1, save_best_only=True, mode='auto')\n    #]","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:31.401321Z","iopub.execute_input":"2022-04-28T22:12:31.402150Z","iopub.status.idle":"2022-04-28T22:12:31.405828Z","shell.execute_reply.started":"2022-04-28T22:12:31.402110Z","shell.execute_reply":"2022-04-28T22:12:31.405237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = [20, 50, 100, 256, 512]\nepochs = [10, 20, 50, 100, 150]\n#optimizer = ['RMSprop', 'Adam','SGD','Adadelta','Adagrad','Adamax','Nadam','Ftrl']\noptimizer = ['RMSprop', 'Adam']\n#neurons = [50,100]\n\ndef create_model(neurons=neurons, epochs=epochs, optimizer=optimizer, batch_size=batch_size):\n    model = Sequential()\n    model.add(Conv2D(128, 3, padding='same', activation='relu', input_shape=[32,32,3]))\n    model.add(MaxPooling2D(3))\n    model.add(Conv2D(128, 3, padding='same', activation='relu'))\n    model.add(Conv2D(128, 3, padding='same', activation='relu'))\n    model.add(MaxPooling2D(2))\n    model.add(Flatten())\n    model.add(Dense(256, activation='softmax'))\n    model.add(BatchNormalization())\n    model.add(Dense(128, activation='softmax'))\n    model.add(Dropout(0.2))\n    model.add(Dense(26, activation='softmax'))\n\n    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n\n    return model\n\n#model_BayesianOptimization = KerasClassifier(build_fn=create_model, verbose=0)\nmodel_GridSearch = KerasClassifier(build_fn=create_model, verbose=0)\n#model_RandomSearch = KerasClassifier(build_fn=create_model, verbose=0)\n\n#param_opt = dict(batch_size=batch_size, epochs=epochs, neurons=neurons, optimizer = optimizer)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:31.407139Z","iopub.execute_input":"2022-04-28T22:12:31.407758Z","iopub.status.idle":"2022-04-28T22:12:31.441970Z","shell.execute_reply.started":"2022-04-28T22:12:31.407723Z","shell.execute_reply":"2022-04-28T22:12:31.441096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#grid = GridSearchCV(estimator=model_GridSearch, param_grid=param_opt, n_jobs=1, cv=3, verbose = 0)\n#grid_result = grid.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:31.442878Z","iopub.status.idle":"2022-04-28T22:12:31.443601Z","shell.execute_reply.started":"2022-04-28T22:12:31.443355Z","shell.execute_reply":"2022-04-28T22:12:31.443380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print('according to gridsearch the best parameters are : ')\n#print('batch_size : ' + str(grid_result.best_params_['batch_size']))\n#print('epochs : ' + str(grid_result.best_params_['epochs']))\n#print('neurons : ' + str(grid_result.best_params_['neurons']))\n#print('optimizer : ' + str(grid_result.best_params_['optimizer']))","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:12:31.445174Z","iopub.status.idle":"2022-04-28T22:12:31.445828Z","shell.execute_reply.started":"2022-04-28T22:12:31.445595Z","shell.execute_reply":"2022-04-28T22:12:31.445620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#opt = Adam(learning_rate=1e-3)\n#model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy'])\nmodel.compile(loss='categorical_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n\n#history = model.fit(X_train, y_train, batch_size = 20, epochs = 10,validation_data = (X_val, y_val))\nhistory = model.fit(X, y, epochs=50, batch_size=128, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:17:26.511183Z","iopub.execute_input":"2022-04-28T22:17:26.511448Z","iopub.status.idle":"2022-04-28T22:33:17.045761Z","shell.execute_reply.started":"2022-04-28T22:17:26.511419Z","shell.execute_reply":"2022-04-28T22:33:17.044914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['accuracy'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:33:17.049230Z","iopub.execute_input":"2022-04-28T22:33:17.050014Z","iopub.status.idle":"2022-04-28T22:33:17.269941Z","shell.execute_reply.started":"2022-04-28T22:33:17.049972Z","shell.execute_reply":"2022-04-28T22:33:17.269277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.title('Model loss')\nplt.ylabel('loss')\nplt.xlabel('Epoch')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T22:33:17.271109Z","iopub.execute_input":"2022-04-28T22:33:17.271435Z","iopub.status.idle":"2022-04-28T22:33:17.478400Z","shell.execute_reply.started":"2022-04-28T22:33:17.271398Z","shell.execute_reply":"2022-04-28T22:33:17.477739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}