{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n'''for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))'''\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-06-24T01:59:33.769991Z","iopub.status.busy":"2021-06-24T01:59:33.769375Z","iopub.status.idle":"2021-06-24T01:59:33.773194Z","shell.execute_reply":"2021-06-24T01:59:33.773594Z","shell.execute_reply.started":"2021-06-24T01:12:58.658786Z"},"papermill":{"duration":0.053169,"end_time":"2021-06-24T01:59:33.773802","exception":false,"start_time":"2021-06-24T01:59:33.720633","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ntrain_csv = pd.read_csv('/kaggle/input/landmark-recognition-2020/train.csv')\ntrain_csv.head(10)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T01:59:33.858203Z","iopub.status.busy":"2021-06-24T01:59:33.857645Z","iopub.status.idle":"2021-06-24T01:59:35.572263Z","shell.execute_reply":"2021-06-24T01:59:35.57354Z","shell.execute_reply.started":"2021-06-24T01:12:58.675687Z"},"papermill":{"duration":1.761795,"end_time":"2021-06-24T01:59:35.57376","exception":false,"start_time":"2021-06-24T01:59:33.811965","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_csv)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T01:59:35.682953Z","iopub.status.busy":"2021-06-24T01:59:35.682167Z","iopub.status.idle":"2021-06-24T01:59:35.685522Z","shell.execute_reply":"2021-06-24T01:59:35.685939Z","shell.execute_reply.started":"2021-06-24T01:13:00.357098Z"},"papermill":{"duration":0.052466,"end_time":"2021-06-24T01:59:35.686078","exception":false,"start_time":"2021-06-24T01:59:35.633612","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_csv['landmark_id'].unique())","metadata":{"execution":{"iopub.execute_input":"2021-06-24T01:59:35.788378Z","iopub.status.busy":"2021-06-24T01:59:35.7874Z","iopub.status.idle":"2021-06-24T01:59:35.809145Z","shell.execute_reply":"2021-06-24T01:59:35.808687Z","shell.execute_reply.started":"2021-06-24T01:13:00.36549Z"},"papermill":{"duration":0.085248,"end_time":"2021-06-24T01:59:35.809262","exception":false,"start_time":"2021-06-24T01:59:35.724014","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.image as img\nimport matplotlib.pyplot as plt\ndef show_img(file_name):\n    image = img.imread('/kaggle/input/landmark-recognition-2020/train/'+file_name[0]+'/'+file_name[1]+'/'+file_name[2]+'/'+file_name)\n    plt.imshow(image)\n    plt.show()","metadata":{"execution":{"iopub.execute_input":"2021-06-24T01:59:35.912444Z","iopub.status.busy":"2021-06-24T01:59:35.911051Z","iopub.status.idle":"2021-06-24T01:59:35.913989Z","shell.execute_reply":"2021-06-24T01:59:35.91358Z","shell.execute_reply.started":"2021-06-24T01:13:03.22602Z"},"papermill":{"duration":0.05489,"end_time":"2021-06-24T01:59:35.914146","exception":false,"start_time":"2021-06-24T01:59:35.859256","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_img('0000059611c7d079.jpg')","metadata":{"execution":{"iopub.execute_input":"2021-06-24T01:59:36.001112Z","iopub.status.busy":"2021-06-24T01:59:36.000304Z","iopub.status.idle":"2021-06-24T01:59:36.240603Z","shell.execute_reply":"2021-06-24T01:59:36.241047Z","shell.execute_reply.started":"2021-06-24T01:13:03.50415Z"},"papermill":{"duration":0.288948,"end_time":"2021-06-24T01:59:36.241197","exception":false,"start_time":"2021-06-24T01:59:35.952249","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.038049,"end_time":"2021-06-24T01:59:36.318129","exception":false,"start_time":"2021-06-24T01:59:36.28008","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_csv = pd.read_csv('/kaggle/input/landmark-recognition-2020/train.csv')\ntrain_csv.head(10)\n\n# put .jpg into the file name\ndef add_txt(fn):\n    return fn+'.jpg'\n\ntrain_csv['id'] = train_csv['id'].apply(add_txt)\n\n\n\n# choose those labels with more than 200 images, and choose the first 200 images of each label\n# move every training files to the same folder\n%cd /kaggle/working\nif not os.path.exists('training'):\n    os.mkdir('training')\nif not os.path.exists('validation'):\n    os.mkdir('validation')\nif not os.path.exists('testing'):\n    os.mkdir('testing')    \n\nimport shutil\nimport random\n\nlabel_list = train_csv['landmark_id'].unique()\ncnt = 0\nfinal_label_list = []\n\nfor label in list(label_list): # label order by random\n    file_list = list(train_csv['id'][train_csv['landmark_id']==label])\n    if len(file_list) >= 200:\n        final_label_list.append(label)\n        if not os.path.exists('/kaggle/working/training/'+str(label)):\n            os.mkdir('/kaggle/working/training/'+str(label))\n        if not os.path.exists('/kaggle/working/validation/'+str(label)):\n            os.mkdir('/kaggle/working/validation/'+str(label))\n        if not os.path.exists('/kaggle/working/testing/'+str(label)):\n            os.mkdir('/kaggle/working/testing/'+str(label))\n        for file in file_list[:120]:  # 120 files for training\n            src = '/kaggle/input/landmark-recognition-2020/train/'+file[0]+'/'+file[1]+'/'+file[2]+'/'+file\n            dst = '/kaggle/working/training/'+str(label)+'/'+file\n            if not os.path.exists(dst):\n                shutil.copyfile(src, dst)\n        for file in file_list[120:160]: # 40 files for validation\n            src = '/kaggle/input/landmark-recognition-2020/train/'+file[0]+'/'+file[1]+'/'+file[2]+'/'+file\n            dst = '/kaggle/working/validation/'+str(label)+'/'+file\n            if not os.path.exists(dst):\n                shutil.copyfile(src, dst)\n        for file in file_list[160:200]: # 40 files for testing\n            src = '/kaggle/input/landmark-recognition-2020/train/'+file[0]+'/'+file[1]+'/'+file[2]+'/'+file\n            dst = '/kaggle/working/testing/'+str(label)+'/'+file\n            if not os.path.exists(dst):\n                shutil.copyfile(src, dst)\n        cnt += 1\n    if cnt == 100: # only need 100 labels\n        break\n# 20,000 files in total","metadata":{"execution":{"iopub.execute_input":"2021-06-24T01:59:36.421337Z","iopub.status.busy":"2021-06-24T01:59:36.420581Z","iopub.status.idle":"2021-06-24T02:02:30.417671Z","shell.execute_reply":"2021-06-24T02:02:30.416605Z","shell.execute_reply.started":"2021-06-24T01:13:06.352569Z"},"papermill":{"duration":174.061282,"end_time":"2021-06-24T02:02:30.417844","exception":false,"start_time":"2021-06-24T01:59:36.356562","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(final_label_list)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T02:02:30.501447Z","iopub.status.busy":"2021-06-24T02:02:30.500809Z","iopub.status.idle":"2021-06-24T02:02:30.504144Z","shell.execute_reply":"2021-06-24T02:02:30.504525Z","shell.execute_reply.started":"2021-06-24T01:17:23.554165Z"},"papermill":{"duration":0.047319,"end_time":"2021-06-24T02:02:30.504657","exception":false,"start_time":"2021-06-24T02:02:30.457338","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(os.listdir('./training')), len(os.listdir('./validation')), len(os.listdir('./testing')))","metadata":{"execution":{"iopub.execute_input":"2021-06-24T02:02:30.586914Z","iopub.status.busy":"2021-06-24T02:02:30.586385Z","iopub.status.idle":"2021-06-24T02:02:30.592784Z","shell.execute_reply":"2021-06-24T02:02:30.59221Z","shell.execute_reply.started":"2021-06-24T01:17:23.561668Z"},"papermill":{"duration":0.049068,"end_time":"2021-06-24T02:02:30.592934","exception":false,"start_time":"2021-06-24T02:02:30.543866","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=40,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True)\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntrain_dir = '/kaggle/working/training'\nvalidation_dir = '/kaggle/working/validation'\ntest_dir = '/kaggle/working/testing'\n\ntrain_generator = train_datagen.flow_from_directory(\n    train_dir,\n    target_size=(256, 256),\n    batch_size = 32,\n    class_mode='categorical',\n    seed=42)\n\nvalidation_generator = test_datagen.flow_from_directory(\n    validation_dir,\n    target_size=(256, 256),\n    batch_size = 32,\n    class_mode='categorical',\n    seed=42)\n\ntest_generator = test_datagen.flow_from_directory(\n    test_dir,\n    target_size=(256, 256),\n    batch_size = 1,\n    class_mode='categorical',\n    seed=42)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T02:02:30.679008Z","iopub.status.busy":"2021-06-24T02:02:30.678462Z","iopub.status.idle":"2021-06-24T02:02:36.531552Z","shell.execute_reply":"2021-06-24T02:02:36.531135Z","shell.execute_reply.started":"2021-06-24T01:17:23.574346Z"},"papermill":{"duration":5.898743,"end_time":"2021-06-24T02:02:36.531686","exception":false,"start_time":"2021-06-24T02:02:30.632943","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator.class_indices.keys()","metadata":{"execution":{"iopub.execute_input":"2021-06-24T02:02:36.61851Z","iopub.status.busy":"2021-06-24T02:02:36.617333Z","iopub.status.idle":"2021-06-24T02:02:36.621142Z","shell.execute_reply":"2021-06-24T02:02:36.621538Z","shell.execute_reply.started":"2021-06-24T01:17:30.268064Z"},"papermill":{"duration":0.0494,"end_time":"2021-06-24T02:02:36.621674","exception":false,"start_time":"2021-06-24T02:02:36.572274","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import MobileNetV2\n\nconv_base = MobileNetV2(include_top=False,\n                        weights=\"imagenet\",\n                        input_shape=(256, 256, 3)\n)\n\nconv_base.trainable = True\n\nconv_base.summary()","metadata":{"execution":{"iopub.execute_input":"2021-06-24T02:02:36.708571Z","iopub.status.busy":"2021-06-24T02:02:36.707939Z","iopub.status.idle":"2021-06-24T02:02:40.435538Z","shell.execute_reply":"2021-06-24T02:02:40.434873Z","shell.execute_reply.started":"2021-06-23T05:32:28.430288Z"},"papermill":{"duration":3.774233,"end_time":"2021-06-24T02:02:40.435717","exception":false,"start_time":"2021-06-24T02:02:36.661484","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import Dense, Dropout, MaxPooling2D, GlobalAveragePooling2D, Flatten, Conv2D, Input\nfrom keras.models import Sequential\nfrom keras import optimizers\nimport tensorflow as tf\n\nmodel = Sequential()\nmodel.add(conv_base)\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Dense(100, activation='softmax'))\nmodel.compile(optimizer=optimizers.RMSprop(lr=2e-5),\n              loss = 'categorical_crossentropy',\n              metrics=['accuracy'])\n\nmodel.summary()","metadata":{"execution":{"iopub.execute_input":"2021-06-24T02:02:40.538256Z","iopub.status.busy":"2021-06-24T02:02:40.534599Z","iopub.status.idle":"2021-06-24T02:02:40.888236Z","shell.execute_reply":"2021-06-24T02:02:40.888643Z","shell.execute_reply.started":"2021-06-23T05:32:30.056871Z"},"papermill":{"duration":0.402372,"end_time":"2021-06-24T02:02:40.888786","exception":false,"start_time":"2021-06-24T02:02:40.486414","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.042156,"end_time":"2021-06-24T02:02:40.973383","exception":false,"start_time":"2021-06-24T02:02:40.931227","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.042818,"end_time":"2021-06-24T02:02:41.058437","exception":false,"start_time":"2021-06-24T02:02:41.015619","status":"completed"},"tags":[]}},{"cell_type":"code","source":"history = model.fit(\n    train_generator,\n    epochs=100, \n    validation_data=validation_generator,\n    verbose=2\n)\n","metadata":{"execution":{"iopub.execute_input":"2021-06-24T02:02:41.159454Z","iopub.status.busy":"2021-06-24T02:02:41.158656Z","iopub.status.idle":"2021-06-24T04:53:47.501489Z","shell.execute_reply":"2021-06-24T04:53:47.501929Z","shell.execute_reply.started":"2021-06-23T02:40:56.944055Z"},"papermill":{"duration":10266.399951,"end_time":"2021-06-24T04:53:47.502106","exception":false,"start_time":"2021-06-24T02:02:41.102155","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the training results\nimport matplotlib.pyplot as plt\n\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(1, len(acc)+1)\n\nplt.plot(epochs, acc, '#21466C', label='Training acc')\nplt.plot(epochs, val_acc, '#ff0051', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, '#21466C', label='Training loss')\nplt.plot(epochs, val_loss, '#ff0051', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:53:47.676698Z","iopub.status.busy":"2021-06-24T04:53:47.675612Z","iopub.status.idle":"2021-06-24T04:53:48.106406Z","shell.execute_reply":"2021-06-24T04:53:48.105675Z","shell.execute_reply.started":"2021-06-23T02:45:19.256543Z"},"papermill":{"duration":0.542565,"end_time":"2021-06-24T04:53:48.106551","exception":false,"start_time":"2021-06-24T04:53:47.563986","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = model.evaluate(test_generator)\nprint('loss:', scores[0])\nprint('accuracy:', scores[1])","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:53:48.242184Z","iopub.status.busy":"2021-06-24T04:53:48.241311Z","iopub.status.idle":"2021-06-24T04:54:25.498851Z","shell.execute_reply":"2021-06-24T04:54:25.499331Z","shell.execute_reply.started":"2021-06-23T02:45:19.525226Z"},"papermill":{"duration":37.326371,"end_time":"2021-06-24T04:54:25.499494","exception":false,"start_time":"2021-06-24T04:53:48.173123","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list = list(test_generator.class_indices.keys())\nlabel_list[:10]","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:54:26.00578Z","iopub.status.busy":"2021-06-24T04:54:26.005238Z","iopub.status.idle":"2021-06-24T04:54:26.0114Z","shell.execute_reply":"2021-06-24T04:54:26.010574Z","shell.execute_reply.started":"2021-06-23T02:45:57.390905Z"},"papermill":{"duration":0.24672,"end_time":"2021-06-24T04:54:26.011517","exception":false,"start_time":"2021-06-24T04:54:25.764797","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(test_generator)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:54:26.490889Z","iopub.status.busy":"2021-06-24T04:54:26.490157Z","iopub.status.idle":"2021-06-24T04:54:59.311189Z","shell.execute_reply":"2021-06-24T04:54:59.310618Z","shell.execute_reply.started":"2021-06-23T02:45:57.399445Z"},"papermill":{"duration":33.062984,"end_time":"2021-06-24T04:54:59.311349","exception":false,"start_time":"2021-06-24T04:54:26.248365","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred.shape","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:54:59.789607Z","iopub.status.busy":"2021-06-24T04:54:59.788879Z","iopub.status.idle":"2021-06-24T04:54:59.792182Z","shell.execute_reply":"2021-06-24T04:54:59.79254Z","shell.execute_reply.started":"2021-06-23T02:46:31.85112Z"},"papermill":{"duration":0.24304,"end_time":"2021-06-24T04:54:59.792672","exception":false,"start_time":"2021-06-24T04:54:59.549632","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max(y_pred[0])","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:00.271069Z","iopub.status.busy":"2021-06-24T04:55:00.270097Z","iopub.status.idle":"2021-06-24T04:55:00.273331Z","shell.execute_reply":"2021-06-24T04:55:00.273758Z","shell.execute_reply.started":"2021-06-23T02:46:31.858996Z"},"papermill":{"duration":0.245362,"end_time":"2021-06-24T04:55:00.273901","exception":false,"start_time":"2021-06-24T04:55:00.028539","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_ls = y_pred.tolist()","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:00.770012Z","iopub.status.busy":"2021-06-24T04:55:00.769192Z","iopub.status.idle":"2021-06-24T04:55:00.772025Z","shell.execute_reply":"2021-06-24T04:55:00.771499Z","shell.execute_reply.started":"2021-06-23T02:46:31.873248Z"},"papermill":{"duration":0.262035,"end_time":"2021-06-24T04:55:00.772173","exception":false,"start_time":"2021-06-24T04:55:00.510138","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.243105,"end_time":"2021-06-24T04:55:01.262325","exception":false,"start_time":"2021-06-24T04:55:01.01922","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = []\n\nfor i in range(len(y_pred_ls)):\n    max_value = max(y_pred_ls[i])\n    max_index = y_pred_ls[i].index(max_value)\n    y.append(max_index)\n    \ny = np.array(y)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:01.779197Z","iopub.status.busy":"2021-06-24T04:55:01.778459Z","iopub.status.idle":"2021-06-24T04:55:01.781401Z","shell.execute_reply":"2021-06-24T04:55:01.780933Z","shell.execute_reply.started":"2021-06-23T02:46:31.913147Z"},"papermill":{"duration":0.256938,"end_time":"2021-06-24T04:55:01.781526","exception":false,"start_time":"2021-06-24T04:55:01.524588","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\nfor i in range(len(y_pred_ls)):\n    scores.append(max(y_pred[i]))\n    \nscores = np.array(scores)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:02.289793Z","iopub.status.busy":"2021-06-24T04:55:02.284595Z","iopub.status.idle":"2021-06-24T04:55:02.349849Z","shell.execute_reply":"2021-06-24T04:55:02.349399Z","shell.execute_reply.started":"2021-06-23T02:46:31.931133Z"},"papermill":{"duration":0.3311,"end_time":"2021-06-24T04:55:02.349962","exception":false,"start_time":"2021-06-24T04:55:02.018862","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:02.829799Z","iopub.status.busy":"2021-06-24T04:55:02.829016Z","iopub.status.idle":"2021-06-24T04:55:02.832472Z","shell.execute_reply":"2021-06-24T04:55:02.832965Z","shell.execute_reply.started":"2021-06-23T02:46:32.026354Z"},"papermill":{"duration":0.245284,"end_time":"2021-06-24T04:55:02.833102","exception":false,"start_time":"2021-06-24T04:55:02.587818","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:03.314528Z","iopub.status.busy":"2021-06-24T04:55:03.313705Z","iopub.status.idle":"2021-06-24T04:55:03.317553Z","shell.execute_reply":"2021-06-24T04:55:03.317064Z","shell.execute_reply.started":"2021-06-23T02:46:32.033525Z"},"papermill":{"duration":0.247591,"end_time":"2021-06-24T04:55:03.317661","exception":false,"start_time":"2021-06-24T04:55:03.07007","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, roc_auc_score\nfpr, tpr, thresholds = roc_curve(y, scores, pos_label=0)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:03.801343Z","iopub.status.busy":"2021-06-24T04:55:03.800735Z","iopub.status.idle":"2021-06-24T04:55:04.35689Z","shell.execute_reply":"2021-06-24T04:55:04.355771Z","shell.execute_reply.started":"2021-06-23T02:46:32.045085Z"},"papermill":{"duration":0.79921,"end_time":"2021-06-24T04:55:04.357043","exception":false,"start_time":"2021-06-24T04:55:03.557833","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_0_1 = []\nauc=[]\nfor j in y:\n    if j != 10:\n        y_0_1.append(0)\n    else:\n        y_0_1.append(1)\ny_0_1 = np.array(y_0_1)\nauc_of_the_label = roc_auc_score(y_0_1, scores)\nauc.append(auc_of_the_label)\nauc","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:05.035524Z","iopub.status.busy":"2021-06-24T04:55:05.034708Z","iopub.status.idle":"2021-06-24T04:55:05.045243Z","shell.execute_reply":"2021-06-24T04:55:05.046191Z","shell.execute_reply.started":"2021-06-23T03:13:07.044941Z"},"papermill":{"duration":0.433678,"end_time":"2021-06-24T04:55:05.046376","exception":false,"start_time":"2021-06-24T04:55:04.612698","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.metrics import roc_curve, roc_auc_score\n\nauc=[]\n\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.legend()\nplt.title('ROC')\n\nfor i in range(100):\n    fpr, tpr, _ = roc_curve(y, scores, pos_label=i)\n    plt.plot(fpr, tpr, label=i)\n    \n    y_0_1 = []\n    for j in y:\n        if j != i:\n            y_0_1.append(0)\n        else:\n            y_0_1.append(1)\n    y_0_1 = np.array(y_0_1)\n    auc_of_the_label = roc_auc_score(y_0_1, scores)\n    auc.append(auc_of_the_label)\n\n\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.legend()\nplt.title('ROC')\n\nplt.show()\n\n","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:05.567307Z","iopub.status.busy":"2021-06-24T04:55:05.566389Z","iopub.status.idle":"2021-06-24T04:55:07.570975Z","shell.execute_reply":"2021-06-24T04:55:07.57139Z","shell.execute_reply.started":"2021-06-23T03:18:03.536188Z"},"papermill":{"duration":2.255989,"end_time":"2021-06-24T04:55:07.571562","exception":false,"start_time":"2021-06-24T04:55:05.315573","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:08.06194Z","iopub.status.busy":"2021-06-24T04:55:08.061166Z","iopub.status.idle":"2021-06-24T04:55:08.064662Z","shell.execute_reply":"2021-06-24T04:55:08.06508Z","shell.execute_reply.started":"2021-06-23T02:51:25.721659Z"},"papermill":{"duration":0.251225,"end_time":"2021-06-24T04:55:08.065215","exception":false,"start_time":"2021-06-24T04:55:07.81399","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max(auc)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:08.556186Z","iopub.status.busy":"2021-06-24T04:55:08.555372Z","iopub.status.idle":"2021-06-24T04:55:08.558794Z","shell.execute_reply":"2021-06-24T04:55:08.559214Z"},"papermill":{"duration":0.25073,"end_time":"2021-06-24T04:55:08.559355","exception":false,"start_time":"2021-06-24T04:55:08.308625","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(auc)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:09.098266Z","iopub.status.busy":"2021-06-24T04:55:09.097464Z","iopub.status.idle":"2021-06-24T04:55:09.101338Z","shell.execute_reply":"2021-06-24T04:55:09.100943Z"},"papermill":{"duration":0.299752,"end_time":"2021-06-24T04:55:09.101449","exception":false,"start_time":"2021-06-24T04:55:08.801697","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list = list(test_generator.class_indices.keys())","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:09.590914Z","iopub.status.busy":"2021-06-24T04:55:09.59022Z","iopub.status.idle":"2021-06-24T04:55:09.593478Z","shell.execute_reply":"2021-06-24T04:55:09.593079Z","shell.execute_reply.started":"2021-06-24T01:40:10.277916Z"},"papermill":{"duration":0.249044,"end_time":"2021-06-24T04:55:09.593589","exception":false,"start_time":"2021-06-24T04:55:09.344545","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list[:10]","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:10.086983Z","iopub.status.busy":"2021-06-24T04:55:10.086187Z","iopub.status.idle":"2021-06-24T04:55:10.090125Z","shell.execute_reply":"2021-06-24T04:55:10.089652Z","shell.execute_reply.started":"2021-06-24T01:41:33.972587Z"},"papermill":{"duration":0.254724,"end_time":"2021-06-24T04:55:10.090254","exception":false,"start_time":"2021-06-24T04:55:09.83553","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_original_label(new_label): # 從新的label對應回原本資料的label\n    return int(label_list[new_label])","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:10.584226Z","iopub.status.busy":"2021-06-24T04:55:10.583494Z","iopub.status.idle":"2021-06-24T04:55:10.586365Z","shell.execute_reply":"2021-06-24T04:55:10.585854Z","shell.execute_reply.started":"2021-06-24T01:56:08.459624Z"},"papermill":{"duration":0.251492,"end_time":"2021-06-24T04:55:10.586473","exception":false,"start_time":"2021-06-24T04:55:10.334981","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_auc_label = to_original_label(auc.index(max(auc))) # label with the highest AUC\nmin_auc_label = to_original_label(auc.index(min(auc))) # label with the lowest AUC\nprint('max auc label', max_auc_label)\nprint('min auc label', min_auc_label)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:11.079091Z","iopub.status.busy":"2021-06-24T04:55:11.07821Z","iopub.status.idle":"2021-06-24T04:55:11.082274Z","shell.execute_reply":"2021-06-24T04:55:11.081799Z"},"papermill":{"duration":0.252945,"end_time":"2021-06-24T04:55:11.082399","exception":false,"start_time":"2021-06-24T04:55:10.829454","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test by eyes","metadata":{"papermill":{"duration":0.243854,"end_time":"2021-06-24T04:55:11.568796","exception":false,"start_time":"2021-06-24T04:55:11.324942","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"First, images with the highest AUC score","metadata":{"papermill":{"duration":0.244462,"end_time":"2021-06-24T04:55:12.056785","exception":false,"start_time":"2021-06-24T04:55:11.812323","status":"completed"},"tags":[]}},{"cell_type":"code","source":"file_list = train_csv[train_csv['landmark_id']==max_auc_label]['id']\nfile_list","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:12.551921Z","iopub.status.busy":"2021-06-24T04:55:12.551088Z","iopub.status.idle":"2021-06-24T04:55:12.55893Z","shell.execute_reply":"2021-06-24T04:55:12.558414Z","shell.execute_reply.started":"2021-06-24T01:50:38.3978Z"},"papermill":{"duration":0.258443,"end_time":"2021-06-24T04:55:12.559049","exception":false,"start_time":"2021-06-24T04:55:12.300606","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in file_list[:20]:\n    show_img(file)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:13.052371Z","iopub.status.busy":"2021-06-24T04:55:13.051693Z","iopub.status.idle":"2021-06-24T04:55:53.998037Z","shell.execute_reply":"2021-06-24T04:55:53.998443Z","shell.execute_reply.started":"2021-06-24T01:53:26.22139Z"},"papermill":{"duration":41.195371,"end_time":"2021-06-24T04:55:53.99859","exception":false,"start_time":"2021-06-24T04:55:12.803219","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then, images with the lowest AUC score","metadata":{"papermill":{"duration":0.954843,"end_time":"2021-06-24T04:55:55.917431","exception":false,"start_time":"2021-06-24T04:55:54.962588","status":"completed"},"tags":[]}},{"cell_type":"code","source":"file_list = train_csv[train_csv['landmark_id']==min_auc_label]['id']\nfile_list","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:57.823897Z","iopub.status.busy":"2021-06-24T04:55:57.823102Z","iopub.status.idle":"2021-06-24T04:55:57.831571Z","shell.execute_reply":"2021-06-24T04:55:57.831061Z"},"papermill":{"duration":0.967981,"end_time":"2021-06-24T04:55:57.831714","exception":false,"start_time":"2021-06-24T04:55:56.863733","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in file_list[:20]:\n    show_img(file)","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:55:59.820609Z","iopub.status.busy":"2021-06-24T04:55:59.819683Z","iopub.status.idle":"2021-06-24T04:58:36.340018Z","shell.execute_reply":"2021-06-24T04:58:36.340431Z"},"papermill":{"duration":157.5534,"end_time":"2021-06-24T04:58:36.340591","exception":false,"start_time":"2021-06-24T04:55:58.787191","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Just view some beautiful images","metadata":{"papermill":{"duration":3.249877,"end_time":"2021-06-24T04:58:42.831302","exception":false,"start_time":"2021-06-24T04:58:39.581425","status":"completed"},"tags":[]}},{"cell_type":"code","source":"image_index = 0 # choose a image (0-3999)\nimage = test_generator[image_index][0] # [0][0]: the first 0 stands for the 0th image, the second 0 stands for the image array\nimage = image.reshape((256,256,3))\nplt.imshow(image)\nplt.show()\nprint(test_generator[0][1][0]) # array of the image's label","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:58:49.926499Z","iopub.status.busy":"2021-06-24T04:58:49.925699Z","iopub.status.idle":"2021-06-24T04:58:50.076917Z","shell.execute_reply":"2021-06-24T04:58:50.077312Z","shell.execute_reply.started":"2021-06-24T01:45:08.774424Z"},"papermill":{"duration":3.528864,"end_time":"2021-06-24T04:58:50.077451","exception":false,"start_time":"2021-06-24T04:58:46.548587","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_label(arr):\n    for i in range(len(arr)):\n        if arr[i] == max(arr):\n            return label_list[i]","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:58:56.820134Z","iopub.status.busy":"2021-06-24T04:58:56.819299Z","iopub.status.idle":"2021-06-24T04:58:56.822164Z","shell.execute_reply":"2021-06-24T04:58:56.821721Z","shell.execute_reply.started":"2021-06-24T01:45:09.496898Z"},"papermill":{"duration":3.446484,"end_time":"2021-06-24T04:58:56.822277","exception":false,"start_time":"2021-06-24T04:58:53.375793","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label from original data\nget_label(test_generator[image_index][1][0])","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:59:03.39597Z","iopub.status.busy":"2021-06-24T04:59:03.395208Z","iopub.status.idle":"2021-06-24T04:59:03.407079Z","shell.execute_reply":"2021-06-24T04:59:03.406556Z","shell.execute_reply.started":"2021-06-24T01:45:21.347823Z"},"papermill":{"duration":3.329493,"end_time":"2021-06-24T04:59:03.407197","exception":false,"start_time":"2021-06-24T04:59:00.077704","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label by prediction\nget_label(model.predict(test_generator[image_index][0])[0])","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:59:10.26726Z","iopub.status.busy":"2021-06-24T04:59:10.266483Z","iopub.status.idle":"2021-06-24T04:59:10.933408Z","shell.execute_reply":"2021-06-24T04:59:10.932525Z","shell.execute_reply.started":"2021-06-23T01:24:32.330806Z"},"papermill":{"duration":4.162179,"end_time":"2021-06-24T04:59:10.933549","exception":false,"start_time":"2021-06-24T04:59:06.77137","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = get_label(model.predict(test_generator[image_index][0])[0])\nimage_list = [] # image with the same label\nfor i in range(4000):\n    if get_label(test_generator[i][1][0]) == label:\n        image_list.append(i)\n","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:59:17.49123Z","iopub.status.busy":"2021-06-24T04:59:17.490391Z","iopub.status.idle":"2021-06-24T04:59:44.583534Z","shell.execute_reply":"2021-06-24T04:59:44.582576Z","shell.execute_reply.started":"2021-06-23T01:24:37.636788Z"},"papermill":{"duration":30.369218,"end_time":"2021-06-24T04:59:44.583678","exception":false,"start_time":"2021-06-24T04:59:14.21446","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(image_list) > 10:\n    for i in range(10):\n        if i != image_index:\n            image = test_generator[image_list[i]][0]\n            image = image.reshape((256,256,3))\n            plt.imshow(image)\n            plt.show()\n        \nelse:\n    for i in range(len(image_list)):\n        if i != image_index:\n            image = test_generator[image_list[i]][0]\n            image = image.reshape((256,256,3))\n            plt.imshow(image)\n            plt.show()\n","metadata":{"execution":{"iopub.execute_input":"2021-06-24T04:59:51.290843Z","iopub.status.busy":"2021-06-24T04:59:51.290067Z","iopub.status.idle":"2021-06-24T04:59:52.952894Z","shell.execute_reply":"2021-06-24T04:59:52.952402Z","shell.execute_reply.started":"2021-06-23T01:25:03.691168Z"},"papermill":{"duration":5.105395,"end_time":"2021-06-24T04:59:52.953042","exception":false,"start_time":"2021-06-24T04:59:47.847647","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":3.289273,"end_time":"2021-06-24T04:59:59.558142","exception":false,"start_time":"2021-06-24T04:59:56.268869","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":3.327527,"end_time":"2021-06-24T05:00:06.359189","exception":false,"start_time":"2021-06-24T05:00:03.031662","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}