{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n'''for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))'''\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.053169,"end_time":"2021-06-24T01:59:33.773802","exception":false,"start_time":"2021-06-24T01:59:33.720633","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:44:59.761720Z","iopub.execute_input":"2021-11-09T06:44:59.762428Z","iopub.status.idle":"2021-11-09T06:44:59.776562Z","shell.execute_reply.started":"2021-11-09T06:44:59.762323Z","shell.execute_reply":"2021-11-09T06:44:59.775633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ntrain_csv = pd.read_csv('/kaggle/input/landmark-recognition-2020/train.csv')\ntrain_csv.head(10)","metadata":{"papermill":{"duration":1.761795,"end_time":"2021-06-24T01:59:35.57376","exception":false,"start_time":"2021-06-24T01:59:33.811965","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:44:59.778166Z","iopub.execute_input":"2021-11-09T06:44:59.778730Z","iopub.status.idle":"2021-11-09T06:45:01.293318Z","shell.execute_reply.started":"2021-11-09T06:44:59.778694Z","shell.execute_reply":"2021-11-09T06:45:01.292512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_csv)","metadata":{"papermill":{"duration":0.052466,"end_time":"2021-06-24T01:59:35.686078","exception":false,"start_time":"2021-06-24T01:59:35.633612","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:45:01.295137Z","iopub.execute_input":"2021-11-09T06:45:01.295492Z","iopub.status.idle":"2021-11-09T06:45:01.302159Z","shell.execute_reply.started":"2021-11-09T06:45:01.295454Z","shell.execute_reply":"2021-11-09T06:45:01.301291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_csv['landmark_id'].unique())","metadata":{"papermill":{"duration":0.085248,"end_time":"2021-06-24T01:59:35.809262","exception":false,"start_time":"2021-06-24T01:59:35.724014","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:45:01.303833Z","iopub.execute_input":"2021-11-09T06:45:01.304368Z","iopub.status.idle":"2021-11-09T06:45:01.329465Z","shell.execute_reply.started":"2021-11-09T06:45:01.304331Z","shell.execute_reply":"2021-11-09T06:45:01.328550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.image as img\nimport matplotlib.pyplot as plt\ndef show_img(file_name):\n    image = img.imread('/kaggle/input/landmark-recognition-2020/train/'+file_name[0]+'/'+file_name[1]+'/'+file_name[2]+'/'+file_name)\n    plt.imshow(image)\n    plt.show()","metadata":{"papermill":{"duration":0.05489,"end_time":"2021-06-24T01:59:35.914146","exception":false,"start_time":"2021-06-24T01:59:35.859256","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:45:01.330687Z","iopub.execute_input":"2021-11-09T06:45:01.331049Z","iopub.status.idle":"2021-11-09T06:45:01.335984Z","shell.execute_reply.started":"2021-11-09T06:45:01.331014Z","shell.execute_reply":"2021-11-09T06:45:01.335082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_img('0000059611c7d079.jpg')","metadata":{"papermill":{"duration":0.288948,"end_time":"2021-06-24T01:59:36.241197","exception":false,"start_time":"2021-06-24T01:59:35.952249","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:45:01.337384Z","iopub.execute_input":"2021-11-09T06:45:01.337867Z","iopub.status.idle":"2021-11-09T06:45:01.565552Z","shell.execute_reply.started":"2021-11-09T06:45:01.337824Z","shell.execute_reply":"2021-11-09T06:45:01.564692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.038049,"end_time":"2021-06-24T01:59:36.318129","exception":false,"start_time":"2021-06-24T01:59:36.28008","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_csv = pd.read_csv('/kaggle/input/landmark-recognition-2020/train.csv')\ntrain_csv.head(10)\n\n# put .jpg into the file name\ndef add_txt(fn):\n    return fn+'.jpg'\n\ntrain_csv['id'] = train_csv['id'].apply(add_txt)\n\n\n\n# choose those labels with more than 200 images, and choose the first 200 images of each label\n# move every training files to the same folder\n%cd /kaggle/working\nif not os.path.exists('training'):\n    os.mkdir('training')\nif not os.path.exists('validation'):\n    os.mkdir('validation')\nif not os.path.exists('testing'):\n    os.mkdir('testing')    \n\nimport shutil\nimport random\n\nlabel_list = train_csv['landmark_id'].unique()\ncnt = 0\nfinal_label_list = []\n\nfor label in list(label_list): # label order by random\n    file_list = list(train_csv['id'][train_csv['landmark_id']==label])\n    if len(file_list) >= 200:\n        final_label_list.append(label)\n        if not os.path.exists('/kaggle/working/training/'+str(label)):\n            os.mkdir('/kaggle/working/training/'+str(label))\n        if not os.path.exists('/kaggle/working/validation/'+str(label)):\n            os.mkdir('/kaggle/working/validation/'+str(label))\n        if not os.path.exists('/kaggle/working/testing/'+str(label)):\n            os.mkdir('/kaggle/working/testing/'+str(label))\n        for file in file_list[:120]:  # 120 files for training\n            src = '/kaggle/input/landmark-recognition-2020/train/'+file[0]+'/'+file[1]+'/'+file[2]+'/'+file\n            dst = '/kaggle/working/training/'+str(label)+'/'+file\n            if not os.path.exists(dst):\n                shutil.copyfile(src, dst)\n        for file in file_list[120:160]: # 40 files for validation\n            src = '/kaggle/input/landmark-recognition-2020/train/'+file[0]+'/'+file[1]+'/'+file[2]+'/'+file\n            dst = '/kaggle/working/validation/'+str(label)+'/'+file\n            if not os.path.exists(dst):\n                shutil.copyfile(src, dst)\n        for file in file_list[160:200]: # 40 files for testing\n            src = '/kaggle/input/landmark-recognition-2020/train/'+file[0]+'/'+file[1]+'/'+file[2]+'/'+file\n            dst = '/kaggle/working/testing/'+str(label)+'/'+file\n            if not os.path.exists(dst):\n                shutil.copyfile(src, dst)\n        cnt += 1\n    if cnt == 100: # only need 100 labels\n        break\n# 20,000 files in total","metadata":{"papermill":{"duration":174.061282,"end_time":"2021-06-24T02:02:30.417844","exception":false,"start_time":"2021-06-24T01:59:36.356562","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:45:01.566659Z","iopub.execute_input":"2021-11-09T06:45:01.566975Z","iopub.status.idle":"2021-11-09T06:48:47.462099Z","shell.execute_reply.started":"2021-11-09T06:45:01.566941Z","shell.execute_reply":"2021-11-09T06:48:47.461182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(final_label_list)","metadata":{"papermill":{"duration":0.047319,"end_time":"2021-06-24T02:02:30.504657","exception":false,"start_time":"2021-06-24T02:02:30.457338","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:48:47.467572Z","iopub.execute_input":"2021-11-09T06:48:47.468423Z","iopub.status.idle":"2021-11-09T06:48:47.475181Z","shell.execute_reply.started":"2021-11-09T06:48:47.468381Z","shell.execute_reply":"2021-11-09T06:48:47.474117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(os.listdir('./training')), len(os.listdir('./validation')), len(os.listdir('./testing')))","metadata":{"papermill":{"duration":0.049068,"end_time":"2021-06-24T02:02:30.592934","exception":false,"start_time":"2021-06-24T02:02:30.543866","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:48:47.494481Z","iopub.execute_input":"2021-11-09T06:48:47.495175Z","iopub.status.idle":"2021-11-09T06:48:47.502553Z","shell.execute_reply.started":"2021-11-09T06:48:47.495140Z","shell.execute_reply":"2021-11-09T06:48:47.501508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=40,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True)\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntrain_dir = '/kaggle/working/training'\nvalidation_dir = '/kaggle/working/validation'\ntest_dir = '/kaggle/working/testing'\n\ntrain_generator = train_datagen.flow_from_directory(\n    train_dir,\n    target_size=(256, 256),\n    batch_size = 32,\n    class_mode='categorical',\n    seed=42)\n\nvalidation_generator = test_datagen.flow_from_directory(\n    validation_dir,\n    target_size=(256, 256),\n    batch_size = 32,\n    class_mode='categorical',\n    seed=42)\n\ntest_generator = test_datagen.flow_from_directory(\n    test_dir,\n    target_size=(256, 256),\n    batch_size = 1,\n    class_mode='categorical',\n    seed=42)","metadata":{"papermill":{"duration":5.898743,"end_time":"2021-06-24T02:02:36.531686","exception":false,"start_time":"2021-06-24T02:02:30.632943","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:48:47.504689Z","iopub.execute_input":"2021-11-09T06:48:47.505585Z","iopub.status.idle":"2021-11-09T06:48:52.768293Z","shell.execute_reply.started":"2021-11-09T06:48:47.505548Z","shell.execute_reply":"2021-11-09T06:48:52.764874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator.class_indices.keys()","metadata":{"papermill":{"duration":0.0494,"end_time":"2021-06-24T02:02:36.621674","exception":false,"start_time":"2021-06-24T02:02:36.572274","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:48:52.769888Z","iopub.execute_input":"2021-11-09T06:48:52.770246Z","iopub.status.idle":"2021-11-09T06:48:52.776964Z","shell.execute_reply.started":"2021-11-09T06:48:52.770208Z","shell.execute_reply":"2021-11-09T06:48:52.775874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import MobileNetV2\n\nconv_base = MobileNetV2(include_top=False,\n                        weights=\"imagenet\",\n                        input_shape=(256, 256, 3)\n)\n\nconv_base.trainable = True\n\nconv_base.summary()","metadata":{"papermill":{"duration":3.774233,"end_time":"2021-06-24T02:02:40.435717","exception":false,"start_time":"2021-06-24T02:02:36.661484","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:48:52.778875Z","iopub.execute_input":"2021-11-09T06:48:52.779592Z","iopub.status.idle":"2021-11-09T06:48:56.207881Z","shell.execute_reply.started":"2021-11-09T06:48:52.779557Z","shell.execute_reply":"2021-11-09T06:48:56.207003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import Dense, Dropout, MaxPooling2D, GlobalAveragePooling2D, Flatten, Conv2D, Input\nfrom keras.models import Sequential\nfrom keras import optimizers\nimport tensorflow as tf\n\nmodel = Sequential()\nmodel.add(conv_base)\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Dense(100, activation='softmax'))\nmodel.compile(optimizer=optimizers.RMSprop(lr=2e-5),\n              loss = 'categorical_crossentropy',\n              metrics=['accuracy'])\n\nmodel.summary()","metadata":{"papermill":{"duration":0.402372,"end_time":"2021-06-24T02:02:40.888786","exception":false,"start_time":"2021-06-24T02:02:40.486414","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:48:56.209198Z","iopub.execute_input":"2021-11-09T06:48:56.209525Z","iopub.status.idle":"2021-11-09T06:48:56.712181Z","shell.execute_reply.started":"2021-11-09T06:48:56.209486Z","shell.execute_reply":"2021-11-09T06:48:56.711296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.042156,"end_time":"2021-06-24T02:02:40.973383","exception":false,"start_time":"2021-06-24T02:02:40.931227","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.042818,"end_time":"2021-06-24T02:02:41.058437","exception":false,"start_time":"2021-06-24T02:02:41.015619","status":"completed"},"tags":[]}},{"cell_type":"code","source":"history = model.fit(\n    train_generator,\n    epochs=10, \n    validation_data=validation_generator,\n    verbose=2\n)\n","metadata":{"papermill":{"duration":10266.399951,"end_time":"2021-06-24T04:53:47.502106","exception":false,"start_time":"2021-06-24T02:02:41.102155","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T06:48:56.713429Z","iopub.execute_input":"2021-11-09T06:48:56.713949Z","iopub.status.idle":"2021-11-09T07:35:01.930150Z","shell.execute_reply.started":"2021-11-09T06:48:56.713908Z","shell.execute_reply":"2021-11-09T07:35:01.929116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the training results\nimport matplotlib.pyplot as plt\n\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(1, len(acc)+1)\n\nplt.plot(epochs, acc, '#21466C', label='Training acc')\nplt.plot(epochs, val_acc, '#ff0051', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, '#21466C', label='Training loss')\nplt.plot(epochs, val_loss, '#ff0051', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"papermill":{"duration":0.542565,"end_time":"2021-06-24T04:53:48.106551","exception":false,"start_time":"2021-06-24T04:53:47.563986","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:35:01.931970Z","iopub.execute_input":"2021-11-09T07:35:01.932367Z","iopub.status.idle":"2021-11-09T07:35:02.220656Z","shell.execute_reply.started":"2021-11-09T07:35:01.932324Z","shell.execute_reply":"2021-11-09T07:35:02.219699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = model.evaluate(test_generator)\nprint('loss:', scores[0])\nprint('accuracy:', scores[1])","metadata":{"papermill":{"duration":37.326371,"end_time":"2021-06-24T04:54:25.499494","exception":false,"start_time":"2021-06-24T04:53:48.173123","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:35:02.222197Z","iopub.execute_input":"2021-11-09T07:35:02.222550Z","iopub.status.idle":"2021-11-09T07:35:42.502997Z","shell.execute_reply.started":"2021-11-09T07:35:02.222514Z","shell.execute_reply":"2021-11-09T07:35:42.502103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list = list(test_generator.class_indices.keys())\nlabel_list[:10]","metadata":{"papermill":{"duration":0.24672,"end_time":"2021-06-24T04:54:26.011517","exception":false,"start_time":"2021-06-24T04:54:25.764797","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:35:42.504724Z","iopub.execute_input":"2021-11-09T07:35:42.505362Z","iopub.status.idle":"2021-11-09T07:35:42.511767Z","shell.execute_reply.started":"2021-11-09T07:35:42.505323Z","shell.execute_reply":"2021-11-09T07:35:42.510769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(test_generator)","metadata":{"papermill":{"duration":33.062984,"end_time":"2021-06-24T04:54:59.311349","exception":false,"start_time":"2021-06-24T04:54:26.248365","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:35:42.518722Z","iopub.execute_input":"2021-11-09T07:35:42.519466Z","iopub.status.idle":"2021-11-09T07:36:17.343212Z","shell.execute_reply.started":"2021-11-09T07:35:42.519429Z","shell.execute_reply":"2021-11-09T07:36:17.342347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred.shape","metadata":{"papermill":{"duration":0.24304,"end_time":"2021-06-24T04:54:59.792672","exception":false,"start_time":"2021-06-24T04:54:59.549632","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:17.348347Z","iopub.execute_input":"2021-11-09T07:36:17.348878Z","iopub.status.idle":"2021-11-09T07:36:17.354541Z","shell.execute_reply.started":"2021-11-09T07:36:17.348834Z","shell.execute_reply":"2021-11-09T07:36:17.353763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max(y_pred[0])","metadata":{"papermill":{"duration":0.245362,"end_time":"2021-06-24T04:55:00.273901","exception":false,"start_time":"2021-06-24T04:55:00.028539","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:17.355760Z","iopub.execute_input":"2021-11-09T07:36:17.356226Z","iopub.status.idle":"2021-11-09T07:36:17.367533Z","shell.execute_reply.started":"2021-11-09T07:36:17.356190Z","shell.execute_reply":"2021-11-09T07:36:17.366638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_ls = y_pred.tolist()","metadata":{"papermill":{"duration":0.262035,"end_time":"2021-06-24T04:55:00.772173","exception":false,"start_time":"2021-06-24T04:55:00.510138","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:17.368814Z","iopub.execute_input":"2021-11-09T07:36:17.369292Z","iopub.status.idle":"2021-11-09T07:36:17.402091Z","shell.execute_reply.started":"2021-11-09T07:36:17.369254Z","shell.execute_reply":"2021-11-09T07:36:17.401196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.243105,"end_time":"2021-06-24T04:55:01.262325","exception":false,"start_time":"2021-06-24T04:55:01.01922","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = []\n\nfor i in range(len(y_pred_ls)):\n    max_value = max(y_pred_ls[i])\n    max_index = y_pred_ls[i].index(max_value)\n    y.append(max_index)\n    \ny = np.array(y)","metadata":{"papermill":{"duration":0.256938,"end_time":"2021-06-24T04:55:01.781526","exception":false,"start_time":"2021-06-24T04:55:01.524588","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:17.404869Z","iopub.execute_input":"2021-11-09T07:36:17.405670Z","iopub.status.idle":"2021-11-09T07:36:17.428303Z","shell.execute_reply.started":"2021-11-09T07:36:17.405630Z","shell.execute_reply":"2021-11-09T07:36:17.427479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\nfor i in range(len(y_pred_ls)):\n    scores.append(max(y_pred[i]))\n    \nscores = np.array(scores)","metadata":{"papermill":{"duration":0.3311,"end_time":"2021-06-24T04:55:02.349962","exception":false,"start_time":"2021-06-24T04:55:02.018862","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:17.429275Z","iopub.execute_input":"2021-11-09T07:36:17.429516Z","iopub.status.idle":"2021-11-09T07:36:17.526445Z","shell.execute_reply.started":"2021-11-09T07:36:17.429493Z","shell.execute_reply":"2021-11-09T07:36:17.525670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y","metadata":{"papermill":{"duration":0.245284,"end_time":"2021-06-24T04:55:02.833102","exception":false,"start_time":"2021-06-24T04:55:02.587818","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:17.527896Z","iopub.execute_input":"2021-11-09T07:36:17.528277Z","iopub.status.idle":"2021-11-09T07:36:17.534469Z","shell.execute_reply.started":"2021-11-09T07:36:17.528243Z","shell.execute_reply":"2021-11-09T07:36:17.533468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"papermill":{"duration":0.247591,"end_time":"2021-06-24T04:55:03.317661","exception":false,"start_time":"2021-06-24T04:55:03.07007","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:17.536134Z","iopub.execute_input":"2021-11-09T07:36:17.536514Z","iopub.status.idle":"2021-11-09T07:36:17.544077Z","shell.execute_reply.started":"2021-11-09T07:36:17.536477Z","shell.execute_reply":"2021-11-09T07:36:17.543193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, roc_auc_score\nfpr, tpr, thresholds = roc_curve(y, scores, pos_label=0)","metadata":{"papermill":{"duration":0.79921,"end_time":"2021-06-24T04:55:04.357043","exception":false,"start_time":"2021-06-24T04:55:03.557833","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:17.545806Z","iopub.execute_input":"2021-11-09T07:36:17.546661Z","iopub.status.idle":"2021-11-09T07:36:18.066222Z","shell.execute_reply.started":"2021-11-09T07:36:17.546626Z","shell.execute_reply":"2021-11-09T07:36:18.065378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_0_1 = []\nauc=[]\nfor j in y:\n    if j != 10:\n        y_0_1.append(0)\n    else:\n        y_0_1.append(1)\ny_0_1 = np.array(y_0_1)\nauc_of_the_label = roc_auc_score(y_0_1, scores)\nauc.append(auc_of_the_label)\nauc","metadata":{"papermill":{"duration":0.433678,"end_time":"2021-06-24T04:55:05.046376","exception":false,"start_time":"2021-06-24T04:55:04.612698","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:18.067640Z","iopub.execute_input":"2021-11-09T07:36:18.067962Z","iopub.status.idle":"2021-11-09T07:36:18.085951Z","shell.execute_reply.started":"2021-11-09T07:36:18.067928Z","shell.execute_reply":"2021-11-09T07:36:18.085072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.metrics import roc_curve, roc_auc_score\n\nauc=[]\n\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.legend()\nplt.title('ROC')\n\nfor i in range(100):\n    fpr, tpr, _ = roc_curve(y, scores, pos_label=i)\n    plt.plot(fpr, tpr, label=i)\n    \n    y_0_1 = []\n    for j in y:\n        if j != i:\n            y_0_1.append(0)\n        else:\n            y_0_1.append(1)\n    y_0_1 = np.array(y_0_1)\n    auc_of_the_label = roc_auc_score(y_0_1, scores)\n    auc.append(auc_of_the_label)\n\n\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.legend()\nplt.title('ROC')\n\nplt.show()\n\n","metadata":{"papermill":{"duration":2.255989,"end_time":"2021-06-24T04:55:07.571562","exception":false,"start_time":"2021-06-24T04:55:05.315573","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:18.087666Z","iopub.execute_input":"2021-11-09T07:36:18.088281Z","iopub.status.idle":"2021-11-09T07:36:20.096867Z","shell.execute_reply.started":"2021-11-09T07:36:18.088246Z","shell.execute_reply":"2021-11-09T07:36:20.096052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"papermill":{"duration":0.251225,"end_time":"2021-06-24T04:55:08.065215","exception":false,"start_time":"2021-06-24T04:55:07.81399","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.098073Z","iopub.execute_input":"2021-11-09T07:36:20.098504Z","iopub.status.idle":"2021-11-09T07:36:20.104610Z","shell.execute_reply.started":"2021-11-09T07:36:20.098471Z","shell.execute_reply":"2021-11-09T07:36:20.103793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max(auc)","metadata":{"papermill":{"duration":0.25073,"end_time":"2021-06-24T04:55:08.559355","exception":false,"start_time":"2021-06-24T04:55:08.308625","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.106120Z","iopub.execute_input":"2021-11-09T07:36:20.106805Z","iopub.status.idle":"2021-11-09T07:36:20.115067Z","shell.execute_reply.started":"2021-11-09T07:36:20.106760Z","shell.execute_reply":"2021-11-09T07:36:20.114188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(auc)","metadata":{"papermill":{"duration":0.299752,"end_time":"2021-06-24T04:55:09.101449","exception":false,"start_time":"2021-06-24T04:55:08.801697","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.116727Z","iopub.execute_input":"2021-11-09T07:36:20.117497Z","iopub.status.idle":"2021-11-09T07:36:20.126001Z","shell.execute_reply.started":"2021-11-09T07:36:20.117462Z","shell.execute_reply":"2021-11-09T07:36:20.124961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list = list(test_generator.class_indices.keys())","metadata":{"papermill":{"duration":0.249044,"end_time":"2021-06-24T04:55:09.593589","exception":false,"start_time":"2021-06-24T04:55:09.344545","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.128859Z","iopub.execute_input":"2021-11-09T07:36:20.129649Z","iopub.status.idle":"2021-11-09T07:36:20.136118Z","shell.execute_reply.started":"2021-11-09T07:36:20.129583Z","shell.execute_reply":"2021-11-09T07:36:20.135173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list[:10]","metadata":{"papermill":{"duration":0.254724,"end_time":"2021-06-24T04:55:10.090254","exception":false,"start_time":"2021-06-24T04:55:09.83553","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.137722Z","iopub.execute_input":"2021-11-09T07:36:20.138509Z","iopub.status.idle":"2021-11-09T07:36:20.147950Z","shell.execute_reply.started":"2021-11-09T07:36:20.138428Z","shell.execute_reply":"2021-11-09T07:36:20.146762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_original_label(new_label): # 從新的label對應回原本資料的label\n    return int(label_list[new_label])","metadata":{"papermill":{"duration":0.251492,"end_time":"2021-06-24T04:55:10.586473","exception":false,"start_time":"2021-06-24T04:55:10.334981","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.149739Z","iopub.execute_input":"2021-11-09T07:36:20.150732Z","iopub.status.idle":"2021-11-09T07:36:20.158880Z","shell.execute_reply.started":"2021-11-09T07:36:20.150691Z","shell.execute_reply":"2021-11-09T07:36:20.157857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_auc_label = to_original_label(auc.index(max(auc))) # label with the highest AUC\nmin_auc_label = to_original_label(auc.index(min(auc))) # label with the lowest AUC\nprint('max auc label', max_auc_label)\nprint('min auc label', min_auc_label)","metadata":{"papermill":{"duration":0.252945,"end_time":"2021-06-24T04:55:11.082399","exception":false,"start_time":"2021-06-24T04:55:10.829454","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.160572Z","iopub.execute_input":"2021-11-09T07:36:20.161402Z","iopub.status.idle":"2021-11-09T07:36:20.171207Z","shell.execute_reply.started":"2021-11-09T07:36:20.161360Z","shell.execute_reply":"2021-11-09T07:36:20.170104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test by eyes","metadata":{"papermill":{"duration":0.243854,"end_time":"2021-06-24T04:55:11.568796","exception":false,"start_time":"2021-06-24T04:55:11.324942","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"First, images with the highest AUC score","metadata":{"papermill":{"duration":0.244462,"end_time":"2021-06-24T04:55:12.056785","exception":false,"start_time":"2021-06-24T04:55:11.812323","status":"completed"},"tags":[]}},{"cell_type":"code","source":"file_list = train_csv[train_csv['landmark_id']==max_auc_label]['id']\nfile_list","metadata":{"papermill":{"duration":0.258443,"end_time":"2021-06-24T04:55:12.559049","exception":false,"start_time":"2021-06-24T04:55:12.300606","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.173030Z","iopub.execute_input":"2021-11-09T07:36:20.173956Z","iopub.status.idle":"2021-11-09T07:36:20.188810Z","shell.execute_reply.started":"2021-11-09T07:36:20.173913Z","shell.execute_reply":"2021-11-09T07:36:20.187923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in file_list[:20]:\n    show_img(file)","metadata":{"papermill":{"duration":41.195371,"end_time":"2021-06-24T04:55:53.99859","exception":false,"start_time":"2021-06-24T04:55:12.803219","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:20.190199Z","iopub.execute_input":"2021-11-09T07:36:20.190858Z","iopub.status.idle":"2021-11-09T07:36:24.692734Z","shell.execute_reply.started":"2021-11-09T07:36:20.190822Z","shell.execute_reply":"2021-11-09T07:36:24.691779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then, images with the lowest AUC score","metadata":{"papermill":{"duration":0.954843,"end_time":"2021-06-24T04:55:55.917431","exception":false,"start_time":"2021-06-24T04:55:54.962588","status":"completed"},"tags":[]}},{"cell_type":"code","source":"file_list = train_csv[train_csv['landmark_id']==min_auc_label]['id']\nfile_list","metadata":{"papermill":{"duration":0.967981,"end_time":"2021-06-24T04:55:57.831714","exception":false,"start_time":"2021-06-24T04:55:56.863733","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:24.694210Z","iopub.execute_input":"2021-11-09T07:36:24.694529Z","iopub.status.idle":"2021-11-09T07:36:24.705622Z","shell.execute_reply.started":"2021-11-09T07:36:24.694496Z","shell.execute_reply":"2021-11-09T07:36:24.704565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in file_list[:20]:\n    show_img(file)","metadata":{"papermill":{"duration":157.5534,"end_time":"2021-06-24T04:58:36.340591","exception":false,"start_time":"2021-06-24T04:55:58.787191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:24.707215Z","iopub.execute_input":"2021-11-09T07:36:24.707980Z","iopub.status.idle":"2021-11-09T07:36:28.528517Z","shell.execute_reply.started":"2021-11-09T07:36:24.707938Z","shell.execute_reply":"2021-11-09T07:36:28.527556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Just view some beautiful images","metadata":{"papermill":{"duration":3.249877,"end_time":"2021-06-24T04:58:42.831302","exception":false,"start_time":"2021-06-24T04:58:39.581425","status":"completed"},"tags":[]}},{"cell_type":"code","source":"image_index = 0 # choose a image (0-3999)\nimage = test_generator[image_index][0] # [0][0]: the first 0 stands for the 0th image, the second 0 stands for the image array\nimage = image.reshape((256,256,3))\nplt.imshow(image)\nplt.show()\nprint(test_generator[0][1][0]) # array of the image's label","metadata":{"papermill":{"duration":3.528864,"end_time":"2021-06-24T04:58:50.077451","exception":false,"start_time":"2021-06-24T04:58:46.548587","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:28.529989Z","iopub.execute_input":"2021-11-09T07:36:28.530335Z","iopub.status.idle":"2021-11-09T07:36:28.719384Z","shell.execute_reply.started":"2021-11-09T07:36:28.530299Z","shell.execute_reply":"2021-11-09T07:36:28.718506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_label(arr):\n    for i in range(len(arr)):\n        if arr[i] == max(arr):\n            return label_list[i]","metadata":{"papermill":{"duration":3.446484,"end_time":"2021-06-24T04:58:56.822277","exception":false,"start_time":"2021-06-24T04:58:53.375793","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:28.720919Z","iopub.execute_input":"2021-11-09T07:36:28.721675Z","iopub.status.idle":"2021-11-09T07:36:28.727225Z","shell.execute_reply.started":"2021-11-09T07:36:28.721617Z","shell.execute_reply":"2021-11-09T07:36:28.726305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label from original data\nget_label(test_generator[image_index][1][0])","metadata":{"papermill":{"duration":3.329493,"end_time":"2021-06-24T04:59:03.407197","exception":false,"start_time":"2021-06-24T04:59:00.077704","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:28.728952Z","iopub.execute_input":"2021-11-09T07:36:28.729717Z","iopub.status.idle":"2021-11-09T07:36:28.752162Z","shell.execute_reply.started":"2021-11-09T07:36:28.729667Z","shell.execute_reply":"2021-11-09T07:36:28.751315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label by prediction\nget_label(model.predict(test_generator[image_index][0])[0])","metadata":{"papermill":{"duration":4.162179,"end_time":"2021-06-24T04:59:10.933549","exception":false,"start_time":"2021-06-24T04:59:06.77137","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:28.753588Z","iopub.execute_input":"2021-11-09T07:36:28.754259Z","iopub.status.idle":"2021-11-09T07:36:29.448347Z","shell.execute_reply.started":"2021-11-09T07:36:28.754221Z","shell.execute_reply":"2021-11-09T07:36:29.447515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = get_label(model.predict(test_generator[image_index][0])[0])\nimage_list = [] # image with the same label\nfor i in range(4000):\n    if get_label(test_generator[i][1][0]) == label:\n        image_list.append(i)\n","metadata":{"papermill":{"duration":30.369218,"end_time":"2021-06-24T04:59:44.583678","exception":false,"start_time":"2021-06-24T04:59:14.21446","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:29.449716Z","iopub.execute_input":"2021-11-09T07:36:29.450074Z","iopub.status.idle":"2021-11-09T07:36:57.049986Z","shell.execute_reply.started":"2021-11-09T07:36:29.450038Z","shell.execute_reply":"2021-11-09T07:36:57.049007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(image_list) > 10:\n    for i in range(10):\n        if i != image_index:\n            image = test_generator[image_list[i]][0]\n            image = image.reshape((256,256,3))\n            plt.imshow(image)\n            plt.show()\n        \nelse:\n    for i in range(len(image_list)):\n        if i != image_index:\n            image = test_generator[image_list[i]][0]\n            image = image.reshape((256,256,3))\n            plt.imshow(image)\n            plt.show()\n","metadata":{"papermill":{"duration":5.105395,"end_time":"2021-06-24T04:59:52.953042","exception":false,"start_time":"2021-06-24T04:59:47.847647","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-09T07:36:57.051310Z","iopub.execute_input":"2021-11-09T07:36:57.051687Z","iopub.status.idle":"2021-11-09T07:36:58.647321Z","shell.execute_reply.started":"2021-11-09T07:36:57.051650Z","shell.execute_reply":"2021-11-09T07:36:58.646360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":3.289273,"end_time":"2021-06-24T04:59:59.558142","exception":false,"start_time":"2021-06-24T04:59:56.268869","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":3.327527,"end_time":"2021-06-24T05:00:06.359189","exception":false,"start_time":"2021-06-24T05:00:03.031662","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}