{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport random\nimport seaborn as sns\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport tensorflow as tf #ignore error if no GPU set up on machine\nfrom tensorflow import keras\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.python.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.python.keras import layers\n#from tensorflow.python.keras.layers import Input, Conv2D, MaxPo","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:06:51.914351Z","iopub.execute_input":"2021-11-24T10:06:51.914899Z","iopub.status.idle":"2021-11-24T10:06:56.948755Z","shell.execute_reply.started":"2021-11-24T10:06:51.914808Z","shell.execute_reply":"2021-11-24T10:06:56.948031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#landmark_id = class labels representing the different landmarks \n#id = img_id \n#img_paths = []\nmainpath = '../input/landmark-recognition-2021/train'\ntraindf = pd.read_csv(\"../input/landmark-recognition-2021/train.csv\")#, index_col=False)\ntraindf['img_path'] = traindf['id'].apply(lambda r: os.path.join(mainpath,r[0], r[1], r[2], r + '.jpg'))\nlandmark_unique = traindf['landmark_id'].unique() \n#img_paths.append(lambda r: os.path.join('../input/landmark-recognition-2021/train',r[0], r[1], r[2], r + '.jpg'))\ntraindf.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:06:56.950349Z","iopub.execute_input":"2021-11-24T10:06:56.950611Z","iopub.status.idle":"2021-11-24T10:07:03.978613Z","shell.execute_reply.started":"2021-11-24T10:06:56.950577Z","shell.execute_reply":"2021-11-24T10:07:03.977945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defining labels \nimg_paths = [] \nlabels = [] \ntemp_labels = [] #later used for random image selection\nclassSize = 1000#1000&700 classes allocated too much memory and notebook was restarted\nimageSize = 224\nminImgs = 100\nmaxImgs = 1000\ni=0\n\nfor u_id in landmark_unique[:classSize]: \n    if(len(traindf['img_path'][traindf['landmark_id'] == u_id].value_counts()) > minImgs):# and\n       #len(traindf['img_path'][traindf['landmark_id'] == u_id].value_counts()) < maxImgs):\n        for path in traindf['img_path'][traindf['landmark_id'] == u_id]:\n            img_paths.append(path)\n            labels.append(u_id)\n            temp_labels.append(i)\n        i = i+1\n\n#defining paths to train images   \ntransformed_imgs = []\nfor img_path in img_paths:\n    img_pix = cv2.imread(img_path,1) #img_pix stores images in finalpath in green-channel\n    transformed_imgs.append(cv2.resize(img_pix, (imageSize,imageSize))) #universally resized images for training\n\n#creation of training data:\n#new indexation to avoid IndexError in building the train dataset\nlbl_path_set = list(zip(img_paths,temp_labels))\n\n#Mix up training data so it is not organised in classes anymore \nrandom.shuffle(lbl_path_set)\n\nimg_data, labels_data = zip(*lbl_path_set) \n\ntrain_data = []\nfor img in transformed_imgs[:len(img_data)]:\n    train_data.append(img)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:07:03.979925Z","iopub.execute_input":"2021-11-24T10:07:03.980211Z","iopub.status.idle":"2021-11-24T10:07:51.110424Z","shell.execute_reply.started":"2021-11-24T10:07:03.980177Z","shell.execute_reply":"2021-11-24T10:07:51.109689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1. How many images does the dataset consist of?\ntraindf.shape   \nprint(\"1. Dataset shape: \",traindf.shape)\n# The training dataset consists of 1580470 images\n#---------------------------------------------------\n# 2.How many classes?\n#Counting unique labels - nr. of classes \nprint(\"2. Total Nr. classes: \",len(landmark_unique))\n# The training dataset consists of 81313 classes\n#----------------------------------------------------\n# 2.How many images per class?\nprint(\"3. Images per class: \",traindf['landmark_id'].value_counts()) #counts values per landmark_id\n# Number of images per class varies - from only 2 images/class up to 6272 images/class\n#----------------------------------------------------\n#2.ii. How many classes have less than 5 training samples?\nnewdf = pd.DataFrame(traindf.landmark_id.value_counts().reset_index().values, columns=[\"landmark_id\", \"images\"])\nvals = []\nfor i in newdf['images']:\n    if i < 5:\n        vals.append(i)\nprint(\"4. \",len(vals),\"categories with <5 samples\")\n\n# 17297 classes have less than 5 training samples\n#-----------------------------------------------------------\n#2.iii. How many classes have between 5 and 10 training samples?\nvals = []\nfor i in newdf['images']:\n    if i >= 5 and i <=10:\n        vals.append(i)\nprint(\"5. \", len(vals),\" categories with 5-10 samples\")\n# 27349 classes have between 5 and 10 training samples\n#----------------------------------------------------------------\n#Plot Data distribution\n#plt.figure(figsize = (8, 8))\n#binsize = 500#len(traindf.landmark_id.unique())\n#plt.title('Landmark id density plot')\n#sns.histplot(traindf['landmark_id'], color=\"tomato\", kde=True, bins=binsize)\n#plt.show()\n#----------------------------------------------------------------\n#Print 4 random images from 4 random classes\nfig = plt.gcf()\nfig.set_size_inches(10, 10)\n\nimages = []\nfor i in range(0,4):\n    rand_nr = np.random.randint(0,len(landmark_unique))\n    rand_landmark = traindf[traindf['landmark_id']==traindf['landmark_id'].value_counts().iloc[[rand_nr]].index[0]]\n    for j in range(0,4):\n        rand_nr = np.random.randint(0, len(rand_landmark))\n        rand_img = rand_landmark.iloc[rand_nr]\n        images.append(rand_img)\n        \n\nfor i in range(len(images)):\n    img = images[i]['id']\n    cl = images[i]['landmark_id']\n    new_path = mainpath+\"/\"+img[:1]+\"/\"+img[1]+\"/\"+img[2]+\"/\"+img+\".jpg\"\n    sp = plt.subplot(5, 4, i + 1)\n    sp.axis('Off')\n    imag = cv2.imread(new_path) \n    plt.imshow(imag)\n    \nprint(\"6. Samples from 4 different categories\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:07:51.112538Z","iopub.execute_input":"2021-11-24T10:07:51.112793Z","iopub.status.idle":"2021-11-24T10:07:53.003556Z","shell.execute_reply.started":"2021-11-24T10:07:51.11276Z","shell.execute_reply":"2021-11-24T10:07:53.00192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare training data - convert from integer to floating point\nX_data = np.array(train_data)#img_data)\nY_data =  to_categorical(labels_data, num_classes = classSize) #to_categorical - converts class vector (integer) to binary matrix\n\n# split train set into training and validation  (70 to 30 ratio)\ntestSize = 0.3\nX_train, X_val, Y_train, Y_val = train_test_split(X_data, Y_data, test_size = testSize, random_state=42) # include fixed randomstate for reproduction\n\nprint(\"X train data : \", X_train.shape)\nprint(\"X label data : \", Y_train.shape)\nprint(\"Y test data : \",  X_val.shape)\nprint(\"Y label data : \", Y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:07:53.004605Z","iopub.execute_input":"2021-11-24T10:07:53.004862Z","iopub.status.idle":"2021-11-24T10:07:53.436737Z","shell.execute_reply.started":"2021-11-24T10:07:53.004823Z","shell.execute_reply":"2021-11-24T10:07:53.436016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# experiment with Hyper-parameters:\n\n#Batch size\nbatchSize = 64#,128\nkernelSize = 3\nstrideSize = 2 \nfilterSize = 7 #3\npaddingSize = 'same'\nactivation_fct = 'relu'\nepochSize = 10\npoolSize = (2,2)#2x2 maxpooling\noptimizer = tf.keras.optimizers.Adam(learning_rate=0.001)#0.0001\n# Preprocessing \ndatagen = ImageDataGenerator(horizontal_flip=True, #ImageDataGenerator class to perform image augmentation during training\n                             vertical_flip=True,\n                             rotation_range=20,\n                             zoom_range=0.2,\n                             width_shift_range=0.2,\n                             height_shift_range=0.2,\n                             shear_range=0.2,\n                             fill_mode=\"nearest\")\n\n#train_data_gen = datagen.flow_from_directory(directory=mainpath, subset=\"training\")\n#val_dat_gen = datagen.flow_from_directory(directory=mainpath,subset=\"validation\")\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:13:59.131869Z","iopub.execute_input":"2021-11-24T10:13:59.132152Z","iopub.status.idle":"2021-11-24T10:13:59.139512Z","shell.execute_reply.started":"2021-11-24T10:13:59.13212Z","shell.execute_reply":"2021-11-24T10:13:59.138366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Sequential(name=\"my_model\")\nmodel.add(layers.Conv2D(64, (3, 3),input_shape = (244,244,3),activation = 'relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Conv2D(128, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.Conv2D(128, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Conv2D(256, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.Conv2D(256, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Conv2D(512, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.Conv2D(512, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Conv2D(512, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.Conv2D(512, (3, 3), padding=\"same\", activation=\"relu\"))\nmodel.add(layers.MaxPooling2D(pool_size=(2, 2)))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(4096, activation=\"relu\")) \nmodel.add(layers.Dense(4096, activation=\"relu\"))\nmodel.add(layers.Dense(classSize, activation='softmax'))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:19:36.22368Z","iopub.execute_input":"2021-11-24T10:19:36.224385Z","iopub.status.idle":"2021-11-24T10:19:36.366971Z","shell.execute_reply.started":"2021-11-24T10:19:36.224347Z","shell.execute_reply":"2021-11-24T10:19:36.366288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model leaning on VGG16 architecture\n#model = keras.Sequential(name=\"my_model\")\n#first conv Block\n#model.add(layers.Conv2D(filters=64,kernel_size=(3,3), padding = \"same\",activation = activation_fct, input_shape = (imageSize,imageSize,3)))#filters, kernel_size, stide(x,x),padding, \n#model.add(layers.Dropout(0.1))\n#model.add(layers.Conv2D(filters=64,kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Dropout(0.1))\n#model.add(layers.MaxPooling2D(pool_size=poolSize,strides=(2,2),padding=\"same\"))\n#second conv Block\n#model.add(layers.Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Dropout(0.1))\n#model.add(layers.Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Dropout(0.2))\n#model.add(layers.MaxPooling2D(pool_size=poolSize,strides=(2,2),padding=\"same\"))\n#third Conv Block\n#model.add(layers.BatchNormalization())\n#model.add(layers.Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.MaxPooling2D(pool_size=poolSize,strides=(2,2),padding=\"same\"))\n#fourth Conv Block\n#model.add(layers.BatchNormalization())\n#model.add(layers.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Dropout(0.1))\n#model.add(layers.BatchNormalization())\n#model.add(layers.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Dropout(0.2))\n#model.add(layers.BatchNormalization())\n#model.add(layers.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Dropout(0.2))\n#model.add(layers.MaxPooling2D(pool_size=poolSize,strides=(2,2),padding=\"same\"))\n#fifth Conv Block\n#model.add(layers.BatchNormalization())\n#model.add(layers.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=activation_fct))\n#model.add(layers.MaxPooling2D(pool_size=poolSize,strides=(2,2),padding=\"same\"))\n\n          \n#Fully Connected Layers\n#model.add(layers.Flatten())\n#model.add(layers.BatchNormalization())\n#model.add(layers.Dense(4096, activation=activation_fct)) #4096\n#odel.add(layers.Dropout(0.5))\n#model.add(layers.BatchNormalization())\n#model.add(layers.Dense(4096, activation=activation_fct))\n#model.add(layers.Dropout(0.5))\n#model.add(layers.BatchNormalization())\n#model.add(layers.Dense(classSize, activation='softmax'))\n\nmodel.summary()\n#model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy']) \n","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:21:35.124391Z","iopub.status.idle":"2021-11-24T10:21:35.126476Z","shell.execute_reply.started":"2021-11-24T10:21:35.126225Z","shell.execute_reply":"2021-11-24T10:21:35.126252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel.compile(optimizer=optimizer,loss='categorical_crossentropy',metrics=['accuracy'])\nhistory = model.fit(datagen.flow(X_train,Y_train,batch_size=batchSize),validation_data=(X_val,Y_val),epochs=epochSize) ","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:19:43.863002Z","iopub.execute_input":"2021-11-24T10:19:43.863728Z","iopub.status.idle":"2021-11-24T10:21:35.121745Z","shell.execute_reply.started":"2021-11-24T10:19:43.863691Z","shell.execute_reply":"2021-11-24T10:21:35.117258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\n# plot Trainging/Validation accuracy\nplt.plot(epochs, acc, 'r', label='Training accuracy')\nplt.plot(epochs, val_acc, 'b', label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend(loc=0)\nplt.figure()\n\n# plot Training/Validation loss\nplt.plot(epochs, loss, 'r', label='Training loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend(loc=0)\nplt.figure()\n\nplt.show()\n\n\nloss, acc = model.evaluate(X_train,Y_train,verbose=0)\nval_loss, val_acc = model.evaluate(X_val,Y_val,verbose=0)\n\nprint(f'Train acc: {acc*100:.3f} % || Validation acc: {val_acc*100:.3f} %')\nprint(f'Train loss: {loss*100:.3f} % || Validation loss: {val_loss*100:.3f} %')\n\n#V01:1000 classes (100-300), 5 conv layers, Batchsize 100, 30 epochs, lr 0.0001\n#V001:1000 classes (100-500), 5 conv layers, Batchsize 100, 30 epochs, lr 0.0001 -> something like Baseline Architecture \n#V0001:1000 classes (50-550), 5 conv layers, Batchsize 100, 30 epochs, lr 0.0001 \n#Batchnorm:\n#V02:1000 classes (100-500), 5 conv layers, Batchsize 32, 30 epochs, w/ Batchnormalization after each activationFct lr 0.0075 \n#Layers\n#V03: 1000 classes (100-500), 3 conv layers (64,128,256,2048), Batchsize 32, 40 epochs, lr 0.001\n#V04: 1000 classes (100-500), 8 conv layers (64*2,128*2,256*2,568,568,4096), Batchsize 32, 40 epochs, lr 0.001\n#V05: 1000 classes (100-500), 5 conv layers (reduced stride to 2), Batchsize 64, 30 epochs, lr 0.001\n#V06: 1000 classes (100-500), 5 conv layers (added Dropoutlayer (0.2 after conv-relu and 0.5 after dense)), Batchsize 100, 30 epochs, lr 0.001\n#V07: 1000 classes (100-500), 5 conv layers (added Dropoutlayer (0.2 after conv-relu and 0.5 after dense)), Batchsize 100, 30 epochs, lr 0.001\n#V08: 1000 classes (100-500), 5 conv layers (added Dropoutlayer (only one dropout 0.5 after dense)), Batchsize 100, 50 epochs, lr 0.00005)\n#V09: 1000 classes (100-500), 5 conv layers, Batchsize 100, 40 epochs, lr 0.0001 - horizontal_flip=True, vertical_flip=True, rotation_range=20, zoom_range=0.2, width_shift_range=0.2, height_shift_range=0.2,\n#V23: 1000 classes (100-500), 5 conv layers (added Dropoutlayers after each convolutional layer (0.1-0.2)), Batchsize 32, 40 epochs, lr 0.0001) + imageDataGen applied","metadata":{"execution":{"iopub.status.busy":"2021-11-24T10:08:05.894979Z","iopub.status.idle":"2021-11-24T10:08:05.895415Z","shell.execute_reply.started":"2021-11-24T10:08:05.895195Z","shell.execute_reply":"2021-11-24T10:08:05.895218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#500 classes, Batchsize 32, 100 Epochs, LR 0.001 - 2.30h\n#500 classes, Batchsize 32, 20 Epochs, LR 0.001 - 1 h\n#500 classes, Batchsize 64, 20 Epochs, LR 0.001 - 30 min\n#500 classes, 3 Conv Layers, Batchsize 64, 20 Epochs, LR 0.001 -> ResourceExhaustedError: OOM when allocating tensor with shape[401408,4096] on GPU -> reducing conv layers caused that (maybe jump from 128 to 512 was too much...having 256 inbetween solves the issue )\n#500 classes, 3 Conv Layers, Batchsize 64, 20 Epochs, LR 0.0001 -> Ran out of memory\n#500 classes, 3 Conv Layers, Batchsize 128, 10 Epochs, LR 0.001\n#V6:500 classes (100-300), 5 conv layers, Batchsize 64,10 epochs, lr 0.001\n#V7:500 classes (100-300), 5 conv layers, Batchsize 64,10 epochs, lr 0.0001 -> filtering the classes got rid of the memory issue\n#V8:1000 classes (100-300), 5 conv layers, Batchsize 64,10 epochs, lr 0.0001 \n#V9:10000 classes (100-300), 5 conv layers, Batchsize 285,10 epochs, lr 0.0001 ->  too big for memory\n#V9:3000 classes (100-300), 5 conv layers, Batchsize 285,10 epochs, lr 0.0001 ->  too big for memory when running network\n#V9:1000 classes (100-300), 5 conv layers, Batchsize 128,10 epochs, lr 0.0001 \n#V10:1000 classes (100-300), 5 conv layers, Batchsize 128,10 epochs, lr 0.001 \n#V11:1000 classes (100-300), 5 conv layers, Batchsize 64,10 epochs, lr 0.001 \n#V12:1000 classes (100-300), 5 conv layers, Batchsize 64,10 epochs, lr 0.00001 \n#V13:1000 classes (100-max), 5 conv layers, Batchsize 128, 10 epochs, lr 0.0001 -> ran out of memory\n#V13:1000 classes (100-max), 5 conv layers, Batchsize 64, 10 epochs, lr 0.0001 \n#V14:1000 classes (100-max), 5 conv layers, Batchsize 64, 20 epochs, lr 0.0001 \n#V15:1000 classes (100-max), 5 conv layers, Batchsize 100, 30 epochs, lr 0.00001 \n#V16:1000 classes (100-max), 5 conv layers, Batchsize 100, 30 epochs, lr 0.00001 w/ Batchnormalization -> best yet but ppbl overfitting?\n#V17:1000 classes (100-max), 5 conv layers, Batchsize 128, 40 epochs, lr 0.001 w/ Batchnormalization\n#V18:1000 classes (100-max), 5 conv layers, Batchsize 128, 40 epochs, lr 0.0001 w/ Batchnormalization -> train & test acc = 11%, loss = 11%\n#V19:1000 classes (100-max), 5 conv layers, Batchsize 128, 40 epochs, lr 0.00001 w/ Batchnormalization -> both at 3%\n#V20:1000 classes (100-max), 5 conv layers, Batchsize 80, 40 epochs, lr 0.00001 w/ Batchnormalization -> really bad\n#V21:1000 classes (100-max), 5 conv layers, Batchsize 100, 40 epochs, lr 0.00001 w/ Batchnormalization -> really bad\n#V22:1000 classes (100-max), 5 conv layers, Batchsize 64, 10 epochs, lr 0.0001 w/ Batchnormalization -> max 10%\n#V22:1000 classes (100-max), 5 conv layers, Batchsize 64, 10 epochs, lr 0.0001 w/ Batchnormalization only before dense layers F-B-D-B-D-B-D\n#Batch Norm:\n#V01:1000 classes (100-300), 5 conv layers, Batchsize 100, 30 epochs, lr 0.0001 -> something like Baseline Architecture durchlauf\n#V02:1000 classes (100-300), 5 conv layers, Batchsize 100, 30 epochs, w/ Batchnormalization lr 0.0075 \n#V03:1000 classes (100-300), 5 conv layers, Batchsize 100, 30 epochs, w/ Batchnormalization lr 0.0001 (take V18)\n","metadata":{}}]}