{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import keras\nimport os\nimport pandas as pd\nimport tensorflow as tf\nimport numpy as np\nfrom collections import defaultdict\nfrom keras.models import Model\nfrom keras.applications import InceptionV3\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Lambda\nfrom keras.optimizers import SGD\nfrom keras import backend as K\nfrom keras.callbacks import ModelCheckpoint\nimport random\nimport cv2\n# from ../input/notebooks/loss_functions import *\n%run ../input/notebooks/loss_functions.py\n# from loss_functions import *\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nfrom collections import defaultdict\nimport os\nfrom sklearn import preprocessing\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"class DataGenerator(keras.utils.Sequence):\n    #'Generates data for Keras'\n    def __init__(self, dictionary , classes,labels ,class_per_batch=50, batch_size=4, dim=(96,96), n_channels=3,shuffle=True):\n        # 'Initialization'\n        self.dim = dim\n        self.dictionary = dictionary\n        self.batch_size = batch_size\n        self.class_per_batch = class_per_batch\n        self.labels = labels\n        self.n_channels = n_channels\n        self.classes = classes\n        self.shuffle = shuffle\n        self.on_epoch_end()\n        \n    def __getitem__(self, index):\n        #'Generate one batch of data'\n        # Generate indexes of the batch\n        indexes = self.indexes[index*self.class_per_batch:(index+1)*self.class_per_batch]\n        # Find list of IDs\n        lab = [self.labels[k] for k in indexes]\n        # Generate data\n        X, y = self.__data_generation(lab)\n        return X, y\n    \n    \n    def __len__(self):\n        #'Denotes the number of batches per epoch'\n        return int(np.floor(len(self.labels) / (self.class_per_batch)))        \n\n    def __data_generation(self, lab):\n        #'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n        # Initialization\n        X = np.empty((self.batch_size*self.class_per_batch, *self.dim, self.n_channels))\n        y = np.empty((self.batch_size*self.class_per_batch))\n\n        # Generate data\n        #print(lab)\n        #print(len(lab))\n        for ID in lab:\n            # Store sample\n            X_temp = random.sample(self.dictionary[ID], 4)\n            #print(X_temp)\n            for i, sample in enumerate(X_temp):\n                #print('\\n\\n\\n\\n\\n',sample)\n                #print(ID)\n                img = cv2.imread('../input/dataset/train/train/'+sample)\n                sample = cv2.resize(img  , self.dim)\n                X[i,] = sample\n                y[i] = ID\n\n        return X, y\n    \n    def on_epoch_end(self):\n        #'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.labels))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a395e60ae446a4c293c2e61fa5b6ca3ec0129d61"},"cell_type":"code","source":"# import os\n# os.chdir(\"../input/incept/\")\n# from rgg import *\nprint(os.getcwd())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3e701a59ad37dbe4910af0a34e9214dee5c07d9"},"cell_type":"code","source":"INCEPTIONV3 = InceptionV3(weights='../input/inceptweight/inception_v3_weights_tf_dim_ordering_tf_kernels.h5')\n# INPUT_HEIGHT = 224\n# INPUT_WIDTH = 224\n# INPUT_MEAN = 127.5\n# INPUT_STD = 127.5\ncheckpoint = ModelCheckpoint('weights-best1.hdf5',monitor = \"val_loss\", verbose = 1,\n  save_best_only = False, save_weights_only = False, mode = \"auto\",\n  period = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3eb96c27aeff12f49d2e2426c3a94f432d1c2c1c"},"cell_type":"code","source":"df = pd.read_csv('../input/preprocessed/file_name.csv')\ndf = df.sort_values(by=['Id'])\nlen(df)\ntraindf = df[:20000]\nvaldf = df[20000:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"335bf9d77c06cb6cbaffebf63220bd572238b06f"},"cell_type":"code","source":"# labels = traindf['Id'].unique().tolist()\nle = preprocessing.LabelEncoder()\nclasses = df['Id'].tolist()\nle.fit(classes)\n\nclasses = le.transform(classes) \n# print(type(classes),classes)\nlabels = np.unique(classes).tolist()\n# print(labels)\nimages = df['Image'].tolist()\ndictionary = defaultdict(list)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec56ee948ea5366552d10113a767af3fe7462809"},"cell_type":"code","source":"for label, image in zip(classes, images):\n    dictionary[label].append(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88f01d399d912c5e84fb03a26b8ed9f661b50e7f"},"cell_type":"code","source":"def model(original_model):\n    \n    bottleneck_input  = original_model.get_layer(index=0).input\n    bottleneck_output = original_model.get_layer(index=-2).output\n    bottleneck_model = Model(inputs=bottleneck_input, outputs=bottleneck_output)\n\n    for layer in bottleneck_model.layers:\n        layer.trainable = False\n        \n    new_model = Sequential()\n    new_model.add(bottleneck_model)\n\n    new_model.add(Dense(1024,input_dim  = 2048,activation = 'relu'))\n    new_model.add(Dense(512,activation = 'relu'))\n    new_model.add(Dense(128,activation = 'relu'))\n    new_model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))\n    print(new_model.summary())\n    new_model.save('model.h5')\n\n    return new_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b5db0c361efa3e9e7704ab45c9a6f8248805c48"},"cell_type":"code","source":"training_generator = DataGenerator(dictionary,classes, labels)\n# validation_generator = DataGenerator(dictionary,classes, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c727686fac6c17773209cd99600ff29f02f657b2"},"cell_type":"code","source":"training_model = model(INCEPTIONV3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ecf8ea94f69eee25f34b0fe0f0190d7f599f6d7"},"cell_type":"code","source":"\n\n# import os\n# os.chdir(\"../input/notebooks/\")\n# from loss_function import *\n\n# training_model.load_weights('../input/inceptweight/weightsbest.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74b6ac5213e9c112135522ec87e2d76ce1f606de"},"cell_type":"code","source":"callbacks_list = [checkpoint]\nprint(os.getcwd())\n\ntraining_model.compile(optimizer = 'adam',loss = we_loss ,metrics= [accuracy])\ntraining_model.fit_generator(generator=training_generator,epochs=100 ,callbacks=callbacks_list,verbose=1)\ntraining_model.save_weights('weights1.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"236ae53578ffd48f80578fa37cf28095d1dc83ce"},"cell_type":"code","source":"callbacks_list = [checkpoint]\ntraining_model.load_weights('weights-best.hdf5')\n\ntraining_model.compile(optimizer = 'adam',loss = we_loss ,metrics= [accuracy])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf1628118053ed9b7a0c78874d8309a26bbf5ec8"},"cell_type":"code","source":"newdict = defaultdict(list)\ncount = 1\nfor label, image in zip(classes, images):\n    \n    img = cv2.imread('train/'+image)\n    sample = cv2.resize(img  , (96,96))\n    g = training_model.predict(np.array([sample]))\n    newdict[label].append(g)\n    count+=1\n    print(\"processed :\" ,count,end='\\r')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5de3b2227745acc4018c92020d854782be19a743"},"cell_type":"code","source":"# np.save('my_file.npy', newdict) \nread_dictionary = np.load('my_file.npy').item()\nemmbed = defaultdict(list)\ncount =0 \nfor k in read_dictionary.keys():\n    z = np.array(read_dictionary[k])\n    a = z.shape[0]\n    \n    z  = z.reshape(a,128)\n    z = z.mean(0)\n    emmbed[k].append(z)\n    count+=1\n    print(\"processed :\", count)\n    \n\n    \nnp.save('emmbedings.npy', emmbed) \n# with open('filename.pickle', 'wb') as handle:\n#     pickle.dump(newdict, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n# with open('filename.pickle', 'rb') as handle:\n#     b = pickle.load(handle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd8965c19611137f1f5e19ffed8162d53ac0e80a"},"cell_type":"code","source":"read_dictionary = np.load('emmbedings.npy').item()\nfrom os import listdir\nimglist = listdir('test')\nimport csv \nfor d in imglist:\n    li = []\n    img = cv2.imread('test/'+d)\n    sample = cv2.resize(img  , (96,96))\n    g = training_model.predict(np.array([sample]))\n    for k in read_dictionary.keys():\n        dist = float(np.linalg.norm(g[0] - read_dictionary[k][0]))\n        li.append([dist,k])\n    li.sort()\n#     print(li[0])\n    sd = le.inverse_transform([li[0][1],li[1][1],li[2][1],li[3][1]])\n#     print(sd)\n    str1 = 'new_whale '+sd[0]+' '+sd[1]+' '+sd[2]+' '+sd[3]\n    fields=[d,str1]\n    with open(r'ans.csv', 'a') as f:\n        writer = csv.writer(f)\n        writer.writerow(fields)\n        ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}