{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tfa-nightly\n!pip install efficientnet\n#!pip install tensorflow -U","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport math\nfrom kaggle_datasets import KaggleDatasets\nimport glob\nimport os\nfrom datetime import datetime\nimport numpy as np\nimport pickle\nfrom IPython.display import FileLink\n\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport tensorflow_probability as tfp\n\nfrom tensorflow.keras.layers import Conv2D, Dense, Activation, BatchNormalization, Dropout, Flatten, Layer\nfrom tensorflow.keras.applications import densenet\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom efficientnet.tfkeras import EfficientNetB6 # use tf 2.3 instead\n\nfrom tensorflow.python.keras.layers.preprocessing import image_preprocessing as image_ops\n\nimport matplotlib.pyplot as plt\nimport pickle\ntfa.register_all(custom_kernels=False)\n#tfa.register_all()\n\nprint(tf.__version__)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nheight, width = tf.constant(610, tf.int64), tf.constant(730,tf.int64)\nrgb_mean = tf.constant([152.7203985, 115.66881524, 94.62871587])\nrgb_stddev=tf.constant([73.70250614, 67.17878002, 61.07537483])\nrot_angle=10./180*math.pi\nshear_angle=10./180*math.pi\nscale=0.9 # smaller -> zoom\npad_margin=0.1\n\npadSize, feedSize = 820, 224\npadSizef, feedSizef = float(padSize), float(feedSize)\n\ndef matrix(a,b,c,d):\n return tf.reshape(tf.stack([a,b,c,d],axis=1), [-1,2,2])\n\ndef decodePad(x):\n x=tf.io.read_file(x)\n img=tf.io.decode_jpeg(x)\n imgshape=tf.shape(img)\n h=tf.math.maximum(tf.math.minimum((padSize-imgshape[0])//2,imgshape[0]),0)\n w=tf.math.maximum(tf.math.minimum((padSize-imgshape[1])//2,imgshape[1]),0)\n \n img=tf.pad(img, tf.reshape(tf.stack([h,h,w,w,0,0]),[3,2]),\"SYMMETRIC\")\n imgshape=tf.shape(img)\n h=padSize-imgshape[0]\n w=padSize-imgshape[1]\n img=tf.pad(img, tf.reshape(tf.stack([h//2,h//2+h%2,w//2,w//2+w%2,0,0]),[3,2]),\"CONSTANT\",0)\n imgshape=tf.shape(img)\n return tf.cast(img, tf.float32)\n\ndef get_shape(x):\n x=tf.io.read_file(x)\n img=tf.io.decode_jpeg(x)\n shape=tf.shape(img)\n return tf.cast(shape, tf.float32)#tf.cast(shape[0], tf.float32),tf.cast(shape[1], tf.float32)\n\ndef afterBatchPreprocess(data):\n bs=tf.shape(data)#[0]\n #data=_parse_image_function(data)\n \n img=tf.map_fn(decodePad, data, dtype=tf.float32)#, fn_output_signature=tf.float32)\n #img=tf.cast(img, tf.float32)\n shape=tf.map_fn(get_shape,data,dtype=tf.float32)#,fn_output_signature=tf.float32)\n imgwidth=tf.squeeze(tf.slice(shape,[0,1],[-1,1]))\n imgheight=tf.squeeze(tf.slice(shape,[0,0],[-1,1]))\n\n #imgwidth=tf.cast(data['image/width'], tf.float32)\n #imgheight=tf.cast(data['image/height'], tf.float32)\n height = 610.#tf.cast(height, tf.float32)\n width = 730.#tf.cast(width, tf.float32)\n\n \n # shear -> rotate -> scale\n rot=tf.random.uniform(shape=bs, minval=-rot_angle, maxval=rot_angle)\n shear_x=tf.math.tan(tf.random.uniform(shape=bs, minval=-shear_angle, maxval=shear_angle))\n shear_y=tf.math.tan(tf.random.uniform(shape=bs, minval=-shear_angle, maxval=shear_angle))\n cropScaleX=tf.math.minimum(imgwidth,width)/feedSizef\n cropScaleY=tf.math.minimum(imgheight,height)/feedSizef\n scale_x=tf.random.uniform(shape=bs, minval=scale, maxval=1)*cropScaleX\n scale_y=tf.random.uniform(shape=bs, minval=scale, maxval=1)*cropScaleY\n center_x=feedSizef/2.#padSizef/2.\n center_y=feedSizef/2.#padSizef/2.\n \n diff_x=(imgwidth-feedSizef*scale_x)/2.\n diff_y=(imgheight-feedSizef*scale_y)/2.\n \n rdCropW=tf.math.maximum(diff_x,0)\n rdCropH=tf.math.maximum(diff_y,0)\n rdCropW=tf.random.uniform(bs,minval=-rdCropW, maxval=rdCropW)\n rdCropH=tf.random.uniform(bs,minval=-rdCropH, maxval=rdCropH)\n\n trans=tf.eye(2, batch_shape=bs)\n trans=tf.linalg.matmul(matrix(tf.repeat(1.,bs), shear_x,shear_y, tf.repeat(1.,bs)),trans)\n trans=tf.linalg.matmul(matrix(tf.math.cos(rot), tf.math.sin(rot),-tf.math.sin(rot), tf.math.cos(rot)),trans)\n coor=tf.ones([tf.squeeze(bs),2,1], tf.float32)*padSizef/2.\n coor+=tf.expand_dims(tf.stack([rdCropW,rdCropH],axis=1), axis=2)\n new_coor=tf.linalg.matmul(trans,coor)\n trans=tf.linalg.matmul(matrix(scale_x, tf.repeat(0.,bs),tf.repeat(0.,bs), scale_y),trans)\n\n rdCrop_coor=tf.expand_dims(tf.stack([diff_x+rdCropW,diff_y+rdCropH],axis=1), axis=2)\n shift_coor = tf.expand_dims((padSizef-tf.stack([imgwidth,imgheight],axis=1))/2., axis=2)\n transform=tf.reshape(tf.concat([trans, coor-new_coor+shift_coor+rdCrop_coor], axis=2),[tf.squeeze(bs), -1])\n #transform=tf.reshape(tf.concat([trans, tf.zeros([tf.squeeze(bs),2,1], tf.float32)], axis=2),[tf.squeeze(bs), -1]) \n transform = tf.pad(transform ,[[0,0],[0,2]], 'CONSTANT',0)\n img=tfa.image.transform(img, transform, interpolation='BILINEAR',output_shape=[feedSize,feedSize])\n return img#, data[\"image/class/label\"], tf.gather(weights,data[\"image/class/label\"])\n\nfilename = glob.glob(\"/kaggle/input/landmark-retrieval-2020/train/0/0/0/*\")[0:10]\nimage = tf.constant(filename)\nbefore=tf.io.read_file(image[0])\nbefore=tf.io.decode_jpeg(before)\npreprocessed = afterBatchPreprocess(image)\n\nn=10\nf, axarr = plt.subplots(n,2,figsize=(20,50))\nfor i in range(n):\n before=tf.io.read_file(image[i])\n before=tf.io.decode_jpeg(before)\n axarr[i,0].imshow(before)\n axarr[i,1].imshow(preprocessed[i,...]/255.)\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def _parse_image_function(example_proto):\n # Parse the input tf.Example proto using the dictionary above.\n return tf.io.parse_example(example_proto, {\n 'image/height': tf.io.FixedLenFeature([], tf.int64),\n 'image/width': tf.io.FixedLenFeature([], tf.int64),\n 'image/channels': tf.io.FixedLenFeature([], tf.int64),\n #'weight': tf.io.FixedLenFeature([], tf.float32),\n 'image/class/label': tf.io.FixedLenFeature([], tf.int64),\n 'image/encoded': tf.io.FixedLenFeature([], tf.string),\n })\n\n\n\n\n#610. 730\nheight, width = tf.constant(610, tf.float32), tf.constant(730,tf.float32)\nrgb_mean = tf.constant([152.7203985, 115.66881524, 94.62871587])\nrgb_stddev=tf.constant([73.70250614, 67.17878002, 61.07537483])\nrot_angle=10./180*math.pi\nshear_angle=10./180*math.pi\nscale=0.9 # smaller -> zoom\npad_margin=0.1\npadSize, feedSize = 820, 224\npadSizef, feedSizef = float(padSize), float(feedSize)\n\ndef matrix(a,b,c,d):\n return tf.reshape(tf.stack([a,b,c,d],axis=1), [-1,2,2])\n\ndef decodeCropResize(data):\n \"\"\"\n cropping size dilemma:\n bigger : more distortion more information\n smaller: information loss less distortion\n \"\"\"\n\n data=_parse_image_function(data)\n\n img=tf.io.decode_jpeg(data['image/encoded'])\n img = tf.image.random_crop(img, size=[tf.math.minimum(height, data['image/height']), tf.math.minimum(width, data['image/width']),data['image/channels']])\n img = tf.image.convert_image_dtype(img, tf.float32)\n img= tf.image.resize(img, [224, 224],method='bicubic')\n\n return img, data[\"image/class/label\"]#, data['weight']\n\ndef decodePad(x):\n #x=tf.io.read_file(x)\n img=tf.io.decode_jpeg(x)\n imgshape=tf.shape(img)\n h=tf.math.maximum(tf.math.minimum((padSize-imgshape[0])//2,imgshape[0]),0)\n w=tf.math.maximum(tf.math.minimum((padSize-imgshape[1])//2,imgshape[1]),0)\n \n img=tf.pad(img, tf.reshape(tf.stack([h,h,w,w,0,0]),[3,2]),\"SYMMETRIC\")\n imgshape=tf.shape(img)\n h=padSize-imgshape[0]\n w=padSize-imgshape[1]\n img=tf.pad(img, tf.reshape(tf.stack([h//2,h//2+h%2,w//2,w//2+w%2,0,0]),[3,2]),\"CONSTANT\",0)\n imgshape=tf.shape(img)\n return tf.cast(img, tf.float32)\n\ndef afterBatchPreprocess(data,weights):\n bs=tf.shape(data)#[0]\n data=_parse_image_function(data)\n \n img=tf.map_fn(decodePad, data['image/encoded'],dtype=tf.float32)#, fn_output_signature=tf.float32)\n img = (img-rgb_mean)/rgb_stddev\n #img = img/255\n imgwidth=tf.cast(data['image/width'], tf.float32)\n imgheight=tf.cast(data['image/height'], tf.float32)\n #tf.cast(height, tf.float32)\n #tf.cast(width, tf.float32)\n \n # shear -> rotate -> scale\n rot=tf.random.uniform(shape=bs, minval=-rot_angle, maxval=rot_angle)\n shear_x=tf.math.tan(tf.random.uniform(shape=bs, minval=-shear_angle, maxval=shear_angle))\n shear_y=tf.math.tan(tf.random.uniform(shape=bs, minval=-shear_angle, maxval=shear_angle))\n cropScaleX=tf.math.minimum(imgwidth,width)/feedSizef\n cropScaleY=tf.math.minimum(imgheight,height)/feedSizef\n scale_x=tf.random.uniform(shape=bs, minval=scale, maxval=1)*cropScaleX\n scale_y=tf.random.uniform(shape=bs, minval=scale, maxval=1)*cropScaleY\n\n \n diff_x=(imgwidth-feedSizef*scale_x)/2.\n diff_y=(imgheight-feedSizef*scale_y)/2.\n \n rdCropW=tf.math.maximum(diff_x,0)\n rdCropH=tf.math.maximum(diff_y,0)\n rdCropW=tf.random.uniform(bs,minval=-rdCropW, maxval=rdCropW)\n rdCropH=tf.random.uniform(bs,minval=-rdCropH, maxval=rdCropH)\n\n trans=tf.eye(2, batch_shape=bs)\n trans=tf.linalg.matmul(matrix(tf.repeat(1.,bs), shear_x,shear_y, tf.repeat(1.,bs)),trans)\n trans=tf.linalg.matmul(matrix(tf.math.cos(rot), tf.math.sin(rot),-tf.math.sin(rot), tf.math.cos(rot)),trans)\n coor=tf.ones([tf.squeeze(bs),2,1], tf.float32)*padSizef/2.\n coor+=tf.expand_dims(tf.stack([rdCropW,rdCropH],axis=1), axis=2)\n new_coor=tf.linalg.matmul(trans,coor)\n trans=tf.linalg.matmul(matrix(scale_x, tf.repeat(0.,bs),tf.repeat(0.,bs), scale_y),trans)\n\n rdCrop_coor=tf.expand_dims(tf.stack([diff_x+rdCropW,diff_y+rdCropH],axis=1), axis=2)\n shift_coor = tf.expand_dims((padSizef-tf.stack([imgwidth,imgheight],axis=1))/2., axis=2)\n transform=tf.reshape(tf.concat([trans, coor-new_coor+shift_coor+rdCrop_coor], axis=2),[tf.squeeze(bs), -1])\n transform = tf.pad(transform ,[[0,0],[0,2]], 'CONSTANT',0)\n img=tfa.image.transform(img, transform, interpolation='BILINEAR',output_shape=[feedSize,feedSize])\n img=tf.image.random_flip_left_right(img)\n img = tf.reshape(img, [-1,feedSize,feedSize,3])\n return img, data[\"image/class/label\"], tf.gather(weights,data[\"image/class/label\"])\n \n\ndef decodeCropResizeAugment(data, weights):\n \"\"\"\n cropping size dilemma:\n bigger : more distortion more information\n smaller: information loss less distortion\n data is a scalar, this function doesn't handle batches\n \"\"\"\n\n data=_parse_image_function(data)\n \n img=tf.io.decode_jpeg(data[\"image/encoded\"])\n img = tf.image.random_crop(img, size=[tf.math.minimum(tf.cast(height, tf.int64), data['image/height']), tf.math.minimum(tf.cast(width, tf.int64), data['image/width']),data['image/channels']])\n #img = tf.image.convert_image_dtype(img, tf.float32)\n img = tf.cast(img, tf.float32)\n img= tf.image.resize(img, [feedSize, feedSize],method='bilinear')\n \n\n return img, data[\"image/class/label\"], tf.gather(weights,data[\"image/class/label\"])\n\ndef after_batch(img, label, weight ):\n img = (img-rgb_mean)/rgb_stddev\n #img = img/255\n img=tf.image.random_flip_left_right(img)\n \n #img=tf.image.random_hue(img, 0.2)\n #tf.image.random_saturation(img,0.7,2.)\n #tf.image.random_brightness(img, 0.1)\n \n img = tf.reshape(img, [-1,feedSize,feedSize,3])\n return img, label, weight\n\n#resampler = tf.data.experimental.rejection_resample(lambda features, label: label, target_dist=tf.ones(counts.size, tf.float32)/counts.size)\n#sampled_train_ds=croped_dataset.apply(resampler).map(lambda a,b:b)\n\ndef get_weights(data):\n data.prefetch(-1)\n n=81313\n weights=np.zeros(n)\n length =0\n for i in data:\n weights[i[\"image/class/label\"]]+=1\n length+=1\n if length%100000 == 0: print(length)\n weights=length/float(n)/np.maximum(weights,1)#[length/float(n)/max(1,weight) for weight in weights]\n return weights, length\n\n\ndef load_dataset( bs):\n classweight = np.load(\"/kaggle/input/landmark-retrieval-2020-classweights/weights.npy\")\n datasets = glob.glob(\"/kaggle/input/google*\")#[0:1]\n ignore_order = tf.data.Options()\n ignore_order.experimental_deterministic = False\n kd = KaggleDatasets()\n files=[]\n gcss=[]\n print(\"loading dataset\")\n for i in datasets:\n gcs = kd.get_gcs_path(os.path.basename(i))\n files.append(tf.io.matching_files(gcs+\"/records/*\"))\n gcss.append(gcs)\n #print(gcs)\n #print(gcss)\n files = tf.concat(files, axis=0)\n filenames = tf.data.Dataset.from_tensor_slices(files)\n train = tf.data.TFRecordDataset(filenames, num_parallel_reads=-1)#filenames.interleave(lambda x: tf.data.TFRecordDataset(x), cycle_length=-1, block_length=1, num_parallel_calls=-1)\n train = train.with_options(ignore_order)\n train=train.shuffle(10000).map(lambda x: decodeCropResizeAugment(x, tf.constant(classweight)),num_parallel_calls=-1).batch(bs,drop_remainder=True).map(after_batch,num_parallel_calls=-1).prefetch(-1).repeat()\n #train=train.shuffle(10000).batch(bs,drop_remainder=True).map(lambda x: afterBatchPreprocess(x, tf.constant(classweight)),num_parallel_calls=-1).prefetch(-1).repeat()\n\n print(\"dataset loaded\")\n return train\n\ni=0\nds = load_dataset(64)\nc=4\nr=2\nfor img,l,w in ds.take(1):\n f, axarr = plt.subplots(r,c,figsize=(18, 8))\n for i in range(r):\n for j in range(c):\n axarr[i,j].imshow((img[c*i+j,...]*rgb_stddev+rgb_mean)/255.)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\ndef sigmoid_crossentropy(z,x):\n return tf.math.maximum(x, 0) - x * z + tf.math.log(1 + tf.math.exp(-tf.math.abs(x)))\n\nclass ArcFace(tf.keras.losses.Loss):\n def __init__(self,m=0.4,s=30):\n super(ArcFace, self).__init__( reduction = tf.keras.losses.Reduction.NONE)\n self.m=m\n self.s=s\n self.scce=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True,reduction = tf.keras.losses.Reduction.NONE)# it handles the softmax\n\n def call(self, y, x):\n \"\"\" softmxx version\n x should be equal normalized_W * normalized_embeding_x, thus, all values must be in [-1,1] so acos can be applied\n \"\"\"\n\n \"\"\"\n y=tf.cast(tf.reshape(y, [-1]), tf.int32)\n shape=tf.shape(x)\n select=tf.gather(x, y, axis=1, batch_dims=1)\n select=tf.math.acos(select)+m\n select=self.s*tf.math.cos(select)\n indices=tf.expand_dims(tf.range(shape[0]), axis=1)\n indices=tf.concat([indices, tf.expand_dims(y, axis=1)], axis=1)\n x=tf.tensor_scatter_nd_update(x,indices,select)\"\"\"\n \n #loss = tf.keras.losses.sparse_categorical_crossentropy(y,x, from_logits=True)\n \n return self.scce(y,x)\n #super().call(y,x)\n \n \"\"\" hierarchical version\n y and x are ragged tensor with the exact same shape, same number of element\n x are the nodes values of the path (float)\n y : 0 if left 1 if right\n \n rl=(y+1)/2\n loss = tf.math.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(rl, x)*tf.math.abs(y), axis=1)\n \"\"\"\n #return loss\n def get_config(self):\n config = super(ArcFace, self).get_config()\n config.update({\"m\": self.m, \"s\": self.s})\n return config\n\n\n\nclass MeanAveragePrecision(tf.keras.metrics.Metric):\n\n def __init__(self, name=\"mean average precision\", top_k=100, max_query=1000, **kwargs):\n super(MeanAveragePrecision, self).__init__(name=name, **kwargs)\n self.map = self.add_weight(name='map', initializer='zeros', dtype=tf.float64)\n self.max_query=max_query\n self.top_k=top_k\n\n def update_state(self, embeddings, labels, sample_weight=None):\n \"\"\"def tokeep(to_remove, idx):\n to_keep=tf.gather(y,to_remove)# get the label\n to_keep=tf.reduce_all(idx!=to_keep,0)# mask\n return tf.reshape(tf.where(to_keep),[-1])# get index\"\"\"\n\n\n\n\n tf.debugging.assert_equal(tf.shape(embeddings)[0],tf.shape(labels)[0])\n y, idx, count =tf.unique_with_counts(labels,out_idx=tf.int64)\n\n self.map.assign(tf.cond(tf.math.reduce_all(count==1),lambda: tf.constant((1/0)*0, dtype=tf.float64), lambda: self.computenext(embeddings, labels)))\n\n\n def computenext(self, embeddings, labels):\n #@tf.function\n def threshold(data,value):\n \"\"\"\n data: 1D tensor tf.int 4 | |\n value: the integral to remove 3 | | | (5 strokes are removed)\n returns the threshold value to clip 2 | | | value=5 -> | | | returns 2\n Finds the horizontal line to clip. 1 | | | | | | | |\n\n mean=tf.cast(tf.math.reduce_mean(data), tf.int32)\n step=2\n while tf.cond(mean>step*2, lambda: True, lambda: False):\n step*=2\n thre=step\n one_flag=False\n change_flag=False\n before=True\n ok=True\n\n while ok:# Dichotomy\n #step/=2\n if step!=1: step//=2\n else: one_flag=True\n if tf.cond(tf.math.reduce_sum(tf.math.maximum(data-thre,0))>value, lambda: True, lambda: False):\n if not before:\n change_flag=True\n else:\n change_flag=False\n before=True\n else:\n if before:\n change_flag=True\n else:\n change_flag=False\n before=False\n ok = not(one_flag and change_flag)\n thre+=step if before else -step\n\n return thre+1 if before else thre\"\"\"\n\n mean=tf.cast(tf.math.reduce_mean(data), tf.int64)\n step=tf.constant(2, tf.int64)\n step=tf.while_loop(lambda step: mean>step*2, lambda step: step*2, [step])[0]#while tf.cond(mean>step*2, lambda: True, lambda: False):\n thre=tf.identity(step)\n one_flag, change_flag=tf.constant(False),tf.constant(False)\n before, ok=tf.constant(True),tf.constant(True)\n\n\n def whileloop(t,s,o,c,b,ok):\n def sup():\n c = tf.math.logical_not(b)#tf.cond(not b, lambda: tf.constant(True), lambda: tf.constant(False))\n return c, tf.constant(True)\n def inf():\n c = tf.identity(b)#tf.cond(b, lambda: tf.constant(True), lambda: tf.constant(False))\n return c, tf.constant(False)\n\n s,o=tf.cond(s!=1, lambda: (s//2,tf.constant(False)), lambda: (s, tf.constant(True)))\n c,b=tf.cond(tf.math.reduce_sum(tf.math.maximum(data-t,0))>value, sup, inf)\n ok=tf.math.logical_and(o,c)\n t+=tf.cond(before, lambda: s, lambda: -s)\n return [t,s,o,c,b,ok]\n\n thre,_,_,_,before,_ = tf.while_loop(lambda t,s,o,c,b,ok: ok, whileloop, [thre, step, one_flag, change_flag, before, ok])\n thre=tf.cond(before, lambda: thre+1, lambda: thre)\n return thre\n\n\n\n\n def take(data,classes,to_keep):\n \"\"\"\n data contains unique values listed in classes\n to_keep defines how much we want to keep for each label\n e.g to_keep=[3,2,1]->we keep 3 element of label \"0\", 2 of label \"1\" and 1 of label \"2\"\n thus, indices of to_keep must match to indicies of classes\n returns indices of the removed part and the remaining part (they are a partition)\n \"\"\"\n nclass=tf.shape(classes)[0]\n ldata=tf.shape(data)[0]\n mask=data==tf.expand_dims(classes,axis=1)\n data=tf.ragged.boolean_mask(tf.repeat(tf.expand_dims(tf.range(ldata),axis=0),repeats=nclass,axis=0),mask)\n mask=tf.ragged.range(data.row_lengths())1,1,0), tf.int64)\n def moreClassesThanMaxQuery():\n lastsolos=tf.slice(tf.squeeze(tf.where(count==1)),[0],[self.max_query])\n lastsolos=tf.gather(lastsolos,[self.max_query-1])\n return tf.where(tf.range(ldata)self.max_query, lambda: moreClassesThanMaxQuery(), lambda: lessClassesThanMaxQuery())\n idx_queries, idx = take(labels, y, splitToQueriesAndDatabase)\n embeddings_d, labels_d = tf.gather(embeddings, idx), tf.gather(labels, idx)# update again\n embeddings_q, labels_q = tf.gather(embeddings, idx_queries), tf.gather(labels, idx_queries)\n y, idx, count =tf.unique_with_counts(labels_d)\n #label2count=tf.lookup.StaticHashTable(tf.lookup.KeyValueTensorInitializer(y,count),0)\n\n\n nd=tf.shape(labels_d,out_type=tf.int64)[0]\n nq=tf.shape(labels_q,out_type=tf.int64)[0]\n # sort k-nearest\n a=tf.expand_dims(embeddings_q,axis=1)-embeddings_d\n a=tf.math.reduce_euclidean_norm(a,axis=2)\n a=tf.argsort(a)\n a=tf.gather(labels_d,a)\n rel=a==tf.expand_dims(labels_q,axis=-1)# relevance\n\n\n\n \"\"\"\n create range tensor from mask\n [[True, True, True], [[1,2,3],\n [False,False,True], -> [0,0,1],\n [True, False,True]] [1,1,2]]\n \"\"\"\n rg=tf.RaggedTensor.to_tensor(tf.ragged.range(tf.ones(nq, tf.int64)*nd))\n idx=tf.pad(rg,[[0,0],[0,1]], \"CONSTANT\",constant_values=nd)\n mask=tf.pad(rel,[[0,0],[0,1]], \"CONSTANT\",constant_values=True)\n idx=tf.ragged.boolean_mask(idx,mask)\n tosub=tf.pad(rg,[[0,0],[1,0]], \"CONSTANT\",constant_values=0)\n mask=tf.pad(rel,[[0,0],[1,0]], \"CONSTANT\",constant_values=True)\n tosub=tf.ragged.boolean_mask(tosub,mask)\n torepeat=idx-tosub\n p=tf.ragged.map_flat_values(lambda rg, torepeat: tf.repeat(rg,torepeat, axis=0), tf.RaggedTensor.from_tensor(rg),tf.RaggedTensor.from_tensor(tf.RaggedTensor.to_tensor(torepeat,shape=tf.shape(rg))))\n\n p=tf.RaggedTensor.to_tensor(tf.cast(p,tf.float64)/tf.cast(tf.ragged.range(tf.ones(nq, tf.int64),tf.ones(nq, tf.int64)*(nd+1)),tf.float64))*tf.cast(rel,tf.float64)\n p=tf.cond(nd>self.top_k, lambda: tf.slice(p,tf.zeros(tf.rank(p),dtype=tf.int64),[-1, self.top_k]), lambda: p)\n\n get_query_count = tf.gather(count,tf.squeeze(tf.slice(tf.where(y==tf.expand_dims(labels_q,axis=1)),[0,1],[-1,-1])))\n p=tf.reduce_sum(p,axis=1)/tf.cast(tf.math.minimum(get_query_count,[self.top_k]),tf.float64)#top k\n p=tf.reduce_mean(p)\n return p#self.map.assign(p)\n\n\n def result(self):\n return self.map\n\n def reset_states(self):\n self.map.assign(0)\n\nclass AdamAccu(tf.keras.optimizers.Adam):\n def __init__(self, grads_shape, lr=0.001, accumulation=1, clip_norm=1., **kwargs):\n super(AdamAccu, self).__init__(lr=lr, **kwargs)\n self.accumulation=accumulation\n self.count=0\n self.gradients=[tf.zeros_like(this_var) for this_var in grads_shape]\n self.clip_norm= clip_norm\n def apply_gradients(self,grad_and_var):\n self.count+=1\n it=iter(zip(*grad_and_var))\n grads=next(it)\n var=next(it)\n self.gradients = [accu + grad for accu, grad in zip(self.gradients, grads)]\n if self.count%self.accumulation==0:\n self.count=0\n self.gradients = [tf.clip_by_norm(grad/self.accumulation , self.clip_norm) for grad in self.gradients]\n super(AdamAccu,self).apply_gradients(zip(self.gradients,var))\n self.gradients=[tf.zeros_like(this_var) for this_var in self.gradients]\n \n def get_config(self):\n config = super(AdamAccu, self).get_config()\n config.update({\"accumulation\": self.accumulation, \"clip_norm\":self.clip_norm,\n 'grads_shape': [tf.zeros_like(this_var) for this_var in self.gradients]})\n return config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ClassLayer(tf.keras.layers.Layer):\n def __init__(self, units=81313, **kwargs):\n super(ClassLayer, self).__init__(**kwargs)\n self.units = units\n\n def build(self, input_shape):\n self.w = self.add_weight( name=\"center\",\n shape=(input_shape[-1], self.units),\n initializer=\"he_normal\",\n trainable=True,\n )\n #self.b = self.add_weight(name=\"center_bias\",shape=(self.units,), initializer=tf.keras.initializers.Zeros(), trainable=True)\n\n def call(self, inputs):\n return tf.matmul(inputs, self.w)# + self.b\n \n def get_config(self):\n #config = super(ClassLayer, self).get_config()\n #config.update({\"units\": self.units})\n return {\"units\": self.units}#config\n\n\nclass Densenet2vec(tf.keras.Model):\n inter=1024\n D=512\n s=10\n m=0\n n=81313\n input=224\n def __init__(self):\n \"\"\"\n train: {symbol:code,...} (use huffman)\n validation: batched tf.data.Dataset ((,224,224,3),())\n \"\"\"\n super(Densenet2vec,self).__init__()\n\n #self.dense_net=densenet.DenseNet121(input_shape=(224,224,3), weights='../input/densenet-keras/DenseNet-BC-121-32-no-top.h5', include_top=False, pooling=\"avg\")\n #self.dense_net=densenet.DenseNet169(input_shape=(224,224,3), weights=None if load_weights else \"imagenet\", include_top=False, pooling=\"avg\")\n self.dense_net=EfficientNetB6(include_top=False,weights=None ,input_shape=(self.input,self.input,3),pooling='avg')\n\n for layer in self.dense_net.layers:\n layer.trainable = True\n\n self.dense1 = Dense(self.inter, activation=tf.keras.layers.LeakyReLU(), kernel_initializer=\"he_normal\")\n #, kernel_regularizer=regularizers.l1_l2(0.01), activity_regularizer=regularizers.l2(0.01))(x)\n self.flatten=Flatten()\n self.bn = BatchNormalization(fused=None)\n #self.relu1 = Activation('relu')\n self.drop=Dropout(0.1)\n #self.dense2 = Dense(768)#, kernel_regularizer=regularizers.l1_l2(0.01), activity_regularizer=regularizers.l2(0.01))(x)\n #self.relu2 = Activation('relu')\n self.dense3 = Dense(self.D, kernel_initializer=\"he_normal\")\n self.norm=tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(x, axis=1))\n self.bn_train = BatchNormalization(fused=None)\n self.classLayer = ClassLayer( self.n, dtype='float32')\n\n\n #if load_weights:\n self.initLayers()\n #self.load_weights(load_weights)\n\n def call(self, image):\n\n X=self.dense_net(image)\n X=self.flatten(X)\n X=self.dense1(X)\n X=self.bn(X)\n #X=self.relu1(X)\n #X=self.drop(X)\n X=self.dense3(X)\n #X=self.norm(X)\n\n\n return X\n\n def train_step(self, data):\n\n if len(data) == 3:\n x, y, sample_weight = data\n else:\n x, y = data\n sample_weight=1\n\n tf.debugging.check_numerics(x,\"input is wrong\")\n\n with tf.GradientTape() as tape:\n x = self.call(x) # Forward pass\n tf.debugging.check_numerics(x,\"call is wrong\")\n x = self.bn_train(x)\n\n #x=self.relu1(x)\n #x = self.norm(x)*self.s\n x = self.classLayer(x)\n\n loss = self.compiled_loss(y, x, sample_weight=sample_weight, regularization_losses=self.losses)\n \n tf.debugging.check_numerics(loss,\"loss1 is wrong\")\n #if isinstance(self.optimizer,mixed_precision.LossScaleOptimizer):\n #loss=self.optimizer.get_scaled_loss(loss)\n #tf.debugging.check_numerics(loss,\"loss2 is wrong\")\n\n # Compute gradients\n \n gradients = tape.gradient(loss, self.trainable_variables)\n gradients = [(tf.clip_by_norm(grad, 1.0)) for grad in gradients]\n # Update weights\n #if isinstance(self.optimizer,mixed_precision.LossScaleOptimizer):\n #gradients=self.optimizer.get_unscaled_gradients(gradients)\n self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))\n\n return {m.name: m.result() for m in self.metrics}#{\"loss\":loss}\n\n\n\n def test_step(self, data):\n\n if len(data) == 3:\n x, y, sample_weight = data\n else:\n x, y = data\n sample_weight=1\n tf.debugging.assert_integer(y)\n x=self(x)\n #self.compiled_metrics.update_state(embedding, label)\n x = self.bn_train(x)\n x = self.classLayer(x)\n #loss = self.compiled_loss(y, x,sample_weight=sample_weight, regularization_losses=self.losses)\n #loss = tf.nn.compute_average_loss(self.loss(y,x), sample_weight=sample_weight, global_batch_size=self.GBS)\n #return {**{\"HAFL loss\": loss}, **{m.name: m.result() for m in self.metrics}}\n return {\"loss\": loss}\n\n\n\n def metrics_valid(self, dataset):\n\n embeddings=tf.constant([], tf.float32)\n labels=tf.constant([], tf.int64)\n loss=tf.constant([], tf.float32)\n length=0\n for img, label in dataset:\n img=self(img)\n embeddings=tf.concat([embeddings, tf.reshape(img,[-1])], axis=0)\n labels=tf.concat([labels, label], axis=0)\n img = self.bn_train(img)\n img = self.norm(img)\n loss=tf.concat([loss, [self.compiled_loss(label, img, regularization_losses=self.losses)]],axis=0)\n length+=tf.shape(label)[0]\n\n embeddings=tf.reshape(embeddings,[length, self.D])#self.validation_length*self.validation_bs, self.D])\n shuffle=tf.random.shuffle(tf.range(length))#self.validation_length*self.validation_bs))\n embeddings, labels = tf.gather(embeddings,shuffle), tf.gather(labels,shuffle)\n self.compiled_metrics.update_state(embeddings, labels)\n return tf.math.reduce_mean(loss)\n \n def initLayers(self):\n x=tf.random.uniform([2,self.input,self.input,3])\n x=self(x)\n x = self.bn_train(x)\n x = self.classLayer(x)\n \n def get_config(self):\n config = super(Densenet2vec, self).get_config()\n config.update({\"units\": self.units})\n return config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntry:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n\n # instantiate a distribution strategy\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n print(strategy.num_replicas_in_sync)\n BS=2*128* strategy.num_replicas_in_sync\n policy='mixed_bfloat16'\n lr=0.0001\nexcept ValueError:\n if tf.test.is_gpu_available() :\n strategy = tf.distribute.MirroredStrategy()\n print('Mirror')\n else:\n strategy = tf.distribute.get_strategy()\n print('Default')\n BS=64\n policy='mixed_float16'\n lr=0.0001\n \nlength_train = 1514470\n\ntrain= load_dataset(BS)\n\npolicy = mixed_precision.Policy(policy)\nmixed_precision.set_policy(policy)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\n\nclass CustomCallback(tf.keras.callbacks.Callback):\n def __init__(self):#, dataset):\n super(CustomCallback,self).__init__()\n #self.model=model\n #self.dataset_valid=dataset\n #logdir = \"logs/retrivial/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n #self.file_writer = tf.summary.create_file_writer(logdir + \"/metrics\")\n #self.file_writer.set_as_default()\n #def on_epoch_end(self, epoch, logs=None):\n #def on_test_end(self, logs=None):\n #print(\"on_test_end callback\")\n\n def on_epoch_end(self, epoch, logs=None):\n print(\"Saving weights\")\n self.model.save_weights('efn2vec.h5', overwrite=True)\n with open('optimizer.pkl', 'wb') as f:\n pickle.dump(self.model.optimizer.get_weights(), f)\n \n #FileLink(r'densenet2vec.h5')\n #print(\"\\nvalidation loss: {}\".format(self.model.metrics_valid(self.dataset_valid)))\n #for m in self.model.metrics:\n #print(\"{} : {}\".format(m.name,m.result()) )\n #tf.summary.scalar(\"{}\".format(m.name), data=m.result(), step=epoch)\n \n #def on_batch_end(self, epoch, logs=None):\n #print(self.model.optimizer.get_weights()[1][0,0,0,0:3])\n\nclass WarmRestart():\n def __init__(self, max, min, cycle):\n self.max=max\n self.min=min\n self.cycle=cycle\n self.count=0\n self.decay=0.9\n def __call__(self,epoch, lr):\n self.count+=1\n if self.count%self.cycle==0:\n self.count=0\n self.max*=self.decay\n self.min*=self.decay\n lr = self.min+(self.max-self.min)/2.*(1+tf.math.cos(self.count*3.14159/self.cycle).numpy())\n print(\"count: {}\\tlr: {}\".format(self.count, lr))\n return lr\n\nschedule = tf.keras.callbacks.LearningRateScheduler(WarmRestart(max=0.0005, min=0.00001, cycle=10).__call__)\n\n#model.evaluate(tf_traindev_database.batch(128, drop_remainder=False))#traindev_images,traindev_labels))\ncp_callback = tf.keras.callbacks.ModelCheckpoint(filepath='efn2vec.h5',\n save_best_only=False, save_weights_only=False, verbose=1)\n#logs = \"logs/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n#tboard_callback = tf.keras.callbacks.TensorBoard(log_dir = logs, profile_batch = '2,50')\n\nreduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.2, patience=3, verbose=1)#, min_lr=0.0000001)\n\n\n# \"\"\"\n\n# \"\"\"\ndef buildFit(path, lr=0.0001, epochs=10, callbacks=None):\n with open(path+'/optimizer.pkl', 'rb') as f:\n weight_values = pickle.load(f)\n with strategy.scope():\n\n model=Densenet2vec()#'/kaggle/input/notebook3eb84c5ce9/efn2vec.h5')\n model.load_weights(path+'/efn2vec.h5')\n tw = model.trainable_weights\n \n optimizer = tf.keras.optimizers.Adam(lr)#, clipnorm=1.\n #optimizer = AdamAccu(grads_shape=tw, lr=lr, accumulation=2)\n #optimizer = optimizer = mixed_precision.LossScaleOptimizer(optimizer, loss_scale='dynamic')\n \n optimizer._create_all_weights(tw)\n #optimizer.set_weights(weight_values)\n \n model.compile(\n optimizer=optimizer,\n loss=ArcFace()\n #,metrics=[MeanAveragePrecision()],run_eagerly=False\n #,experimental_steps_per_execution=length_train//BS # tf 2.3\n )\n \n #print(model.optimizer.get_weights()[1])\n model.optimizer.set_weights(weight_values)\n #print(model.optimizer.get_weights()[1])\n #print(weight_values[1][0,0,0,0:3])\n\n history=model.fit(train,\n epochs=epochs, verbose=2, steps_per_epoch=length_train//BS\n #,validation_data=dev #,validation_steps=\n ,callbacks=callbacks\n )\n\n \n return history\n\nbuildFit(path=\"/kaggle/input/notebook3eb84c5ce9\", lr=lr, epochs=12, callbacks=[CustomCallback(),reduce_lr])\n#buildFit(path=\"/kaggle/input/landmark-retrieval-2020-classweights\", lr=lr, epochs=12, callbacks=[CustomCallback(),reduce_lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a=tf.ragged.constant([[1,2,3],[4,5]])\nfor x in a:\n print(x)\n\n\n \n \n \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}