{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport math\nimport pandas as pd\nimport glob\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_submission_file(input_path, alpha=0.5):\n    files_paths = glob.glob(input_path + 'test/*/*/*/*')\n    mapping = {}\n    for path in files_paths:\n        mapping[path.split('/')[-1].split('.')[0]] = path\n    df = pd.read_csv(input_path + 'sample_submission.csv')\n    df['path'] = df['id'].map(mapping)\n    df['label'] = -1\n    df['prob'] = -1\n    return df\n\ndef read_train_file(input_path, alpha=0.5):\n    files_paths = glob.glob(input_path + 'train/*/*/*/*')\n    mapping = {}\n    for path in files_paths:\n        mapping[path.split('/')[-1].split('.')[0]] = path\n    df = pd.read_csv(input_path + 'train.csv')\n    df['path'] = df['id'].map(mapping)\n    \n    counts_map = dict(df.groupby('landmark_id')['path'].agg(lambda x: len(x)))\n    df['counts'] = df['landmark_id'].map(counts_map)\n    df['prob'] = ((1/df.counts**alpha) / (1/df.counts**alpha).max()).astype(np.float32)\n    uniques = df['landmark_id'].unique()\n    df['label'] = df['landmark_id'].map(dict(zip(uniques, range(len(uniques)))))\n    return df, dict(zip(range(len(uniques)), uniques))\n\n\nsubmission_df = read_submission_file('../input/landmark-recognition-2020/')\ntrain_df, mapping = read_train_file('../input/landmark-recognition-2020/')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"for simplicity only 1000 images are taken","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = train_df.iloc[:1000,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The following functions can be used to convert a value to a type compatible\n# with tf.Example.\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a dictionary with features that may be relevant.\ndef image_example(image_string, label):\n  image_shape = tf.image.decode_jpeg(image_string).shape\n\n  feature = {\n      'height': _int64_feature(image_shape[0]),\n      'width': _int64_feature(image_shape[1]),\n      'depth': _int64_feature(image_shape[2]),\n      'label': _int64_feature(label),\n      'image_raw': _bytes_feature(image_string),\n  }\n\n  return tf.train.Example(features=tf.train.Features(feature=feature))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Tfrecord store the features height, width, depth, label, image","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# code for generating tfrecord file\nrecord_file = 'train00.tfrecords'\nwith tf.io.TFRecordWriter(record_file) as writer:\n  for i in range(len(dataset)):\n    path = dataset.path[dataset.index[i]]\n    label = dataset.label[dataset.index[i]]\n    image_string = tf.io.read_file(path)\n    tf_example = image_example(image_string, label)\n    writer.write(tf_example.SerializeToString())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Reading the tfrecord files**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n    'height': tf.io.FixedLenFeature([], tf.int64),\n    'width': tf.io.FixedLenFeature([], tf.int64),\n    'depth': tf.io.FixedLenFeature([], tf.int64),\n    'label': tf.io.FixedLenFeature([], tf.int64),\n    'image_raw': tf.io.FixedLenFeature([], tf.string),\n        }\n    return tf.io.parse_single_example(example, tfrec_format)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = tf.data.TFRecordDataset('./train00.tfrecords')\nds = ds.map(read_labeled_tfrecord)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import IPython.display as display\nfor feature in ds.take(3):\n  image_raw = feature['image_raw'].numpy()\n  display.display(display.Image(data=image_raw))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}