{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Purpose of this notebook is we have to make dataset. we have large dataset for training by the image take long time to run the task so we are going to make tfrecord from the origin dataset\nref link : https://www.tensorflow.org/tutorials/load_data/tfrecord","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os, json, random, cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf, re, math\nfrom tqdm import tqdm\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:27.440182Z","iopub.execute_input":"2022-02-15T12:07:27.440772Z","iopub.status.idle":"2022-02-15T12:07:33.825616Z","shell.execute_reply.started":"2022-02-15T12:07:27.440733Z","shell.execute_reply":"2022-02-15T12:07:33.824661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\n### Create Kaggle Dataset if not exist\nDATASET_NAME = f'happywhale-tfrecords-v1'\n# LETS CHECK IF ANY DATASET PRESENT IN THE DIRECTORY REMOVE IT\n!rm -r /tmp/{DATASET_NAME}\n\nos.makedirs(f'/tmp/{DATASET_NAME}', exist_ok=True)\n\n\nwith open('../input/kaggle-api-creds/kaggle.json') as f:\n    kaggle_creds = json.load(f)\n    \nos.environ['KAGGLE_USERNAME'] = kaggle_creds['username']\nos.environ['KAGGLE_KEY'] = kaggle_creds['key']\n\n!kaggle datasets init -p /tmp/{DATASET_NAME}\n\n\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json') as f:\n    dataset_meta = json.load(f)\n    \ndataset_meta['id'] = f'nikhiljothiprakash/{DATASET_NAME}'\ndataset_meta['title'] =DATASET_NAME\n\n\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json',\"w\") as outfile:\n    json.dump(dataset_meta, outfile)\nprint(dataset_meta)\n\n!cp /tmp/{DATASET_NAME}/dataset-metadata.json /tmp/{DATASET_NAME}/meta.json\n!ls /tmp/{DATASET_NAME}\n\n!kaggle datasets create -u -p /tmp/{DATASET_NAME} \n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:33.827396Z","iopub.execute_input":"2022-02-15T12:07:33.828132Z","iopub.status.idle":"2022-02-15T12:07:39.040774Z","shell.execute_reply.started":"2022-02-15T12:07:33.828086Z","shell.execute_reply":"2022-02-15T12:07:39.039626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/happywhale/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:39.042680Z","iopub.execute_input":"2022-02-15T12:07:39.042970Z","iopub.status.idle":"2022-02-15T12:07:39.124687Z","shell.execute_reply.started":"2022-02-15T12:07:39.042937Z","shell.execute_reply":"2022-02-15T12:07:39.123725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/happy-whale-and-dolphin/sample_submission.csv')\ntest_df['split'] = test_df.index%10","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:39.126706Z","iopub.execute_input":"2022-02-15T12:07:39.127258Z","iopub.status.idle":"2022-02-15T12:07:39.201286Z","shell.execute_reply.started":"2022-02-15T12:07:39.127220Z","shell.execute_reply":"2022-02-15T12:07:39.200202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_fold(fold):\n    val_df = train_df[train_df.kfold==fold].reset_index(drop=True)\n    val_df['order'] = val_df.index\n    val_df['order'] = val_df.groupby('individual_id').order.rank()\n    val_total_counts = val_df.individual_id.value_counts().to_dict()\n    val_df['total_counts'] = val_df.individual_id.map(val_total_counts)\n    val_df['order'] = val_df['order']/val_df['total_counts']\n    val_df = val_df.sort_values('order',ascending=False).reset_index(drop=True)\n    val_df = val_df[['image','species','individual_id']]\n    return val_df","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:39.203095Z","iopub.execute_input":"2022-02-15T12:07:39.203434Z","iopub.status.idle":"2022-02-15T12:07:39.213164Z","shell.execute_reply.started":"2022-02-15T12:07:39.203390Z","shell.execute_reply":"2022-02-15T12:07:39.211825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image,image_name,target,species):\n    feature = {\n        'image': _bytes_feature(image),\n        'image_name': _bytes_feature(image_name),\n        'target': _int64_feature(target),\n        'species': _int64_feature(species),\n      }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:39.214543Z","iopub.execute_input":"2022-02-15T12:07:39.214826Z","iopub.status.idle":"2022-02-15T12:07:39.233119Z","shell.execute_reply.started":"2022-02-15T12:07:39.214792Z","shell.execute_reply":"2022-02-15T12:07:39.231710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_tf_records(fold  = 0):\n    df = get_fold(fold)\n    tfr_filename = f'/tmp/{DATASET_NAME}/happywhale-2022-train-{fold}-{df.shape[0]}.tfrec'\n    with tf.io.TFRecordWriter(tfr_filename) as writer:\n        for i,row in df.iterrows():\n            image_id = row.image\n            target = row.individual_id\n            species = row.species\n            image_path = f\"../input/happy-whale-and-dolphin/train_images/{image_id}\"\n            image_encoded = tf.io.read_file(image_path)\n            image_name = str.encode(image_id)\n            example = serialize_example(image_encoded,image_name,target,species)\n            writer.write(example)\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:39.235046Z","iopub.execute_input":"2022-02-15T12:07:39.236146Z","iopub.status.idle":"2022-02-15T12:07:39.247688Z","shell.execute_reply.started":"2022-02-15T12:07:39.236100Z","shell.execute_reply":"2022-02-15T12:07:39.246580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\n_ = joblib.Parallel(n_jobs=8)(\n        joblib.delayed(create_tf_records)(fold) for fold in tqdm(range(10))\n    )","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:39.248866Z","iopub.execute_input":"2022-02-15T12:07:39.249381Z","iopub.status.idle":"2022-02-15T12:14:56.202102Z","shell.execute_reply.started":"2022-02-15T12:07:39.249320Z","shell.execute_reply":"2022-02-15T12:14:56.200408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_test_tf_records(fold  = 0):\n    df = test_df[test_df.split==fold]\n    tfr_filename = f'/tmp/{DATASET_NAME}/happywhale-2022-test-{fold}-{df.shape[0]}.tfrec'\n    with tf.io.TFRecordWriter(tfr_filename) as writer:\n        for i,row in df.iterrows():\n            image_id = row.image\n            target = -1\n            species = -1\n            image_path = f\"../input/happy-whale-and-dolphin/test_images/{image_id}\"\n            image_encoded = tf.io.read_file(image_path)\n            image_name = str.encode(image_id)\n            example = serialize_example(image_encoded,image_name,target,species)\n            writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:14:56.206268Z","iopub.execute_input":"2022-02-15T12:14:56.206719Z","iopub.status.idle":"2022-02-15T12:14:56.217193Z","shell.execute_reply.started":"2022-02-15T12:14:56.206672Z","shell.execute_reply":"2022-02-15T12:14:56.216058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\n_ = joblib.Parallel(n_jobs=8)(\n        joblib.delayed(create_test_tf_records)(fold) for fold in tqdm(range(10))\n    )","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:14:56.220112Z","iopub.execute_input":"2022-02-15T12:14:56.220428Z","iopub.status.idle":"2022-02-15T12:20:31.575663Z","shell.execute_reply.started":"2022-02-15T12:14:56.220397Z","shell.execute_reply":"2022-02-15T12:20:31.574528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nversion_name = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\nprint(version_name)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:20:31.577810Z","iopub.execute_input":"2022-02-15T12:20:31.578175Z","iopub.status.idle":"2022-02-15T12:20:31.586544Z","shell.execute_reply.started":"2022-02-15T12:20:31.578130Z","shell.execute_reply":"2022-02-15T12:20:31.585113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!kaggle datasets version -m {version_name} -p /tmp/{DATASET_NAME} -r zip -q","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:20:31.588555Z","iopub.execute_input":"2022-02-15T12:20:31.589176Z","iopub.status.idle":"2022-02-15T12:34:39.100436Z","shell.execute_reply.started":"2022-02-15T12:20:31.589135Z","shell.execute_reply":"2022-02-15T12:34:39.099444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nVerify TFRecords¶","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.image.resize(image,IMAGE_SIZE_)\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        'target': tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = example['target']\n    return image, label # returns a dataset of (image, label) pairs\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:34:39.102759Z","iopub.execute_input":"2022-02-15T12:34:39.103992Z","iopub.status.idle":"2022-02-15T12:34:39.121730Z","shell.execute_reply.started":"2022-02-15T12:34:39.103946Z","shell.execute_reply":"2022-02-15T12:34:39.120565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\nCLASSES = [0,1]\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    #if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n    #    numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef display_single_sample(image, label, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    title = str(label)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch):\n    \"\"\"\n    Display single batch Of images \n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        correct = True\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_single_sample(image, label, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:34:39.123598Z","iopub.execute_input":"2022-02-15T12:34:39.124312Z","iopub.status.idle":"2022-02-15T12:34:39.146200Z","shell.execute_reply.started":"2022-02-15T12:34:39.124265Z","shell.execute_reply":"2022-02-15T12:34:39.145024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = 256\nBATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:34:39.147611Z","iopub.execute_input":"2022-02-15T12:34:39.148067Z","iopub.status.idle":"2022-02-15T12:34:39.164613Z","shell.execute_reply.started":"2022-02-15T12:34:39.148022Z","shell.execute_reply":"2022-02-15T12:34:39.163480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE_ = [IMAGE_SIZE,IMAGE_SIZE]\nAUTO = tf.data.experimental.AUTOTUNE\nTRAINING_FILENAMES = tf.io.gfile.glob(f'/tmp/{DATASET_NAME}/happywhale-2022-train*.tfrec')\nprint(len(TRAINING_FILENAMES))\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True)\ndataset = dataset.repeat()\ndataset = dataset.shuffle(2048)\ndataset = dataset.batch(BATCH_SIZE)\ndataset = dataset.prefetch(AUTO) #This dataset can directly be passed to keras.fit method\nprint(count_data_items(TRAINING_FILENAMES))\n\n# Displaying single batch of TFRecord\ntrain_batch = iter(dataset)\ndisplay_batch_of_images(next(train_batch))\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:34:39.168125Z","iopub.execute_input":"2022-02-15T12:34:39.168470Z","iopub.status.idle":"2022-02-15T12:35:21.641150Z","shell.execute_reply.started":"2022-02-15T12:34:39.168431Z","shell.execute_reply":"2022-02-15T12:35:21.640355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}