{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#

IMPORT

\n****\n","metadata":{}},{"cell_type":"code","source":"import os, json, random, cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf, re, math\nfrom tqdm import tqdm\nimport joblib\nfrom datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:14.723710Z","iopub.execute_input":"2022-04-10T05:49:14.724390Z","iopub.status.idle":"2022-04-10T05:49:20.299710Z","shell.execute_reply.started":"2022-04-10T05:49:14.724349Z","shell.execute_reply":"2022-04-10T05:49:20.298856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

SETUP

\n****","metadata":{}},{"cell_type":"code","source":"print(f\"\\n... ACCELERATOR SETUP STARTING ...\\n\")\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n TPU = tf.distribute.cluster_resolver.TPUClusterResolver() \nexcept ValueError:\n TPU = None\n\nif TPU:\n print(f\"\\n... RUNNING ON TPU - {TPU.master()}...\")\n tf.config.experimental_connect_to_cluster(TPU)\n tf.tpu.experimental.initialize_tpu_system(TPU)\n strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n print(f\"\\n... RUNNING ON CPU/GPU ...\")\n # Yield the default distribution strategy in Tensorflow\n # --> Works on CPU and single GPU.\n strategy = tf.distribute.get_strategy() \n\n# What Is a Replica?\n# --> A single Cloud TPU device consists of FOUR chips, each of which has TWO TPU cores. \n# --> Therefore, for efficient utilization of Cloud TPU, a program should make use of each of the EIGHT (4x2) cores. \n# --> Each replica is essentially a copy of the training graph that is run on each core and \n# trains a mini-batch containing 1/8th of the overall batch size\nN_REPLICAS = strategy.num_replicas_in_sync\n \nprint(f\"... # OF REPLICAS: {N_REPLICAS} ...\\n\")\n\nprint(f\"\\n... ACCELERATOR SETUP COMPLTED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:20.301449Z","iopub.execute_input":"2022-04-10T05:49:20.301705Z","iopub.status.idle":"2022-04-10T05:49:20.317586Z","shell.execute_reply.started":"2022-04-10T05:49:20.301675Z","shell.execute_reply":"2022-04-10T05:49:20.316559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

VERSIONS

\n****\n\nV1-2 : Create TFRecords from [whale2-cropped-dataset](https://www.kaggle.com/datasets/phalanx/whale2-cropped-dataset) by @Phalanx\n\nV3 : Create TFRecords from [happywhale-cropped-removeBackground-v1](https://www.kaggle.com/datasets/phanttan/happywhale-cropped-removebackground-v1) by me @phanttan","metadata":{}},{"cell_type":"code","source":"print(\"\\n... DATA ACCESS SETUP STARTED ...\\n\")\n\nif TPU:\n # Google Cloud Dataset path to training and validation images\n DATA_DIR = KaggleDatasets().get_gcs_path('happy-whale-and-dolphin')\n save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n load_locally = tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')\nelse:\n # Local path to training and validation images\n DATA_DIR = \"../input/happy-whale-and-dolphin\"\n save_locally = None\n load_locally = None\n\n# EXTRA_DATA_DIR = \"../input/whale2-cropped-dataset\"\nEXTRA_DATA_DIR = \"../input/happywhale-cropped-removebackground-v1\"\n\nprint(f\"\\n... DATA DIRECTORY PATH IS:\\n\\t--> {DATA_DIR}\")\nprint(f\"\\n... EXTRA METADATA DIRECTORY PATH IS:\\n\\t--> {EXTRA_DATA_DIR}\")\n\nprint(f\"\\n... IMMEDIATE CONTENTS OF DATA DIRECTORY IS:\")\nfor file in tf.io.gfile.glob(os.path.join(DATA_DIR, \"*\")): print(f\"\\t--> {file}\")\n\nprint(f\"\\n... IMMEDIATE CONTENTS OF EXTRA METADATA DIRECTORY IS:\")\nfor file in tf.io.gfile.glob(os.path.join(EXTRA_DATA_DIR, \"*\")): print(f\"\\t--> {file}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:20.318952Z","iopub.execute_input":"2022-04-10T05:49:20.319637Z","iopub.status.idle":"2022-04-10T05:49:20.338913Z","shell.execute_reply.started":"2022-04-10T05:49:20.319602Z","shell.execute_reply":"2022-04-10T05:49:20.337984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n### Create Kaggle Dataset if not exists \n\nDATASET_NAME = f'happywhale-cropped-removebackground-tfrecords-v1'\n\n!rm -r /tmp/{DATASET_NAME} # remove folder\n\nos.makedirs(f'/tmp/{DATASET_NAME}', exist_ok=True)\n\nwith open('../input/kaggle-json-file/kaggle.json') as f:\n kaggle_creds = json.load(f)\n \nos.environ['KAGGLE_USERNAME'] = kaggle_creds['username']\nos.environ['KAGGLE_KEY'] = kaggle_creds['key']\n\n!kaggle datasets init -p /tmp/{DATASET_NAME}\n\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json') as f:\n dataset_meta = json.load(f)\ndataset_meta['id'] = f'phanttan/{DATASET_NAME}'\ndataset_meta['title'] = DATASET_NAME\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json', \"w\") as outfile:\n json.dump(dataset_meta, outfile)\nprint(dataset_meta)\n\n!cp /tmp/{DATASET_NAME}/dataset-metadata.json /tmp/{DATASET_NAME}/meta.json\n!ls /tmp/{DATASET_NAME}\n\n!kaggle datasets create -u -p /tmp/{DATASET_NAME} ","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:20.340755Z","iopub.execute_input":"2022-04-10T05:49:20.341242Z","iopub.status.idle":"2022-04-10T05:49:25.721958Z","shell.execute_reply.started":"2022-04-10T05:49:20.341208Z","shell.execute_reply":"2022-04-10T05:49:25.720919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

CHECKING DATASET

\n****\nThe original dataset is saved in **'../input/happy-whale-and-dolphin'**\n\nAnd the new preprocessed dataset is created by @phalanx in **'../input/whale2-cropped-dataset'** ([more details](https://www.kaggle.com/competitions/happy-whale-and-dolphin/discussion/305503))","metadata":{}},{"cell_type":"code","source":"ORIG_DIR = \"../input/happy-whale-and-dolphin/\"\nEXTRA_DIR = \"../input/happywhale-cropped-removebackground-v1/\"\nIMAGE_TRAIN_DIR = f\"{EXTRA_DIR}removedBackground_train_images/\"\nIMAGE_TEST_DIR = f\"{EXTRA_DIR}removedBackground_test_image/\"","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:25.723975Z","iopub.execute_input":"2022-04-10T05:49:25.724248Z","iopub.status.idle":"2022-04-10T05:49:25.729670Z","shell.execute_reply.started":"2022-04-10T05:49:25.724217Z","shell.execute_reply":"2022-04-10T05:49:25.728576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_TEST_DIR","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:25.730924Z","iopub.execute_input":"2022-04-10T05:49:25.731180Z","iopub.status.idle":"2022-04-10T05:49:25.747916Z","shell.execute_reply.started":"2022-04-10T05:49:25.731152Z","shell.execute_reply":"2022-04-10T05:49:25.747188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_train_files = os.listdir(ORIG_DIR + \"train_images\")\nextra_train_files = os.listdir(EXTRA_DIR + \"removedBackground_train_images\")\nif orig_train_files == extra_train_files:\n print(\"Training files is OK\")\nelse: \n print(\"Check again Training Files\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:25.749298Z","iopub.execute_input":"2022-04-10T05:49:25.749743Z","iopub.status.idle":"2022-04-10T05:49:27.447568Z","shell.execute_reply.started":"2022-04-10T05:49:25.749711Z","shell.execute_reply":"2022-04-10T05:49:27.446733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_test_files = os.listdir(ORIG_DIR + \"test_images\")\nextra_test_files = os.listdir(EXTRA_DIR + \"removedBackground_test_image\")\nif orig_test_files == extra_test_files:\n print(\"Testing files is OK\")\nelse: \n print(\"Check again Testing Files\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:27.449180Z","iopub.execute_input":"2022-04-10T05:49:27.449755Z","iopub.status.idle":"2022-04-10T05:49:28.490613Z","shell.execute_reply.started":"2022-04-10T05:49:27.449711Z","shell.execute_reply":"2022-04-10T05:49:28.489732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_train_df = pd.read_csv(ORIG_DIR + \"train.csv\")\nif os.path.isdir(EXTRA_DIR + \"train2.csv\"):\n cropped_train_df = pd.read_csv(EXTRA_DIR + \"train2.csv\")\n if all(orig_train_df == cropped_train_df[['image','species','individual_id']]):\n print(\"Training CSV File is OK\")\n else:\n print(\"Check again Training CSV File\")\nelse:\n print('No Training CSV File in Extra Dataset')","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.492269Z","iopub.execute_input":"2022-04-10T05:49:28.492781Z","iopub.status.idle":"2022-04-10T05:49:28.603998Z","shell.execute_reply.started":"2022-04-10T05:49:28.492737Z","shell.execute_reply":"2022-04-10T05:49:28.603113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_ss_df = pd.read_csv(ORIG_DIR + \"sample_submission.csv\")\nif os.path.isdir(EXTRA_DIR + \"test2.csv\"):\n extra_ss_df = pd.read_csv(EXTRA_DIR + \"test2.csv\")\n if all(orig_ss_df == extra_ss_df[['image','predictions']]):\n print(\"Sample Submission CSV File is OK\")\n else:\n print(\"Check again Sample Submission CSV File\")\nelse:\n print(\"No Sample Submission File in Extra Dataset\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.606643Z","iopub.execute_input":"2022-04-10T05:49:28.606874Z","iopub.status.idle":"2022-04-10T05:49:28.675268Z","shell.execute_reply.started":"2022-04-10T05:49:28.606846Z","shell.execute_reply":"2022-04-10T05:49:28.674485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

HELPING FUNCTIONS

\n****","metadata":{}},{"cell_type":"code","source":"def get_fold(fold):\n val_df = train_df[train_df.fold==fold].reset_index(drop=True)\n val_df['order'] = val_df.index\n val_df['order'] = val_df.groupby('individual_id').order.rank()\n val_total_counts = val_df.individual_id.value_counts().to_dict()\n val_df['total_counts'] = val_df.individual_id.map(val_total_counts)\n val_df['order'] = val_df['order']/val_df['total_counts']\n val_df = val_df.sort_values('order',ascending=False).reset_index(drop=True)\n val_df = val_df[['image','species','individual_id']]\n return val_df","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.676503Z","iopub.execute_input":"2022-04-10T05:49:28.677094Z","iopub.status.idle":"2022-04-10T05:49:28.683621Z","shell.execute_reply.started":"2022-04-10T05:49:28.677062Z","shell.execute_reply":"2022-04-10T05:49:28.682908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n if isinstance(value, type(tf.constant(0))):\n value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n \"\"\"Returns a float_list from a float / double.\"\"\"\n return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image,image_name,target,species):\n feature = {\n 'image': _bytes_feature(image),\n 'image_name': _bytes_feature(image_name),\n 'target': _int64_feature(target),\n 'species': _int64_feature(species),\n }\n example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.684657Z","iopub.execute_input":"2022-04-10T05:49:28.685062Z","iopub.status.idle":"2022-04-10T05:49:28.699032Z","shell.execute_reply.started":"2022-04-10T05:49:28.685021Z","shell.execute_reply":"2022-04-10T05:49:28.697968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_tf_records(fold = 0):\n df = get_fold(fold)\n tfr_filename = f'/tmp/{DATASET_NAME}/happywhale-2022-train-{fold}-{df.shape[0]}.tfrec'\n with tf.io.TFRecordWriter(tfr_filename) as writer:\n for i,row in df.iterrows():\n image_id = row.image\n target = row.individual_id\n species = row.species\n image_path = f\"{IMAGE_TRAIN_DIR}/{image_id}\"\n image_encoded = tf.io.read_file(image_path)\n image_name = str.encode(image_id)\n example = serialize_example(image_encoded,image_name,target,species)\n writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.702747Z","iopub.execute_input":"2022-04-10T05:49:28.703818Z","iopub.status.idle":"2022-04-10T05:49:28.717072Z","shell.execute_reply.started":"2022-04-10T05:49:28.703757Z","shell.execute_reply":"2022-04-10T05:49:28.716074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_test_tf_records(fold = 0):\n df = test_df[test_df.split==fold]\n tfr_filename = f'/tmp/{DATASET_NAME}/happywhale-2022-test-{fold}-{df.shape[0]}.tfrec'\n with tf.io.TFRecordWriter(tfr_filename) as writer:\n for i,row in df.iterrows():\n image_id = row.image\n target = -1\n species = -1\n image_path = f\"{IMAGE_TEST_DIR}{image_id}\"\n image_encoded = tf.io.read_file(image_path)\n image_name = str.encode(image_id)\n example = serialize_example(image_encoded,image_name,target,species)\n writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.718575Z","iopub.execute_input":"2022-04-10T05:49:28.719042Z","iopub.status.idle":"2022-04-10T05:49:28.731367Z","shell.execute_reply.started":"2022-04-10T05:49:28.718986Z","shell.execute_reply":"2022-04-10T05:49:28.730396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify TFRecords\n\ndef decode_image(image_data):\n image = tf.image.decode_jpeg(image_data, channels=3)\n image = tf.cast(image, tf.float32) / 255.0 # convert image to floats in [0, 1] range\n image = tf.image.resize(image,IMAGE_SIZE_)\n return image\n\ndef read_labeled_tfrecord(example):\n LABELED_TFREC_FORMAT = {\n \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n \"image_name\": tf.io.FixedLenFeature([], tf.string), # shape [] means single element\n 'target': tf.io.FixedLenFeature([], tf.int64),\n }\n example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n image = decode_image(example['image'])\n label = example['target']\n return image, label # returns a dataset of (image, label) pairs\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n # Read from TFRecords. For optimal performance, reading from multiple files at once and\n # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n ignore_order = tf.data.Options()\n if not ordered:\n ignore_order.experimental_deterministic = False # disable order, increase speed\n\n dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n dataset = dataset.map(read_labeled_tfrecord)\n # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n return dataset\n\ndef get_training_dataset():\n dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n dataset = dataset.repeat() # the training dataset must repeat for several epochs\n dataset = dataset.shuffle(2048)\n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n return dataset\n\ndef count_data_items(filenames):\n # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.732571Z","iopub.execute_input":"2022-04-10T05:49:28.732796Z","iopub.status.idle":"2022-04-10T05:49:28.750616Z","shell.execute_reply.started":"2022-04-10T05:49:28.732767Z","shell.execute_reply":"2022-04-10T05:49:28.749969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\nCLASSES = [0,1]\n\ndef batch_to_numpy_images_and_labels(data):\n images, labels = data\n numpy_images = images.numpy()\n numpy_labels = labels.numpy()\n #if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n # numpy_labels = [None for _ in enumerate(numpy_images)]\n # If no labels, only image IDs, return None for labels (this is the case for test data)\n return numpy_images, numpy_labels\n\ndef display_single_sample(image, label, subplot, red=False, titlesize=16):\n plt.subplot(*subplot)\n plt.axis('off')\n plt.imshow(image)\n title = str(label)\n if len(title) > 0:\n plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n return (subplot[0], subplot[1], subplot[2]+1)\n \ndef display_batch_of_images(databatch):\n \"\"\"\n Display single batch Of images \n \"\"\"\n # data\n images, labels = batch_to_numpy_images_and_labels(databatch)\n if labels is None:\n labels = [None for _ in enumerate(images)]\n \n # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n rows = int(math.sqrt(len(images)))\n cols = len(images)//rows\n \n # size and spacing\n FIGSIZE = 13.0\n SPACING = 0.1\n subplot=(rows,cols,1)\n if rows < cols:\n plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n else:\n plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n \n # display\n for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n correct = True\n dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n subplot = display_single_sample(image, label, subplot, not correct, titlesize=dynamic_titlesize)\n \n #layout\n plt.tight_layout()\n if label is None and predictions is None:\n plt.subplots_adjust(wspace=0, hspace=0)\n else:\n plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.751839Z","iopub.execute_input":"2022-04-10T05:49:28.752279Z","iopub.status.idle":"2022-04-10T05:49:28.771781Z","shell.execute_reply.started":"2022-04-10T05:49:28.752235Z","shell.execute_reply":"2022-04-10T05:49:28.771088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Create TFRecords

\n****","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/happywhale-splits/skf_species_10folds.csv')\ntest_df = pd.read_csv('../input/happy-whale-and-dolphin/sample_submission.csv')\ntest_df['split'] = test_df.index%10\ntrain_df.agg(['min','max','count','nunique'])","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.773228Z","iopub.execute_input":"2022-04-10T05:49:28.773656Z","iopub.status.idle":"2022-04-10T05:49:28.967817Z","shell.execute_reply.started":"2022-04-10T05:49:28.773625Z","shell.execute_reply":"2022-04-10T05:49:28.966989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = joblib.Parallel(n_jobs=8)(joblib.delayed(create_tf_records)(fold) for fold in tqdm(range(10)))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:49:28.969274Z","iopub.execute_input":"2022-04-10T05:49:28.969605Z","iopub.status.idle":"2022-04-10T05:50:30.117721Z","shell.execute_reply.started":"2022-04-10T05:49:28.969561Z","shell.execute_reply":"2022-04-10T05:50:30.116359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = joblib.Parallel(n_jobs=8)(joblib.delayed(create_test_tf_records)(fold) for fold in tqdm(range(10)))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:50:30.120599Z","iopub.execute_input":"2022-04-10T05:50:30.121078Z","iopub.status.idle":"2022-04-10T05:50:55.740565Z","shell.execute_reply.started":"2022-04-10T05:50:30.120999Z","shell.execute_reply":"2022-04-10T05:50:55.739820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /tmp/{DATASET_NAME}","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:50:55.741885Z","iopub.execute_input":"2022-04-10T05:50:55.742240Z","iopub.status.idle":"2022-04-10T05:50:56.525698Z","shell.execute_reply.started":"2022-04-10T05:50:55.742198Z","shell.execute_reply":"2022-04-10T05:50:56.524484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"version_name = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n!kaggle datasets version -m {version_name} -p /tmp/{DATASET_NAME} -r zip -q","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:50:56.527798Z","iopub.execute_input":"2022-04-10T05:50:56.528734Z","iopub.status.idle":"2022-04-10T05:51:18.051250Z","shell.execute_reply.started":"2022-04-10T05:50:56.528672Z","shell.execute_reply":"2022-04-10T05:51:18.049930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Verify TFRecords

\n****","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = 256\nBATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:51:18.053162Z","iopub.execute_input":"2022-04-10T05:51:18.053423Z","iopub.status.idle":"2022-04-10T05:51:18.057995Z","shell.execute_reply.started":"2022-04-10T05:51:18.053391Z","shell.execute_reply":"2022-04-10T05:51:18.057047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE_ = [IMAGE_SIZE, IMAGE_SIZE]\nAUTO = tf.data.experimental.AUTOTUNE\nTRAINING_FILENAMES = tf.io.gfile.glob(f'/tmp/{DATASET_NAME}/happywhale-2022-train*.tfrec')\nprint(len(TRAINING_FILENAMES))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:51:18.059658Z","iopub.execute_input":"2022-04-10T05:51:18.060048Z","iopub.status.idle":"2022-04-10T05:51:18.075044Z","shell.execute_reply.started":"2022-04-10T05:51:18.059978Z","shell.execute_reply":"2022-04-10T05:51:18.074112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\ndataset = dataset.repeat()\ndataset = dataset.shuffle(2048)\ndataset = dataset.batch(BATCH_SIZE)\ndataset = dataset.prefetch(AUTO) #This dataset can directly be passed to keras.fit method\nprint(count_data_items(TRAINING_FILENAMES))\n\n# Displaying single batch of TFRecord\ntrain_batch = iter(dataset)\ndisplay_batch_of_images(next(train_batch))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:51:18.076684Z","iopub.execute_input":"2022-04-10T05:51:18.077100Z","iopub.status.idle":"2022-04-10T05:51:23.376286Z","shell.execute_reply.started":"2022-04-10T05:51:18.076977Z","shell.execute_reply":"2022-04-10T05:51:23.375347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE_ = [IMAGE_SIZE,IMAGE_SIZE]\nAUTO = tf.data.experimental.AUTOTUNE\nTRAINING_FILENAMES = tf.io.gfile.glob(f'/tmp/{DATASET_NAME}/happywhale-2022-test*.tfrec')\nprint(len(TRAINING_FILENAMES))\ndataset = load_dataset(TRAINING_FILENAMES, labeled=True)\ndataset = dataset.repeat()\ndataset = dataset.shuffle(2048)\ndataset = dataset.batch(BATCH_SIZE)\ndataset = dataset.prefetch(AUTO) #This dataset can directly be passed to keras.fit method\nprint(count_data_items(TRAINING_FILENAMES))\n\n# Displaying single batch of TFRecord\ntrain_batch = iter(dataset)\ndisplay_batch_of_images(next(train_batch))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:51:23.377580Z","iopub.execute_input":"2022-04-10T05:51:23.378143Z","iopub.status.idle":"2022-04-10T05:51:27.581715Z","shell.execute_reply.started":"2022-04-10T05:51:23.378104Z","shell.execute_reply":"2022-04-10T05:51:27.577924Z"},"trusted":true},"execution_count":null,"outputs":[]}]}