{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\nimport gc\nimport glob\nimport json\nimport string\nimport random\nimport pprint\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom functools import partial\nfrom argparse import Namespace\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import *\n\nfrom sklearn.model_selection import StratifiedKFold\n\nimport wandb\nfrom wandb.keras import WandbCallback\n\n# Imports for augmentations. \nfrom albumentations import Compose, RandomResizedCrop, Cutout, Rotate, HorizontalFlip, VerticalFlip,\\\n                           RandomBrightnessContrast, ShiftScaleRotate, CenterCrop, Resize, Normalize","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-19T18:55:31.335773Z","iopub.execute_input":"2022-03-19T18:55:31.336102Z","iopub.status.idle":"2022-03-19T18:55:41.715139Z","shell.execute_reply.started":"2022-03-19T18:55:31.336016Z","shell.execute_reply":"2022-03-19T18:55:41.714064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Create Kaggle Dataset if not exists \n\nDATASET_NAME = f'sorghum-100-tfrecords'\n\n!rm -r ../tmp/{DATASET_NAME}\n\nos.makedirs(f'../tmp/{DATASET_NAME}', exist_ok=True)\n\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"KAGGLE_KEY\")\n    \nos.environ['KAGGLE_USERNAME'] = 'ayuraj'\nos.environ['KAGGLE_KEY'] = secret_value_0\n\n!kaggle datasets init -p ../tmp/{DATASET_NAME}\n\nwith open(f'../tmp/{DATASET_NAME}/dataset-metadata.json') as f:\n    dataset_meta = json.load(f)\ndataset_meta['id'] = f'ayuraj/{DATASET_NAME}'\ndataset_meta['title'] = DATASET_NAME\nwith open(f'../tmp/{DATASET_NAME}/dataset-metadata.json', \"w\") as outfile:\n    json.dump(dataset_meta, outfile)\nprint(dataset_meta)\n\n!cp ../tmp/{DATASET_NAME}/dataset-metadata.json ../tmp/{DATASET_NAME}/meta.json\n!ls ../tmp/{DATASET_NAME}\n\n!kaggle datasets create -u -p ../tmp/{DATASET_NAME}","metadata":{"execution":{"iopub.status.busy":"2022-03-19T18:58:59.888189Z","iopub.execute_input":"2022-03-19T18:58:59.888508Z","iopub.status.idle":"2022-03-19T18:59:14.021767Z","shell.execute_reply.started":"2022-03-19T18:58:59.888474Z","shell.execute_reply":"2022-03-19T18:59:14.020459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_PATH = '../input/sorghum-id-fgvc-9/'\nTRAIN_PATH = ROOT_PATH+'train_images/'\nTEST_PATH = ROOT_PATH+'test/'\n\ndef add_train_path(row):\n    return TRAIN_PATH+row.image\n\ndef add_test_path(row):\n    return TEST_PATH+row.image\n\ndef parse_label(row):\n    target = row.target\n    return int(label2ids[target])\n\ntrain_files = glob.glob(TRAIN_PATH+'*')\ntest_files = glob.glob(TEST_PATH+'*')\n\n# Prep Train CSV\ndf = pd.read_csv(ROOT_PATH+'train_cultivar_mapping.csv')\ndf.rename(columns={'cultivar': 'target'}, inplace=True)\ndf = df.sample(frac=1).reset_index(drop=True)\n\nlabels = df.target.unique()\nlabel2ids = {label:idx for idx, label in enumerate(labels)}\ndf['img_path'] = df.apply(lambda row: add_train_path(row), axis=1)\ndf['target'] = df.apply(lambda row: parse_label(row), axis=1)\n\n# Prep Test CSV\ntest_df = pd.read_csv(ROOT_PATH+'sample_submission.csv')\ntest_df.rename(columns={'filename': 'image'}, inplace=True)\ntest_df['img_path'] = test_df.apply(lambda row: add_test_path(row), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T18:59:36.262255Z","iopub.execute_input":"2022-03-19T18:59:36.262646Z","iopub.status.idle":"2022-03-19T18:59:39.234777Z","shell.execute_reply.started":"2022-03-19T18:59:36.262607Z","shell.execute_reply":"2022-03-19T18:59:39.233780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T18:59:39.236763Z","iopub.execute_input":"2022-03-19T18:59:39.237023Z","iopub.status.idle":"2022-03-19T18:59:39.254752Z","shell.execute_reply.started":"2022-03-19T18:59:39.236989Z","shell.execute_reply":"2022-03-19T18:59:39.254019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T18:59:39.255845Z","iopub.execute_input":"2022-03-19T18:59:39.256376Z","iopub.status.idle":"2022-03-19T18:59:39.268626Z","shell.execute_reply.started":"2022-03-19T18:59:39.256342Z","shell.execute_reply":"2022-03-19T18:59:39.267661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove TRAIN missing file\nmissing_files = []\nfor idx, tmp_df in tqdm(df.iterrows()):\n    img_path = tmp_df.img_path\n    if img_path not in train_files:\n        missing_files.append(img_path)\nprint('missing files num: ', len(missing_files))\n\ndf = df[~df.img_path.isin(missing_files)].reset_index(drop=True)\ndf = df.sample(frac=1).reset_index(drop=True)\n\ndf.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T18:59:46.475644Z","iopub.execute_input":"2022-03-19T18:59:46.475922Z","iopub.status.idle":"2022-03-19T18:59:55.407096Z","shell.execute_reply.started":"2022-03-19T18:59:46.475892Z","shell.execute_reply":"2022-03-19T18:59:55.406127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove TEST missing file\nmissing_files = []\nfor idx, tmp_df in tqdm(test_df.iterrows()):\n    img_path = tmp_df.img_path\n    if img_path not in test_files:\n        missing_files.append(img_path)\nprint('missing files num: ', len(missing_files))\n\ntest_df = test_df[~test_df.img_path.isin(missing_files)].reset_index(drop=True)\n\ntest_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T18:59:55.409253Z","iopub.execute_input":"2022-03-19T18:59:55.409577Z","iopub.status.idle":"2022-03-19T19:00:04.039235Z","shell.execute_reply.started":"2022-03-19T18:59:55.409531Z","shell.execute_reply":"2022-03-19T19:00:04.038181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_stratified_k_fold(df, target, num_folds):\n    \"\"\"\n    Add fold numbers to the given dataframe\n    \n    Arguments:\n    df: Dataframe\n    target: List of target to stratify on\n    num_folds: Number of folds\n    \"\"\"\n    kfold = StratifiedKFold(num_folds, shuffle=True, random_state=42)\n\n    for fold, (train_indices, valid_indices) in enumerate(kfold.split(df, target)):\n        df.loc[valid_indices, 'fold'] = fold\n        \n    return df.astype({'fold': int})\n\ndf = get_stratified_k_fold(df, df.target.values, 5)\ndf.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:00:04.040647Z","iopub.execute_input":"2022-03-19T19:00:04.040862Z","iopub.status.idle":"2022-03-19T19:00:04.078427Z","shell.execute_reply.started":"2022-03-19T19:00:04.040836Z","shell.execute_reply":"2022-03-19T19:00:04.077526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_samples = 4096\nnum_tfrecords = len(test_df) // num_samples\nprint(num_tfrecords)\n\nnum_folds = 5","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:00:06.754715Z","iopub.execute_input":"2022-03-19T19:00:06.755723Z","iopub.status.idle":"2022-03-19T19:00:06.762088Z","shell.execute_reply.started":"2022-03-19T19:00:06.755676Z","shell.execute_reply":"2022-03-19T19:00:06.761124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(\n        bytes_list=tf.train.BytesList(value=[tf.io.encode_jpeg(value).numpy()])\n    )\n\n\ndef bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value.encode()]))\n\n\ndef int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef create_train_example(image, target, fold, index, img_name):\n    feature = {\n        \"image\": image_feature(image),\n        \"target\": int64_feature(target),\n        \"fold\": int64_feature(fold),\n        \"csv_index\": int64_feature(index),\n        'image_name': bytes_feature(img_name)\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\ndef parse_train_tfrecord_fn(example):\n    feature_description = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n        \"fold\": tf.io.FixedLenFeature([], tf.int64),\n        \"csv_index\": tf.io.FixedLenFeature([], tf.int64),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, feature_description)\n    example[\"image\"] = tf.io.decode_jpeg(example[\"image\"], channels=3)\n    return example\n\n\ndef create_test_example(image, index, img_name):\n    feature = {\n        \"image\": image_feature(image),\n        \"csv_index\": int64_feature(index),\n        'image_name': bytes_feature(img_name)\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\ndef parse_test_tfrecord_fn(example):\n    feature_description = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"csv_index\": tf.io.FixedLenFeature([], tf.int64),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, feature_description)\n    example[\"image\"] = tf.io.decode_jpeg(example[\"image\"], channels=3)\n    return example","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:00:12.833262Z","iopub.execute_input":"2022-03-19T19:00:12.833971Z","iopub.status.idle":"2022-03-19T19:00:12.850249Z","shell.execute_reply.started":"2022-03-19T19:00:12.833918Z","shell.execute_reply":"2022-03-19T19:00:12.849089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../tmp","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:00:25.565834Z","iopub.execute_input":"2022-03-19T19:00:25.566287Z","iopub.status.idle":"2022-03-19T19:00:26.327574Z","shell.execute_reply.started":"2022-03-19T19:00:25.566255Z","shell.execute_reply":"2022-03-19T19:00:26.326546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tfrecords_dir = f'../tmp/{DATASET_NAME}/train'\ntest_tfrecords_dir = f'../tmp/{DATASET_NAME}/test'\n\nos.makedirs(train_tfrecords_dir, exist_ok=True)\nos.makedirs(test_tfrecords_dir, exist_ok=True)\n\n!ls ../tmp/{DATASET_NAME}","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:01:53.847929Z","iopub.execute_input":"2022-03-19T19:01:53.849195Z","iopub.status.idle":"2022-03-19T19:01:54.631248Z","shell.execute_reply.started":"2022-03-19T19:01:53.849127Z","shell.execute_reply":"2022-03-19T19:01:54.629748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(num_folds):\n    print(fold)\n    tmp_df = df[df.fold == fold]\n\n    with tf.io.TFRecordWriter(\n        train_tfrecords_dir + \"/train_fold_%.2i-%i.tfrec\" % (fold, len(tmp_df))\n    ) as writer:\n        for idx, row in tqdm(tmp_df.iterrows()):\n            image = tf.io.decode_jpeg(tf.io.read_file(row.img_path))\n            example = create_train_example(\n                image, row.target, row.fold, idx, row.image\n            )\n            writer.write(example.SerializeToString())","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:02:25.822459Z","iopub.execute_input":"2022-03-19T19:02:25.822845Z","iopub.status.idle":"2022-03-19T19:02:26.086271Z","shell.execute_reply.started":"2022-03-19T19:02:25.822810Z","shell.execute_reply":"2022-03-19T19:02:26.085185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../tmp/{DATASET_NAME}/train","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:02:28.157735Z","iopub.execute_input":"2022-03-19T19:02:28.158152Z","iopub.status.idle":"2022-03-19T19:02:28.925559Z","shell.execute_reply.started":"2022-03-19T19:02:28.158097Z","shell.execute_reply":"2022-03-19T19:02:28.924703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = glob.glob(f\"{train_tfrecords_dir}/*.tfrec\")\nraw_dataset = tf.data.TFRecordDataset(train_files[0])\nparsed_dataset = raw_dataset.map(parse_train_tfrecord_fn)\n\nfor features in parsed_dataset.take(1):\n    for key in features.keys():\n        if key != \"image\":\n            print(f\"{key}: {features[key]}\")\n\n    print(f\"Image shape: {features['image'].shape}\")\n    plt.figure(figsize=(7, 7))\n    plt.imshow(features[\"image\"].numpy())\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:02:36.581727Z","iopub.execute_input":"2022-03-19T19:02:36.582301Z","iopub.status.idle":"2022-03-19T19:02:37.356304Z","shell.execute_reply.started":"2022-03-19T19:02:36.582260Z","shell.execute_reply":"2022-03-19T19:02:37.355478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for tfrec_num in range(num_tfrecords):\n    print(tfrec_num)\n    tmp_df = test_df.loc[(tfrec_num * num_samples) : ((tfrec_num + 1) * num_samples)]\n\n    with tf.io.TFRecordWriter(\n        test_tfrecords_dir + \"/test_%.2i-%i.tfrec\" % (tfrec_num, len(tmp_df))\n    ) as writer:\n        for idx, row in tqdm(tmp_df.iterrows()):\n            image = tf.io.decode_jpeg(tf.io.read_file(row.img_path))\n            example = create_test_example(\n                image, idx, row.image\n            )\n            writer.write(example.SerializeToString())","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:05:20.764782Z","iopub.execute_input":"2022-03-19T19:05:20.765075Z","iopub.status.idle":"2022-03-19T19:05:21.071440Z","shell.execute_reply.started":"2022-03-19T19:05:20.765046Z","shell.execute_reply":"2022-03-19T19:05:21.070203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../tmp/{DATASET_NAME}/test","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:04:17.732016Z","iopub.execute_input":"2022-03-19T19:04:17.732317Z","iopub.status.idle":"2022-03-19T19:04:18.505939Z","shell.execute_reply.started":"2022-03-19T19:04:17.732281Z","shell.execute_reply":"2022-03-19T19:04:18.504669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files = glob.glob(f\"{test_tfrecords_dir}/*.tfrec\")\nraw_dataset = tf.data.TFRecordDataset(test_files[0])\nparsed_dataset = raw_dataset.map(parse_test_tfrecord_fn)\n\nfor features in parsed_dataset.take(1):\n    for key in features.keys():\n        if key != \"image\":\n            print(f\"{key}: {features[key]}\")\n\n    print(f\"Image shape: {features['image'].shape}\")\n    plt.figure(figsize=(7, 7))\n    plt.imshow(features[\"image\"].numpy())\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:05:38.943031Z","iopub.execute_input":"2022-03-19T19:05:38.943320Z","iopub.status.idle":"2022-03-19T19:05:39.462564Z","shell.execute_reply.started":"2022-03-19T19:05:38.943289Z","shell.execute_reply":"2022-03-19T19:05:39.461774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nversion_name = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\nprint(version_name)","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:07:27.869507Z","iopub.execute_input":"2022-03-19T19:07:27.869950Z","iopub.status.idle":"2022-03-19T19:07:27.876616Z","shell.execute_reply.started":"2022-03-19T19:07:27.869907Z","shell.execute_reply":"2022-03-19T19:07:27.875675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!kaggle datasets version -m {version_name} -p ../tmp/{DATASET_NAME} -r zip -q","metadata":{"execution":{"iopub.status.busy":"2022-03-19T19:07:46.854282Z","iopub.execute_input":"2022-03-19T19:07:46.854611Z","iopub.status.idle":"2022-03-19T19:08:10.735922Z","shell.execute_reply.started":"2022-03-19T19:07:46.854566Z","shell.execute_reply":"2022-03-19T19:08:10.734812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}