{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, json, random, cv2\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf, re, math\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-13T09:22:47.415185Z","iopub.execute_input":"2022-04-13T09:22:47.416123Z","iopub.status.idle":"2022-04-13T09:22:53.463993Z","shell.execute_reply.started":"2022-04-13T09:22:47.415982Z","shell.execute_reply":"2022-04-13T09:22:53.463012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n### Create Kaggle Dataset if not exists \n\nDATASET_NAME = f'backfins_pseudo_tfrecords'\n\n!rm -r /tmp/{DATASET_NAME}\n\nos.makedirs(f'/tmp/{DATASET_NAME}', exist_ok=True)\n\nwith open('../input/kaggleapicreds/kaggle.json') as f:\n    kaggle_creds = json.load(f)\n    \nos.environ['KAGGLE_USERNAME'] = kaggle_creds['username']\nos.environ['KAGGLE_KEY'] = kaggle_creds['key']\n\n!kaggle datasets init -p /tmp/{DATASET_NAME}\n\n\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json') as f:\n    dataset_meta = json.load(f)\ndataset_meta['id'] = f'qi0239/{DATASET_NAME}'\ndataset_meta['title'] = DATASET_NAME\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json', \"w\") as outfile:\n    json.dump(dataset_meta, outfile)\nprint(dataset_meta)\n\n!cp /tmp/{DATASET_NAME}/dataset-metadata.json /tmp/{DATASET_NAME}/meta.json\n!ls /tmp/{DATASET_NAME}\n\n!kaggle datasets create -u -p /tmp/{DATASET_NAME}","metadata":{"execution":{"iopub.status.busy":"2022-04-13T08:45:15.080401Z","iopub.execute_input":"2022-04-13T08:45:15.080719Z","iopub.status.idle":"2022-04-13T08:45:25.048639Z","shell.execute_reply.started":"2022-04-13T08:45:15.080689Z","shell.execute_reply":"2022-04-13T08:45:25.047416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /tmp/{DATASET_NAME}","metadata":{"execution":{"iopub.status.busy":"2022-04-13T08:45:25.050921Z","iopub.execute_input":"2022-04-13T08:45:25.051227Z","iopub.status.idle":"2022-04-13T08:45:25.817526Z","shell.execute_reply.started":"2022-04-13T08:45:25.051195Z","shell.execute_reply":"2022-04-13T08:45:25.816336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/happywhale-splits/skf_species_10folds.csv')\ntest_df = pd.read_csv('../input/happy-whale-and-dolphin/sample_submission.csv')\ntest_df['split'] = test_df.index%10\ntrain_df.agg(['min','max','count','nunique'])","metadata":{"execution":{"iopub.status.busy":"2022-04-13T08:45:39.155139Z","iopub.execute_input":"2022-04-13T08:45:39.155821Z","iopub.status.idle":"2022-04-13T08:45:39.401007Z","shell.execute_reply.started":"2022-04-13T08:45:39.155781Z","shell.execute_reply":"2022-04-13T08:45:39.399975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nf = open ('../input/happywhale-splits/individual_ids.json', \"r\")\ntarget_encodings = json.loads(f.read())\ntarget_encodings = {target_encodings[x]:x for x in target_encodings}\ntarget_decoding = dict(zip(target_encodings.values(), target_encodings.keys()))\ntarget_decoding","metadata":{"execution":{"iopub.status.busy":"2022-04-13T08:48:57.503798Z","iopub.execute_input":"2022-04-13T08:48:57.504671Z","iopub.status.idle":"2022-04-13T08:48:57.572842Z","shell.execute_reply.started":"2022-04-13T08:48:57.50462Z","shell.execute_reply":"2022-04-13T08:48:57.572157Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_train.loc[9933]","metadata":{"execution":{"iopub.status.busy":"2022-04-13T08:55:30.861241Z","iopub.execute_input":"2022-04-13T08:55:30.861567Z","iopub.status.idle":"2022-04-13T08:55:30.869467Z","shell.execute_reply.started":"2022-04-13T08:55:30.861534Z","shell.execute_reply":"2022-04-13T08:55:30.868815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_train = pd.read_csv('../input/pseudo-th08/third_th0.8_only_top1.csv')\npseudo_train = pseudo_train.loc[0:9933]","metadata":{"execution":{"iopub.status.busy":"2022-04-13T08:56:02.200796Z","iopub.execute_input":"2022-04-13T08:56:02.201285Z","iopub.status.idle":"2022-04-13T08:56:02.236159Z","shell.execute_reply.started":"2022-04-13T08:56:02.201251Z","shell.execute_reply":"2022-04-13T08:56:02.235256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([train_df,pseudo_train],axis = 0).reset_index(drop=True)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-04-13T09:06:19.586257Z","iopub.execute_input":"2022-04-13T09:06:19.586973Z","iopub.status.idle":"2022-04-13T09:06:19.612969Z","shell.execute_reply.started":"2022-04-13T09:06:19.586794Z","shell.execute_reply":"2022-04-13T09:06:19.612021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_train['species'] = 300\npseudo_train['individual_id'] = pseudo_train['predictions'].map(target_decoding)\npseudo_train['fold'] = 0","metadata":{"execution":{"iopub.status.busy":"2022-04-13T09:02:18.681695Z","iopub.execute_input":"2022-04-13T09:02:18.682135Z","iopub.status.idle":"2022-04-13T09:02:18.704716Z","shell.execute_reply.started":"2022-04-13T09:02:18.682092Z","shell.execute_reply":"2022-04-13T09:02:18.703412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_train = pseudo_train.loc[:,['image','species','individual_id','fold']]","metadata":{"execution":{"iopub.status.busy":"2022-04-13T09:03:35.884367Z","iopub.execute_input":"2022-04-13T09:03:35.884923Z","iopub.status.idle":"2022-04-13T09:03:35.894282Z","shell.execute_reply.started":"2022-04-13T09:03:35.884887Z","shell.execute_reply":"2022-04-13T09:03:35.89317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_fold(fold):\n    val_df = train_df[train_df.fold==fold].reset_index(drop=True)\n    val_df['order'] = val_df.index\n    val_df['order'] = val_df.groupby('individual_id').order.rank()\n    val_total_counts = val_df.individual_id.value_counts().to_dict()\n    val_df['total_counts'] = val_df.individual_id.map(val_total_counts)\n    val_df['order'] = val_df['order']/val_df['total_counts']\n    val_df = val_df.sort_values('order',ascending=False).reset_index(drop=True)\n    val_df = val_df[['image','species','individual_id']]\n    return val_df","metadata":{"execution":{"iopub.status.busy":"2022-04-13T08:47:06.762113Z","iopub.execute_input":"2022-04-13T08:47:06.762448Z","iopub.status.idle":"2022-04-13T08:47:06.769648Z","shell.execute_reply.started":"2022-04-13T08:47:06.762412Z","shell.execute_reply":"2022-04-13T08:47:06.768955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image,image_name,target,species):\n    feature = {\n        'image': _bytes_feature(image),\n        'image_name': _bytes_feature(image_name),\n        'target': _int64_feature(target),\n        'species': _int64_feature(species),\n      }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-04-13T08:47:14.268989Z","iopub.execute_input":"2022-04-13T08:47:14.269406Z","iopub.status.idle":"2022-04-13T08:47:14.285384Z","shell.execute_reply.started":"2022-04-13T08:47:14.269362Z","shell.execute_reply":"2022-04-13T08:47:14.283888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_tf_records(fold  = 0):\n    df = get_fold(fold)\n    tfr_filename = f'/tmp/{DATASET_NAME}/happywhale-2022-train-{fold}-{df.shape[0]}.tfrec'\n    with tf.io.TFRecordWriter(tfr_filename) as writer:\n        for i,row in df.iterrows():\n            image_id = row.image\n            target = row.individual_id\n            species = row.species\n            if species != 300:\n                try:\n                    image_path = f\"../input/backfin-cropped/train_images/{image_id}\"\n                    image_encoded = tf.io.read_file(image_path)\n                except:\n                    pass\n            else:\n                try:\n                    image_path = f\"../input/backfin-cropped/test_images/{image_id}\"\n                    image_encoded = tf.io.read_file(image_path)\n                except:\n                    pass\n            image_name = str.encode(image_id)\n            example = serialize_example(image_encoded,image_name,target,species)\n            writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2022-04-13T09:10:36.493597Z","iopub.execute_input":"2022-04-13T09:10:36.494521Z","iopub.status.idle":"2022-04-13T09:10:36.503496Z","shell.execute_reply.started":"2022-04-13T09:10:36.494466Z","shell.execute_reply":"2022-04-13T09:10:36.502641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\n_ = joblib.Parallel(n_jobs=8)(\n        joblib.delayed(create_tf_records)(fold) for fold in tqdm(range(10))\n    )","metadata":{"execution":{"iopub.status.busy":"2022-04-13T09:10:39.626506Z","iopub.execute_input":"2022-04-13T09:10:39.627026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_test_tf_records(fold  = 0):\n    df = test_df[test_df.split==fold]\n    tfr_filename = f'/tmp/{DATASET_NAME}/happywhale-2022-test-{fold}-{df.shape[0]}.tfrec'\n    with tf.io.TFRecordWriter(tfr_filename) as writer:\n        for i,row in df.iterrows():\n            image_id = row.image\n            target = -1\n            species = -1\n            try:\n                image_path = f\"../input/backfin-cropped/test_images/{image_id}\"\n                image_encoded = tf.io.read_file(image_path)\n            image_name = str.encode(image_id)\n            example = serialize_example(image_encoded,image_name,target,species)\n            writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2022-02-24T09:14:25.951651Z","iopub.execute_input":"2022-02-24T09:14:25.952086Z","iopub.status.idle":"2022-02-24T09:14:25.961939Z","shell.execute_reply.started":"2022-02-24T09:14:25.952036Z","shell.execute_reply":"2022-02-24T09:14:25.961118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\n_ = joblib.Parallel(n_jobs=8)(\n        joblib.delayed(create_test_tf_records)(fold) for fold in tqdm(range(10))\n    )","metadata":{"execution":{"iopub.status.busy":"2022-02-24T09:14:25.963707Z","iopub.execute_input":"2022-02-24T09:14:25.963957Z","iopub.status.idle":"2022-02-24T09:20:08.780895Z","shell.execute_reply.started":"2022-02-24T09:14:25.963926Z","shell.execute_reply":"2022-02-24T09:20:08.779959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nversion_name = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\nprint(version_name)","metadata":{"execution":{"iopub.status.busy":"2022-02-24T09:20:08.783009Z","iopub.execute_input":"2022-02-24T09:20:08.78362Z","iopub.status.idle":"2022-02-24T09:20:08.79054Z","shell.execute_reply.started":"2022-02-24T09:20:08.783577Z","shell.execute_reply":"2022-02-24T09:20:08.789555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /tmp/{DATASET_NAME}","metadata":{"execution":{"iopub.status.busy":"2022-02-24T09:20:08.792381Z","iopub.execute_input":"2022-02-24T09:20:08.792979Z","iopub.status.idle":"2022-02-24T09:20:24.210722Z","shell.execute_reply.started":"2022-02-24T09:20:08.792936Z","shell.execute_reply":"2022-02-24T09:20:24.209582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!kaggle datasets version -m {version_name} -p /tmp/{DATASET_NAME} -r zip -q","metadata":{"execution":{"iopub.status.busy":"2022-02-24T09:20:24.214845Z","iopub.execute_input":"2022-02-24T09:20:24.215804Z","iopub.status.idle":"2022-02-24T09:49:39.313778Z","shell.execute_reply.started":"2022-02-24T09:20:24.215735Z","shell.execute_reply":"2022-02-24T09:49:39.310196Z"},"trusted":true},"execution_count":null,"outputs":[]}]}