{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n!pip install tensorflow_addons","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nimport tensorflow as tf\n#import efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, train_test_split\n#from tensorflow.keras import backend as K\nimport tensorflow_addons as tfa\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport pickle\nimport json\nimport tensorflow_hub as tfhub\nfrom datetime import datetime","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_dir = '.'\nEXPERIMENT = 0\nrun_ts = datetime.now().strftime('%Y%m%d-%H%M%S')\nprint(run_ts)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n        \n    SEED = 56\n    N_CLASSES = 15587\n    FOLD_TO_RUN = 5\n    FOLDS = 10\n    IMAGE_SIZE = 768\n    BATCH_SIZE = 32 * strategy.num_replicas_in_sync\n    model_type = 'effnetv1'  \n    EFF_NET = 7\n    EFF_NETV2 = 's-21k-ft1k'\n    FREEZE_BATCH_NORM = False\n    head = 'arcface' \n    save_dir = save_dir\n    KNN = 1000\n    \ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAME = None\nif config.model_type == 'effnetv1':\n    MODEL_NAME = f'effnetv1_b{config.EFF_NET}'\n\nconfig.MODEL_NAME = MODEL_NAME\nprint(MODEL_NAME)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('backfintfrecords')\n    \ntrain_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/happywhale-2022-train*.tfrec')))\ntest_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/happywhale-2022-test*.tfrec')))\nprint(GCS_PATH)\nprint(len(train_files),len(test_files),count_data_items(train_files),count_data_items(test_files))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(config.save_dir+'/config.json', 'w') as fp:\n    json.dump({x:dict(config.__dict__)[x] for x in dict(config.__dict__) if not x.startswith('_')}, fp)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def better_than_median(inputs, axis):\n    \"\"\"Compute the mean of the predictions if there are no outliers, or the median if there are outliers.\n    Parameter: inputs = ndarray of shape (n_samples, n_folds)\n    \"\"\"\n    spread = inputs.max(axis=axis) - inputs.min(axis=axis) \n    spread_lim = 0.45\n    print(f\"Inliers:  {(spread < spread_lim).sum():7} -> compute mean\")\n    print(f\"Outliers: {(spread >= spread_lim).sum():7} -> compute median\")\n    print(f\"Total:    {len(inputs):7}\")\n    return np.where(spread < spread_lim,\n                    np.mean(inputs, axis=axis),\n                    np.median(inputs, axis=axis))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\"image_name\": tf.io.FixedLenFeature([], tf.string),\n                            \"image\": tf.io.FixedLenFeature([], tf.string),\n                            \"target\": tf.io.FixedLenFeature([], tf.int64),}\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    posting_id = example['image_name']\n    image = decode_image(example['image'])\n    label_group = tf.cast(example['target'], tf.int32)\n    matches = 1\n    return posting_id, image, label_group, matches\n\ndef load_dataset(filenames, ordered = False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\ndef get_training_dataset(filenames):\n    dataset = load_dataset(filenames, ordered = False)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_val_dataset(filenames):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_eval_dataset(filenames, get_targets = True):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_eval_format, num_parallel_calls = AUTO)\n    if not get_targets:\n        dataset = dataset.map(lambda image, target: image)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(filenames, get_names = True):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_inference_format, num_parallel_calls = AUTO)\n    if not get_names:\n        dataset = dataset.map(lambda image, posting_id: image)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, [config.IMAGE_SIZE,config.IMAGE_SIZE])\n    image = tf.cast(image, tf.float32) / 255.0\n    return image","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_ids(filename):\n    ds = get_test_dataset([filename],get_names=True).map(lambda image, image_name: image_name).unbatch()\n    NUM_IMAGES = count_data_items([filename])\n    ids = next(iter(ds.batch(NUM_IMAGES))).numpy().astype('U')\n    return ids\n\ndef get_targets(filename):\n    ds = get_eval_dataset([filename],get_targets=True).map(lambda image, target: target).unbatch()\n    NUM_IMAGES = count_data_items([filename])\n    ids = next(iter(ds.batch(NUM_IMAGES))).numpy()\n    return ids\n\ndef get_embeddings_np(filename,data_types='train',kfold_list=[1,4,5,8], dataset='../input/finlasts'):             \n    val_train={'train':'val','val':'train','test':'test'}\n    embeddings=[]\n    for kfold in kfold_list:\n        path=f'{dataset}/{data_types}_{filename.split(\"/\")[-1]}_{kfold}.npy'\n        if os.path.exists(path):\n            print(path)\n            with open(path, 'rb') as f:\n                embeddings.append(np.load(f))\n        else:\n            path=f'{dataset}/{val_train[data_types]}_{filename.split(\"/\")[-1]}_{kfold}.npy'\n            if os.path.exists(path):\n                print(path)\n                with open(path, 'rb') as f:\n                    embeddings.append(np.load(f))\n                \n                \n    print (len(embeddings))\n    #embeddings = np.mean(np.stack(embeddings), axis=0)\n    embeddings = np.median(np.stack(embeddings), axis=0)\n    #embeddings = better_than_median(np.stack(embeddings), axis=0)\n    return embeddings\n\ndef get_predictions(test_df,threshold=0.2):\n    predictions = {}\n    for i,row in tqdm(test_df.iterrows()):\n        if row.image in predictions:\n            if len(predictions[row.image])==5:\n                continue\n            predictions[row.image].append(row.target)\n        elif row.confidence>threshold:\n            predictions[row.image] = [row.target,'new_individual']\n        else:\n            predictions[row.image] = ['new_individual',row.target]\n\n    for x in tqdm(predictions):\n        if len(predictions[x])<5:\n            remaining = [y for y in sample_list if y not in predictions]\n            predictions[x] = predictions[x]+remaining\n            predictions[x] = predictions[x][:5]\n        \n    return predictions\n\ndef map_per_image(label, predictions):\n    try:\n        return 1 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0\n    \nf = open ('../input/hwsplits/individual_ids.json', \"r\")\ntarget_encodings = json.loads(f.read())\ntarget_encodings = {target_encodings[x]:x for x in target_encodings}\nsample_list = ['938b7e931166', '5bf17305f073', '7593d2aee842', '7362d7a01d00','956562ff2888']\n\nTRAINING_FILENAMES = [x for i,x in enumerate(train_files) if i%config.FOLDS!=config.FOLD_TO_RUN]\nVALIDATION_FILENAMES = [x for i,x in enumerate(train_files) if i%config.FOLDS==config.FOLD_TO_RUN]\nprint(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),\n      count_data_items(VALIDATION_FILENAMES))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def arcface_format(posting_id, image, label_group, matches):\n    return posting_id, {'inp1': image, 'inp2': label_group}, label_group, matches\n\ndef arcface_inference_format(posting_id, image, label_group, matches):\n    return image,posting_id\n\ndef arcface_eval_format(posting_id, image, label_group, matches):\n    return image,label_group\n\ndef data_augment(posting_id, image, label_group, matches):\n    image = tf.image.random_flip_left_right(image, 6)\n    image = tf.image.random_hue(image, 0.01, 3)\n    image = tf.image.random_saturation(image, 0.65, 1.15)\n    image = tf.image.random_contrast(image, 0.7, 1.10, 4)\n    image = tf.image.random_brightness(image, 0.1, 5)\n    return posting_id, image, label_group, matches","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets = []\ntrain_embeddings = []\nfor filename in tqdm(TRAINING_FILENAMES):\n    embeddings = get_embeddings_np(filename)\n    targets = get_targets(filename)\n    train_embeddings.append(embeddings)\n    train_targets.append(targets)\ntrain_embeddings = np.concatenate(train_embeddings)            # \ntrain_targets = np.concatenate(train_targets)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import NearestNeighbors\nneigh = NearestNeighbors(n_neighbors=config.KNN,metric='cosine')\nneigh.fit(train_embeddings)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ids = []\ntest_nn_distances = []\ntest_nn_idxs = []\nval_targets = []\nval_embeddings = []\nfor filename in tqdm(VALIDATION_FILENAMES):\n    embeddings = get_embeddings_np(filename,'val')\n    targets = get_targets(filename)\n    ids = get_ids(filename)\n    distances,idxs = neigh.kneighbors(embeddings, config.KNN, return_distance=True)\n    test_ids.append(ids)\n    test_nn_idxs.append(idxs)\n    test_nn_distances.append(distances)\n    val_embeddings.append(embeddings)\n    val_targets.append(targets)\ntest_nn_distances = np.concatenate(test_nn_distances)\ntest_nn_idxs = np.concatenate(test_nn_idxs)\ntest_ids = np.concatenate(test_ids)\nval_embeddings = np.concatenate(val_embeddings)\nval_targets = np.concatenate(val_targets)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allowed_targets = set([target_encodings[x] for x in np.unique(train_targets)])\nval_targets_df = pd.DataFrame(np.stack([test_ids,val_targets],axis=1),columns=['image','target'])\nval_targets_df['target'] = val_targets_df['target'].astype(int).map(target_encodings)\nval_targets_df.loc[~val_targets_df.target.isin(allowed_targets),'target'] = 'new_individual'\nval_targets_df.target.value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = []\nfor i in tqdm(range(len(test_ids))):\n    id_ = test_ids[i]\n    targets = train_targets[test_nn_idxs[i]]\n    distances = test_nn_distances[i]\n    subset_preds = pd.DataFrame(np.stack([targets,distances],axis=1),columns=['target','distances'])\n    subset_preds['image'] = id_\n    test_df.append(subset_preds)\ntest_df = pd.concat(test_df).reset_index(drop=True)\ntest_df['confidence'] = 1-test_df['distances']\ntest_df = test_df.groupby(['image','target']).confidence.max().reset_index()\ntest_df = test_df.sort_values('confidence',ascending=False).reset_index(drop=True)\ntest_df['target'] = test_df['target'].map(target_encodings)\ntest_df.to_csv('val_neighbors.csv')\ntest_df.image.value_counts().value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_th = 0\nbest_cv = 0\nfor th in [0.1*x for x in range(11)]:\n    all_preds = get_predictions(test_df,threshold=th)\n    cv = 0\n    for i,row in val_targets_df.iterrows():\n        target = row.target\n        preds = all_preds[row.image]\n        val_targets_df.loc[i,th] = map_per_image(target,preds)\n    cv = val_targets_df[th].mean()\n    print(f\"CV at threshold {th}: {cv}\")\n    if cv>best_cv:\n        best_th = th\n        best_cv = cv","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best threshold\",best_th)\nprint(\"Best cv\",best_cv)\nval_targets_df.describe()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_targets_df['is_new_individual'] = val_targets_df.target=='new_individual'\nprint(val_targets_df.is_new_individual.value_counts().to_dict())\nval_scores = val_targets_df.groupby('is_new_individual').mean().T\nval_scores['adjusted_cv'] = val_scores[True]*0.1+val_scores[False]*0.9\nbest_threshold_adjusted = val_scores['adjusted_cv'].idxmax()\nprint(\"best_threshold\",best_threshold_adjusted)\nval_scores","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeddings = np.concatenate([train_embeddings,val_embeddings])\ntrain_targets = np.concatenate([train_targets,val_targets])\nprint(train_embeddings.shape,train_targets.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import NearestNeighbors\nneigh = NearestNeighbors(n_neighbors=config.KNN,metric='cosine')\nneigh.fit(train_embeddings)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ids = []\ntest_nn_distances = []\ntest_nn_idxs = []\nfor filename in tqdm(test_files):\n    embeddings = get_embeddings_np(filename,'test')\n    ids = get_ids(filename)\n    distances,idxs = neigh.kneighbors(embeddings, config.KNN, return_distance=True)\n    test_ids.append(ids)\n    test_nn_idxs.append(idxs)\n    test_nn_distances.append(distances)\ntest_nn_distances = np.concatenate(test_nn_distances)\ntest_nn_idxs = np.concatenate(test_nn_idxs)\ntest_ids = np.concatenate(test_ids)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/happy-whale-and-dolphin/sample_submission.csv',index_col='image')\nprint(len(test_ids),len(sample_submission))\ntest_df = []\nfor i in tqdm(range(len(test_ids))):\n    id_ = test_ids[i]\n    targets = train_targets[test_nn_idxs[i]]\n    distances = test_nn_distances[i]\n    subset_preds = pd.DataFrame(np.stack([targets,distances],axis=1),columns=['target','distances'])\n    subset_preds['image'] = id_\n    test_df.append(subset_preds)\ntest_df = pd.concat(test_df).reset_index(drop=True)\ntest_df['confidence'] = 1-test_df['distances']\ntest_df = test_df.groupby(['image','target']).confidence.max().reset_index()\ntest_df = test_df.sort_values('confidence',ascending=False).reset_index(drop=True)\ntest_df['target'] = test_df['target'].map(target_encodings)\ntest_df.to_csv('test_neighbors.csv')\ntest_df.image.value_counts().value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_list = ['938b7e931166', '5bf17305f073', '7593d2aee842', '7362d7a01d00','956562ff2888']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = {}\nfor i,row in tqdm(test_df.iterrows()):\n    if row.image in predictions:\n        if len(predictions[row.image])==5:\n            continue\n        predictions[row.image].append(row.target)\n    elif row.confidence>best_threshold_adjusted:\n        predictions[row.image] = [row.target,'new_individual']\n    else:\n        predictions[row.image] = ['new_individual',row.target]\n        \nfor x in tqdm(predictions):\n    if len(predictions[x])<5:\n        remaining = [y for y in sample_list if y not in predictions]\n        predictions[x] = predictions[x]+remaining\n        predictions[x] = predictions[x][:5]\n    predictions[x] = ' '.join(predictions[x])\n    \npredictions = pd.Series(predictions).reset_index()\npredictions.columns = ['image','predictions']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = pd.read_csv(\"/kaggle/input/forsubs/submission796best.csv\")\nids = np.load(\"/kaggle/input/bafins/ids_without_backfin.npy\", allow_pickle = True) \nids2 = df2[\"image\"][~df2[\"image\"].isin(predictions[\"image\"])] ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([predictions[~(predictions[\"image\"].isin(ids))], df2[df2[\"image\"].isin(ids)],\n                        df2[df2[\"image\"].isin(ids2)]])\nsubmission = submission.drop_duplicates()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)\nsubmission.head()","metadata":{},"execution_count":null,"outputs":[]}]}