{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, sys, cv2, random\nfrom tqdm.notebook import tqdm\nimport numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter, defaultdict\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.utils import check_random_state\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\n\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nseed=2001","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-08T15:16:44.360579Z","iopub.execute_input":"2022-02-08T15:16:44.361012Z","iopub.status.idle":"2022-02-08T15:16:44.372130Z","shell.execute_reply.started":"2022-02-08T15:16:44.360958Z","shell.execute_reply":"2022-02-08T15:16:44.371246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df= pd.read_csv('../input/happy-whale-and-dolphin/train.csv')\ndf['id_freq'] = df['individual_id'].map(df['individual_id'].value_counts())\nprint(len(df.species.unique()), len(df.individual_id.unique()))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:03.439170Z","iopub.execute_input":"2022-02-08T15:18:03.439475Z","iopub.status.idle":"2022-02-08T15:18:03.583513Z","shell.execute_reply.started":"2022-02-08T15:18:03.439442Z","shell.execute_reply":"2022-02-08T15:18:03.582695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:03.655632Z","iopub.execute_input":"2022-02-08T15:18:03.655929Z","iopub.status.idle":"2022-02-08T15:18:03.683779Z","shell.execute_reply.started":"2022-02-08T15:18:03.655893Z","shell.execute_reply":"2022-02-08T15:18:03.683132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.species= df.species.apply(lambda x: 'killer_whale' if x=='kiler_whale' else x)\ndf.species= df.species.apply(lambda x: 'bottlenose_dolphin' if x=='bottlenose_dolpin' else x)\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:03.831306Z","iopub.execute_input":"2022-02-08T15:18:03.832169Z","iopub.status.idle":"2022-02-08T15:18:03.889570Z","shell.execute_reply.started":"2022-02-08T15:18:03.832115Z","shell.execute_reply":"2022-02-08T15:18:03.888628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var= df.species.value_counts()\nx,y= var.index, var.values\nplt.figure(figsize=(15,8))\nplt.title(\"Species Distribution\", size=16)\nsns.barplot(y, x, palette='Blues_d')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:04.069878Z","iopub.execute_input":"2022-02-08T15:18:04.070651Z","iopub.status.idle":"2022-02-08T15:18:04.490533Z","shell.execute_reply.started":"2022-02-08T15:18:04.070590Z","shell.execute_reply":"2022-02-08T15:18:04.489874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#target\nsns.histplot(df.id_freq, bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:04.491835Z","iopub.execute_input":"2022-02-08T15:18:04.492234Z","iopub.status.idle":"2022-02-08T15:18:04.817030Z","shell.execute_reply.started":"2022-02-08T15:18:04.492193Z","shell.execute_reply":"2022-02-08T15:18:04.816099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=1\ni, (len(df[df.id_freq<=i])/51033)*100, (len(df[df.id_freq>i])/51033)*100","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:04.818259Z","iopub.execute_input":"2022-02-08T15:18:04.818479Z","iopub.status.idle":"2022-02-08T15:18:04.831021Z","shell.execute_reply.started":"2022-02-08T15:18:04.818450Z","shell.execute_reply":"2022-02-08T15:18:04.830041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=150\ni, (len(df[df.id_freq<=i])/51033)*100, (len(df[df.id_freq>i])/51033)*100","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:28:53.024481Z","iopub.execute_input":"2022-02-08T14:28:53.024706Z","iopub.status.idle":"2022-02-08T14:28:53.036547Z","shell.execute_reply.started":"2022-02-08T14:28:53.024679Z","shell.execute_reply":"2022-02-08T14:28:53.035734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df= df[df.id_freq<=150]","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:09.045755Z","iopub.execute_input":"2022-02-08T15:18:09.046035Z","iopub.status.idle":"2022-02-08T15:18:09.053584Z","shell.execute_reply.started":"2022-02-08T15:18:09.046004Z","shell.execute_reply":"2022-02-08T15:18:09.052908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:09.768780Z","iopub.execute_input":"2022-02-08T15:18:09.769088Z","iopub.status.idle":"2022-02-08T15:18:09.780176Z","shell.execute_reply.started":"2022-02-08T15:18:09.769054Z","shell.execute_reply":"2022-02-08T15:18:09.779356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Triplate Loss","metadata":{}},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:14.941137Z","iopub.execute_input":"2022-02-08T15:18:14.941430Z","iopub.status.idle":"2022-02-08T15:18:14.947230Z","shell.execute_reply.started":"2022-02-08T15:18:14.941398Z","shell.execute_reply":"2022-02-08T15:18:14.946427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate(df):\n    random.seed(seed)\n    df_grp= dict(list(df.groupby('individual_id')))\n    \n    def aux(row):\n        single=False\n        anchor= row.image\n        ids= df_grp[row.individual_id]['image'].tolist()\n        if not len(ids)-1:\n            single=True\n            positive= ids[0]\n        else:\n            ids.remove(anchor)\n            positive= random.choice(ids)\n            \n        if not single:\n            otherId= list(df_grp.keys())\n            otherId.remove(row.individual_id)\n            neg_group = random.choice(otherId)\n            negative = random.choice(df_grp[neg_group].image.tolist())\n\n            return anchor, positive, negative, None, None\n        else:\n            otherId= list(df_grp.keys())\n            otherId.remove(row.individual_id)\n            #3x\n            k=3\n            negative=[]\n            for _ in range(k):\n                neg_group = random.choice(otherId)\n                negative.append(random.choice(df_grp[neg_group].image.tolist()))\n            single=False\n            return anchor, positive, negative[0], negative[1], negative[2]\n    \n    return aux","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:17.582701Z","iopub.execute_input":"2022-02-08T15:18:17.583527Z","iopub.status.idle":"2022-02-08T15:18:17.594498Z","shell.execute_reply.started":"2022-02-08T15:18:17.583473Z","shell.execute_reply":"2022-02-08T15:18:17.593696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_triplets = df.progress_apply(generate(df), axis=1).tolist()\ntrain_triplets_df = pd.DataFrame(train_triplets, columns=['anchor', 'positive', 'negative0', 'negative1', 'negative2'])\ntrain_triplets_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:24.787033Z","iopub.execute_input":"2022-02-08T15:18:24.787854Z","iopub.status.idle":"2022-02-08T15:18:28.833656Z","shell.execute_reply.started":"2022-02-08T15:18:24.787811Z","shell.execute_reply":"2022-02-08T15:18:28.832697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_triplets_df.to_csv('tripletData.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T15:18:59.727448Z","iopub.execute_input":"2022-02-08T15:18:59.727749Z","iopub.status.idle":"2022-02-08T15:18:59.759022Z","shell.execute_reply.started":"2022-02-08T15:18:59.727719Z","shell.execute_reply":"2022-02-08T15:18:59.758244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('data.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:28:53.119404Z","iopub.execute_input":"2022-02-08T14:28:53.119822Z","iopub.status.idle":"2022-02-08T14:28:53.322047Z","shell.execute_reply.started":"2022-02-08T14:28:53.119775Z","shell.execute_reply":"2022-02-08T14:28:53.321207Z"},"trusted":true},"execution_count":null,"outputs":[]}]}