{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport copy\nimport sys\nfrom PIL import Image\nimport time \nfrom tqdm.autonotebook import tqdm\nimport random\nimport gc\nimport cv2\nimport scipy\nimport math\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import fbeta_score\n\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.optim.optimizer import Optimizer\nimport torch.backends.cudnn as cudnn\nfrom torch.autograd import Variable\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR, _LRScheduler\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# We get all classes of labels "},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy.special\n\nSEED = 42\nbase_dir = '../input/'\ndef seed_everything(seed=SEED):\n    random.seed(seed)\n    os.environ['PYHTONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\nlabels_df = pd.read_csv('../input/labels.csv')\ntest_df = pd.read_csv('../input/sample_submission.csv')\n\ntr, val = train_test_split(train_df['id'], test_size=0.15, random_state=SEED)\n\nimg_class_dict = {k:v for k, v in zip(train_df.id, train_df.attribute_ids)}\n\ndef get_label(attribute_ids):\n    attribute_ids = attribute_ids.split()\n    one_hot = np.zeros(1103, dtype=np.int)\n    for _,ids in enumerate(attribute_ids):\n        one_hot[int(ids)] = 1\n    return one_hot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df.columns)\nprint(labels_df.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes =train_df['attribute_ids'].value_counts().to_frame().reset_index()\nclasses.rename(columns={'index': 'classes', 'attribute_ids':'counts'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#classes['classes'] = classes['classes'].apply(get_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes['ratio'] = classes['counts']/train_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**WE have 50238 classes(too many) which means the samples in one class are few comparing to the whole dataset.**<br/>\n**If we use triplet-loss an anchor may hardly find a positive sample in a batch.**"},{"metadata":{},"cell_type":"markdown","source":"# Let's see the samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_label_name(attribute_ids):\n    attribute_ids = attribute_ids.split()\n    attribute_name = []\n    for _,ids in enumerate(attribute_ids):\n        attribute_name.append(labels_df.loc[labels_df['attribute_id']==int(ids)])\n    return attribute_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df['attribute_name'] = train_df['attribute_ids'].apply(get_label_name)\n#too slow","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['count'] = train_df.groupby(['attribute_ids'])['id'].transform('count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.sort_values(by='attribute_ids')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df['attribute_ids'] = train_df['attribute_ids'].apply(get_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grouped_id = train_df.groupby('attribute_ids')['id']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **We collect one image from one class**"},{"metadata":{"trusted":true},"cell_type":"code","source":"collect_image_names = {}\n\nfor key in classes['classes']:\n    name = grouped_id.get_group(key).values[0]\n    count = grouped_id.get_group(key).values.shape[0]\n    collect_image_names[name] = count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator\nsorted_collect_image_names = sorted(collect_image_names.items(), key=operator.itemgetter(1))\nsorted_collect_image_names.reverse()\nprint(len(sorted_collect_image_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(sorted_collect_image_names[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_name = sorted_collect_image_names[0][0]\nattribute_ids = train_df.loc[train_df['id']==image_name]['attribute_ids'].values[0]\nprint(attribute_ids.split())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Let's see the top 10 most frequent classes (one image per class)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"c = 1\nplt.figure(figsize=[20, 20])\nfor idx in range(10):\n    image_name = sorted_collect_image_names[idx][0]\n    img = cv2.imread(\"../input/train/{}.png\".format(image_name))[...,[2,1,0]]\n    plt.subplot(5,2,c)\n    plt.imshow(img)\n    \n    attribute_ids = train_df.loc[train_df['id']==image_name]['attribute_ids'].values[0].split()\n    attribute_name = []\n    for _,ids in enumerate(attribute_ids):\n        attribute_name.append(labels_df.loc[labels_df['attribute_id']==int(ids)]['attribute_name'].values[0])\n    plt.title(\"train image {} count {}\".format(attribute_name, sorted_collect_image_names[idx][1]))\n    c += 1\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# **Let's see the top 10 least frequent classes (one image per class)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"c = 1\nplt.figure(figsize=[20,20])\n\nsize = len(sorted_collect_image_names)\n\nfor idx in range(size-10, size):\n    image_name = sorted_collect_image_names[idx][0]\n    img = cv2.imread(\"../input/train/{}.png\".format(image_name))[...,[2,1,0]]\n    plt.subplot(5,2,c)\n    plt.imshow(img)\n    \n    attribute_ids = train_df.loc[train_df['id']==image_name]['attribute_ids'].values[0].split()\n    attribute_name = []\n    for _,ids in enumerate(attribute_ids):\n        attribute_name.append(labels_df.loc[labels_df['attribute_id']==int(ids)]['attribute_name'].values[0])\n    plt.title(\"train image {} count {}\".format(attribute_name, sorted_collect_image_names[idx][1]))\n    c += 1\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Let's see 10 images from the most frequent class (one class)**<br/>\n**the most frequent class is 'culture::american'+'tag::actresses'+'tag::portaits'+'tag::women'**"},{"metadata":{"trusted":true},"cell_type":"code","source":"name = grouped_id.get_group(classes['classes'][0]).values[0]\ncount = grouped_id.get_group(classes['classes'][0]).values.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c = 1\nplt.figure(figsize=[20,20])\n\nmost_frequent_class_top_10 = {}\n\nfor i in range(10):\n    name = grouped_id.get_group(classes['classes'][0]).values[i]\n    count = grouped_id.get_group(classes['classes'][0]).values.shape[0]\n    most_frequent_class_top_10[name] = count\n\nsize = len(most_frequent_class_top_10)\n\nfor element in most_frequent_class_top_10:\n    image_name = element\n    img = cv2.imread(\"../input/train/{}.png\".format(image_name))[...,[2,1,0]]\n    plt.subplot(5,2,c)\n    plt.imshow(img)\n    \n    attribute_ids = train_df.loc[train_df['id']==image_name]['attribute_ids'].values[0].split()\n    attribute_name = []\n    for _,ids in enumerate(attribute_ids):\n        attribute_name.append(labels_df.loc[labels_df['attribute_id']==int(ids)]['attribute_name'].values[0])\n    plt.title(\"train image {} count {}\".format(attribute_name, most_frequent_class_top_10[element]))\n    c += 1\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's get sorted categories count"},{"metadata":{"trusted":true},"cell_type":"code","source":"category_count = {}\n\nfor i in range(1103):\n    category_count[i] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for key in classes['classes']:\n    category_name = key.split()\n    count = grouped_id.get_group(key).values.shape[0]\n    for element in category_name:\n        category_count[int(element)] += count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_category_count = sorted(category_count.items(), key=operator.itemgetter(1))\nsorted_category_count.reverse()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_category_count_frame = pd.DataFrame.from_dict(sorted_category_count)\nsorted_category_count_frame.columns=['attribute_id', 'count']\nsorted_category_count_frame['ratio'] = sorted_category_count_frame['count']/train_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_category_count_frame.head(30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"category_name_count = {}\n\nfor element in sorted_category_count:\n    key = element[0]\n    name = labels_df[labels_df['attribute_id']==key]['attribute_name'].values[0]\n    category_name_count[name] = element[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_category_name_count = sorted(category_name_count.items(), key=operator.itemgetter(1))\nsorted_category_name_count.reverse()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_sorted_category_name_count_frame = pd.DataFrame.from_dict(sorted_category_name_count)\nsorted_sorted_category_name_count_frame.columns=['attribute_name', 'count']\nsorted_sorted_category_name_count_frame['ratio'] = sorted_sorted_category_name_count_frame['count']/train_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_sorted_category_name_count_frame.head(30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**We can see that the most frequent categories are 'tag::men'(ratio 0.18), 'tag::women'(ratio 0.13), 'tag::flowers'(ratio 0.07), 'culture::...'(...)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_category_count_frame.to_csv('sorted_category_count_frame.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}