{"cells":[{"metadata":{"_uuid":"06d1df304ea449b4b7b14c59cb987f9ce31ff998"},"cell_type":"markdown","source":"# A Brief Data Exploration\n\nSummary:\n1. There are 23 classes but about 9 of them are not in the training set. \n2. Highly unbalanced data"},{"metadata":{"_uuid":"a7da785cb56f725c5987aac75e29630e5bd464f9"},"cell_type":"markdown","source":"## The data we obtain"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"sample_submission_csv = pd.read_csv(\"../input/sample_submission.csv\")\nsample_submission_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b1dc2d8684072d2a259bce0ccc8a8c7bf9fcb03"},"cell_type":"code","source":"test_csv = pd.read_csv(\"../input/test.csv\")\ntest_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d572e52c658122d9d5b35761618410891475448"},"cell_type":"code","source":"train_csv = pd.read_csv(\"../input/train.csv\")\ntrain_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dce1ba44757cb438d8e8cbefad24d7835df6197"},"cell_type":"code","source":"print(\"There are {} trian images and {} test images\".format(len(train_csv.file_name.values), len(test_csv.file_name.values)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aa4901832d55d498998c3a837c435e8b39fc8a6d"},"cell_type":"markdown","source":"# Classes"},{"metadata":{"trusted":true,"_uuid":"e2243282da8dfe0b12b669d3ebc7da3d4292c60b"},"cell_type":"code","source":"dictionary = {0:'empty',\n              1:'deer',\n              2:'moose',\n              3:'squirrel',\n              4:'rodent',\n              5:'small_mammal',\n              6:'elk',\n              7:'pronghorn_antelope',\n              8:'rabbit',\n              9:'bighorn_sheep',\n              10:'fox',\n              11:'coyote',\n              12:'black_bear',\n              13:'raccoon',\n              14:'skunk',\n              15:'wolf',\n              16:'bobcat',\n              17:'cat',\n              18:'dog',\n              19:'opossum',\n              20:'bison',\n              21:'mountrain_goat',\n              22:'mountain_lion',\n             }\ndictionary_reverse = dict((i,j) for j,i in dictionary.items())\n\ntrain_ids = set(train_csv.category_id.values)\nprint(\"There are the following classes: {} in train\".format(train_ids))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e10efb4463318826c40169dfbf3777a1a6a68fb"},"cell_type":"code","source":"train_ids = set(dictionary[item] for item in train_ids)\nprint(\"There are the following classes: {} in train\".format(train_ids))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a67a4ecd7cee08265befbadc0cad06902f288b31"},"cell_type":"code","source":"print(\"There are some classes missing: {}\".format(set(dictionary.values())-train_ids))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bc7eb0ec6ca4ad3877e29e70d952c952be8d36b"},"cell_type":"code","source":"id_table = pd.read_csv(\"../input/train.csv\")\n\nappearance_dict = dict()\nfor i in range(23):\n    appearance = len([val for val in id_table.category_id.values if val == i])\n    appearance_dict[i] = appearance\n    \nprint(appearance_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7cd07061d43801b0caa014e5b5937830d87adc5"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_bar_x(index, values, x_label, y_label, title, y_lim=None):\n    plt.bar(index, values)\n    plt.xlabel(x_label, fontsize=5)\n    plt.ylabel(y_label, fontsize=5)\n    plt.xticks(range(len(index)), index, fontsize=5, rotation=90)\n    plt.title(title)\n    \n    if y_lim != None:\n        plt.ylim(top=y_lim)\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f015c0e67690cbe8cfd28617e2cd5220089f70dd"},"cell_type":"code","source":"plot_bar_x(dictionary.values(), appearance_dict.values(), \"Names\", \"Frequency\", \"Data Distribution Betwen Classes\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"304004dc7a6e3dd11ab278082781634c8c5d9d6b"},"cell_type":"code","source":"plot_bar_x(dictionary.values(), appearance_dict.values(), \"Names\", \"Frequency\", \"Data Distribution Betwen Classes Without Class 0\", y_lim=15000)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9740d92ecbeefed8ea335b3c40a129de1e53f0b4"},"cell_type":"markdown","source":"# Heights"},{"metadata":{"trusted":true,"_uuid":"386c3e9efbefb1755cbe93772f9ce210a9d6f2b7"},"cell_type":"code","source":"print(\"There are only one widths: {}\".format(set(train_csv.width.values)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd6bad04bbde7064cda48cb23f12dd11d97eb24a"},"cell_type":"code","source":"print(\"But there are three heights: {}\".format(set(train_csv.height.values)))\n\nheight_dict = dict()\nfor i in set(train_csv.height.values):\n    height = len([val for val in id_table.height.values if val == i])\n    height_dict[i] = height\n    \nprint(\"Here is the distribution: {}\".format(height_dict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0bbf1b8bca66bc3da9925353091d85d8f86d2b0"},"cell_type":"code","source":"plot_bar_x(['768','747','748'], height_dict.values(), \"Hights\", \"Frequency\", \"Data Distribution of Heights\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"298ab1c100fc9ef6fd91926ae2efa139a23de3af"},"cell_type":"markdown","source":"## The Structure of the Public LB\nF1 Score in each class: [0.089, ...]\n"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}