{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n# basically libraries\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# image libaries\nimport cv2\nimport matplotlib.pyplot as plt\n\n# for split train and test\nfrom sklearn.model_selection import train_test_split\n\n# for model\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense, Dropout, BatchNormalization, Flatten\nfrom tensorflow.keras.layers import Add, Concatenate, GlobalAvgPool2D\nfrom tensorflow.keras.layers import MaxPooling2D, SeparableConv2D \nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### First of all...  \n I am not good at English. So, I think my description is difficult to read and understand.   \n Everyone, Please pardon."},{"metadata":{},"cell_type":"markdown","source":"# Confirm Input Data\nRead 'labels.csv' and confirm contents.  \nThere are 1,103 attibutes. "},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"# Label.csv\nlabels_ds = pd.read_csv(filepath_or_buffer='../input/labels.csv', dtype={'attribute_id':np.object, 'attribute_name':np.object})\nprint(labels_ds.head())\nprint(labels_ds.tail())\nprint(\"\")\nprint(labels_ds.info())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Read train.csv  \nRead train.csv to pandas data frame.  \ntrain.csv contains no n/a data.  \nattribute_ids contains multi values, so need to split."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# train.csv\ntrain_ds = pd.read_csv(filepath_or_buffer='../input/train.csv')\nprint(train_ds.head())\nprint(\"\")\nprint(train_ds.info())\nprint(\"\")\nprint(train_ds.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Check image files  \nImage files are exist in '../input/train/' folder.  \nImage file name is represented by '<id>.png'."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input/train/\")[0:12])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Show image files and image attibutes  \nShow first 12 images, image height, width, and relative attributes."},{"metadata":{"trusted":true},"cell_type":"code","source":"# image data \n# Check image data size and image by first 12 files\nimage_file_list = os.listdir(\"../input/train/\")[0:12]\nimage_data_list = []\n\nfig = plt.figure(figsize=(10, 15))\nfor image_index in range(12):\n    image_file_name = train_ds.iloc[image_index, 0]\n    image_np = cv2.imread(\"../input/train/\" + image_file_name + \".png\")\n\n    image_label = \"{}\\n height:{} width:{}\\nattr:{}\".format(\n        image_file_name, image_np.shape[0], image_np.shape[1], train_ds.iloc[image_index, 1]\n    )\n    image_area = fig.add_subplot(4,3,image_index + 1, title=image_label)\n    image_area.imshow(cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB))\n    \nfig.tight_layout()\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### One-hot encoding image attributes  \nEncoding the image attibutes, number to binary. In here, use original simple function, because MultiLabelBinarizer don't work expectly..."},{"metadata":{"trusted":true},"cell_type":"code","source":"# One hot encoding for multi labels.\ndef OneHotEncoding(rec):\n    attribute_id_list = rec[\"attribute_ids\"].split()\n    for attribute_id in attribute_id_list:\n        rec[attribute_id] = 1\n    \n    return rec\n\n# Append new columns from list\ndef AppendColumns(df, columnList):\n    for newColumn in columnList:\n        df[newColumn] = 0\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create MultiLabelBinarizer instance and fit to attibute id in labels.csv\ntrain_ds_encoded = AppendColumns(train_ds, labels_ds['attribute_id'])\ntrain_ds_encoded = train_ds_encoded.apply(OneHotEncoding, axis=1)\ntrain_ds_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Append filename column\ntrain_ds_encoded[\"filename\"] = train_ds_encoded[\"id\"] + \".png\" \ntrain_ds_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Check frequency  \nCheck frequency of attribute.  \n81 attributes (about 7% attributes) are less than 5. So, we need to increase data that is infrequent."},{"metadata":{"trusted":true},"cell_type":"code","source":"summary_df = pd.DataFrame(data={'id':labels_ds['attribute_id'], 'attibute':labels_ds['attribute_name'], 'count':np.array(train_ds_encoded.iloc[:, 2:].sum(numeric_only=True))})\nsummary_df = summary_df.sort_values(by='count')\nprint(summary_df.head())\nprint(summary_df.tail(20))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rare_attr_df = summary_df.sort_values(by='count').loc[summary_df['count'] <= 5]\nrare_data_df = train_ds_encoded.loc[train_ds_encoded.apply(lambda x: set(x['attribute_ids'].split(' ')).isdisjoint(rare_attr_df['id']) == False, axis=1)]\nrare_data_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_2 = train_ds_encoded\nfor count in range(10):\n    train_df_2 = train_df_2.append(rare_data_df)\n\ntrain_df_2.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Separate data and label\ntrain_df_X = train_df_2.iloc[:, 0]\ntrain_df_y = train_df_2.iloc[:, 2:]\n\n# Split train and test\nX_train, X_test, y_train, y_test = train_test_split(train_df_X, train_df_y, test_size=0.10, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_train = train_df_2.sample(frac=0.9, random_state=42)\ntrain_df_test = train_df_2.drop(train_df_train.index)\nprint(\"{} {} {}\".format(len(train_df_2), len(train_df_train), len(train_df_test)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check Labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split lable\nsplitted_attr = labels_ds['attribute_name'].str.split('::', expand = True)\nsplitted_attr.columns = ['main', 'sub']\nsplitted_attr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(splitted_attr['main'].drop_duplicates())\nprint('culture : {}; tag : {}'.format(len(splitted_attr.loc[splitted_attr.main == 'culture']), len(splitted_attr.loc[splitted_attr.main == 'tag'])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(splitted_attr['sub'].drop_duplicates())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"main category is 2, 'culture' and 'tag'.\nsub category is 1103, not duplicated."},{"metadata":{},"cell_type":"markdown","source":"### Check corrolation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds_encoded.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corr_df = train_ds_encoded.iloc[:, 2:-1].corr()\ncorr_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corr_df2 = corr_df.replace(1, 0).abs()\ncorr_df2['id'] = corr_df2.index\ncorr_df2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corr_df3 = corr_df2.loc[lambda x: x[0:-1].max() > 0.4]\nmax_values = corr_df3.iloc[:, 0:-1].max(axis=1)\nmax_index1 = corr_df3.iloc[:, 0:-1].idxmax(axis=1)\nmax_index2 = corr_df3['id']\ncorr_df4 = pd.DataFrame(np.stack((max_values, max_index1, max_index2), axis=-1), columns=['value', 'id1', 'id2'])\ncorr_df4 = corr_df4.merge(labels_ds, left_on = 'id1', right_on = 'attribute_id')\ncorr_df4 = corr_df4.merge(labels_ds, left_on = 'id2', right_on = 'attribute_id', suffixes=('_1', '_2'))\ncorr_df4 = corr_df4.drop(columns=['attribute_id_1', 'attribute_id_2'])\ncorr_df4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}