{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\npd.options.display.max_columns = 100\npd.options.display.max_colwidth = 200\npd.options.display.max_rows = 500\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nimport json\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\nimport gc\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/sample_submission.csv\")\nlabels_df = pd.read_csv(\"../input/labels.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(\"[train]\")\nprint(len(train_df))\nprint(Counter(map(lambda x: x.split(\".\")[-1], os.listdir(\"../input/train/\"))))\n\nprint(\"[test]\")\nprint(len(test_df))\nprint(Counter(map(lambda x: x.split(\".\")[-1], os.listdir(\"../input/test/\"))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## check labels"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"len(labels_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_df[\"attribute_type\"] = labels_df.attribute_name.apply(lambda x: x.split(\"::\")[0])\nprint(labels_df[\"attribute_type\"].value_counts())\nlabels_df.attribute_type.value_counts().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_df.query(\"attribute_type == 'culture'\").index ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_df.query(\"attribute_type == 'tag'\").index ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_df[:398].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_df[398:].head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## check attibure_id's frequency in train"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_attr_ohot = np.zeros((len(train_df), len(labels_df)), dtype=int)\n\nfor idx, attr_arr in enumerate(train_df.attribute_ids.str.split(\" \").apply(lambda l: list(map(int, l))).values):\n    train_attr_ohot[idx, attr_arr] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"names_arr = labels_df.attribute_name.values\ntrain_df[\"attribute_names\"] = [\", \".join(names_arr[arr == 1]) for arr in train_attr_ohot]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"attr_num\"] = train_attr_ohot.sum(axis=1)\ntrain_df[\"culture_attr_num\"] = train_attr_ohot[:, :398].sum(axis=1)\ntrain_df[\"tag_attr_num\"] = train_attr_ohot[:, 398:].sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### number of attributes each art has"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.attr_num.value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### number of _\"culture\"_ attributes each art has"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.culture_attr_num.value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### number of _\"tag\"_ attributes each art has"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.tag_attr_num.value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"markdown","source":"### plots"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize=(15, 10))\nfig.subplots_adjust(hspace=0.4)\nax1 = fig.add_subplot(3,1,1)\nsns.countplot(train_df.attr_num,)\nax1.set_title(\"number of attributes each art has\")\nax2 = fig.add_subplot(3,1,2,)\nsns.countplot(train_df.culture_attr_num, ax=ax2)\nax2.set_title(\"number of 'culture' attributes each art has\")\nax3 = fig.add_subplot(3,1,3,)\nax3.set_title(\"number of 'tag' attributes each art has\")\nsns.countplot(train_df.tag_attr_num, ax=ax3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.pivot_table(\n    train_df, index='culture_attr_num', columns='tag_attr_num', values='attr_num',\n    aggfunc=len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.culture_attr_num.value_counts(normalize=True).sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.tag_attr_num.value_counts(normalize=True).sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"There is difference between the distribution of number of culture attributes and one of tag attributes.  \n\nThe number of _culture_ attributes 99% of arts have is 0 or 1 or 2, moreover, **80% is 1**.  \nOn the other hands, the number of _tag_ attributes shows a **gentler** slope from 1 to 5. Very few of arts have no tag attribute.\n\nI think these observations may be useful for deciding thresholds ofclassifiers. \n\n\nNext, I show the arts which have many culuture or tag attributes. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.sort_values(by=\"culture_attr_num\", ascending=False).head(15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.sort_values(by=\"tag_attr_num\", ascending=False).head(15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It is difficult for me to find somthing from these tables. Let's show images of arts in these tables."},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### arts with many _culture_ attributes"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize=(5 * 5, 5 * 6))\nfig.subplots_adjust(wspace=0.6, hspace=0.6)\nfor i, (art_id, attr_names) in enumerate(train_df.sort_values(by=\"culture_attr_num\", ascending=False)[[\"id\", \"attribute_names\"]].values[:15]):\n    ax = fig.add_subplot(5, 3, i // 3 * 3 + i % 3 + 1)\n    im = Image.open(\"../input/train/{}.png\".format(art_id))\n    ax.imshow(im)\n    im.close()\n    attr_split = attr_names.split(\", \")\n    attr_culture = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:7] == \"culture\", attr_split)))\n    attr_tag = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:3] == \"tag\", attr_split)))\n    ax.set_title(\"art id: {}\\nculture: {}\\ntag: {}\".format(art_id, attr_culture, attr_tag))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"markdown","source":"### arts with many _tag_ attirbutes"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize=(5 * 6, 5 * 5))\nfig.subplots_adjust(wspace=0.6, hspace=0.6)\nfor i, (art_id, attr_names) in enumerate(train_df.sort_values(by=\"tag_attr_num\", ascending=False)[[\"id\", \"attribute_names\"]].values[:15]):\n    ax = fig.add_subplot(5, 3, i // 3 * 3 + i % 3 + 1)\n    im = Image.open(\"../input/train/{}.png\".format(art_id))\n    ax.imshow(im)\n    im.close()\n    attr_split = attr_names.split(\", \")\n    attr_culture = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:7] == \"culture\", attr_split)))\n    attr_tag = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:3] == \"tag\", attr_split)))\n    ax.set_title(\"art id: {}\\nculture: {}\\ntag: {}\".format(art_id, attr_culture, attr_tag))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since I have poor knowledge of art, cannot validate _culture_ attributes.\n\nHow about _tag_ attributes? They are relatively interpretable, but may be splitable into some types:\n\n* objects painted (or carved) on arts: animals, humans, places, ...\n* type of arts: 'cups', 'coat of arms', 'textiles'. 'dishes', 'vines'...\n* special situations: 'nativity', 'last jugement', 'crucifixion',...\n* actions: 'hourse riding', 'reading', 'suffering', ...\n* ...\n\nTherefore, I think it may be useful for classification to consider type of _tag_ attribute.\n<br>\n<br>\n<br>\nWith respect to number, pictures tend to have more _tag_ attributes because of painted objects on them.  \nI have one assumption that number of tag attributes depends on type of arts. Then, check several examples."},{"metadata":{},"cell_type":"markdown","source":"### arts with 1 _tag_ attribute"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize=(5 * 8, 5 * 7))\nfig.subplots_adjust(wspace=0.6, hspace=0.6)\nfor i, (art_id, attr_names) in enumerate(train_df[train_df.tag_attr_num == 1][[\"id\", \"attribute_names\"]].values[:49]):\n    ax = fig.add_subplot(7, 7, i // 7 * 7 + i % 7 + 1)\n    im = Image.open(\"../input/train/{}.png\".format(art_id))\n    ax.imshow(im)\n    im.close()\n    attr_split = attr_names.split(\", \")\n    attr_culture = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:7] == \"culture\", attr_split)))\n    attr_tag = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:3] == \"tag\", attr_split)))\n    ax.set_title(\"art id: {}\\nculture: {}\\ntag: {}\".format(art_id, attr_culture, attr_tag))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"markdown","source":"### arts with 2 _tag_ attributes"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize=(5 * 8, 5 * 7))\nfig.subplots_adjust(wspace=0.6, hspace=0.6)\nfor i, (art_id, attr_names) in enumerate(train_df[train_df.tag_attr_num == 2][[\"id\", \"attribute_names\"]].values[:49]):\n    ax = fig.add_subplot(7, 7, i // 7 * 7 + i % 7 + 1)\n    im = Image.open(\"../input/train/{}.png\".format(art_id))\n    ax.imshow(im)\n    im.close()\n    attr_split = attr_names.split(\", \")\n    attr_culture = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:7] == \"culture\", attr_split)))\n    attr_tag = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:3] == \"tag\", attr_split)))\n    ax.set_title(\"art id: {}\\nculture: {}\\ntag: {}\".format(art_id, attr_culture, attr_tag))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"markdown","source":"### arts with 3 _tag_ attributes"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize=(5 * 8, 5 * 7))\nfig.subplots_adjust(wspace=0.6, hspace=0.6)\nfor i, (art_id, attr_names) in enumerate(train_df[train_df.tag_attr_num == 3][[\"id\", \"attribute_names\"]].values[:49]):\n    ax = fig.add_subplot(7, 7, i // 7 * 7 + i % 7 + 1)\n    im = Image.open(\"../input/train/{}.png\".format(art_id))\n    ax.imshow(im)\n    im.close()\n    attr_split = attr_names.split(\", \")\n    attr_culture = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:7] == \"culture\", attr_split)))\n    attr_tag = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:3] == \"tag\", attr_split)))\n    ax.set_title(\"art id: {}\\nculture: {}\\ntag: {}\".format(art_id, attr_culture, attr_tag))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"markdown","source":"### arts with 4 _tag_ attributes"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize=(5 * 8, 5 * 7))\nfig.subplots_adjust(wspace=0.6, hspace=0.6)\nfor i, (art_id, attr_names) in enumerate(train_df[train_df.tag_attr_num == 4][[\"id\", \"attribute_names\"]].values[:49]):\n    ax = fig.add_subplot(7, 7, i // 7 * 7 + i % 7 + 1)\n    im = Image.open(\"../input/train/{}.png\".format(art_id))\n    ax.imshow(im)\n    im.close()\n    attr_split = attr_names.split(\", \")\n    attr_culture = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:7] == \"culture\", attr_split)))\n    attr_tag = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:3] == \"tag\", attr_split)))\n    ax.set_title(\"art id: {}\\nculture: {}\\ntag: {}\".format(art_id, attr_culture, attr_tag))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"markdown","source":"### arts with 5 _tag_ attributes"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(5 * 8, 5 * 7))\nfig.subplots_adjust(wspace=0.6, hspace=0.6)\nfor i, (art_id, attr_names) in enumerate(train_df[train_df.tag_attr_num == 5][[\"id\", \"attribute_names\"]].values[:49]):\n    ax = fig.add_subplot(7, 7, i // 7 * 7 + i % 7 + 1)\n    im = Image.open(\"../input/train/{}.png\".format(art_id))\n    ax.imshow(im)\n    im.close()\n    attr_split = attr_names.split(\", \")\n    attr_culture = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:7] == \"culture\", attr_split)))\n    attr_tag = list(map(lambda x: x.split(\"::\")[-1], filter(lambda x: x[:3] == \"tag\", attr_split)))\n    ax.set_title(\"art id: {}\\nculture: {}\\ntag: {}\".format(art_id, attr_culture, attr_tag))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It seems that more _tag_ attributes arts have, more complex they are.\n\nMost of arts with one _tag_ attributes are single objects such as ornaments.  \nIn contrast, most of arts with five ones are pictures or objects with complex design.\n\nMaybe, we can predict number of _tag_ attributes by **_complexity_** of arts ?"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}