{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This code is made for if you download the metadata in your PC\n### I hope that it will be helpful for beginner :)\n---\n"},{"metadata":{},"cell_type":"markdown","source":"# Install and import package"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install requests wget wordcloud","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport os.path as pth\n\nimport wget\nfrom multiprocessing import Pool\n\nimport pandas as pd\nfrom collections import Counter\nfrom wordcloud import WordCloud\nfrom matplotlib import pyplot  as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Download data for visualization\n### This part based on web crawling.\n### If you wonder more detail, Please to visit my another kernel ([Download metadata and segmentation images](https://www.kaggle.com/bbchip/download-metadata-and-segmentation-images))"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_metadata(url, base_path='./'):\n    filename = url.split('/')[-1]\n    full_filename = pth.join(base_path, filename)\n    if pth.exists(full_filename):\n        return full_filename, 1\n    wget.download(url, out=base_path)\n    ### If you can't use wget, you can use below blocked code\n    ### But It's much slower than wget...\n    # data = requests.get(url).data\n    # with open(full_filename, 'wb') as f:\n    #     f.write(data)\n    return full_filename, 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_base_path = 'metadata'\nmetadata_path = pth.join(data_base_path, 'Metadata')\nbox_path = pth.join(data_base_path, 'Boxes')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metadata_url = 'https://storage.googleapis.com/openimages/v5/class-descriptions-boxable.csv'\ntrain_box_url = 'https://storage.googleapis.com/openimages/2018_04/train/train-annotations-bbox.csv'\nvalidation_box_url = 'https://storage.googleapis.com/openimages/v5/validation-annotations-bbox.csv'\ntest_box_url = 'https://storage.googleapis.com/openimages/v5/test-annotations-bbox.csv'\n\nurl_list = [metadata_url, train_box_url, validation_box_url, test_box_url]\npath_list = [metadata_path] + [box_path]*3\n\nos.makedirs(metadata_path, exist_ok=True)\nos.makedirs(box_path, exist_ok=True)\npool = Pool(8)\nfor filename, status in pool.starmap(get_metadata, zip(url_list, path_list)):\n    if status == 0:\n        print(filename + ' is saved.')\n    elif status == 1:\n        print(filename + ' is already exist.')\n    else:\n        print('???')\npool.close()\npool.join()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make the label table. {LabelName:RealName}\n###  ['Pandas'](http://zeldahagoshipda.com) is useful package for data analysis.\n### I recommand you to get used to this package for data analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"label_filename = pth.join(metadata_path, 'class-descriptions-boxable.csv')\ndf = pd.read_csv(label_filename, header=None, index_col=None)\nlabel_dict = dict(df.values)\ndict(list(label_dict.items())[:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Count the number of label\n### ['Collections'](https://docs.python.org/3/library/collections.html) has a lot of useful function."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_box_filename = pth.join(box_path, 'train-annotations-bbox.csv')\nval_box_filename = pth.join(box_path, 'validation-annotations-bbox.csv')\ntest_box_filename = pth.join(box_path, 'test-annotations-bbox.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(train_box_filename)\nlabels = df['LabelName'].values\ntrain_cnt = Counter(labels)\n    \ndf = pd.read_csv(val_box_filename)\nlabels = df['LabelName'].values\nval_cnt = Counter(labels)\n\ndf = pd.read_csv(test_box_filename)\nlabels = df['LabelName'].values\ntest_cnt = Counter(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cnt.most_common(5), val_cnt.most_common(5), test_cnt.most_common(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cnt_dict = {label_dict[k]:v for k, v in train_cnt.items()}\nval_cnt_dict = {label_dict[k]:v for k, v in val_cnt.items()}\ntest_cnt_dict = {label_dict[k]:v for k, v in test_cnt.items()}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make a word cloud\n### It made by wordcloud module ([And there is a lot of exameple!](https://amueller.github.io/word_cloud/auto_examples/index.html#example-gallery))\n\n### You can make your wordcloud (Just tune the \"max_word, color, mask image, iamge size\")\n### I recommend you to apply mask first, Because it makes word-cloud more beautiful."},{"metadata":{"trusted":true},"cell_type":"code","source":"wc = WordCloud(max_words=300\n                , background_color='white'\n#                 , width=1920, height=1080\n#                 , mask=mask\n#                 , color_func=MakeColor\n                )\nwc.generate_from_frequencies(train_cnt_dict)\n# wc.to_file(pth.join(your_directory, wordcloud_filename))\n\nplt.figure()\nplt.axis(\"off\")\nplt.title('Train')\nplt.imshow(wc, interpolation='bilinear')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wc = WordCloud(max_words=300\n                , background_color='white'\n                )\nwc.generate_from_frequencies(val_cnt_dict)\n\nplt.figure()\nplt.axis(\"off\")\nplt.title('Validation')\nplt.imshow(wc, interpolation='bilinear')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wc = WordCloud(max_words=300\n                , background_color='white'\n                )\nwc.generate_from_frequencies(val_cnt_dict)\n\nplt.figure()\nplt.axis(\"off\")\nplt.title('Test')\nplt.imshow(wc, interpolation='bilinear')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.0"}},"nbformat":4,"nbformat_minor":1}