{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This kernel prepares the csv file needed for another kernel : [EDA - Weird Images](https://www.kaggle.com/chewzy/eda-weird-images-with-new-updates/)\n\n### The csv contains:\n* Image level meta data (width, height, pixel statistics etc)\n* One-hot encoded image labels"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport PIL\nimport gc\n\nfrom PIL import ImageOps, ImageFilter\nfrom multiprocessing import Pool","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def get_img_properties(img_id, path):\n    im = PIL.Image.open(f'{path}{img_id}.png')\n    \n    width = im.size[0]\n    height = im.size[1]\n    \n    r, g, b = im.split()\n    r_arr, g_arr, b_arr = np.array(r), np.array(g), np.array(b)\n    r_mean, r_std = np.mean(r_arr), np.std(r_arr)\n    g_mean, g_std = np.mean(g_arr), np.std(g_arr)\n    b_mean, b_std = np.mean(b_arr), np.std(b_arr)\n    \n    edges_arr = np.array(im.filter(ImageFilter.FIND_EDGES))\n    r_edge_arr, g_edge_arr, b_edge_arr = edges_arr[:,:,0], edges_arr[:,:,1], edges_arr[:,:,2]\n    r_edge_mean, r_edge_std = np.mean(r_edge_arr), np.std(r_edge_arr)\n    g_edge_mean, g_edge_std = np.mean(g_edge_arr), np.std(g_edge_arr)\n    b_edge_mean, b_edge_std = np.mean(b_edge_arr), np.std(b_edge_arr)\n    \n    hist = im.histogram()\n    peak_index = np.argmax(hist)\n    peak_val = np.max(hist) / (width * height) # normalize this as images have different size\n    \n    return np.array([width, height, \\\n                     r_mean, r_std, g_mean, g_std, b_mean, b_std, \\\n                     r_edge_mean, r_edge_std, g_edge_mean, g_edge_std, b_edge_mean, b_edge_std, \\\n                     peak_index, peak_val])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_cols = ['width', 'height', \\\n             'r_mean', 'r_std', 'g_mean', 'g_std', 'b_mean', 'b_std', \\\n             'r_edge_mean', 'r_edge_std', 'g_edge_mean', 'g_edge_std', 'b_edge_mean', 'b_edge_std', \\\n             'peak_index', 'peak_val']\n\n# allocate some memory first\nfor col in meta_cols:\n    df_train[col] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_partitions = 12\nn_workers = 12\ntrain_path = '../input/train/'\n\ndef parallelize_dataframe(df, func):\n    df_split = np.array_split(df, n_partitions)\n    pool = Pool(n_workers)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df\n\n\ndef get_meta_data(data):\n    \n    meta_data = np.zeros((data.shape[0], 16))\n    \n    for index, file_id in enumerate(data['id'].values):\n        meta_data[index] = get_img_properties(file_id, train_path)\n    \n    data[meta_cols] = meta_data\n    \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndf_train = parallelize_dataframe(df_train, get_meta_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_df = pd.read_csv('../input/labels.csv')\nlabel_names = label_df['attribute_name'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_labels = np.zeros((df_train.shape[0], len(label_names)))\n\nfor row_index, row in enumerate(df_train['attribute_ids']):\n    for label in row.split():\n        train_labels[row_index, int(label)] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in label_names:\n    df_train[col] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndf_train[label_names] = train_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.to_csv('weird_images_w_labels.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}