{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom zipfile import ZipFile\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nfrom nltk.stem.snowball import SnowballStemmer\nimport os\nprint(os.listdir(\"../input\"))\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"train_zip  = ZipFile('../input/avito-demand-prediction/train_jpg.zip')","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57dcb1c77146a2242665fb00ee0a55017cc28677","collapsed":true},"cell_type":"code","source":"filenames = train_zip.namelist()[1:]\nlen(filenames)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65a8b67090c09fd07d4cdc8b15efc3bc0f15b20d","collapsed":true},"cell_type":"code","source":"train_zip.close()\ndel train_zip","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7385b954d7e07b2b17a6578ea3cf2aa0daf0836d","collapsed":true},"cell_type":"code","source":"train_zip  = ZipFile('../input/avito-demand-prediction/test_jpg.zip')\nfilenames = train_zip.namelist()[1:]\nprint(len(filenames))\ntrain_zip.close()\ndel train_zip","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6d7f781905afc1eabad11abe39c0164c7f4530d9"},"cell_type":"markdown","source":"# image base features"},{"metadata":{"trusted":true,"_uuid":"e7c7b1b4664ea65c3e0af84314f1fd8001f78af4","collapsed":true},"cell_type":"code","source":"train_image_info = pd.read_csv('../input/extract-image-features-train-files/train_img_feat.csv',index_col=0)\ntrain_image_info.shape","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b865bfdea99435907aab6e5a78212c6d7c7b948c","collapsed":true},"cell_type":"code","source":"train_image_size = pd.read_csv('../input/extract-image-features-train-files/train_img_files_info.csv',index_col=0)\ntrain_image_size.shape","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"915f212a334988d7aae2a2dbbbebd580c889068b","collapsed":true},"cell_type":"code","source":"%%time\nimage_features = train_image_info.merge(train_image_size,on='image',how='left')","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2730c2eb1993301ad8731544ee0a9e80245620ff","collapsed":true},"cell_type":"code","source":"image_features.head()","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a8727502ebffc046e7763bb58848181b35735c9","collapsed":true},"cell_type":"code","source":"image_features['dim'] = image_features['dim'].apply(lambda x:eval(str(x)))","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c83b7e0ec3705a46c31b7ca6eb51cfd226e48963","collapsed":true},"cell_type":"code","source":"image_features['colors'] = image_features['colors'].apply(lambda x: eval(str(x)))","execution_count":11,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e4a58b352a84813f98b6046780910398496dafd","collapsed":true},"cell_type":"code","source":"image_features['width'] = image_features['dim'].apply(lambda x: x[0])\nimage_features['height'] = image_features['dim'].apply(lambda x: x[1])\nimage_features['red_avg'] = image_features['colors'].apply(lambda x: x[0])\nimage_features['green_avg'] = image_features['colors'].apply(lambda x: x[1])\nimage_features['blue_avg'] = image_features['colors'].apply(lambda x: x[2])","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f5ba4854ae99b6c2ae04ab8a23974159fa7daa96"},"cell_type":"code","source":"image_features.drop(columns=['dim','colors','csize'],inplace=True)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a16ed35075fab79b2a37f31208c1c24650fe001","collapsed":true},"cell_type":"code","source":"image_features.head()","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"868a96216c8547ba76f3eb958d3108f6bcd884be"},"cell_type":"code","source":"def make_features(image_features):\n    image_features['width_height_diff'] = image_features[['width','height']].diff(axis=1)['height']\n    image_features['green_blue_diff'] = image_features[['green_avg','blue_avg']].diff(axis=1)['blue_avg']\n    image_features['green_red_diff'] = image_features[['green_avg','red_avg']].diff(axis=1)['red_avg']\n    image_features['red_blue_diff'] = image_features[['red_avg','blue_avg']].diff(axis=1)['blue_avg']\n    image_features['width_height_ratio'] = image_features['width']/image_features['height']\n    image_features['total_pixel'] = image_features['width']*image_features['height']\n    return image_features","execution_count":15,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"002b89d63836826c02f1e656939e9359bc0fc021","collapsed":true},"cell_type":"code","source":"image_features = make_features(image_features)","execution_count":16,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7e6bfcced0e93e866e16133bc28eea14aa133f6","collapsed":true},"cell_type":"code","source":"image_features.head()","execution_count":17,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1af346671c4d48de65b973f50cf4d3903110af7","collapsed":true},"cell_type":"code","source":"def show_corr(df):\n    f, ax = plt.subplots(figsize=[10,7])\n    sns.heatmap(df.corr(),\n                annot=False, fmt=\".2f\",cbar_kws={'label': 'Correlation Coefficient'},cmap=\"plasma\",ax=ax, linewidths=.5)\n    ax.set_title(\"Dense Features Correlation Matrix\")\n    plt.savefig('correlation_matrix.png')\nshow_corr(image_features)","execution_count":18,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a26a391be707d210183712ec9c6a1ab26e6d4b28","collapsed":true},"cell_type":"code","source":"image_features.columns","execution_count":19,"outputs":[]},{"metadata":{"_uuid":"f62710081e3f2c2ab8adfa745a97522d76ad1ff8"},"cell_type":"markdown","source":"# yuv features"},{"metadata":{"trusted":true,"_uuid":"5577b11f47aec6cccc88103db276b0696dad8cfb","_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"test_image_yuv = pd.read_csv('../input/script-image-features-test-yuv-multproces/test_jpg_img_feat_saturation.csv',index_col=0)\ntrain_image_yuv = pd.read_csv('../input/script-image-features-train-yuv-multprocess/train_jpg_img_feat_saturation.csv',index_col=0)","execution_count":122,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a8d758682b1f9fd2b0c29bc982a5718f9f7d33b","collapsed":true},"cell_type":"code","source":"yuv_col = [ 'bright_avg', 'u_avg', 'yuv_v_avg', 'bright_std', 'bright_min','birght_max', 'bright_diff']","execution_count":123,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"613c64ea0261d33ed8345468664c4f83a7811dd9","collapsed":true},"cell_type":"code","source":"test_image_yuv.columns","execution_count":124,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a6347ddaf077d4546c41db4d4743a41cf3a0c54","collapsed":true},"cell_type":"code","source":"test_image_yuv.rename(columns={'v_avg':'yuv_v_avg'},inplace=True,index=str)\ntrain_image_yuv.rename(columns={'v_avg':'yuv_v_avg'},inplace=True,index=str)","execution_count":137,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cec389d827df6c31ecabbaffd3de6d89dc76992","collapsed":true},"cell_type":"code","source":"test_image_yuv.columns,train_image_yuv.columns","execution_count":138,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbd3b5dbd20b6ec9e6d195f73e8d409d58ea91cc","collapsed":true},"cell_type":"code","source":"test_image_yuv.head()","execution_count":32,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fce456367f16bad06639a0f478d3d3605bcedcf","collapsed":true},"cell_type":"code","source":"image_features.shape,test_image_yuv.shape,train_image_yuv.shape","execution_count":54,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a5721fbee377319f3fe65d5a6bd406ccb72250f7"},"cell_type":"code","source":"# test_image_yuv= test_image_yuv[using_col]\n# train_image_yuv= train_image_yuv[using_col]","execution_count":34,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d8ee80d8daaec2b7a0c8caf4cb3ba6c8c2d42da","collapsed":true},"cell_type":"code","source":"train_image_yuv.head()","execution_count":55,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4a0ed2afa683991e5549499876e4a38f700ce199"},"cell_type":"code","source":"image_features = image_features.merge(train_image_yuv,on='image',how='left')","execution_count":56,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a62955c1d2d984afed8d7a113735ae60a4a0c1db","collapsed":true},"cell_type":"code","source":"image_features.head()","execution_count":57,"outputs":[]},{"metadata":{"_uuid":"f1c8e6311c49ccf810c96599f8fe59311ca0a5c1"},"cell_type":"markdown","source":"# hsv features"},{"metadata":{"trusted":true,"_uuid":"27287eb53a4b771653c36e8d7c1bec59d8aa9904","collapsed":true},"cell_type":"code","source":"train_path_ = '../input/script-image-features-train-hsv-batch%s/train_jpg_img_feat_saturation.csv'\ntrain_ids = [1,15,2,25,3,4]\ntest_path_ = '../input/script-image-features-test-hsv-batch%s/test_jpg_img_feat_saturation.csv'\ntest_ids = [1,2]\ntrain_hsv_df = pd.DataFrame()\ntest_hsv_df = pd.DataFrame()\nfor i in train_ids:\n    p = train_path_%i\n    train_hsv_df = train_hsv_df.append(pd.read_csv(p,index_col=0))\n    print(p,train_hsv_df.shape)\n\nfor i in test_ids:\n    p = test_path_%i\n    test_hsv_df = test_hsv_df.append(pd.read_csv(p,index_col=0))\n    print(p,test_hsv_df.shape)\ntrain_hsv_df.shape,test_hsv_df.shape","execution_count":59,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c0ab41e9c698ffbffd31212e8ceab7ce09a54d1a"},"cell_type":"code","source":"train_hsv_df.rename(columns={'v_avg':'hsv_v_avg'},inplace=True,index=str)\ntest_hsv_df.rename(columns={'v_avg':'hsv_v_avg'},inplace=True,index=str)","execution_count":139,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e582e21a34505059a99b7d4671123fd8eda94ca5","collapsed":true},"cell_type":"code","source":"image_features = image_features.merge(train_hsv_df,on='image',how='left')\nimage_features.head()","execution_count":60,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d7191f6316c16f87073c0a81ef5ececb2417c08","collapsed":true},"cell_type":"code","source":"train_hsv_df.columns","execution_count":66,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"81ff41b98d1d932ce67a83deea36a0cb8fe4ba07"},"cell_type":"code","source":"hsv_col = ['hue_avg', 'sst_avg', 'hsv_v_avg', 'sat_std', 'sat_min', 'sat_max', 'sat_diff']","execution_count":67,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cea77d3832097b99b4877749f7c9b3b6d6cb7fd7","collapsed":true},"cell_type":"code","source":"show_corr(image_features)","execution_count":61,"outputs":[]},{"metadata":{"_uuid":"3edc91e7d02ab4c45c9f37bc5cf626ab0fc0c1a5"},"cell_type":"markdown","source":"# colorfull"},{"metadata":{"trusted":true,"_uuid":"7bbfd45fdd4860afa833452d337c1ba24d6aeedb","collapsed":true},"cell_type":"code","source":"train_path_ = '../input/script-image-features-train-color-batch%s/train_jpg_img_feat_saturation.csv'\ntest_path_ = '../input/script-image-features-test-color-multprocess/test_jpg_img_feat_saturation.csv'\ntrain_ids= [1,2,3,4,5,6]\n# train_ids= [1,2,3,5,6]\ntest_color_df = pd.read_csv(test_path_,index_col=0)\ntrain_color_df = pd.DataFrame()\nfor i in train_ids:\n    p = train_path_%i\n    x = pd.read_csv(p,index_col=0)\n    print(p,x.shape)\n    train_color_df = train_color_df.append(x)\ntrain_color_df.shape,test_color_df.shape","execution_count":70,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"968b752ca74575d1618d997e53393d772d16876e","collapsed":true},"cell_type":"code","source":"del train_image_info,train_image_size\nimport gc\ngc.collect()","execution_count":39,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2dabf0fb8302430e20c27ef7e8ab467e0fa057b","collapsed":true},"cell_type":"code","source":"image_features = image_features.merge(train_color_df,on='image',how='left')\nimage_features.head()","execution_count":71,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d3d45f925d285a31b870634685271906711709fb"},"cell_type":"code","source":"color = ['colorfull']","execution_count":73,"outputs":[]},{"metadata":{"_uuid":"eb1fe1abb8357ca6edd95bb4ae6b71b3c9cc556e"},"cell_type":"markdown","source":"# std"},{"metadata":{"trusted":true,"_uuid":"c3dec9c601a7bb6700b0257702d1963b691fcc08","collapsed":true},"cell_type":"code","source":"train_std_df = pd.read_csv(\"../input/script-image-features-train-pil-std-batch1/train_jpg_img_feat_saturation.csv\",index_col=0)\ntest_std_df = pd.read_csv(\"../input/script-image-features-test-pil-std-batch1/test_jpg_img_feat_saturation.csv\",index_col=0)\nstd_cols = ['r_std', 'g_std', 'b_std', 'r_md', 'g_md', 'b_md']\nimage_features = image_features.merge(train_std_df,on='image',how='left')","execution_count":30,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b11551eaee9dd440f57a9ca6550e9ba1a6f0f86c","collapsed":true},"cell_type":"code","source":"image_features.head()","execution_count":31,"outputs":[]},{"metadata":{"_uuid":"83d6b0d719c0657dfa44b0210f49b1306f8cda2c"},"cell_type":"markdown","source":"# xception"},{"metadata":{"trusted":true,"_uuid":"3225ff37f968cc62991788eb1158395afd61481f","collapsed":true},"cell_type":"code","source":"train_xception_df = pd.read_csv('../input/xception-train-features-starter-include-top/train_xception.csv',index_col=0)\ntest_xception_df = pd.read_csv('../input/xception-test-features-starter-include-top/test_xception.csv',index_col=0)","execution_count":25,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc75615b2859dfb165372ed6ce0f0473ebc411cc","collapsed":true},"cell_type":"code","source":"train_xception_df.columns = ['image','item_label', 'xception_prob', 'xception_var',\n       'xception_nonzero']\ntest_xception_df.columns = ['image','item_label', 'xception_prob', 'xception_var',\n       'xception_nonzero']","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"692d80e63e71be9511011064c69f5c34168999db","collapsed":true},"cell_type":"code","source":"train_xception_df.head()","execution_count":34,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"71701670a2ae4c50c918dd6c0a92a7a127f329f1"},"cell_type":"code","source":"cols = ['item_label', 'xception_prob', 'xception_var',\n       'xception_nonzero']","execution_count":35,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7da365bc3ee615760341b0cc3eb38604fa522ce7","collapsed":true},"cell_type":"code","source":"image_features = image_features.merge(train_xception_df,on='image',how='left')\nimage_features.head()","execution_count":36,"outputs":[]},{"metadata":{"_uuid":"0c84a785fe853b2355c30698ec3710b66173cb8b"},"cell_type":"markdown","source":"# nima"},{"metadata":{"trusted":true,"_uuid":"f622c1c182c8b952ed4c92c5f3e6611e4de9cef2","collapsed":true},"cell_type":"code","source":"train_nima_df = pd.read_csv('../input/neural-image-assessment-train-features-starter/train_xception.csv',index_col=0)\ntest_nima_df = pd.read_csv('../input/neural-image-assessment-test-feature-starter/test_xception.csv',index_col=0)","execution_count":28,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e3c89a4a68809844c3bc01c6e7d5dcef0315720","collapsed":true},"cell_type":"code","source":"train_nima_df.shape,test_nima_df.shape","execution_count":29,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5485eec11dcb0521b5945423aa32ba8cbb7299bd"},"cell_type":"code","source":"image_features = image_features.merge(train_nima_df,on='image',how='left')","execution_count":30,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3c108733d2dff75bfdf6d4c1dc3fe88bf8ec350c"},"cell_type":"code","source":"nima_cols = ['mean_nima','std_nima_']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"801357aae3d0acb3a75139798bcc353aebed1b1c","collapsed":true},"cell_type":"code","source":"image_features.head()","execution_count":31,"outputs":[]},{"metadata":{"_uuid":"f9a83b6bef42b4c4922ac19c19455b47cf5cd900"},"cell_type":"markdown","source":"# blurr"},{"metadata":{"trusted":true,"_uuid":"0b38f8fb96a32b6d465b1d784dd0a2c73af5d6e4"},"cell_type":"code","source":"train_blurr_df = pd.read_csv('../input/blurr-features-train-part1/train_jpg_img_feat_blurr.csv',index_col=0)\ntrain_blurr_df=train_blurr_df.append(pd.read_csv('../input/blurr-features-train-part2/train_jpg_img_feat_blurr.csv',index_col=0))\ntrain_blurr_df=train_blurr_df.append(pd.read_csv('../input/fork-of-blurr-features-train-part/train_jpg_img_feat_blurr.csv',index_col=0))\n\ntest_blurr_df = pd.read_csv('../input/blurr-image-features-test/test_jpg_img_feat_blurr.csv',index_col=0)\ntrain_blurr_df.shape,test_blurr_df.shape","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2646a3bc317148de6e29d32cd847764973d3ba7d"},"cell_type":"code","source":"train_blurr_df.columns","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34394072bb951a3b990babb8f1d7c9ccf3c1c7de"},"cell_type":"code","source":"image_features = image_features.merge(train_blurr_df,on='image',how='left')\nimage_features.head()","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b9330df10f98f7d83b1ce3ff29c558cce345771"},"cell_type":"code","source":"show_corr(image_features)","execution_count":16,"outputs":[]},{"metadata":{"_uuid":"67b69b94736a910e0edb2389a56a374c3695b1c1"},"cell_type":"markdown","source":"# train"},{"metadata":{"trusted":true,"_uuid":"f10583308384db0f79283c55bb161d3a0b1ac85b","collapsed":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/avito-demand-prediction/train.csv',usecols=['image','item_id',],index_col=0)","execution_count":26,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9b72e50d8a94b59a5dd7cfbe28df911c41928cc","collapsed":true},"cell_type":"code","source":"%%time\ntrain_df = train_df.reset_index().merge(image_features,on='image',how='left').set_index('item_id')","execution_count":78,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6148f711b965fe96e3cda397eb03742e886008c0","collapsed":true},"cell_type":"code","source":"train_df.head()","execution_count":79,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e034c1c8957a35fa2d67ef6224730a00396c2bc2"},"cell_type":"code","source":"image_cols = image_features.columns","execution_count":80,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"451dc14468945ed68865edbde0ac1f7a7b3bfdc6","collapsed":true},"cell_type":"code","source":"image_cols","execution_count":81,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"03fb51f550c6da0719047ad036bc2de3b19eaf24","collapsed":true},"cell_type":"code","source":"train_image_features=image_features","execution_count":82,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8a5e6f54691c4b8922d99b7f60ed7de9ecb5300","collapsed":true},"cell_type":"code","source":"image_features.shape","execution_count":93,"outputs":[]},{"metadata":{"_uuid":"2ddfe80e9a2204dbbd940977010fd42066ec8294"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"04f5fbc3220fceabc2949e042b9a1622a2eb38f5"},"cell_type":"markdown","source":"# test"},{"metadata":{"trusted":true,"_uuid":"3f1e7c60432291ecc6e3af67cc9bcacf9a1392dc","collapsed":true},"cell_type":"code","source":"test_image_info = pd.read_csv('../input/extract-image-features-test-file/train_img_feat.csv',index_col=0)\ntest_file_info = pd.read_csv('../input/extract-image-features-test-file/test_img_files_info.csv',index_col=0)\ntest_imge_features = test_image_info.merge(test_file_info,on='image',how='left')","execution_count":38,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ed34802731fdc3b5abdbc581dca4350db543440","collapsed":true},"cell_type":"code","source":"def proce_data(image_features):\n    image_features['dim'] = image_features['dim'].apply(lambda x:eval(x))\n    image_features['colors'] = image_features['colors'].apply(lambda x: eval(x))\n    image_features['width'] = image_features['dim'].apply(lambda x: x[0])\n    image_features['height'] = image_features['dim'].apply(lambda x: x[1])\n    image_features['red_avg'] = image_features['colors'].apply(lambda x: x[0])\n    image_features['green_avg'] = image_features['colors'].apply(lambda x: x[1])\n    image_features['blue_avg'] = image_features['colors'].apply(lambda x: x[2])\n    return image_features\n\ntest_imge_features = proce_data(test_imge_features)\ntest_imge_features = make_features(test_imge_features)","execution_count":39,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"600f3a90e3463895ed081f0e5628233b662233f8","collapsed":true},"cell_type":"code","source":"test_imge_features.shape","execution_count":40,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd00d71c5f8121d0d7d39e16fbb5d97a55553600","collapsed":true},"cell_type":"code","source":"test_imge_features.head()","execution_count":96,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ccf9414ab3a73cbd978fd2b96bc0323fb2f0f6a5"},"cell_type":"code","source":"test_imge_features.drop(columns=['dim','colors','csize'],inplace=True)","execution_count":87,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60a71dea0ca6a4e440317db8d43ad52c5745c1d1","collapsed":true},"cell_type":"code","source":"test_imge_features = test_imge_features.merge(test_hsv_df,on='image',how='left')\ntest_imge_features = test_imge_features.merge(test_color_df,on='image',how='left')\ntest_imge_features.shape","execution_count":94,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b7b5925e5bbf552b7fdb654314725b0d62787ab","collapsed":true},"cell_type":"code","source":"test_imge_features = test_imge_features.merge(test_image_yuv,on='image',how='left')\ntest_imge_features.shape","execution_count":95,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ae6dee4fd1714b3af5e12b07dc42ba677cdc38e6"},"cell_type":"code","source":"test_imge_features = test_imge_features.merge(test_std_df,on='image',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"020216cfd078158ef73c64a243f5f395e9e64ea0"},"cell_type":"code","source":"test_imge_features = test_imge_features.merge(test_xception_df,on='image',how='left')","execution_count":41,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"408b11ba799d5a42ed120c25fd94ced01b1f25c6"},"cell_type":"code","source":"test_imge_features = test_imge_features.merge(test_nima_df,on='image',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff6a5421b4fbaad309c401e02a89d705041c5e55"},"cell_type":"code","source":"test_imge_features = test_imge_features.merge(test_blurr_df,on='image',how='left')","execution_count":18,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"163bdb03efb6bada58cc8b11494548f74b0bb687","collapsed":true},"cell_type":"code","source":"show_corr(test_imge_features)","execution_count":42,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ef5ef0097c9ac954cf2d558614ed452b80cf9e80"},"cell_type":"code","source":"test_df = pd.read_csv('../input/avito-demand-prediction/test.csv',usecols=['image','item_id','image_top_1','description','title'],index_col=0)","execution_count":100,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c3cecd533ab6aac6d3cd6ba7c4cc84d345294d86"},"cell_type":"code","source":"test_df  = test_df.reset_index().merge(test_imge_features,on='image',how='left').set_index('item_id')","execution_count":101,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a8558ff44fa6cfbb51d389db7fb6efb9d0e900b","collapsed":true},"cell_type":"code","source":"test_df.shape","execution_count":102,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"collapsed":true,"_uuid":"cc127ff56f6bba15a3dec0de78775927f0d22266","_kg_hide-input":true},"cell_type":"code","source":"test_df[train_image_features.columns].drop(columns=['image']).to_csv('test_image_features.csv.gzip',compression='gzip')\ntrain_df[image_features.columns].drop(columns=['image']).to_csv('train_image_features.csv.gzip',compression='gzip')","execution_count":141,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"313807aafdd5087fb00c98e669f88bca2ee60947"},"cell_type":"code","source":"# submission = pd.read_csv('../input/avito-demand-prediction/sample_submission.csv',index_col=0)","execution_count":83,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c0625cc650c1e4ae03bea5da67823b23d93957a1","_kg_hide-output":true},"cell_type":"code","source":"# submission['deal_probability'] = predict_y\n# submission.to_csv('rf_image.csv',index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}