{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\nplt.rcParams['figure.figsize'] = (30, 15)\nplt.rcParams['font.size'] = 25","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e2d5672400ad14ba72b1e066679c1e103d219d9"},"cell_type":"code","source":"print(os.listdir(\"../input/avito-demand-prediction\"))\nprint(os.listdir(\"../input/keras-pretrained-models\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c2825dd172c52fa84f28565c37160f2c36cad32"},"cell_type":"code","source":"train = pd.read_csv(\"../input/avito-demand-prediction/train.csv\", parse_dates=[\"activation_date\"])\ntest = pd.read_csv(\"../input/avito-demand-prediction/test.csv\", parse_dates=[\"activation_date\"])\ndata = pd.concat([train, test])\ndel train, test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d0f625791d61d36225c0f7813a02e22fae48b53"},"cell_type":"code","source":"pd.read_csv?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9480440b8325964c85d82ea7f1eb9b59c71c090c"},"cell_type":"code","source":"data.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4752dd63f4ef9bfbc74a5894681294875c9a9a35"},"cell_type":"code","source":"data.dtypes.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e02d06c549c282984092ed7fcbf348892daa5790"},"cell_type":"code","source":"data.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6932dad9c77ba72259e5474ed84793972022aae1"},"cell_type":"code","source":"parent_category_name_map = {\"Личные вещи\" : \"Personal belongings\",\n                            \"Для дома и дачи\" : \"For the home and garden\",\n                            \"Бытовая электроника\" : \"Consumer electronics\",\n                            \"Недвижимость\" : \"Real estate\",\n                            \"Хобби и отдых\" : \"Hobbies & leisure\",\n                            \"Транспорт\" : \"Transport\",\n                            \"Услуги\" : \"Services\",\n                            \"Животные\" : \"Animals\",\n                            \"Для бизнеса\" : \"For business\"}\n\nregion_map = {\"Свердловская область\" : \"Sverdlovsk oblast\",\n            \"Самарская область\" : \"Samara oblast\",\n            \"Ростовская область\" : \"Rostov oblast\",\n            \"Татарстан\" : \"Tatarstan\",\n            \"Волгоградская область\" : \"Volgograd oblast\",\n            \"Нижегородская область\" : \"Nizhny Novgorod oblast\",\n            \"Пермский край\" : \"Perm Krai\",\n            \"Оренбургская область\" : \"Orenburg oblast\",\n            \"Ханты-Мансийский АО\" : \"Khanty-Mansi Autonomous Okrug\",\n            \"Тюменская область\" : \"Tyumen oblast\",\n            \"Башкортостан\" : \"Bashkortostan\",\n            \"Краснодарский край\" : \"Krasnodar Krai\",\n            \"Новосибирская область\" : \"Novosibirsk oblast\",\n            \"Омская область\" : \"Omsk oblast\",\n            \"Белгородская область\" : \"Belgorod oblast\",\n            \"Челябинская область\" : \"Chelyabinsk oblast\",\n            \"Воронежская область\" : \"Voronezh oblast\",\n            \"Кемеровская область\" : \"Kemerovo oblast\",\n            \"Саратовская область\" : \"Saratov oblast\",\n            \"Владимирская область\" : \"Vladimir oblast\",\n            \"Калининградская область\" : \"Kaliningrad oblast\",\n            \"Красноярский край\" : \"Krasnoyarsk Krai\",\n            \"Ярославская область\" : \"Yaroslavl oblast\",\n            \"Удмуртия\" : \"Udmurtia\",\n            \"Алтайский край\" : \"Altai Krai\",\n            \"Иркутская область\" : \"Irkutsk oblast\",\n            \"Ставропольский край\" : \"Stavropol Krai\",\n            \"Тульская область\" : \"Tula oblast\"}\n\n\ncategory_map = {\"Одежда, обувь, аксессуары\":\"Clothing, shoes, accessories\",\n\"Детская одежда и обувь\":\"Children's clothing and shoes\",\n\"Товары для детей и игрушки\":\"Children's products and toys\",\n\"Квартиры\":\"Apartments\",\n\"Телефоны\":\"Phones\",\n\"Мебель и интерьер\":\"Furniture and interior\",\n\"Предложение услуг\":\"Offer services\",\n\"Автомобили\":\"Cars\",\n\"Ремонт и строительство\":\"Repair and construction\",\n\"Бытовая техника\":\"Appliances\",\n\"Товары для компьютера\":\"Products for computer\",\n\"Дома, дачи, коттеджи\":\"Houses, villas, cottages\",\n\"Красота и здоровье\":\"Health and beauty\",\n\"Аудио и видео\":\"Audio and video\",\n\"Спорт и отдых\":\"Sports and recreation\",\n\"Коллекционирование\":\"Collecting\",\n\"Оборудование для бизнеса\":\"Equipment for business\",\n\"Земельные участки\":\"Land\",\n\"Часы и украшения\":\"Watches and jewelry\",\n\"Книги и журналы\":\"Books and magazines\",\n\"Собаки\":\"Dogs\",\n\"Игры, приставки и программы\":\"Games, consoles and software\",\n\"Другие животные\":\"Other animals\",\n\"Велосипеды\":\"Bikes\",\n\"Ноутбуки\":\"Laptops\",\n\"Кошки\":\"Cats\",\n\"Грузовики и спецтехника\":\"Trucks and buses\",\n\"Посуда и товары для кухни\":\"Tableware and goods for kitchen\",\n\"Растения\":\"Plants\",\n\"Планшеты и электронные книги\":\"Tablets and e-books\",\n\"Товары для животных\":\"Pet products\",\n\"Комнаты\":\"Room\",\n\"Фототехника\":\"Photo\",\n\"Коммерческая недвижимость\":\"Commercial property\",\n\"Гаражи и машиноместа\":\"Garages and Parking spaces\",\n\"Музыкальные инструменты\":\"Musical instruments\",\n\"Оргтехника и расходники\":\"Office equipment and consumables\",\n\"Птицы\":\"Birds\",\n\"Продукты питания\":\"Food\",\n\"Мотоциклы и мототехника\":\"Motorcycles and bikes\",\n\"Настольные компьютеры\":\"Desktop computers\",\n\"Аквариум\":\"Aquarium\",\n\"Охота и рыбалка\":\"Hunting and fishing\",\n\"Билеты и путешествия\":\"Tickets and travel\",\n\"Водный транспорт\":\"Water transport\",\n\"Готовый бизнес\":\"Ready business\",\n\"Недвижимость за рубежом\":\"Property abroad\"}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1ac0ef21137a97bde454ab265a546a265e649b7"},"cell_type":"code","source":"data['region_en'] = data['region'].apply(lambda x : region_map[x])\ndata['parent_category_name_en'] = data['parent_category_name'].apply(lambda x : parent_category_name_map[x])\ndata['category_name_en'] = data['category_name'].apply(lambda x : category_map[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"761b47155e398ac1eebaae673abd1888e220257c"},"cell_type":"code","source":"data.head(5).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9dc3b30bd0071a01434fc259475f9e30b9ce8f25"},"cell_type":"code","source":"data.isnull().sum() / data.shape[0] * 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"009508fe6366f90b423cf4fcaad8c2e6ba5f110c"},"cell_type":"code","source":"sns.distplot(data[\"deal_probability\"].dropna().values, bins=120)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55557e9ebcd2552e125ee9852d41ac957775149a"},"cell_type":"code","source":"pd.cut(data[\"deal_probability\"].dropna(), 10).value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb24beaf54a00acb7def738814e589a2105697c7"},"cell_type":"code","source":"pd.cut(data[\"deal_probability\"].dropna(), 10).value_counts(True).sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"782777d8b836c15a2eba5d3dbdf2cf0e90ed43fd"},"cell_type":"code","source":"idx = range(data.deal_probability.notnull().sum())\nplt.scatter(idx, np.sort(data['deal_probability'].dropna().values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2958b6a41e379977fddda0d36bf7b10d36ec2e8"},"cell_type":"code","source":"pd.concat([data.region_en.value_counts().rename(\"abs\"), data.region_en.value_counts(True).rename(\"rel\")], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"882777fa60054aae43564f4746c1f96aca822254"},"cell_type":"code","source":"train = data[data.deal_probability.notnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"286451f0fbcda38657fd1196f8c6d1e1c9b03b03"},"cell_type":"code","source":"reg_label = pd.crosstab(train.region_en, pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\nreg_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"819fd38b91020d2491ac7b755c0f21a4433b6a97"},"cell_type":"code","source":"sns.heatmap(reg_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da11bf08f6e66a0e6ab4534c640f568c52705669"},"cell_type":"code","source":"sns.boxplot(x=\"region_en\", y=\"deal_probability\", data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf553508936eac3d182095596e6c738a946c39fe"},"cell_type":"code","source":"cat_label = pd.crosstab(train.category_name_en, pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\ncat_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8586c27946c0196a8d14417a47f918067187f954"},"cell_type":"code","source":"train.category_name_en.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9dc483601259a9373788c9eb905c17727815416a"},"cell_type":"code","source":"sns.heatmap(cat_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"365da70f433076a832ce016faecd96ddbc3992c4"},"cell_type":"code","source":"sns.boxplot(x=\"category_name_en\", y=\"deal_probability\", data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acbf4d9fe78e36d2ed2605cd2adcf0cd146e51f1"},"cell_type":"code","source":"cat_label = pd.crosstab(train.parent_category_name_en, pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\ncat_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b659bfb890ef56fb1d7afd592dc22067ff9cec3"},"cell_type":"code","source":"data.groupby(\"parent_category_name_en\").category_name_en.apply(set)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81894e2e4078761097d706a1080f66a7693183a6"},"cell_type":"code","source":"sns.heatmap(cat_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c89fb632ab65d9c6dfca95378cd39b2ec26bf1d"},"cell_type":"code","source":"data.parent_category_name_en.value_counts(True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d4f00d8846e501867a933f81b46518315c464ec"},"cell_type":"code","source":"sns.boxplot(x=\"parent_category_name_en\", y=\"deal_probability\", data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f11e7762e6740555c138efc3fa8f5eb21c8c4e4c"},"cell_type":"code","source":"data.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eef9f8756d35d2c6ecfe8e97776b63dfa1d41ead"},"cell_type":"code","source":"data.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8202b445ea101c66cb52f275d85b42418b23bd0"},"cell_type":"code","source":"data[data.deal_probability.notnull()].activation_date.min(), data[data.deal_probability.notnull()].activation_date.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"310bce1791e110f242e88bfcfdee972b19589a39"},"cell_type":"code","source":"data[data.deal_probability.isnull()].activation_date.min(), data[data.deal_probability.isnull()].activation_date.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d72f1b556a046f224e5a3e10333bd88610368c27"},"cell_type":"code","source":"data.apply(lambda x: x.unique().shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36f4a0414bff7143056ea68d3436b48f5bd5a97b"},"cell_type":"code","source":"itemsq_label = pd.crosstab(pd.qcut(train.item_seq_number, 10), pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\nitemsq_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07d08864ff0b93f81d681fc70b285709c1fe17a9"},"cell_type":"code","source":"sns.heatmap(itemsq_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37b951f17d5fdb17bfbad685a6ab70ef5403e288"},"cell_type":"code","source":"train.groupby(pd.qcut(train.item_seq_number, 15)).deal_probability.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c369161de1af0f113580a03230d47316ebf71d30"},"cell_type":"code","source":"train.groupby(pd.qcut(train.item_seq_number, 10)).deal_probability.mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60117ba8d27dfd0c4fa016ca19f7d6088cb7eeb7"},"cell_type":"code","source":"plt.scatter(\"item_seq_number\", \"deal_probability\", data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bbcd163202a86fe50744a2344a3df8dfd15531ce"},"cell_type":"code","source":"pd.crosstab?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b22e161b5a1a2b382091efe05a1ac0c1a2bdd4d3"},"cell_type":"code","source":"cross = pd.crosstab(pd.qcut(train.item_seq_number, 10), train.parent_category_name_en,\n            values=train.deal_probability, aggfunc=np.mean)\ncross","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf58e407164fd370744afbd59fa20f30d96ee82c"},"cell_type":"code","source":"sns.heatmap(cross)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3751f6c73d42bc7a4a616c56ea82206c49a5b8df"},"cell_type":"code","source":"price_label = pd.crosstab(pd.qcut(train.price, 10), pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\nprice_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c58cb6d94baf2cdf9817fcda3f6171be1ad78f0"},"cell_type":"code","source":"sns.heatmap(price_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff5554ff09e865b6341c3276b7cbd645fa157730"},"cell_type":"code","source":"user_label = pd.crosstab(train.user_type, pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\nuser_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a20c30800238e203dc586d5b216aa8368a8edf7"},"cell_type":"code","source":"sns.heatmap(user_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a597d3c8a37fac434a632d0b0d5e5bb418d5ac4"},"cell_type":"code","source":"train.user_type.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ce71675fe8fb9523252b29068777c3278ddbab8"},"cell_type":"code","source":"# variables de fecha\ndata['weekday'] = data.activation_date.dt.weekday\ndata['month'] = data.activation_date.dt.month\ndata['day'] = data.activation_date.dt.day\ndata['week'] = data.activation_date.dt.week \ntrain = data[data.deal_probability.notnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d695250290a535b06fc0f973f5d4953746c89d82"},"cell_type":"code","source":"temp = pd.crosstab(train.weekday, pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0c97b2efce88d27c8f9ad3a5eb7664610f73307"},"cell_type":"code","source":"sns.heatmap(temp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa7f9c0d1d2d919a4e9491aca3c26ee5b7597399"},"cell_type":"code","source":"sns.boxplot(x=\"weekday\", y=\"deal_probability\", data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cccdd12dc0204bf9d828392921fbd35fb9404922"},"cell_type":"code","source":"temp = pd.crosstab(train.day, pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"995ea89b95a5a022f3e0695ec606001dee3636e7"},"cell_type":"code","source":"train.day.value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea507fd5f4d0e3859404682a4f91473ee58e905e"},"cell_type":"code","source":"train.month.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1601cd8ca4bc506c6f31ff3da3b1cd41f47bb4b5"},"cell_type":"code","source":"data['description'] = data['description'].fillna(\"\")\ndata['description_len'] = data['description'].apply(lambda x : len(x.split()))\ntrain = data[data.deal_probability.notnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5723f3f316a4bcd5c5ef44bcbf3b3579e0425b4f"},"cell_type":"code","source":"temp = pd.crosstab(pd.qcut(train.description_len, 10), pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"947d8ee721e8697130261cad7337df22c6eb0f27"},"cell_type":"code","source":"sns.heatmap(temp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd493b6e81c3a23b981f3b77f050e14a4048c101"},"cell_type":"code","source":"data['title'] = data['title'].fillna(\" \")\ndata['title_len'] = data['title'].apply(lambda x : len(x.split()))\ntrain = data[data.deal_probability.notnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dff0d8627fc978646b1fff3dbb102800bc9659a"},"cell_type":"code","source":"train.title_len.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e6836b5efabee114f1af88c8b1f21696171501f"},"cell_type":"code","source":"temp = pd.crosstab(train.title_len, pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"508893e348ad6ecc8434497ef78a81684f6b977b"},"cell_type":"code","source":"sns.heatmap(temp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"400a021f23c27c2a75e5cb486425f5ac9125ff86"},"cell_type":"code","source":"data['title_len_char'] = data['title'].str.len()\ntrain = data[data.deal_probability.notnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af6de572dcb9419dcc59b1424f7f4141dcb9c6b0"},"cell_type":"code","source":"train.title_len_char.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0bedeb891b37ab574949fd01ae7c89f021f3edd8"},"cell_type":"code","source":"temp = pd.crosstab(train.title_len_char, pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"314a2237f20a776d86c240ac5a5ad82a629fe8d0"},"cell_type":"code","source":"sns.heatmap(temp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"beb24801d2ed41b048f06f17f40c7a1f19109c9d"},"cell_type":"code","source":"train.groupby(\"title_len_char\").deal_probability.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2747d0e02acf423d8c129d4acf331023650dbb0"},"cell_type":"code","source":"_.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3528cda12e2677876d119059276619be934d9fb4"},"cell_type":"code","source":"train.groupby(\"title_len\").deal_probability.mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f01e4c16998426d16235189f4f815ed5e601135"},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31d3054e8c710e5468e83ba740322e453878ac8b"},"cell_type":"code","source":"data.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"824c3b6e765d84d4f555b3e52f03a53ad6a1f3dd"},"cell_type":"code","source":"text_cols = ['description', 'param_1', 'param_2', 'param_3','title']\ndata[\"total_text\"] = data[text_cols].fillna(\"\").sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3073287de305a6d5b9a56b3972ffe0719c71078"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n\ntfidf = TfidfVectorizer(ngram_range=(1,1)).fit_transform(data.total_text)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b914d12f82a841f9af164f5822ece29c0abc97e1"},"cell_type":"code","source":"tfidf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a41b19060dacfb0576cc5fbf8e84d526a0c0bc91"},"cell_type":"code","source":"data.index = range(data.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63bbc1e25d10c262082e0cfb9d23340b52d5c3ba"},"cell_type":"code","source":"from sklearn.decomposition import TruncatedSVD\n\nn_comps = 10\ntsvd = pd.DataFrame(TruncatedSVD(n_components=n_comps, algorithm='arpack').fit_transform(tfidf),\n                    columns=[\"svd_comp_\" + str(i) for i in range(n_comps)], index=data.index)\ntsvd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62b307104caa3dba338f25fd1850380d683ffe20"},"cell_type":"code","source":"data = data.join(tsvd)\ntrain = data[data.deal_probability.notnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67c3625ee27dc0abaa8b1b55ad08043fd4d415e1"},"cell_type":"code","source":"temp = pd.crosstab(pd.qcut(train.svd_comp_0, 10), pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\nsns.heatmap(temp)\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7761b302e045cd20de1da6fd51555d068dc7d09a"},"cell_type":"code","source":"temp = pd.crosstab(pd.qcut(train.svd_comp_1, 10), pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\nsns.heatmap(temp)\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ac860840a38adc627e462c333be9898313c62dc"},"cell_type":"code","source":"temp = pd.crosstab(pd.qcut(train.svd_comp_2, 10), pd.cut(train.deal_probability, 10)).apply(lambda x: x/x.sum(), axis=1)\nsns.heatmap(temp)\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dabb228260b101c3075d05b651bcce061233cbc7"},"cell_type":"code","source":"[train.groupby(pd.qcut(train[\"svd_comp_\" + str(i)], 10)).deal_probability.mean().plot() for i in range(n_comps)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6ad75e8c2bf2c5dc0bc0458438ee37b4e94cf4c"},"cell_type":"code","source":"print(os.listdir(\"../input/keras-pretrained-models\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b91563f068317b9e00b8b1a1aac3788fd346fb6"},"cell_type":"code","source":"from keras.preprocessing import image\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.applications.xception import Xception \nfrom keras.applications.inception_v3 import InceptionV3\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8703d44b5ede425d8e2d7f5c1419382212bc28e6"},"cell_type":"code","source":"model = VGG16(weights=None, include_top=False)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0787a6a82dc6919e14abaa32c6fa853dd77be771"},"cell_type":"code","source":"SVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc38e436e2d123b23bd3043852aa1a820fb6e6fc"},"cell_type":"code","source":"model = ResNet50(weights=None, include_top=False)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec9a7ca13dd38e0ab781eabe5adcae243d9f03c5"},"cell_type":"code","source":"SVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"137e0d788d809fbd792591d0ae1107da4d12b8b8"},"cell_type":"code","source":"model = Xception(weights=None, include_top=False)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c10453b571b29d5bbfae28d5a56b807a1862f0ea"},"cell_type":"code","source":"SVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"470f4c0e00e6a3920a90f5c22c3857caf96d7c1a"},"cell_type":"code","source":"model = InceptionV3(weights=None, include_top=False)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc84ad190dbcd03a6fa70923708d5739e761a9ca"},"cell_type":"code","source":"SVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"797ad0d49d4f96fee49502a69513fba2019e0fe7"},"cell_type":"code","source":"model = VGG16(weights=None, include_top=False)\nmodel.load_weights('../input/keras-pretrained-models/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91c13313b811693781ea8d76649b8e1cb5fd0e7d"},"cell_type":"code","source":"model.input_shape, model.output_shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc9e54ea048aade486f03cb6c48b396c7185a0f5"},"cell_type":"code","source":"from keras.applications.vgg16 import preprocess_input\nimport zipfile\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f82aa3d6c10cd53199fada639b052cc053810eb"},"cell_type":"code","source":"myzip = zipfile.ZipFile('../input/avito-demand-prediction/train_jpg.zip')\nfiles_in_zip = myzip.namelist()\nprint(files_in_zip[:5])\nprint(\"total\", len(files_in_zip))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91816cfb06e992a2cd900c0374411a6f444350ed"},"cell_type":"code","source":"myzip.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b36b0a3cef3eb0a0594b999dc85f55249c21319"},"cell_type":"code","source":"from time import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f1e6084b5d360a72d320e052cb574b69ab164ef"},"cell_type":"code","source":"# bach_size  = 100\n# im_dim = 63\n# names = []\n# embeddings = []\n# with zipfile.ZipFile('../input/avito-demand-prediction/train_jpg.zip') as f:\n#     bach = np.zeros((bach_size, im_dim, im_dim, 3))\n#     j = 0\n#     start = time()\n#     for i, name in enumerate(f.namelist()):\n#         if i % 1000 == 0:\n#             print(\"done\", i, \"in\", time() - start)\n#         if not name.endswith('.jpg'): continue\n#         names.append(name.split(\"/\")[-1].split(\".\")[0])\n#         img = cv2.imdecode(np.frombuffer(f.read(name), dtype='uint8'), cv2.IMREAD_COLOR)\n#         height, width, _ = img.shape\n#         if height > width:\n#             new_dim = (width*im_dim//height, im_dim)\n#         else:\n#             new_dim = (im_dim, height*im_dim//width)\n#         img = cv2.resize(img, new_dim, interpolation=cv2.INTER_AREA)\n#         h, w = img.shape[:2]\n\n#         off_x = (im_dim-w)//2\n#         off_y = (im_dim-h)//2\n#         bach[j, off_y:off_y+h, off_x:off_x+w] = img\n#         j += 1\n#         if j == bach_size:\n#             j = 0\n#             embedding = model.predict(preprocess_input(bach))\n#             embeddings.append(embedding.reshape(bach_size, embedding.shape[-1]))\n#             bach = np.zeros((bach_size, im_dim, im_dim,3))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db1ccdce64b1d1ee0face4117b05f6ecb3de6c5a"},"cell_type":"code","source":"# with zipfile.ZipFile('../input/avito-demand-prediction/test_jpg.zip') as f:\n#     bach = np.zeros((bach_size, im_dim, im_dim, 3))\n#     j = 0\n#     start = time()\n#     for i, name in enumerate(f.namelist()):\n#         if i % 1000 == 0:\n#             print(\"done\", i, \"in\", time() - start)\n#         if not name.endswith('.jpg'): continue\n#         names.append(name.split(\"/\")[-1].split(\".\")[0])\n#         img = cv2.imdecode(np.frombuffer(f.read(name), dtype='uint8'), cv2.IMREAD_COLOR)\n#         height, width, _ = img.shape\n#         if height > width:\n#             new_dim = (width*im_dim//height, im_dim)\n#         else:\n#             new_dim = (im_dim, height*im_dim//width)\n#         img = cv2.resize(img, new_dim, interpolation=cv2.INTER_AREA)\n#         h, w = img.shape[:2]\n\n#         off_x = (im_dim-w)//2\n#         off_y = (im_dim-h)//2\n#         bach[j, off_y:off_y+h, off_x:off_x+w] = img\n#         j += 1\n#         if j == bach_size:\n#             j = 0\n#             embedding = model.predict(preprocess_input(bach))\n#             embeddings.append(embedding.reshape(bach_size, embedding.shape[-1]))\n#             bach = np.zeros((bach_size, im_dim, im_dim,3))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4dcf024d03002540f4a75052815ebab5ba75698"},"cell_type":"code","source":"# embeddings = np.concatenate(embeddings)\n# embeddings = pd.DataFrame(embeddings, index=names[:embeddings.shape[0]], columns=[\"img_emb_\" + str(i) for i in range(embeddings.shape[1])])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"ad9185bdead99ad6b1246009aa5d1bacf060c3cb"},"cell_type":"code","source":"# embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80b4341bd7e7dfd3919b81c2a73a57f8b5a4818c"},"cell_type":"code","source":"# data = data.join(embeddings, on=\"image\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85929fa0cde63fc033fc173c2338b07d6fc90ef1"},"cell_type":"code","source":"data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27b189ea3289fce0a2e091119d028d258932d8c1"},"cell_type":"code","source":"data.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4218a8a0abfd64e0a66e7297d9ec649d68c7c7bf"},"cell_type":"code","source":"data = data.drop(['activation_date', 'image', 'title', 'description', 'region_en',\n                  'parent_category_name_en', 'category_name_en', 'total_text'], axis=1)\ndata.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"833e64adbd2c60d1375536b74949aaf7d0c950c3"},"cell_type":"code","source":"data.set_index(\"item_id\", inplace=True)\nobj_columns = data.select_dtypes(\"object\").columns\nobj_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d96c8a134e038ddc7f0a2a1f1f92f1750d3f804d"},"cell_type":"code","source":"for c in obj_columns:\n    data[c] = pd.factorize(data[c])[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b067f753fcf55445218117dda252809c3ee49fdb"},"cell_type":"code","source":"data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53fb2ceede849fcc85ff8bbee75b5db24afabe8e"},"cell_type":"code","source":"data.isnull().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bf739c12df0de91c7313142ae3c50000fa9c729"},"cell_type":"code","source":"test = data[data.deal_probability.isnull()].drop(\"deal_probability\", axis=1)\ntrain = data.drop(test.index)\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4eb1ee51c95852e9cb93cdc2a2723a4b2152da7c"},"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom lightgbm import LGBMRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f082ead436fd6886490989b1d15b2acc90533612"},"cell_type":"code","source":"LGBMRegressor.fit?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ddf584e6a3d6633946880afae7fccca52e2ee64"},"cell_type":"code","source":"def train_CV(train, test, folds, **params):\n    test_preds = []\n    train_preds = []\n    for i, (train_idx, test_idx) in enumerate(folds):\n        X_train = train.drop(\"deal_probability\", axis=1).iloc[train_idx]\n        y_train = train.deal_probability.iloc[train_idx]\n        X_valid = train.drop(\"deal_probability\", axis=1).iloc[test_idx]\n        y_valid = train.deal_probability.iloc[test_idx]\n        learner = LGBMRegressor(n_estimators=10000, **params)\n        learner.fit(X_train, y_train, early_stopping_rounds=10,\n                    eval_metric=\"rmse\", verbose=100,\n                    eval_set=[(X_train, y_train),\n                              (X_valid, y_valid)]\n                   )\n        preds = pd.Series(learner.predict(X_valid), index=X_valid.index, name=\"deal_probability\")\n        train_preds.append(preds)\n        preds = pd.Series(learner.predict(test), index=test.index, name=\"fold_\" + str(i))\n        test_preds.append(preds)\n    return pd.concat(train_preds).clip(0, 1),\\\n           pd.concat(test_preds, axis=1).mean(axis=1).clip(0, 1).rename(\"deal_probability\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"2e5aa7068579d85b442b81c221e228215f21a59c"},"cell_type":"code","source":"folds = list(KFold(n_splits=5, shuffle=True).split(train))\nres = pd.Series(index=[2 ** i for i in range(5, 11)])\nbest_res = 10\nfor nl in res.index:\n    print(\"*\" * 20)\n    print(\"doing\", nl)\n    train_preds, test_preds = train_CV(train, test, folds, num_leaves=nl)\n    res = np.power(np.power(train.deal_probability - train_preds.loc[train.index], 2).mean(), 0.5)\n    if res < best_res:\n        print(\"*\" * 20)\n        print(\"*\" * 20)\n        print(\"got best with nl {}: {}\".format(nl, res))\n        print(\"*\" * 20)\n        print(\"*\" * 20)\n        best_res = res\n        best_test_preds = test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25e0c8e004388af2c53a3de0a184391be14ed3ef"},"cell_type":"code","source":"train.deal_probability.mean(), best_test_preds.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e7cce2dc87001ee81bbcf2c4c8d7e9742e3f18f"},"cell_type":"code","source":"best_test_preds.to_csv(\"preds.csv\", header=True, index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}