{"cells":[{"metadata":{"_uuid":"7db4c352d30e4b5a133e0b94b1e5d81ccef22592","_cell_guid":"770216ec-28d1-47e3-af12-019d4cf96c5b"},"cell_type":"markdown","source":"# General information\nThis kernel is dedicated to extensive EDA of Avito Demand Prediction Challenge competition as well as feature engineering. Only a simple model is used due to kernel memory constraint."},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","collapsed":true,"trusted":false},"cell_type":"code","source":"import os\nimport pandas_profiling as pp\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nplt.style.use('ggplot')\n\nimport datetime\nimport pandas_profiling as pp\nfrom scipy import stats\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.model_selection import train_test_split\nfrom wordcloud import WordCloud\nfrom collections import Counter\nfrom nltk.corpus import stopwords\nfrom nltk.util import ngrams\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nstop = set(stopwords.words('russian'))\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.model_selection import KFold\nkf = KFold(n_splits=5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\nperiods_train = pd.read_csv('../input/periods_train.csv')\nsub = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb706366a50b9643ff4e7b1d516478d6963ff90c","_cell_guid":"0e716b2e-6f4f-427e-9557-47760319d071"},"cell_type":"markdown","source":"## Data overview"},{"metadata":{"_uuid":"a2d711a1330f6b01145896c2a1a026758188f38a","collapsed":true,"_cell_guid":"6e24d639-96ab-4c0c-b2f8-ffc8e8e26f2a","trusted":false},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c1f964599451ca5130c6a53ec781df3806768b6","collapsed":true,"_cell_guid":"562e39ef-ae6c-443d-9752-a291caf1b30c","trusted":false},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb51f1c4a38f3d7486e3e915d09a19341a847834","collapsed":true,"_cell_guid":"4135d212-3b0f-4984-a911-368168627104","trusted":false},"cell_type":"code","source":"train.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f27537b45e40d129d8469207c40235412b01f93","collapsed":true,"_cell_guid":"c6cf16f9-fa4f-4113-a9f7-472562c49585","trusted":false},"cell_type":"code","source":"periods_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e55120b1b1f751ec6a5006f5d135199fc0b0ab5e","collapsed":true,"_cell_guid":"1e926d69-aa47-47a5-ae0b-17ef8bf3f0b8","trusted":false},"cell_type":"code","source":"pp.ProfileReport(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d6cad0d19d5a2c61491141985f2ab430469e708","_cell_guid":"2350ded3-51c1-472b-a685-299177b67b8d"},"cell_type":"markdown","source":"We can see several things from this overview:\n\n* There is a variety of features: numerical, categorical, text and date;\n* Some columns have missing values: not all users define additional parameters of items or descriptions, sometimes they don't even provide descriptions. In some cases there are no photos of the wares;\n* As expected, there are a lot of unique users and most of them don't post a lot of ads, but there are outliers with 600+ ads;\n* There are 9 categories in parent_category_name and \"Личные вещи\" (Private things) have 46,4% of values;\n* price will require a careful processing - the values are skewered and there are some outliers with huge values;\n* It is possible to use images in the analysis, but I'll simply use the fact whether there was image or not;"},{"metadata":{"_uuid":"69891ecc975796de5724a9fb9ac700c0e620dc0c","_cell_guid":"3392fa97-6123-4153-8d33-c52ff1200a2e"},"cell_type":"markdown","source":"## Feature analysis\nWe saw a lot of information about features, so let's now analyze each of them in more details."},{"metadata":{"_uuid":"7d1bc90005715a985bde8b882f6c52ae46dc8144","_cell_guid":"f3eac6ed-fb59-4cdb-a0e5-5de9553102f8"},"cell_type":"markdown","source":"### activation_date\n\nAt first let's create new features based on activation_date: date, weekday and day of month."},{"metadata":{"_kg_hide-input":true,"_uuid":"6e71b96d49487ddc201bc104dbc80ff6da7ca565","_cell_guid":"f0e05603-3e9b-4ab3-a14d-c50b46503af5","collapsed":true,"trusted":false},"cell_type":"code","source":"train['activation_date'] = pd.to_datetime(train['activation_date'])\ntrain['date'] = train['activation_date'].dt.date\ntrain['weekday'] = train['activation_date'].dt.weekday\ntrain['day'] = train['activation_date'].dt.day\ncount_by_date_train = train.groupby('date')['deal_probability'].count()\nmean_by_date_train = train.groupby('date')['deal_probability'].mean()\n\ntest['activation_date'] = pd.to_datetime(test['activation_date'])\ntest['date'] = test['activation_date'].dt.date\ntest['weekday'] = test['activation_date'].dt.weekday\ntest['day'] = test['activation_date'].dt.day\ncount_by_date_test = test.groupby('date')['item_id'].count()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"5fdbc2ddb845742a953128f1318402f54d19faf1","_cell_guid":"beb3a3d1-6988-49fc-8be8-ef7afa1e4062","collapsed":true,"trusted":false},"cell_type":"code","source":"fig, (ax1, ax3) = plt.subplots(figsize=(26, 8), ncols=2, sharey=True)\ncount_by_date_train.plot(ax=ax1, legend=False, label='Ads count')\nax1.set_ylabel('Ads count', color='b')\nax2 = ax1.twinx()\nmean_by_date_train.plot(ax=ax2, color='g', legend=False, label='Mean deal_probability')\nax2.set_ylabel('Mean deal_probability', color='g')\ncount_by_date_test.plot(ax=ax3, color='r', legend=False, label='Ads count test')\nplt.grid(False)\n\nax1.title.set_text('Trends of deal_probability and number of ads')\nax3.title.set_text('Trends of number of ads for test data')\nax1.legend(loc=(0.8, 0.35))\nax2.legend(loc=(0.8, 0.2))\nax3.legend(loc=\"upper right\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06a3fcea3d22be79293f6cef43640d0e48e4a7e3","_cell_guid":"07204a80-dcec-4130-aefa-9c3d696b4f39"},"cell_type":"markdown","source":"As we can see, we don't only several weeks of data in train and a little more than a week in test\n\n* For most of the data in train the number of ads is quite high (100 000 or more) and mean deal_probability is around 0.14, but after March 28 the number of ads drastically falls so deal_probability fluctuates. I wonder if decreased number of ads is intentional;\n* In test we have a reasonable number of ads up to April 18 and then number of ads become negligible - 64 and 1;\n* As a result I'd suggest not to use train data with too low number of ads (since 2017-03-29);"},{"metadata":{"_kg_hide-input":true,"_uuid":"90888b1a2d77ee3ed72ff57c16e4b375cb993817","_cell_guid":"8f58a6c9-8339-419c-98d6-147426b52c58","collapsed":true,"trusted":false},"cell_type":"code","source":"fig, ax1 = plt.subplots(figsize=(16, 8))\nplt.title(\"Ads count and deal_probability by day of week.\")\nsns.countplot(x='weekday', data=train, ax=ax1)\nax1.set_ylabel('Ads count', color='b')\nplt.legend(['Projects count'])\nax2 = ax1.twinx()\nsns.pointplot(x=\"weekday\", y=\"deal_probability\", data=train, ci=99, ax=ax2, color='black')\nax2.set_ylabel('deal_probability', color='g')\nplt.legend(['deal_probability'], loc=(0.875, 0.9))\nplt.grid(False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"013ef52e8a19188c3a917bde5f9cac6d52e1a5a2","_cell_guid":"eea01205-a7b4-4000-82dc-a68ca3572866"},"cell_type":"markdown","source":"We can see that there is a little difference in deal_probability if we look at deal_probability by weekday."},{"metadata":{"_uuid":"d66f5ea6cc958e2b6ebad058b1ed35b6ef5e65cc","_cell_guid":"fe7eadce-106d-4297-b9c9-4cc9d25d1b00"},"cell_type":"markdown","source":"## Categories\n"},{"metadata":{"_uuid":"dba7cce49a810c991848410c38b6801cee23cbb6","collapsed":true,"_cell_guid":"92436504-b0b9-4654-a08a-cbe0143e95b7","trusted":false},"cell_type":"code","source":"a = train.groupby(['parent_category_name', 'category_name']).agg({'deal_probability': ['mean', 'count']}).reset_index().sort_values([('deal_probability', 'mean')], ascending=False).reset_index(drop=True)\na","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7958ef091941b0faafdf05695455e51dc6c1af8","_cell_guid":"dbe77992-9a17-4c1f-ab5e-80da3259d2cd"},"cell_type":"markdown","source":"We can see that \"Услуги\" (services) is the category with the highest deal_probability. Other \"good\" categories are about animals or electronics/cars.\nLeast successful are various accessories or expensive things."},{"metadata":{"_uuid":"c3a4984022cf168df00d8fc6bcc71463750e711b","_cell_guid":"77423ad1-3331-49c5-9be2-924e426d4545"},"cell_type":"markdown","source":"## city"},{"metadata":{"_kg_hide-input":true,"_uuid":"cb8ded0a9d4f3eb104ca84c5a2f5bd71677b5d66","_cell_guid":"3258f1d3-343d-43a1-9b90-0e8c4b2d2dd2","collapsed":true,"trusted":false},"cell_type":"code","source":"city_ads = train.groupby('city').agg({'deal_probability': ['mean', 'count']}).reset_index().sort_values([('deal_probability', 'mean')], ascending=False).reset_index(drop=True)\nprint('There are {0} cities in total.'.format(len(train.city.unique())))\nprint('There are {1} cities with more that {0} ads.'.format(100, city_ads[city_ads['deal_probability']['count'] > 100].shape[0]))\nprint('There are {1} cities with more that {0} ads.'.format(1000, city_ads[city_ads['deal_probability']['count'] > 1000].shape[0]))\nprint('There are {1} cities with more that {0} ads.'.format(10000, city_ads[city_ads['deal_probability']['count'] > 10000].shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb040dab765e20613eb4188908548b1567e1b7fa","_cell_guid":"2f9e0c9a-bba6-4975-ab5f-b8f2a8778fb8"},"cell_type":"markdown","source":"It seems that most of the cities have little ads posted and only in 33 of them there a lot of ads. Let's see the best and the worst cities by mean deal_probability."},{"metadata":{"_kg_hide-input":true,"_uuid":"32c9e1fdd181b3a7c8ba4e3f064c05460c658caa","_cell_guid":"24e49c90-c4e4-43f7-841f-3883ed890ab0","collapsed":true,"trusted":false},"cell_type":"code","source":"city_ads[city_ads['deal_probability']['count'] > 1000].head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"530211f36debb6dcc0f686776c74a9e025aa5266","_cell_guid":"7c8f3241-8c6f-4218-abf1-3c2af2e790e4","collapsed":true,"trusted":false},"cell_type":"code","source":"city_ads[city_ads['deal_probability']['count'] > 1000].tail()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b0e4deb27235d274e09b8f75956393aa739b3ae","_cell_guid":"05cc9b71-01fe-4b5e-9bf7-54a111bed321"},"cell_type":"markdown","source":"I think that it could be interesting to see what is sold in Лабинск and Миллерово"},{"metadata":{"_kg_hide-input":true,"_uuid":"f9131af25bd13aadf96e9608a1e5b14332ac4cb8","_cell_guid":"e634f2c8-0fe6-453c-aa9b-aeb24a47c83a","collapsed":true,"trusted":false},"cell_type":"code","source":"print('Лабинск')\ntrain.loc[train.city == 'Лабинск'].groupby('category_name').agg({'deal_probability': ['mean', 'count']}).reset_index().sort_values([('deal_probability', 'count')], ascending=False).reset_index(drop=True).head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"317debf1d0d9ee65e4dfbb2a4e4bc1a388f1a963","_cell_guid":"795e9dd6-e5fb-4193-9532-d908fae9f24d"},"cell_type":"markdown","source":"Most popular categories are \"Автомобили\" (cars) and \"Телефоны\" (telephones)."},{"metadata":{"_kg_hide-input":true,"_uuid":"833d6c8f02c69268c2559a5e5b89a099a95d5b13","_cell_guid":"e7f27f5d-fc92-4e77-b353-ae08c28fe47b","collapsed":true,"trusted":false},"cell_type":"code","source":"print('Миллерово')\ntrain.loc[train.city == 'Миллерово'].groupby('category_name').agg({'deal_probability': ['mean', 'count']}).reset_index().sort_values([('deal_probability', 'count')], ascending=False).reset_index(drop=True).head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e7928feee3a6aecc31fde51dd08934c0f571667","_cell_guid":"3b2eba96-22e5-41e7-a90c-a785189cba72"},"cell_type":"markdown","source":"Most popular categories are \"Автомобили\" (cars). And it seems that second-hand wares are least popular."},{"metadata":{"_uuid":"84c9b9505caa53755de908ea6594426635e6d63a","_cell_guid":"837b735a-15b5-43e5-958c-14546486938a"},"cell_type":"markdown","source":"## deal_probability"},{"metadata":{"_kg_hide-input":true,"_uuid":"074c6669842189faccb533b664bfb5d820261872","_cell_guid":"235e39e2-4aff-4405-a085-d936cbd8f496","collapsed":true,"trusted":false},"cell_type":"code","source":"plt.hist(train['deal_probability']);\nplt.title('deal_probability');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"130cac27fbfb831e651896591ec1f0589e1c578c","_cell_guid":"d6a16998-2b7c-4f8d-b17d-733a40e9f800"},"cell_type":"markdown","source":"On the one hand the distribution of the target value is highly skewered towards zero, on the other hand, there is a spike at about 0.8."},{"metadata":{"_uuid":"0ec56e179192cb54bbde32811f585dc69833b919","_cell_guid":"a6765818-a87f-4f1d-b783-9287c230a9db"},"cell_type":"markdown","source":"## title"},{"metadata":{"_kg_hide-input":true,"_uuid":"56404b91e32976af2dad183626d5485cad469752","_cell_guid":"f90e9226-23d5-4069-9c08-2310193ad5e6","collapsed":true,"trusted":false},"cell_type":"code","source":"text = ' '.join(train['title'].values)\nwordcloud = WordCloud(max_font_size=None, stopwords=stop, background_color='white',\n                      width=1200, height=1000).generate(text)\nplt.figure(figsize=(12, 8))\nplt.imshow(wordcloud)\nplt.title('Top words for title')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d697d54d81993c0d570a0469de06986a11fa61d1","_cell_guid":"913446f0-a1c5-4c62-8a3c-a62475f090c9"},"cell_type":"markdown","source":"## description"},{"metadata":{"_uuid":"5072991eba1c3001cb6a0fe2f6a738000d22ea3f","collapsed":true,"_cell_guid":"16729c3a-04b9-4ed4-95e9-23bde4901933","trusted":false},"cell_type":"code","source":"train['description'] = train['description'].apply(lambda x: str(x).replace('/\\n', ' ').replace('\\xa0', ' '))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"a15b6f1bf6d214b5f053efc1216936fc5bb1bcb9","_cell_guid":"099281c4-8562-4bcc-b87b-d8f0e3a019bb","collapsed":true,"trusted":false},"cell_type":"code","source":"text = ' '.join(train['description'].values)\ntext = [i for i in ngrams(text.lower().split(), 3)]\nprint('Common trigrams.')\nCounter(text).most_common(40)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7f886deda9408667b19b86e27c2113ad3c6b684","_cell_guid":"8fd06868-86cf-4972-9b3d-8a6f0f43879e"},"cell_type":"markdown","source":"We can see that sellers try to tell buyers that their wares are great and also tell about possibilities of delivery. But there are some strange values, let's have a look..."},{"metadata":{"_uuid":"99a0126e9266acdd15b6d4115aa0ea0afffa8b0d","collapsed":true,"_cell_guid":"bef615ed-5ea4-4fc3-9892-8a4e966c2307","trusted":false},"cell_type":"code","source":"train[train.description.str.contains('↓')]['description'].head(10).values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e298399f2bb072261169a17dd322f1ce23c9465","_cell_guid":"c9e0987f-551a-4cb2-9f70-a269bd133975"},"cell_type":"markdown","source":"It seems that some authors really like emotional text with a lot of symbols or even words in upper case! We will use these features."},{"metadata":{"_uuid":"abd760b5c250cfc71bcdf4639ee75c9097ec8b61","_cell_guid":"a46285d1-586f-4bb8-a53f-aeb5ce7c2294"},"cell_type":"markdown","source":"## image\nIn this kernel I won't use the images themselves, but I'll create a feature showing wheather there is an image or not"},{"metadata":{"_kg_hide-input":true,"_uuid":"fa04990e4061e65a91aeeb25b3f1d86a211c18ec","_cell_guid":"9f72ef26-bb95-4198-bf7a-77dfcb0f3042","collapsed":true,"trusted":false},"cell_type":"code","source":"train['has_image'] = 1\ntrain.loc[train['image'].isnull(),'has_image'] = 0\nprint('There are {} ads with images. Mean deal_probability is {:.3}.'.format(len(train.loc[train['has_image'] == 1]), train.loc[train['has_image'] == 1, 'deal_probability'].mean()))\nprint('There are {} ads without images. Mean deal_probability is {:.3}.'.format(len(train.loc[train['has_image'] == 0]), train.loc[train['has_image'] == 0, 'deal_probability'].mean()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7016c7d0ebafb60ee02c02708732c23cdd2407dd","_cell_guid":"df57d81e-4fcc-4895-8597-718a9dee5408"},"cell_type":"markdown","source":"It is interesting, but ads without images are more likely to be bought."},{"metadata":{"_uuid":"159964ec3f4ba50b2aa59916b46a35d3066e5bb2","_cell_guid":"136ece67-f4e9-4bac-b8af-8b157d8f4778"},"cell_type":"markdown","source":"## item_seq_number"},{"metadata":{"_kg_hide-input":true,"_uuid":"d9e88b2eb5fd5f684b156d0d65e41c5e6f32f19f","_cell_guid":"28119f04-f57c-46d3-884f-6e9d183c4dd9","collapsed":true,"trusted":false},"cell_type":"code","source":"plt.scatter(train.item_seq_number, train.deal_probability, label='item_seq_number vs deal_probability');\nplt.xlabel('item_seq_number');\nplt.ylabel('deal_probability');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a26a8ceff2339ccd5d13ca05261c69b70c6fe8b","_cell_guid":"b71d3acf-3603-4c60-82cd-2b4e5bd769ba"},"cell_type":"markdown","source":"It seems that there are many users who post a lot of ads and number of ads posted isn't really correlated with deal_probability. There is some descreading trend, but we can't be sure."},{"metadata":{"_uuid":"271130f868e5d530d5ebc1d041a94a42fefc8df7","_cell_guid":"9828823b-059c-4760-bc2e-602a85cb6158"},"cell_type":"markdown","source":"## Params\nThere are three fields with additional information, let's combine it into one. Technically it is possible to treat these features as categorical, but there would be too many of them"},{"metadata":{"_kg_hide-input":true,"_uuid":"0c40a6d8d6a8297e745805c9cd5d19da14299730","_cell_guid":"9d1247d7-63a1-4e0d-a994-62a888090b06","collapsed":true,"trusted":false},"cell_type":"code","source":"train['params'] = train['param_1'].fillna('') + ' ' + train['param_2'].fillna('') + ' ' + train['param_3'].fillna('')\ntrain['params'] = train['params'].str.strip()\ntext = ' '.join(train['params'].values)\ntext = [i for i in ngrams(text.lower().split(), 3)]\nprint('Common trigrams.')\nCounter(text).most_common(40)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c70a95b25cca8c90494e1eddcd1d1e0119a964e","collapsed":true,"_cell_guid":"8b2b37d1-753f-4249-a432-41cc87cf1083"},"cell_type":"markdown","source":"Most of params belong to clothes or cars."},{"metadata":{"_uuid":"9c665c6df3c535ebfbbb3657d6a8ceb93d4029a5","_cell_guid":"bd70a5fd-a3cf-49e6-a0c8-dd6857668cf8"},"cell_type":"markdown","source":"## user_type \nThere are three main user_types. Let's see prices of their wares, where prices are below 100000."},{"metadata":{"_kg_hide-input":true,"_uuid":"7d368340c57a4dba63ae48678d12ce63dd3c9846","_cell_guid":"ba13c1ed-f362-4743-800e-a3486cb15fae","collapsed":true,"trusted":false},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(15, 8)})\ntrain_ = train[train.price.isnull() == False]\ntrain_ = train.loc[train.price < 100000.0]\nsns.boxplot(x=\"parent_category_name\", y=\"price\", hue=\"user_type\",  data=train_)\nplt.title(\"Price by parent category and user type\")\nplt.xticks(rotation='vertical')\nplt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d18bedb95b534a11ba953076ec8590e324e5ec5","_cell_guid":"0d56c0d5-e5d9-4737-ba4e-de0ab002c6cb"},"cell_type":"markdown","source":"We can see that shops usually have higher prices than companies and private sellers usually have the lowest price - maybe because they are usually second-hand."},{"metadata":{"_uuid":"b4a46329e83fc82d1148e37886fdbac32ef25a98","_cell_guid":"1018c79e-8191-4a44-a988-eb49acdb56a2"},"cell_type":"markdown","source":"## price\n\nThe first question is how to deal with missing values.\nI have decided to do the following:\n\n- at first fill missing values with median by city and category;\n- then missing values which are left are filled with region by region and category;\n- the remaining missing values are filled with median by category;"},{"metadata":{"_uuid":"5c4c364de000694ffe89f2347a677d6b9030770f","collapsed":true,"_cell_guid":"10b61343-53fd-46aa-b5d0-997a0d7bf78b","trusted":false},"cell_type":"code","source":"train['price'] = train.groupby(['city', 'category_name'])['price'].apply(lambda x: x.fillna(x.median()))\ntrain['price'] = train.groupby(['region', 'category_name'])['price'].apply(lambda x: x.fillna(x.median()))\ntrain['price'] = train.groupby(['category_name'])['price'].apply(lambda x: x.fillna(x.median()))\nplt.hist(train['price']);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bcea327441782cd572580a2c9d4232da5a54d5f8","_cell_guid":"e768b8c5-04b0-4e03-9654-e2c8fa5d58d5"},"cell_type":"markdown","source":"Let's use boxcox transformation to get rid of skewness"},{"metadata":{"_uuid":"c0b8a8caed9e3c0127cfed427166f92a8b5c72af","collapsed":true,"_cell_guid":"2c1cab76-987e-4643-b28e-9136f29055d4","trusted":false},"cell_type":"code","source":"plt.hist(stats.boxcox(train['price'] + 1)[0]);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36bf42544f1896c17d737652d66d924a1a8e40b0","_cell_guid":"7c855855-b5ec-4af7-a13c-78dcde895aae"},"cell_type":"markdown","source":"## Feature engineering\n"},{"metadata":{"_kg_hide-input":true,"_uuid":"ca2116825ae481f1637ebc033a001d80a66b30d2","_cell_guid":"05b136ab-bea7-465b-b45f-66a8fde5cca2","collapsed":true,"trusted":false},"cell_type":"code","source":"#Let's transform test in the same way as train.\ntest['params'] = test['param_1'].fillna('') + ' ' + test['param_2'].fillna('') + ' ' + test['param_3'].fillna('')\ntest['params'] = test['params'].str.strip()\n\ntest['description'] = test['description'].apply(lambda x: str(x).replace('/\\n', ' ').replace('\\xa0', ' '))\ntest['has_image'] = 1\ntest.loc[test['image'].isnull(),'has_image'] = 0\n\ntest['price'] = test.groupby(['city', 'category_name'])['price'].apply(lambda x: x.fillna(x.median()))\ntest['price'] = test.groupby(['region', 'category_name'])['price'].apply(lambda x: x.fillna(x.median()))\ntest['price'] = test.groupby(['category_name'])['price'].apply(lambda x: x.fillna(x.median()))\ntrain['price'] = stats.boxcox(train.price + 1)[0]\ntest['price'] = stats.boxcox(test.price + 1)[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d9bb5082c365f31e39da195fc69558170448e26","_cell_guid":"6d62ab7e-e0bf-4e25-a660-d7d3bedc44b3"},"cell_type":"markdown","source":"## Aggregate features\nI'll create a number of aggregate features."},{"metadata":{"_uuid":"980c407894b5350892d7c610ee44ce0a63c42f25","collapsed":true,"_cell_guid":"d286cc29-d179-47b2-8531-72f1547f1114","trusted":false},"cell_type":"code","source":"train['user_price_mean'] = train.groupby('user_id')['price'].transform('mean')\ntrain['user_ad_count'] = train.groupby('user_id')['price'].transform('sum')\n\ntrain['region_price_mean'] = train.groupby('region')['price'].transform('mean')\ntrain['region_price_median'] = train.groupby('region')['price'].transform('median')\ntrain['region_price_max'] = train.groupby('region')['price'].transform('max')\n\ntrain['region_price_mean'] = train.groupby('region')['price'].transform('mean')\ntrain['region_price_median'] = train.groupby('region')['price'].transform('median')\ntrain['region_price_max'] = train.groupby('region')['price'].transform('max')\n\ntrain['city_price_mean'] = train.groupby('city')['price'].transform('mean')\ntrain['city_price_median'] = train.groupby('city')['price'].transform('median')\ntrain['city_price_max'] = train.groupby('city')['price'].transform('max')\n\ntrain['parent_category_name_price_mean'] = train.groupby('parent_category_name')['price'].transform('mean')\ntrain['parent_category_name_price_median'] = train.groupby('parent_category_name')['price'].transform('median')\ntrain['parent_category_name_price_max'] = train.groupby('parent_category_name')['price'].transform('max')\n\ntrain['category_name_price_mean'] = train.groupby('category_name')['price'].transform('mean')\ntrain['category_name_price_median'] = train.groupby('category_name')['price'].transform('median')\ntrain['category_name_price_max'] = train.groupby('category_name')['price'].transform('max')\n\ntrain['user_type_category_price_mean'] = train.groupby(['user_type', 'parent_category_name'])['price'].transform('mean')\ntrain['user_type_category_price_median'] = train.groupby(['user_type', 'parent_category_name'])['price'].transform('median')\ntrain['user_type_category_price_max'] = train.groupby(['user_type', 'parent_category_name'])['price'].transform('max')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f419c0ed1d3cfd5fe00677e54cb861fa74ace1fa","collapsed":true,"_cell_guid":"8045e939-b097-4f1c-8a7f-7e2db44c5c0d","trusted":false},"cell_type":"code","source":"test['user_price_mean'] = test.groupby('user_id')['price'].transform('mean')\ntest['user_ad_count'] = test.groupby('user_id')['price'].transform('sum')\n\ntest['region_price_mean'] = test.groupby('region')['price'].transform('mean')\ntest['region_price_median'] = test.groupby('region')['price'].transform('median')\ntest['region_price_max'] = test.groupby('region')['price'].transform('max')\n\ntest['region_price_mean'] = test.groupby('region')['price'].transform('mean')\ntest['region_price_median'] = test.groupby('region')['price'].transform('median')\ntest['region_price_max'] = test.groupby('region')['price'].transform('max')\n\ntest['city_price_mean'] = test.groupby('city')['price'].transform('mean')\ntest['city_price_median'] = test.groupby('city')['price'].transform('median')\ntest['city_price_max'] = test.groupby('city')['price'].transform('max')\n\ntest['parent_category_name_price_mean'] = test.groupby('parent_category_name')['price'].transform('mean')\ntest['parent_category_name_price_median'] = test.groupby('parent_category_name')['price'].transform('median')\ntest['parent_category_name_price_max'] = test.groupby('parent_category_name')['price'].transform('max')\n\ntest['category_name_price_mean'] = test.groupby('category_name')['price'].transform('mean')\ntest['category_name_price_median'] = test.groupby('category_name')['price'].transform('median')\ntest['category_name_price_max'] = test.groupby('category_name')['price'].transform('max')\n\ntest['user_type_category_price_mean'] = test.groupby(['user_type', 'parent_category_name'])['price'].transform('mean')\ntest['user_type_category_price_median'] = test.groupby(['user_type', 'parent_category_name'])['price'].transform('median')\ntest['user_type_category_price_max'] = test.groupby(['user_type', 'parent_category_name'])['price'].transform('max')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4b5e8319f404a9c01b664a2353b57891c95fdbd","_cell_guid":"6cabe34f-9edb-4f5f-95cf-a9e9bba7594a"},"cell_type":"markdown","source":"## Categorical features\n\nI'll use target encoding to deal with categorical features."},{"metadata":{"_kg_hide-input":true,"_uuid":"4d013c09fc94ebd61345d82935f5d654118c30a8","_cell_guid":"390750c8-607e-4a82-854a-442db3c0d509","collapsed":true,"trusted":false},"cell_type":"code","source":"def target_encode(trn_series=None, \n                  tst_series=None, \n                  target=None, \n                  min_samples_leaf=1, \n                  smoothing=1,\n                  noise_level=0):\n    \"\"\"\n    \n    https://www.kaggle.com/ogrellier/python-target-encoding-for-categorical-features\n    Smoothing is computed like in the following paper by Daniele Micci-Barreca\n    https://kaggle2.blob.core.windows.net/forum-message-attachments/225952/7441/high%20cardinality%20categoricals.pdf\n    trn_series : training categorical feature as a pd.Series\n    tst_series : test categorical feature as a pd.Series\n    target : target data as a pd.Series\n    min_samples_leaf (int) : minimum samples to take category average into account\n    smoothing (int) : smoothing effect to balance categorical average vs prior  \n    \"\"\" \n    assert len(trn_series) == len(target)\n    assert trn_series.name == tst_series.name\n    temp = pd.concat([trn_series, target], axis=1)\n    # Compute target mean \n    averages = temp.groupby(by=trn_series.name)[target.name].agg([\"mean\", \"count\"])\n    # Compute smoothing\n    smoothing = 1 / (1 + np.exp(-(averages[\"count\"] - min_samples_leaf) / smoothing))\n    # Apply average function to all target data\n    prior = target.mean()\n    # The bigger the count the less full_avg is taken into account\n    averages[target.name] = prior * (1 - smoothing) + averages[\"mean\"] * smoothing\n    averages.drop([\"mean\", \"count\"], axis=1, inplace=True)\n    # Apply averages to trn and tst series\n    ft_trn_series = pd.merge(\n        trn_series.to_frame(trn_series.name),\n        averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),\n        on=trn_series.name,\n        how='left')['average'].rename(trn_series.name + '_mean').fillna(prior)\n    # pd.merge does not keep the index so restore it\n    ft_trn_series.index = trn_series.index \n    ft_tst_series = pd.merge(\n        tst_series.to_frame(tst_series.name),\n        averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),\n        on=tst_series.name,\n        how='left')['average'].rename(trn_series.name + '_mean').fillna(prior)\n    # pd.merge does not keep the index so restore it\n    ft_tst_series.index = tst_series.index\n    return ft_trn_series, ft_tst_series","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b097f7fc22be10792a09ed46ba88b4d6ef448d7d","collapsed":true,"_cell_guid":"b874718f-b3c7-4807-9445-7df9c7827b27","trusted":false},"cell_type":"code","source":"train['parent_category_name'], test['parent_category_name'] = target_encode(train['parent_category_name'], test['parent_category_name'], train['deal_probability'])\ntrain['category_name'], test['category_name'] = target_encode(train['category_name'], test['category_name'], train['deal_probability'])\ntrain['region'], test['region'] = target_encode(train['region'], test['region'], train['deal_probability'])\ntrain['image_top_1'], test['image_top_1'] = target_encode(train['image_top_1'], test['image_top_1'], train['deal_probability'])\ntrain['city'], test['city'] = target_encode(train['city'], test['city'], train['deal_probability'])\ntrain['param_1'], test['param_1'] = target_encode(train['param_1'], test['param_1'], train['deal_probability'])\ntrain['param_2'], test['param_2'] = target_encode(train['param_2'], test['param_2'], train['deal_probability'])\ntrain['param_3'], test['param_3'] = target_encode(train['param_3'], test['param_3'], train['deal_probability'])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"7a22cab24aa2b0179fde66f8758d73ad142cabc7","_cell_guid":"aa82855a-1a82-4f8e-923b-656ea13f7153","collapsed":true,"trusted":false},"cell_type":"code","source":"train.drop(['date', 'day', 'user_id'], axis=1, inplace=True)\ntest.drop(['date', 'day', 'user_id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ab7099606375bc6f217a68df23e5b076f1708de","_cell_guid":"808a5edf-4546-454b-8c9f-ebe06e8da5a4"},"cell_type":"markdown","source":"## Text features\n\nWe have several features with text data and they need to be processed in different ways. But at first let's create new features based on texts: length of text (symbols) and number of words. Also let's calculate counts of punctuation and some of strange symbols."},{"metadata":{"_uuid":"ff45d0154ea23e3fabcd587e560c60b881f90b08","collapsed":true,"_cell_guid":"d5c0a7af-71a4-4d0f-83e5-ea4d93bf3383","trusted":false},"cell_type":"code","source":"train['len_title'] = train['title'].apply(lambda x: len(x))\ntrain['words_title'] = train['title'].apply(lambda x: len(x.split()))\ntrain['len_description'] = train['description'].apply(lambda x: len(x))\ntrain['words_description'] = train['description'].apply(lambda x: len(x.split()))\ntrain['len_params'] = train['params'].apply(lambda x: len(x))\ntrain['words_params'] = train['params'].apply(lambda x: len(x.split()))\n\ntrain['symbol1_count'] = train['description'].str.count('↓')\ntrain['symbol2_count'] = train['description'].str.count('\\*')\ntrain['symbol3_count'] = train['description'].str.count('✔')\ntrain['symbol4_count'] = train['description'].str.count('❀')\ntrain['symbol5_count'] = train['description'].str.count('➚')\ntrain['symbol6_count'] = train['description'].str.count('ஜ')\ntrain['symbol7_count'] = train['description'].str.count('.')\ntrain['symbol8_count'] = train['description'].str.count('!')\ntrain['symbol9_count'] = train['description'].str.count('\\?')\ntrain['symbol10_count'] = train['description'].str.count('  ')\ntrain['symbol11_count'] = train['description'].str.count('-')\ntrain['symbol12_count'] = train['description'].str.count(',')\n\ntest['len_title'] = test['title'].apply(lambda x: len(x))\ntest['words_title'] = test['title'].apply(lambda x: len(x.split()))\ntest['len_description'] = test['description'].apply(lambda x: len(x))\ntest['words_description'] = test['description'].apply(lambda x: len(x.split()))\ntest['len_params'] = test['params'].apply(lambda x: len(x))\ntest['words_params'] = test['params'].apply(lambda x: len(x.split()))\n\ntest['symbol1_count'] = test['description'].str.count('↓')\ntest['symbol2_count'] = test['description'].str.count('\\*')\ntest['symbol3_count'] = test['description'].str.count('✔')\ntest['symbol4_count'] = test['description'].str.count('❀')\ntest['symbol5_count'] = test['description'].str.count('➚')\ntest['symbol6_count'] = test['description'].str.count('ஜ')\ntest['symbol7_count'] = test['description'].str.count('.')\ntest['symbol8_count'] = test['description'].str.count('!')\ntest['symbol9_count'] = test['description'].str.count('\\?')\ntest['symbol10_count'] = test['description'].str.count('  ')\ntest['symbol11_count'] = test['description'].str.count('-')\ntest['symbol12_count'] = test['description'].str.count(',')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"605c33a70825d4b1fcb25d1144ca407b0df525d4","_cell_guid":"020cfd8b-0587-4f71-a6db-e9ef82e9e1d1"},"cell_type":"markdown","source":"Now let's start transforming texts. Titles have little number of unique words, so we can use default values for TfidfVectorizer (only add stopwords). I have to limit max_features due to memory constraints. I won't use descriptions and parameters due to kernel limits."},{"metadata":{"_uuid":"95590c2935b391a6988e6f74b83844a9bdd555c2","collapsed":true,"_cell_guid":"a62ca089-2310-433f-b5eb-1a5036d25eeb","trusted":false},"cell_type":"code","source":"vectorizer=TfidfVectorizer(stop_words=stop, max_features=2000)\nvectorizer.fit(train['title'])\ntrain_title = vectorizer.transform(train['title'])\ntest_title = vectorizer.transform(test['title'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"460f015166866cb02cef5176246b44b44e84ce0a","collapsed":true,"_cell_guid":"6914ecc6-d876-422f-bf4a-5b0b5f751551","trusted":false},"cell_type":"code","source":"train.drop(['title', 'params', 'description', 'user_type', 'activation_date'], axis=1, inplace=True)\ntest.drop(['title', 'params', 'description', 'user_type', 'activation_date'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3738ce7ca32215e1e32e1db8bff14fb853cc3def","collapsed":true,"_cell_guid":"b920e40c-80a0-449f-b6f3-94cfba902ae1","trusted":false},"cell_type":"code","source":"pd.set_option('max_columns', 60)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a0ba9040fb7711c5ce61eeec5011d413c541a59","_cell_guid":"fda994ab-ee59-437b-b202-e68c2bbea3dd"},"cell_type":"markdown","source":"One of possible ideas is creating meta-features. It means that we use some features to build a model and use the predictions in another model. I'll use ridge regression to create a new feature based on tokenized title and then I'll combine it with other features."},{"metadata":{"_uuid":"c15d146ae089a09572e6656a0487abe2f3e7cca8","collapsed":true,"_cell_guid":"8f59bcad-9312-4579-879d-e40425936f01","trusted":false},"cell_type":"code","source":"%%time\nX_meta = np.zeros((train_title.shape[0], 1))\nX_test_meta = []\nfor fold_i, (train_i, test_i) in enumerate(kf.split(train_title)):\n    print(fold_i)\n    model = Ridge()\n    model.fit(train_title.tocsr()[train_i], train['deal_probability'][train_i])\n    X_meta[test_i, :] = model.predict(train_title.tocsr()[test_i]).reshape(-1, 1)\n    X_test_meta.append(model.predict(test_title))\n    \nX_test_meta = np.stack(X_test_meta)\nX_test_meta_mean = np.mean(X_test_meta, axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ecbb0efb047cec8540ed08eb40cf6bb32b899f19","collapsed":true,"_cell_guid":"5083f63f-7112-4e90-be69-1eae74ae4aa5","trusted":false},"cell_type":"code","source":"X_full = csr_matrix(hstack([train.drop(['item_id', 'deal_probability', 'image'], axis=1), X_meta]))\nX_test_full = csr_matrix(hstack([test.drop(['item_id', 'image'], axis=1), X_test_meta_mean.reshape(-1, 1)]))\n\nX_train, X_valid, y_train, y_valid = train_test_split(X_full, train['deal_probability'], test_size=0.20, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72ab3f54a8e608358f9d652ec734f9f661ddf523","_cell_guid":"a2724bf0-e84a-42c1-8ff1-9f6091b511e7"},"cell_type":"markdown","source":"## Building a simple model\n"},{"metadata":{"_uuid":"bf2727d3e4e02abd9a6e07829156be86f744c88a","collapsed":true,"_cell_guid":"e8624daa-6b2d-4b3c-b7e2-e3c9dbc1994b","trusted":false},"cell_type":"code","source":"def rmse(predictions, targets):\n    return np.sqrt(((predictions - targets) ** 2).mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc684f8424f0991932eed7e8c0cf923057f766ae","collapsed":true,"_cell_guid":"e73dabe3-d8e1-4389-aa45-1f06e33cd1ad","trusted":false},"cell_type":"code","source":"#took parameters from this kernel: https://www.kaggle.com/the1owl/beep-beep\nparams = {'learning_rate': 0.05, 'max_depth': 6, 'boosting': 'gbdt', 'objective': 'regression', 'metric': ['auc','rmse'], 'is_training_metric': True, 'seed': 19, 'num_leaves': 63, 'feature_fraction': 0.9, 'bagging_fraction': 0.8, 'bagging_freq': 5}\nmodel = lgb.train(params, lgb.Dataset(X_train, label=y_train), 2000, lgb.Dataset(X_valid, label=y_valid), verbose_eval=50, early_stopping_rounds=20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f0405c58015a76db2191c550bbd138fdc285832","collapsed":true,"_cell_guid":"74af618b-1f3a-4680-8c88-94c7fa8c0a36","trusted":false},"cell_type":"code","source":"pred = model.predict(X_test_full)\n#clipping is necessary.\nsub['deal_probability'] = np.clip(pred, 0, 1)\nsub.to_csv('sub.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8dbcb34719a4da386dcd3e424ef8916d3fd73050","_cell_guid":"360367b4-0c63-4ec9-8c5d-2e76a69985cf"},"cell_type":"markdown","source":"## Additional ideas\n- It is really necessary to use a machine with more CPU. I used tf-idf on descriptions and params and got 0.227 on leaderboard with this lgb model;\n- Price is a tricky feature and needs careful preprocessing;\n- Texts are really interesting. I'm sure there are a lot of features which can be created based on them. Also russian embeddings can be used;\n- And of course there are pictures. We can try extract some interesting features, or build CNN models on  "}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}