{"cells":[{"metadata":{"_uuid":"9c5fad5603799bbcac23a5799b4e8cff17edd4ba","_cell_guid":"ca0871fd-2d30-49d4-8ef2-541296844b69"},"cell_type":"markdown","source":"This kernel is based on the following notes.\n\nhttps://www.kaggle.com/sudalairajkumar/simple-exploration-baseline-notebook-avito\n\nhttps://www.kaggle.com/bguberfain/naive-lgb-with-text-images/"},{"metadata":{"_uuid":"b588970d9f1d6304ee9e363ce8580507b389814f","_cell_guid":"357cf094-2ea2-4a44-8aeb-d0717696bd2f"},"cell_type":"markdown","source":"**Import Libraries:**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn import preprocessing, model_selection, metrics\nimport lightgbm as lgb\n\nfrom nltk.corpus import stopwords\nfrom pathlib import PurePath\nfrom scipy import sparse\nimport gc\nimport gzip\n\n%matplotlib inline\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72a37b584b83e1296647bb168dcc4294dd9f1873","_cell_guid":"997c8acb-b10d-4b86-abe5-f7b50c70138f"},"cell_type":"markdown","source":"**Read Data:**"},{"metadata":{"_uuid":"67e924346e31d55b07a38a1038da815d76e28356","_cell_guid":"02081c71-977c-4f5a-b990-b1d0dc8bb858","trusted":false,"collapsed":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/avito-demand-prediction/train.csv\", parse_dates=[\"activation_date\"])\ntest_df = pd.read_csv(\"../input/avito-demand-prediction/test.csv\", parse_dates=[\"activation_date\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b080dd322618632376658e7c6075f79a7150c3e","_cell_guid":"101e25f2-3dde-4f42-a2bf-1b39872d64a6"},"cell_type":"markdown","source":"** Image Features:**"},{"metadata":{"_uuid":"789ef5c235beb3aa38867bcc7e3fa02887bdc139","_cell_guid":"ca68eb3d-4e50-4a1e-ab97-a354d6a46a71","trusted":false,"collapsed":true},"cell_type":"code","source":"### Image features ###\ndef load_imfeatures(folder):\n    path = PurePath(folder)\n    features = sparse.load_npz(str(path / 'features.npz'))\n    return features\n\nftrain = load_imfeatures('../input/vgg16-train-features/')\nftest = load_imfeatures('../input/vgg16-test-features/')\n\n\n### Create both dataframe ###\ndf_both = pd.concat([train_df, test_df])\n\nfboth = sparse.vstack([ftrain, ftest])\ndel ftrain, ftest\ngc.collect()\nfboth.shape\n\n### Categorical image feature (max and min VGG16 feature) ###\ndf_both['im_max_feature'] = fboth.argmax(axis=1)  # This will be categorical\ndf_both['im_min_feature'] = fboth.argmin(axis=1)  # This will be categorical\n\ndf_both['im_n_features'] = fboth.getnnz(axis=1)\ndf_both['im_mean_features'] = fboth.mean(axis=1)\ndf_both['im_meansquare_features'] = fboth.power(2).mean(axis=1)\n\n### Let`s reduce 512 VGG16 featues into 32 ###\ntsvd = TruncatedSVD(32)\nftsvd = tsvd.fit_transform(fboth)\ndel fboth\ngc.collect()\n\n### Merge image features into df_both ###\ndf_ftsvd = pd.DataFrame(ftsvd, index=df_both.index).add_prefix('im_tsvd_')\n\ndf_both = pd.concat([df_both, df_ftsvd], axis=1)\n\ndel df_ftsvd, ftsvd\ngc.collect();\n\n###Split df_both in train and test ###\nn_train = train_df.shape[0]\ntrain_df = df_both.iloc[:n_train]\ntest_df = df_both.iloc[n_train:]\n\ndel df_both\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02c773d25acfe2e751959cee4fde706beb152333","_cell_guid":"448a7a3e-7530-4438-a927-423952de0f14"},"cell_type":"markdown","source":"** \"title\" Feature:**"},{"metadata":{"_uuid":"b8e6cc225eb2087bbddf5fac90568404974e9099","collapsed":true,"_cell_guid":"db2a92d6-c85c-400a-bcf7-f94724ab5711","trusted":false},"cell_type":"code","source":"### Stop Words ###\nrussian_stop = set(stopwords.words('russian'))\n\n### TFIDF Vectorizer ###\ntfidf_vec = TfidfVectorizer(ngram_range=(1,1), stop_words=russian_stop)\nfull_tfidf = tfidf_vec.fit_transform(train_df['title'].values.tolist() + test_df['title'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['title'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['title'].values.tolist())\n\n### SVD Components ###\nn_comp = 3\nsvd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\nsvd_obj.fit(full_tfidf)\ntrain_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\ntest_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\ntrain_svd.columns = ['svd_title_'+str(i+1) for i in range(n_comp)]\ntest_svd.columns = ['svd_title_'+str(i+1) for i in range(n_comp)]\ntrain_df = pd.concat([train_df, train_svd], axis=1)\ntest_df = pd.concat([test_df, test_svd], axis=1)\ndel full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cea3dffe744cf251fe0427f8fea81a051beeaabc","_cell_guid":"80ce1a70-f3f8-4c44-8290-bcc9fa5ad45b"},"cell_type":"markdown","source":"** \"description\" Feature:**"},{"metadata":{"_uuid":"cce3a9d56fd12d3858bebd18fe32f1a6f808a1ec","collapsed":true,"_cell_guid":"733354be-6725-40fa-afe8-4624c4c1a475","trusted":false},"cell_type":"code","source":"### Filling missing values ###\ntrain_df[\"description\"].fillna(\"NA\", inplace=True)\ntest_df[\"description\"].fillna(\"NA\", inplace=True)\n\n### TFIDF Vectorizer ###\ntfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000, stop_words=russian_stop)\nfull_tfidf = tfidf_vec.fit_transform(train_df['description'].values.tolist() + test_df['description'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['description'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['description'].values.tolist())\n\n### SVD Components ###\nn_comp = 3\nsvd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\nsvd_obj.fit(full_tfidf)\ntrain_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\ntest_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\ntrain_svd.columns = ['svd_desc_'+str(i+1) for i in range(n_comp)]\ntest_svd.columns = ['svd_desc_'+str(i+1) for i in range(n_comp)]\ntrain_df = pd.concat([train_df, train_svd], axis=1)\ntest_df = pd.concat([test_df, test_svd], axis=1)\ndel full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32e280660bbc5a0258fd4f4ee2ede004a41bcfcf","_cell_guid":"000dde77-a41a-4ed0-b502-c16510d26549"},"cell_type":"markdown","source":"** \"activation_date\" Feature:**"},{"metadata":{"_uuid":"156d30e719b6eb3dea0c181f7e21092475a3d30e","collapsed":true,"_cell_guid":"6c8ce622-388c-4419-a9c4-b42422266e3a","trusted":false},"cell_type":"code","source":"### Date Variables ###\ntrain_df[\"activation_weekday\"] = train_df[\"activation_date\"].dt.weekday\ntrain_df[\"activation_weekofyear\"] = train_df[\"activation_date\"].dt.week\ntrain_df[\"activation_weekofmonth\"] = train_df[\"activation_date\"].dt.day\ntest_df[\"activation_weekday\"] = test_df[\"activation_date\"].dt.weekday\ntest_df[\"activation_weekofyear\"] = test_df[\"activation_date\"].dt.week\ntest_df[\"activation_weekofmonth\"] = test_df[\"activation_date\"].dt.day","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0fabd2e64e7045073b46a129d323c3f73c0ded58","_cell_guid":"0b13d03a-e129-4d89-822e-cab9cc4f1a0c"},"cell_type":"markdown","source":"** Features Selection: **"},{"metadata":{"_uuid":"c484e796af93f56fcae063b73c22326937c95d9b","collapsed":true,"_cell_guid":"54b73c02-4e4a-4d1d-bd4c-4a7bf340effd","trusted":false},"cell_type":"code","source":"### Target and ID variables ###\ntrain_y = train_df[\"deal_probability\"].values\ntest_id = test_df[\"item_id\"].values\n    \n### Label encode the categorical variables ###\ncat_vars = [ \"region\", \"city\", \"parent_category_name\", \"category_name\", \"user_type\", \"param_1\", \"param_2\", \"param_3\"]\nfor col in cat_vars:\n    lbl = preprocessing.LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))\n\ncols_to_drop = [\"item_id\", \"user_id\", \"description\", \"title\", \"activation_date\", \"image\"]\ntrain_X = train_df.drop(cols_to_drop + [\"deal_probability\"], axis=1)\ntest_X = test_df.drop(cols_to_drop, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"413f7a12938345695f89cc298d3913ef8895f916","_cell_guid":"74c83d35-e72b-46b9-86f5-fb1f734bfc23"},"cell_type":"markdown","source":"** LightGBM Model: **"},{"metadata":{"_uuid":"d9e7d6deb3cae10ecdfb01058325e1ae240f6510","collapsed":true,"_cell_guid":"bb2a95b1-a2c2-43c1-81c0-0472664a508d","trusted":false},"cell_type":"code","source":"### LightGBM model ###\ndef run_lgb(train_X, train_y, val_X, val_y, test_X):\n\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\",\n        \"num_leaves\" : 30,\n        \"learning_rate\" : 0.1,\n        \"bagging_fraction\" : 0.7,\n        \"feature_fraction\" : 0.7,\n        \"bagging_frequency\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1\n    }\n\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    evals_result = {}\n    model = lgb.train(params, lgtrain, 5000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=20, evals_result=evals_result)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, evals_result","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"62f821910bede6e456a1fa2d08650647db4482ba","_cell_guid":"14057738-8630-49aa-82f4-f4c943e0f9bc"},"cell_type":"markdown","source":"** Model Training: **"},{"metadata":{"_uuid":"135a8e08158017253adf5a46dcb46fcb0d1a5bbd","_cell_guid":"26320351-3ec9-4ba8-8531-d0adaedb777e","trusted":false,"collapsed":true},"cell_type":"code","source":"### Splitting the data for model training ###\ndev_X = train_X.iloc[:-200000,:]\nval_X = train_X.iloc[-200000:,:]\ndev_y = train_y[:-200000]\nval_y = train_y[-200000:]\nprint(dev_X.shape, val_X.shape, test_X.shape)\n\n### Training the model ###\npred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)\n\n### Making a submission file ###\npred_test[pred_test>1] = 1\npred_test[pred_test<0] = 0\nsub_df = pd.DataFrame({\"item_id\":test_id})\nsub_df[\"deal_probability\"] = pred_test\nsub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3788c1605ef990a0bfe17887b878e8e313bb770c","_cell_guid":"a8303efa-519b-4b4a-92e5-f55ae8d0dc08"},"cell_type":"markdown","source":"** Feature Importance : **\n\n"},{"metadata":{"_uuid":"2c31a6370706f5c0630e565bb32cfdd8e0d1fa9f","_cell_guid":"a5ada78e-6a18-4cad-8a34-a91725d6d4d9","trusted":false,"collapsed":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,18))\nlgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax)\nax.grid(False)\nplt.title(\"LightGBM - Feature Importance\", fontsize=15)\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}