{"cells":[{"metadata":{"_uuid":"eb59ba957119aaebc240c48f6d986c993302db74"},"cell_type":"markdown","source":"## Import all the requirements"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn import preprocessing, model_selection, metrics\nfrom sklearn.decomposition import TruncatedSVD\nfrom datetime import date\n\ncolor = sns.color_palette()\n%matplotlib inline\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0156787da5137b8d5c43919607492e296f494fef"},"cell_type":"code","source":"## Import necessary files","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\", parse_dates=[\"activation_date\"])\ntest_df = pd.read_csv(\"../input/test.csv\", parse_dates=[\"activation_date\"])\nprint(\"Train file rows and columns are : \", train_df.shape)\nprint(\"Test file rows and columns are : \", test_df.shape)\n\ntrain_prd = pd.read_csv(\"../input/periods_train.csv\", parse_dates=[\"activation_date\",\"date_from\", \"date_to\"])\ntest_prd = pd.read_csv(\"../input/periods_test.csv\", parse_dates=[\"activation_date\",\"date_from\", \"date_to\"])\nprint(\"Period Train file rows and columns are : \", train_prd.shape)\nprint(\"Period Test file rows and columns are : \", test_prd.shape)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29844b1e6f08510549cf3ec431a6974280c1ce81"},"cell_type":"code","source":"train_prd.dtypes","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b702534b16ad97dfca029cf483ce20be40e1e4e","collapsed":true},"cell_type":"code","source":"#Number of days an ad was active on the portal\ntrain_prd['days'] = (train_prd['date_to'] - train_prd['date_from']).dt.days\ntest_prd['days'] = (test_prd['date_to'] - test_prd['date_from']).dt.days\n\nenc = train_prd.groupby('item_id')['days'].agg('sum').astype(np.float32).reset_index()\nenc.head(5)\n\ntrain_df = pd.merge(train_df, enc, how='left', on='item_id')\ntest_df = pd.merge(test_df, enc, how='left', on='item_id')","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12ab80657fa592a5917faed2bffe6e636c3f0cc0"},"cell_type":"code","source":"train_df.head()","execution_count":12,"outputs":[]},{"metadata":{"_uuid":"1057760c07fee56176267711396b09fbbec543aa"},"cell_type":"markdown","source":"# IMPUTE Missing Values for days active\n"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c01e6f097c01dbf0db282b7db5103626aeeed03c"},"cell_type":"markdown","source":"## Create new variables and process the existing ones (train.cs)"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a5fad3bb6c39309f7060158966a55135a93bde48"},"cell_type":"code","source":"# New variables #\ntrain_df[\"activation_weekday\"] = train_df[\"activation_date\"].dt.weekday\ntest_df[\"activation_weekday\"] = test_df[\"activation_date\"].dt.weekday\n\ntrain_df[\"activation_month\"] = train_df[\"activation_date\"].dt.month\ntest_df[\"activation_month\"] = test_df[\"activation_date\"].dt.month\n\ntrain_df[\"title_nwords\"] = train_df[\"title\"].apply(lambda x: len(x.split()))\ntest_df[\"title_nwords\"] = test_df[\"title\"].apply(lambda x: len(x.split()))\n\ntrain_df[\"description\"].fillna(\"NA\", inplace=True)\ntest_df[\"description\"].fillna(\"NA\", inplace=True)\ntrain_df[\"desc_nwords\"] = train_df[\"description\"].apply(lambda x: len(x.split()))\ntest_df[\"desc_nwords\"] = test_df[\"description\"].apply(lambda x: len(x.split()))\n\ntrain_df['param123'] = train_df['param_1'].fillna('') + \" \" + train_df['param_2'].fillna('') + \" \" + train_df['param_3'].fillna('') \ntest_df['param123'] = test_df['param_1'].fillna('') + \" \" + test_df['param_2'].fillna('') + \" \" + test_df['param_3'].fillna('') \n\n#Impute image_top_1\nenc = train_df.groupby('category_name')['image_top_1'].agg(lambda x:x.value_counts().index[0]).astype(np.float32).reset_index()\nenc.columns = ['category_name' ,'image_top_1_impute']\n#Cross Check values\n#enc = train_df.loc[train_df['category_name'] == 'Аквариум'].groupby('image_top_1').agg('count')\n#enc.sort_values(['item_id'], ascending=False).head(2)\n\ntrain_df = pd.merge(train_df, enc, how='left', on='category_name')\ntest_df = pd.merge(test_df, enc, how='left', on='category_name')\n\ntrain_df['image_top_1'].fillna(train_df['image_top_1_impute'], inplace=True)\ntest_df['image_top_1'].fillna(test_df['image_top_1_impute'], inplace=True)\n\n#Impute Days diff\nenc = train_df.groupby('category_name')['days'].agg('median').astype(np.float32).reset_index()\nenc.columns = ['category_name' ,'days_impute']\n#Cross Check values\n#enc = train_df.loc[train_df['category_name'] == 'Аквариум'].groupby('image_top_1').agg('count')\n#enc.sort_values(['item_id'], ascending=False).head(2)\n\ntrain_df = pd.merge(train_df, enc, how='left', on='category_name')\ntest_df = pd.merge(test_df, enc, how='left', on='category_name')\n\ntrain_df['days'].fillna(train_df['days_impute'], inplace=True)\ntest_df['days'].fillna(test_df['days_impute'], inplace=True)\n\n\n#Create image flag \ntest_df['image'] = test_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)\ntrain_df['image'] = train_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)\n\n# City names are duplicated across region, HT: Branden Murray \n#https://www.kaggle.com/c/avito-demand-prediction/discussion/55630#321751\ntrain_df['city'] = train_df['city'] + \"_\" + train_df['region']\ntest_df['city'] = test_df['city'] + \"_\" + test_df['region']\n\ntrain_df['price'].fillna(0, inplace=True)\ntest_df['price'].fillna(0, inplace=True)\ntrain_df['price'] = np.log1p(train_df['price'])\ntest_df['price'] = np.log1p(test_df['price'])\n\nprice_mean = train_df['price'].mean()\nprice_std = train_df['price'].std()\ntrain_df['price'] = (train_df['price'] - price_mean) / price_std\ntest_df['price'] = (test_df['price'] - price_mean) / price_std\n\ncat_cols = ['category_name', 'image_top_1']\nnum_cols = ['price', 'deal_probability']\n\nfor c in cat_cols:\n    for c2 in num_cols:\n        enc = train_df.groupby(c)[c2].agg(['median']).astype(np.float32).reset_index()\n        enc.columns = ['_'.join([str(c), str(c2), str(c3)]) if c3 != c else c for c3 in enc.columns]\n        train_df = pd.merge(train_df, enc, how='left', on=c)\n        test_df = pd.merge(test_df, enc, how='left', on=c)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89082c16fd96cb7bf2ed68d74eaa022656724949","collapsed":true},"cell_type":"code","source":"### TFIDF Vectorizer ###\ntfidf_vec = TfidfVectorizer(ngram_range=(1,1))\n#ngram_range defines how you want to have words in your dictionary. \n#(min,max) = (1,2) will mean you will have unigrams and bigrms in your vocabulary. \n#Example String: \"The old fox\"\n#Vocabulary: \"The\", \"old\", \"fox\", \"The old\", \"old fox\"\n\nfull_tfidf = tfidf_vec.fit_transform(train_df['title'].values.tolist() + test_df['title'].values.tolist())\n#train_df['title'].values.tolist() this converts all the values in the title column into a list. '+' appends two lists","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71277def8861c0ad7d25c12eaaa8fe3965857db2","collapsed":true},"cell_type":"code","source":"train_tfidf = tfidf_vec.transform(train_df['title'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['title'].values.tolist())\n\n### SVD Components ###\nn_comp = 5\nsvd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\nsvd_obj.fit(full_tfidf)\ntrain_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\ntest_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\ntrain_svd.columns = ['svd_title_'+str(i+1) for i in range(n_comp)]\ntest_svd.columns = ['svd_title_'+str(i+1) for i in range(n_comp)]\ntrain_df = pd.concat([train_df, train_svd], axis=1)\ntest_df = pd.concat([test_df, test_svd], axis=1)\ndel full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b458b180ae81b26bb604830004925927cc1a2016","collapsed":true},"cell_type":"code","source":"tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)\nfull_tfidf = tfidf_vec.fit_transform(train_df['description'].values.tolist() + test_df['description'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['description'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['description'].values.tolist())\n\n### SVD Components ###\nn_comp = 5\nsvd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\nsvd_obj.fit(full_tfidf)\ntrain_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\ntest_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\ntrain_svd.columns = ['svd_desc_'+str(i+1) for i in range(n_comp)]\ntest_svd.columns = ['svd_desc_'+str(i+1) for i in range(n_comp)]\ntrain_df = pd.concat([train_df, train_svd], axis=1)\ntest_df = pd.concat([test_df, test_svd], axis=1)\ndel full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"018bbff9eb2065c4406d96694af3addac6894180"},"cell_type":"code","source":"tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)\nfull_tfidf = tfidf_vec.fit_transform(train_df['param123'].values.tolist() + test_df['param123'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['param123'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['param123'].values.tolist())\n\n### SVD Components ###\nn_comp = 5\nsvd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\nsvd_obj.fit(full_tfidf)\ntrain_svd = pd.DataFrame(svd_obj.transform(train_tfidf))\ntest_svd = pd.DataFrame(svd_obj.transform(test_tfidf))\ntrain_svd.columns = ['svd_params_'+str(i+1) for i in range(n_comp)]\ntest_svd.columns = ['svd_params_'+str(i+1) for i in range(n_comp)]\ntrain_df = pd.concat([train_df, train_svd], axis=1)\ntest_df = pd.concat([test_df, test_svd], axis=1)\ndel full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44a4dfec9ac46ab8a366e77ad230b01e3e229927","collapsed":true},"cell_type":"code","source":"train = train_df\ntest = test_df\n\n# Label encode the categorical variables #\ncat_vars = [\"region\", \"city\", \"parent_category_name\", \"category_name\", \"user_type\", \"param_1\"]\nfor col in cat_vars:\n    lbl = preprocessing.LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))\n\ncols_to_drop = [\"item_id\", \"user_id\", \"title\", \"description\", \"activation_date\", \"image\", \"param_2\", \"param_3\"\n                , \"param123\", \"image_top_1_impute\", \"days_impute\"]\ntrain_X = train_df.drop(cols_to_drop + [\"deal_probability\"], axis=1)\ntest_X = test_df.drop(cols_to_drop, axis=1)\n\ntrain_y = train_df[\"deal_probability\"].values\ntest_id = test_df[\"item_id\"].values","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17ef5dc0db3ef85fdf2d906c7f4d46c4ca4ead49"},"cell_type":"code","source":"train_X.head()","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77c1749da1b867a06db437ddb79ab59c598706a2"},"cell_type":"code","source":"#split the train into development and validation sample. Take the last 100K rows as validation sample.\n# Splitting the data for model training#\ndev_X = train_X.iloc[:-100000,:]\nval_X = train_X.iloc[-100000:,:]\ndev_y = train_y[:-100000]\nval_y = train_y[-100000:]\nprint(dev_X.shape, val_X.shape, test_X.shape)","execution_count":11,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6d6cddeb77e5f3571a6bc156d5d7fd885903e6f","collapsed":true},"cell_type":"code","source":"#custom function to build the LightGBM model.\ndef run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\",\n        \"num_leaves\" : 1000,\n        \"learning_rate\" : 0.02,\n        \"bagging_fraction\" : 0.75,\n        \"feature_fraction\" : 0.6,\n        \"bagging_freq\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1,\n        \"max_depth\": 18,\n        \"min_child_samples\":100\n       # ,\"boosting\":\"rf\"\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    evals_result = {}\n    model = lgb.train(params, lgtrain, 2500, valid_sets=[lgval], early_stopping_rounds=50, verbose_eval=50, evals_result=evals_result)\n    \n    #model = lgb.cv(params, lgtrain, 1000, early_stopping_rounds=20, verbose_eval=20, stratified=False )\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, evals_result","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b0862faf1afadc32ad6bb5d7d616b11ae32dc5d","scrolled":true},"cell_type":"code","source":"# Training the model #\nimport lightgbm as lgb\npred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2751f8058a23655f15f081923db9b35307aecd3"},"cell_type":"code","source":"# Plot importance\nlgb.plot_importance(model, importance_type=\"split\", title=\"split\")\nplt.show()\n\nlgb.plot_importance(model, importance_type=\"gain\", title='gain')\nplt.show()\n\n# Importance values are also available in:\nprint(model.feature_importance(\"split\"))\nprint(model.feature_importance(\"gain\"))","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d94d8d9e870608e8b2f8fbb0cb1a304b89c3be0","collapsed":true},"cell_type":"code","source":"# Making a submission file #\npred_test[pred_test>1] = 1\npred_test[pred_test<0] = 0\nsub_df = pd.DataFrame({\"item_id\":test_id})\nsub_df[\"deal_probability\"] = pred_test\nsub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":51,"outputs":[]},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":false,"trusted":true,"_uuid":"483dd45bc9560f728bc2e0f7a74759c04d3c9999","collapsed":true},"cell_type":"code","source":"#print(os.listdir(\"../working\"))","execution_count":27,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}