{"cells":[{"metadata":{"_uuid":"eb59ba957119aaebc240c48f6d986c993302db74"},"cell_type":"markdown","source":"## Import all the requirements"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Models Packages\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn import feature_selection\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn import preprocessing, model_selection, metrics\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.pipeline import FeatureUnion\nfrom scipy.sparse import hstack, csr_matrix\nfrom nltk.corpus import stopwords \n\nimport re\nimport string\nimport os\nimport gc\nfrom datetime import date\n\n# Gradient Boosting\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\n\n# Viz\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\ncolor = sns.color_palette()\n%matplotlib inline\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0156787da5137b8d5c43919607492e296f494fef"},"cell_type":"code","source":"## Import necessary files","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\", parse_dates=[\"activation_date\"])\ntest_df = pd.read_csv(\"../input/test.csv\", parse_dates=[\"activation_date\"])\nprint(\"Train file rows and columns are : \", train_df.shape)\nprint(\"Test file rows and columns are : \", test_df.shape)\n\ntestdex = test_df.index\ntraindex = train_df.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c01e6f097c01dbf0db282b7db5103626aeeed03c"},"cell_type":"markdown","source":"## Create new variables and process the existing ones (train.cs)"},{"metadata":{"trusted":true,"_uuid":"a5fad3bb6c39309f7060158966a55135a93bde48","collapsed":true},"cell_type":"code","source":"# New variables #\ntrain_df[\"activation_weekday\"] = train_df[\"activation_date\"].dt.weekday\ntest_df[\"activation_weekday\"] = test_df[\"activation_date\"].dt.weekday\n\ntrain_df[\"activation_month\"] = train_df[\"activation_date\"].dt.month\ntest_df[\"activation_month\"] = test_df[\"activation_date\"].dt.month\n\ntrain_df[\"title_nwords\"] = train_df[\"title\"].apply(lambda x: len(x.split()))\ntest_df[\"title_nwords\"] = test_df[\"title\"].apply(lambda x: len(x.split()))\n\ntrain_df[\"description\"].fillna(\"NA\", inplace=True)\ntest_df[\"description\"].fillna(\"NA\", inplace=True)\ntrain_df[\"desc_nwords\"] = train_df[\"description\"].apply(lambda x: len(x.split()))\ntest_df[\"desc_nwords\"] = test_df[\"description\"].apply(lambda x: len(x.split()))\n\ntrain_df['param123'] = train_df['param_1'].fillna('') + \" \" + train_df['param_2'].fillna('') + \" \" + train_df['param_3'].fillna('') \ntest_df['param123'] = test_df['param_1'].fillna('') + \" \" + test_df['param_2'].fillna('') + \" \" + test_df['param_3'].fillna('') \n\n#Impute image_top_1\nenc = train_df.groupby('category_name')['image_top_1'].agg(lambda x:x.value_counts().index[0]).astype(np.float32).reset_index()\nenc.columns = ['category_name' ,'image_top_1_impute']\n#Cross Check values\n#enc = train_df.loc[train_df['category_name'] == 'Аквариум'].groupby('image_top_1').agg('count')\n#enc.sort_values(['item_id'], ascending=False).head(2)\n\ntrain_df = pd.merge(train_df, enc, how='left', on='category_name')\ntest_df = pd.merge(test_df, enc, how='left', on='category_name')\n\ndel enc\ngc.collect()\n\ntrain_df['image_top_1'].fillna(train_df['image_top_1_impute'], inplace=True)\ntest_df['image_top_1'].fillna(test_df['image_top_1_impute'], inplace=True)\n\n#Impute Days diff\n#enc = train_df.groupby('category_name')['days'].agg('median').astype(np.float32).reset_index()\n#enc.columns = ['category_name' ,'days_impute']\n#Cross Check values\n#enc = train_df.loc[train_df['category_name'] == 'Аквариум'].groupby('image_top_1').agg('count')\n#enc.sort_values(['item_id'], ascending=False).head(2)\n\n#train_df = pd.merge(train_df, enc, how='left', on='category_name')\n#test_df = pd.merge(test_df, enc, how='left', on='category_name')\n\n#train_df['days'].fillna(train_df['days_impute'], inplace=True)\n#test_df['days'].fillna(test_df['days_impute'], inplace=True)\n\n\n#Create image flag \ntest_df['image'] = test_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)\ntrain_df['image'] = train_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)\n\n# City names are duplicated across region, HT: Branden Murray \n#https://www.kaggle.com/c/avito-demand-prediction/discussion/55630#321751\ntrain_df['city'] = train_df['city'] + \"_\" + train_df['region']\ntest_df['city'] = test_df['city'] + \"_\" + test_df['region']\n\ntrain_df['price'].fillna(0, inplace=True)\ntest_df['price'].fillna(0, inplace=True)\ntrain_df['price'] = np.log1p(train_df['price'])\ntest_df['price'] = np.log1p(test_df['price'])\n\nprice_mean = train_df['price'].mean()\nprice_std = train_df['price'].std()\ntrain_df['price'] = (train_df['price'] - price_mean) / price_std\ntest_df['price'] = (test_df['price'] - price_mean) / price_std\ncat_cols = ['category_name', 'image_top_1']\nnum_cols = ['price', 'deal_probability']\n\nfor c in cat_cols:\n    for c2 in num_cols:\n        enc = train_df.groupby(c)[c2].agg(['median']).astype(np.float32).reset_index()\n        enc.columns = ['_'.join([str(c), str(c2), str(c3)]) if c3 != c else c for c3 in enc.columns]\n        train_df = pd.merge(train_df, enc, how='left', on=c)\n        test_df = pd.merge(test_df, enc, how='left', on=c)\n        \ndel cat_cols, num_cols, c, c2, enc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"62cea5b011adec5ebe2499916531471bc3731045"},"cell_type":"code","source":"def cleanName(text):\n    try:\n        textProc = text.lower()\n        # textProc = \" \".join(map(str.strip, re.split('(\\d+)',textProc)))\n        #regex = re.compile(u'[^[:alpha:]]')\n        #textProc = regex.sub(\" \", textProc)\n        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)\n        textProc = \" \".join(textProc.split())\n        return textProc\n    except: \n        return \"name error\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13f6a26acd1e7f4daa69513531af783fe35a85e6","collapsed":true},"cell_type":"code","source":"# Meta Text Features\ntextfeats = [\"description\", \"title\"]\ntrain_df['desc_punc'] = train_df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\ntest_df['desc_punc'] = test_df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n\ntrain_df['title'] = train_df['title'].apply(lambda x: cleanName(x))\ntrain_df[\"description\"]   = train_df[\"description\"].apply(lambda x: cleanName(x))\n\ntest_df['title'] = test_df['title'].apply(lambda x: cleanName(x))\ntest_df[\"description\"]   = test_df[\"description\"].apply(lambda x: cleanName(x))\n\nfor cols in textfeats:\n    train_df[cols] = train_df[cols].astype(str) \n    train_df[cols] = train_df[cols].astype(str).fillna('missing') # FILL NA\n    train_df[cols] = train_df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n    train_df[cols + '_num_words'] = train_df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n    train_df[cols + '_num_unique_words'] = train_df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n    train_df[cols + '_words_vs_unique'] = train_df[cols+'_num_unique_words'] / train_df[cols+'_num_words'] * 100 # Count Unique Words\n    train_df[cols + '_num_letters'] = train_df[cols].apply(lambda comment: len(comment)) # Count number of Letters\n\n    test_df[cols] = test_df[cols].astype(str) \n    test_df[cols] = test_df[cols].astype(str).fillna('missing') # FILL NA\n    test_df[cols] = test_df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n    test_df[cols + '_num_words'] = test_df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n    test_df[cols + '_num_unique_words'] = test_df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n    test_df[cols + '_words_vs_unique'] = test_df[cols+'_num_unique_words'] / test_df[cols+'_num_words'] * 100 # Count Unique Words\n    test_df[cols + '_num_letters'] = test_df[cols].apply(lambda comment: len(comment)) # Count number of Letters\n    \n# Extra Feature Engineering\ntrain_df['title_desc_len_ratio'] = train_df['title_num_letters']/train_df['description_num_letters']\ntest_df['title_desc_len_ratio'] = test_df['title_num_letters']/test_df['description_num_letters']\n\nprint(train_df.head())\nprint(test_df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44a4dfec9ac46ab8a366e77ad230b01e3e229927","collapsed":true},"cell_type":"code","source":"# Label encode the categorical variables #\ncat_vars = [\"region\", \"city\", \"parent_category_name\", \"category_name\", \"user_type\", \"param_1\", \"param123\"]\nfor col in cat_vars:\n    lbl = preprocessing.LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))\n\ncols_to_drop = [\"item_id\", \"user_id\", \"activation_date\", \"image\", \"param_2\", \"param_3\"\n                , \"image_top_1_impute\"] #,\"days_impute\"]\ntrain_X = train_df.drop(cols_to_drop + [\"deal_probability\"], axis=1)\ntest_X = test_df.drop(cols_to_drop, axis=1)\n\ntrain_y = train_df[\"deal_probability\"].values\ntest_id = test_df[\"item_id\"].values\n\ndel train_df, test_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"883100630fb85c52c7b941a3dddb81a9d85af68b","collapsed":true},"cell_type":"code","source":"russian_stop = set(stopwords.words('russian'))\n\ntfidf_para = {\n    \"stop_words\": russian_stop,\n    \"analyzer\": 'word',\n    \"token_pattern\": r'\\w{1,}',\n    \"sublinear_tf\": True,\n    \"dtype\": np.float32,\n    \"norm\": 'l2',\n    #\"min_df\":5,\n    #\"max_df\":.9,\n    \"smooth_idf\":False\n}\ndef get_col(col_name): return lambda x: x[col_name]\n##I added to the max_features of the description. It did not change my score much but it may be worth investigating\nvectorizer = FeatureUnion([\n        ('description',TfidfVectorizer(\n            ngram_range=(1, 2),\n            max_features=17000,\n            **tfidf_para,\n            preprocessor=get_col('description'))),\n        ('title',CountVectorizer(\n            ngram_range=(1, 2),\n            stop_words = russian_stop,\n            #max_features=7000,\n            preprocessor=get_col('title')))\n    ])\n\n#Fit my vectorizer on the entire dataset instead of the training rows\n#Score improved by .0001\ndf = pd.concat([train_X,test_X],axis=0)\nvectorizer.fit(df.to_dict('records'))\nready_df = vectorizer.transform(df.to_dict('records'))\ntfvocab = vectorizer.get_feature_names()\ndf.drop(textfeats, axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9192117ef6777a5be57923c365b80efb9d118e22","collapsed":true},"cell_type":"code","source":"sparse_m = csr_matrix(df.values)\ncomplete = hstack([sparse_m,ready_df])\ntfvocab = df.columns.tolist() + tfvocab\ncomplete = complete.tocsr()\n\nprint(complete.shape)\nprint(ready_df.shape)\n\ndel sparse_m, ready_df, df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"644032c3ef338cbc0db4922046d153345b34a3b2","collapsed":true},"cell_type":"code","source":"train_X = complete[0:traindex.shape[0],]\ntest_X = complete[traindex.shape[0]:,]\n\ndel complete \ngc.collect()\ntrain_X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77c1749da1b867a06db437ddb79ab59c598706a2","collapsed":true},"cell_type":"code","source":"#split the train into development and validation sample. Take the last 100K rows as validation sample.\n# Splitting the data for model training#\ndev_X = train_X[:-200000,:]\nval_X = train_X[-200000:,:]\ndev_y = train_y[:-200000]\nval_y = train_y[-200000:]\nprint(dev_X.shape, val_X.shape, test_X.shape)\n\ndel train_X, train_y\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6d6cddeb77e5f3571a6bc156d5d7fd885903e6f","collapsed":true},"cell_type":"code","source":"#custom function to build the LightGBM model.\ndef run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\",\n        \"num_leaves\" : 1000,\n        \"learning_rate\" : 0.01,\n        \"bagging_fraction\" : 0.75,\n        \"feature_fraction\" : 0.5,\n        \"bagging_freq\" : 2,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1,\n        \"max_depth\": 18,\n        \"min_child_samples\":100\n       # ,\"boosting\":\"rf\"\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y, feature_name=tfvocab)\n    lgval = lgb.Dataset(val_X, label=val_y, feature_name=tfvocab)\n    \n    del train_X, val_X; gc.collect()\n    \n    evals_result = {}\n    model = lgb.train(params, lgtrain, 10000, valid_sets=[lgval], early_stopping_rounds=50, verbose_eval=200, evals_result=evals_result)\n    \n    #model = lgb.cv(params, lgtrain, 1000, early_stopping_rounds=20, verbose_eval=20, stratified=False )\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    del lgtrain, lgval; gc.collect()\n    return pred_test_y, model, evals_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b0862faf1afadc32ad6bb5d7d616b11ae32dc5d","scrolled":false,"collapsed":true},"cell_type":"code","source":"# Training the model #\nimport lightgbm as lgb\npred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2751f8058a23655f15f081923db9b35307aecd3","collapsed":true},"cell_type":"code","source":"# Plot importance\nlgb.plot_importance(model, importance_type=\"split\", title=\"split\",max_num_features=50, figsize=(7,10))\nplt.show()\n\nlgb.plot_importance(model, importance_type=\"gain\", title='gain', max_num_features=50, figsize=(7,10))\nplt.show()\n\n# Importance values are also available in:\nprint(model.feature_importance(\"split\"))\nprint(model.feature_importance(\"gain\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d94d8d9e870608e8b2f8fbb0cb1a304b89c3be0","collapsed":true},"cell_type":"code","source":"# Making a submission file #\npred_test[pred_test>1] = 1\npred_test[pred_test<0] = 0\nsub_df = pd.DataFrame({\"item_id\":test_id})\nsub_df[\"deal_probability\"] = pred_test\nsub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}