{"cells":[{"metadata":{"_uuid":"b5f9408c4daf6db02fc1ce9eabac9d0c61e8ed25"},"cell_type":"markdown","source":"### Simple text analytics with tf-idf \n\n<br> In this notebook we focus on the tf-idf count based text processing method to predict deal probability of a product being sold on the Avito Website.\n<br> Apart from text analytics, the notebook also illustrates use of aggregating behavior on historical data, out-of-fold predictions for a two-step modeling and simple text cleaning strategies"},{"metadata":{"trusted":true,"_uuid":"6a5f1e4df767ba6b821e373c98d807e17dfbf222"},"cell_type":"code","source":"from IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { width:100% !important; }</style>\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef1eb5cdebdd8b6c768505e86ce5bd4e57f1e0ca"},"cell_type":"code","source":"import time\nnotebookstart= time.time()\nfrom io import StringIO\n\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nimport random\nrandom.seed(2018)\nprint(\"Data:\\n\",os.listdir(\"../input\"))\n\n# Models Packages\nfrom sklearn import metrics\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn import feature_selection\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing\n\n# Gradient Boosting\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\nfrom sklearn.cross_validation import KFold\n\n# Tf-Idf\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.pipeline import FeatureUnion\nfrom scipy.sparse import hstack, csr_matrix\nfrom nltk.corpus import stopwords \n\n# Viz\n#import seaborn as sns\nimport matplotlib.pyplot as plt\nimport re\nimport string\n\n#from gby_functions. import *\n\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"518bac844a52d6a178f93674e6d4758cc10b5422"},"cell_type":"code","source":"from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport plotly as py\nimport plotly.graph_objs as go\nimport plotly.figure_factory as ff\n\n\ninit_notebook_mode(connected=True) #do not miss this line\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc6eee3651d08d1d4e7f71a453c24501a2873393"},"cell_type":"code","source":"NFOLDS = 5\nSEED = 2\nVALID = False\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4156f45eeab488b66270e8c730fc1436c572ef2c"},"cell_type":"markdown","source":"#### Helper functions, class instances and methods"},{"metadata":{"trusted":true,"_uuid":"7f0743c29a1300f7bbc75b0df85f202f5a019bff"},"cell_type":"code","source":"class SklearnWrapper(object):\n    def __init__(self, clf, seed=0, params=None, seed_bool = True):\n        if(seed_bool == True):\n            params['random_state'] = seed\n        self.clf = clf(**params)\n\n    def train(self, x_train, y_train):\n        self.clf.fit(x_train, y_train)\n\n    def predict(self, x):\n        return self.clf.predict(x)\n        \ndef get_oof(clf, x_train, y, x_test):\n    oof_train = np.zeros((ntrain,))\n    oof_test = np.zeros((ntest,))\n    oof_test_skf = np.empty((NFOLDS, ntest))\n\n    for i, (train_index, test_index) in enumerate(kf):\n        print('\\nFold {}'.format(i))\n        x_tr = x_train[train_index]\n        y_tr = y[train_index]\n        x_te = x_train[test_index]\n\n        clf.train(x_tr, y_tr)\n\n        oof_train[test_index] = clf.predict(x_te)\n        oof_test_skf[i, :] = clf.predict(x_test)\n\n    oof_test[:] = oof_test_skf.mean(axis=0)\n    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)\n    \ndef cleanName(text):\n    try:\n        textProc = text.lower()\n        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)\n        textProc = \" \".join(textProc.split())\n        return textProc\n    except: \n        return \"name error\"\n    \n    \ndef rmse(y, y0):\n    assert len(y) == len(y0)\n    return np.sqrt(np.mean(np.power((y - y0), 2)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82e93013ffa0451b25d203275e7a0d882e36bcef"},"cell_type":"markdown","source":"#### Data Load Stage\n"},{"metadata":{"trusted":true,"_uuid":"8677d0f7e8aa653af287fd465459c3ee3ebb82b8"},"cell_type":"code","source":"training = pd.read_csv('../input/avito-demand-prediction/train.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\ntraindex = training.index\ntesting = pd.read_csv('../input/avito-demand-prediction/test.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\ntestdex = testing.index\nprint(\"\\n Finished Data Load Stage\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4912bb6facbc6a331a6f4efce151a3e77b5aaa3"},"cell_type":"code","source":"training.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27d52041e6ab8800604f75d3fefba42ca03fee8e"},"cell_type":"code","source":"training.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8db159c8b719063fdfa158e23acc2fcf2b010582"},"cell_type":"code","source":"testing.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f9a1b9075daad0f213fe015c20c447ef864779a"},"cell_type":"code","source":"training.reset_index(inplace = True,drop = False)\ntesting.reset_index(inplace = True,drop = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ecd0b4449406553fd45dad0537933a0c685532d"},"cell_type":"code","source":"#### Quick data exploration","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7d5038a78b916e75e7f84458b90cdfdba98f9a7"},"cell_type":"code","source":"\nfrom io import StringIO\n\nrussian_en1 = StringIO(u\"\"\"\\\nparent_category_name,parent_category_name_en\nЛичные вещи,Personal belongings\nДля дома и дачи,For the home and garden\nБытовая электроника,Consumer electronics\nНедвижимость,Real estate\nХобби и отдых,Hobbies & leisure\nТранспорт,Transport\nУслуги,Services\nЖивотные,Animals\nДля бизнеса,For business\n\"\"\")\n\nrussian_en1_df = pd.read_csv(russian_en1)\ntraining = pd.merge(training, russian_en1_df, on=\"parent_category_name\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7669bd11231d14e244692fb07a7442b4ac8e429"},"cell_type":"code","source":"plt.figure(figsize=(15,5))\nsns.distplot(training[\"deal_probability\"].values, bins=120, color=\"#ff002e\")\nplt.xlabel('Deal Probability', fontsize=14);\nplt.title(\"Distribution of Deal Probability\", fontsize=14);\nplt.style.use('ggplot')\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78551d9b65b8d0a70ca8615adb9af068c8d37cc2"},"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.boxplot(y=\"parent_category_name_en\", x=\"deal_probability\", data=training)\nplt.xlabel('Deal probability', fontsize=12)\nplt.ylabel('Parent Category', fontsize=12)\nplt.title(\"Deal probability by Parent Category\")\nplt.xticks(rotation='vertical')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9262623048e9aeb37b44080d55d837b45e95395"},"cell_type":"code","source":"training.drop('parent_category_name_en',axis = 1,inplace = True)\ntraining.describe()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5be2a7e00b8a2e605dd8906b54b075706463a730"},"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.style.use('ggplot')\n\nplt.scatter(training.loc[(training['param_1'] == 'Samsung')&(training['price']<1e5),'price'],\\\n            training.loc[(training['param_1'] == 'Samsung')&(training['price']<1e5),'deal_probability'])\nplt.xlabel('Price', fontsize=14);\nplt.ylabel('Deal Probability', fontsize=14);\nplt.title(\"Deal Probability sensitivity with Price for Samsung Phones\", fontsize=14);\n#plt.style.use('ggplot')\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ca0c5b394895dc6edbcd9a7f27f4b56e327a54e"},"cell_type":"code","source":"colorscale = ['#7A4579', '#D56073', 'rgb(236,158,105)', (1, 1, 0.2), (0.98,0.98,0.98)]\n\nfig = ff.create_2d_density(\n    training.loc[(training['param_1'] == 'Samsung')&(training['price']<1e5)&\\\n                 (training['deal_probability']>0),'price'], \n    training.loc[(training['param_1'] == 'Samsung')&(training['price']<1e5)&\\\n                 (training['deal_probability']>0),'deal_probability'], colorscale=colorscale,\n    hist_color='rgb(255, 237, 222)', point_size=2\n)\nfig.layout.update({'title': 'Deal Probability sensitivity with Price for Samsung Phones'})\nfig['layout']['yaxis1'].update(title='Deal Probability')\nfig['layout']['xaxis1'].update(title='Price of Phone')\n\n\npy.offline.iplot(fig, filename='histogram_subplots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4d83ef4024c40d74947a74a21e9dd978272625e"},"cell_type":"code","source":"training['dayofweek'] = training['activation_date'].dt.dayofweek","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e042ff85df34bd73e839ae62b6a3fdb226318da5"},"cell_type":"code","source":"dayofweek_count = training['dayofweek'].value_counts()\ntrace = go.Bar(\n    x=dayofweek_count.index,\n    y=dayofweek_count.values,\n    orientation = 'v',\n    marker=dict(\n        color=dayofweek_count.values,\n        colorscale = 'Jet',\n        reversescale = True\n    ),\n)\n\nlayout = dict(\n    title='Ad Counts by day of week',\n    yaxis=dict(\n        title='Counts'\n    ),\n    xaxis=dict(\n        title='Day of Week'\n    ),\n    height=400\n    )\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.offline.iplot(fig, filename=\"category name\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e43892d36c81abb22f3ca3539f91e9081803c5a4"},"cell_type":"code","source":"print(\"Combine Train and Test\")\nntrain = training.shape[0]\nntest = testing.shape[0]\n\nkf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)\n\ny = training.deal_probability.copy()\ntraining.drop(\"deal_probability\",axis=1, inplace=True)\nprint('Train shape: {} Rows, {} Columns'.format(*training.shape))\nprint('Test shape: {} Rows, {} Columns'.format(*testing.shape))\n\n\ndf = pd.concat([training,testing],axis=0)\ndel training, testing\ngc.collect()\nprint('\\nAll Data shape: {} Rows, {} Columns'.format(*df.shape))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4949a64ddfae743a34d7a886b48d0b343fb1eb73"},"cell_type":"markdown","source":"#### Feature Engineering\nAggreagte features"},{"metadata":{"trusted":true,"_uuid":"edc8c81e5db09168f57016342bf558932a64a468"},"cell_type":"code","source":"def do_count( df, group_cols, agg_name, agg_type='uint32', show_max=False, show_agg=True ):\n    if show_agg:\n        print( \"Aggregating by \", group_cols , '...' )\n    gp = df[group_cols][group_cols].groupby(group_cols).size().rename(agg_name).to_frame().reset_index()\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    #df[agg_name] = df[agg_name].astype(agg_type)\n    gc.collect()\n    return( df )\n\n\n# In[ ]:\n\n\ndef do_cumcount( df, group_cols, counted, agg_name, agg_type='uint32', show_max=False, show_agg=True ):\n    if show_agg:\n        print( \"Cumulative count by \", group_cols , '...' )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()\n    df[agg_name]=gp.values\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    #df[agg_name] = df[agg_name].astype(agg_type)\n    gc.collect()\n    return( df )\n\n\n# In[ ]:\n\n\ndef do_countuniq( df, group_cols, counted, agg_name, agg_type='uint32', show_max=False, show_agg=True ):\n    if show_agg:\n        print( \"Counting unqiue \", counted, \" by \", group_cols , '...' )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    #df[agg_name] = df[agg_name].astype(agg_type)\n    gc.collect()\n    return( df )\n\n\n# In[ ]:\n\n\ndef do_mean( df, group_cols, counted, agg_name, agg_type='float32', show_max=False, show_agg=True ):\n    if show_agg:\n        print( \"Calculating mean of \", counted, \" by \", group_cols , '...' )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    #df[agg_name] = df[agg_name].astype(agg_type)\n    gc.collect()\n    return( df )\n\n\n# In[ ]:\n\n\ndef do_median( df, group_cols, counted, agg_name, agg_type='float32', show_max=False, show_agg=True ):\n    if show_agg:\n        print( \"Calculating mean of \", counted, \" by \", group_cols , '...' )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].median().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    #df[agg_name] = df[agg_name].astype(agg_type)\n    gc.collect()\n    return( df )\n\n\ndef do_var( df, group_cols, counted, agg_name, agg_type='float32', show_max=False, show_agg=True ):\n    if show_agg:\n        print( \"Calculating variance of \", counted, \" by \", group_cols , '...' )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c6eec5346d2d2982847ccb466a324f599a9e9e0"},"cell_type":"code","source":"df[\"price\"] = np.log(df[\"price\"]+0.001)\ndf[\"Weekday\"] = df['activation_date'].dt.weekday\ndf[\"Day of Month\"] = df['activation_date'].dt.day\ndf[\"price\"].fillna(-999,inplace=True)\ndf[\"image_top_1\"].fillna(-999,inplace=True)\n\n\ndf = do_count( df, ['region', 'param_1'], 'region_param_1', show_max=True ); gc.collect()\ndf = do_count( df, ['region', 'Day of Month','param_1'], 'region_day_param_1', show_max=True ); gc.collect()\ndf = do_count( df, ['city', 'category_name'], 'city_category_name', show_max=True ); gc.collect()\ndf = do_count( df, ['param_1' ], 'param_1_count', show_max=True ); gc.collect()\n\ndf = do_countuniq( df, ['city'], 'param_1', 'X2', 'uint8', show_max=True ); gc.collect()\ndf = do_countuniq( df, ['city'], 'param_2', 'X3', 'uint8', show_max=True ); gc.collect()\n\n\n\ndf = do_mean( df, [ 'param_1'],'price', 'param_1_mean_price', show_max=True ); gc.collect()\ndf = do_mean( df, [ 'param_2'],'price', 'param_2_mean_price', show_max=True ); gc.collect()\n\n\n\ndf = do_var( df, [ 'param_1'],'price', 'param_1_var_price', show_max=True ); gc.collect()\ndf = do_var( df, [ 'param_2'],'price', 'param_2_var_price', show_max=True ); gc.collect()\n\n\ndf = do_median( df, [ 'param_1'],'item_seq_number', 'param_1_median_seq', show_max=True ); gc.collect()\ndf = do_median( df, [ 'category_name'],'item_seq_number', 'category_median_seq', show_max=True ); gc.collect()\ndf = do_median( df, [ 'param_2'],'item_seq_number', 'param_2_median_seq', show_max=True ); gc.collect()\n\ndf[\"param_1_mean_price\"].fillna(-999,inplace=True)\ndf[\"param_2_mean_price\"].fillna(-999,inplace=True)\ndf[\"region_param_1\"].fillna(-999,inplace=True)\ndf[\"region_day_param_1\"].fillna(-999,inplace=True)\ndf[\"param_1_count\"].fillna(-999,inplace=True)\n\n\ndf['price_minus_mean'] = df['price'] - df['param_1_mean_price']\ndf['price2_minus_mean'] = df['price'] - df['param_2_mean_price']\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b181dcdd64472082dc8c1134ecf0f7e80283f441"},"cell_type":"markdown","source":"#### Text Features"},{"metadata":{"trusted":true,"_uuid":"cf6eea8926e8e934ce9374bf9b1b57300d224937"},"cell_type":"code","source":"df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0eb123e9d41b779d0cb9de22f2b6884f01319317"},"cell_type":"code","source":"\n# Meta Text Features\ntextfeats = [\"description\", \"title\"]\ndf['desc_punc'] = df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n\ndf['title'] = df['title'].apply(lambda x: cleanName(x))\ndf[\"description\"]   = df[\"description\"].apply(lambda x: cleanName(x))\n\nfor cols in textfeats:\n    df[cols] = df[cols].astype(str) \n    df[cols] = df[cols].astype(str).fillna('missing') # FILL NA\n    df[cols] = df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n    df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words\n    df[cols + '_num_letters'] = df[cols].apply(lambda comment: len(comment)) # Count number of Letters\n    df[cols + '_num_alphabets'] = df[cols].apply(lambda comment: (comment.count(r'[a-zA-Z]'))) # Count number of Alphabets\n    df[cols + '_num_alphanumeric'] = df[cols].apply(lambda comment: (comment.count(r'[A-Za-z0-9]'))) # Count number of AlphaNumeric\n    df[cols + '_num_digits'] = df[cols].apply(lambda comment: (comment.count('[0-9]'))) # Count number of Digits\n    \n# Extra Feature Engineering\ndf['title_desc_len_ratio'] = df['title_num_letters']/df['description_num_letters']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8870e7b4c547d4ea06a81785a2d0fa4a06109132"},"cell_type":"markdown","source":"#### Create Validation Index \n"},{"metadata":{"trusted":true,"_uuid":"1905f468df6b5c70fa65c583f26939464659ba15"},"cell_type":"code","source":"training_index = df.loc[df.activation_date<=pd.to_datetime('2017-04-07')].index\nvalidation_index = df.loc[df.activation_date>=pd.to_datetime('2017-04-08')].index\ndf.drop([\"activation_date\",\"image\"],axis=1,inplace=True)\n\nprint(\"\\nEncode Variables\")\ncategorical = [\"user_id\",\"region\",\"city\",\"parent_category_name\",\"category_name\",\"user_type\",\"image_top_1\",\"param_1\",\"param_2\",\"param_3\"]\nprint(\"Encoding :\",categorical)\n\n# Encoder:\nlbl = preprocessing.LabelEncoder()\nfor col in categorical:\n    df[col].fillna('Unknown')\n    df[col] = lbl.fit_transform(df[col].astype(str))\n    \ndf.set_index('item_id',inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29098850ecd91f376993c21b6fcb74a43b78a171"},"cell_type":"markdown","source":"#### TF-IDF Term Frequency Inverse Document Frequency\n"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"cea55bfe8ebd41ce38bedfdf13bec91b4cf2b269"},"cell_type":"code","source":"russian_stop = set(stopwords.words('russian'))\n\ntfidf_para = {\n    \"stop_words\": russian_stop,\n    \"analyzer\": 'word',\n    \"token_pattern\": r'\\w{1,}',\n    \"sublinear_tf\": True,\n    \"dtype\": np.float32,\n    \"norm\": 'l2',\n    #\"min_df\":5,\n    #\"max_df\":.9,\n    \"smooth_idf\":False\n}\n\n\ndef get_col(col_name): return lambda x: x[col_name]\n##I added to the max_features of the description. It did not change my score much but it may be worth investigating\nvectorizer = FeatureUnion([\n        ('description',TfidfVectorizer(\n            ngram_range=(1, 2),\n            max_features=1000,\n            **tfidf_para,\n            preprocessor=get_col('description'))),\n        ('title',CountVectorizer(\n            ngram_range=(1, 2),\n            stop_words = russian_stop,\n            max_features=100,\n            preprocessor=get_col('title')))\n    ])\n    \nstart_vect=time.time()\n\n#Fit my vectorizer on the entire dataset instead of the training rows\n#Score improved by .0001\nvectorizer.fit(df.to_dict('records'))\n\nready_df = vectorizer.transform(df.to_dict('records'))\ntfvocab = vectorizer.get_feature_names()\nprint(\"Vectorization Runtime: %0.2f Minutes\"%((time.time() - start_vect)/60))\n\n# Drop Text Cols\ntextfeats = [\"description\", \"title\",'Day of Month']\ndf.drop(textfeats, axis=1,inplace=True)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d76f5b85d94781105cb84a4d2428acd1d7c16bf4"},"cell_type":"markdown","source":"#### Adding out-of-predictions from Ridge Regression"},{"metadata":{"trusted":true,"_uuid":"00f40fff91e2e2dd0909fbc67ba8a314cfa4d9ad"},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nfrom math import sqrt\n\nridge_params = {'alpha':30.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,\n                'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}\n\n#Ridge oof method from Faron's kernel\n#I was using this to analyze my vectorization, but figured it would be interesting to add the results back into the dataset\n#It doesn't really add much to the score, but it does help lightgbm converge faster\nridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)\nridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])\n\nrms = sqrt(mean_squared_error(y, ridge_oof_train))\nprint('Ridge OOF RMSE: {}'.format(rms))\n\nridge_preds = np.concatenate([ridge_oof_train, ridge_oof_test])\n\ndf['ridge_preds'] = ridge_preds\n\n# Combine Dense Features with Sparse Text Bag of Words Features\nX = hstack([csr_matrix(df.loc[traindex,:].values),ready_df[0:traindex.shape[0]]]) # Sparse Matrix\ntesting = hstack([csr_matrix(df.loc[testdex,:].values),ready_df[traindex.shape[0]:]])\ntfvocab = df.columns.tolist() + tfvocab\nfor shape in [X,testing]:\n    print(\"{} Rows and {} Cols\".format(*shape.shape))\nprint(\"Feature Names Length: \",len(tfvocab))\ndel df\ngc.collect();\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22318cdcf1993a952b4336d2eaf4d0e2d0f0d412"},"cell_type":"markdown","source":"#### Modeling \n"},{"metadata":{"trusted":true,"_uuid":"5ad8810a721628b70c8e6ac82f7d98b513a452ca"},"cell_type":"code","source":"\ndel ridge_preds,vectorizer,ready_df\ngc.collect();\n    \nprint(\"Light Gradient Boosting Regressor\")\nlgbm_params =  {\n    'task': 'train',\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': 'rmse',\n    # 'max_depth': 15,\n    'num_leaves': 30,\n    'feature_fraction': 0.5,\n    'bagging_fraction': 0.75,\n    'bagging_freq': 2,\n    'learning_rate': 0.3,\n    'verbose': 0\n}  \n\n\n\n# LGBM Dataset Formatting \nimport datetime\nprint(datetime.datetime.now())\nlgtrain_cv = lgb.Dataset(X, y,\n                    feature_name=tfvocab,\n                    categorical_feature = categorical)\n    \n   \ncv_results = lgb.cv(\n            lgbm_params,\n            lgtrain_cv,\n            num_boost_round=600,\n            nfold=5,\n            early_stopping_rounds=50,\n            verbose_eval=100,\n            stratified=False\n            )\n\nprint(\"LGB CV modeling complete Runtime: %0.2f Minutes\"%((time.time() - start_vect)/60))\nprint(datetime.datetime.now())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f75fc6060e78490059d1fb05f0365afcde4e4bbe"},"cell_type":"code","source":"print(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"739348f08eb20f5540b8265b88cc907e873a12c5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}