{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import time\nnotebookstart= time.time()\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nprint(\"Data:\\n\",os.listdir(\"../input\"))","execution_count":1,"outputs":[{"output_type":"stream","text":"Data:\n ['test.csv', 'periods_test.csv', 'train_jpg.zip', 'test_active.csv', 'test_jpg.zip', 'train_active.csv', 'train.csv', 'sample_submission.csv', 'periods_train.csv']\n","name":"stdout"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Models Packages\nfrom sklearn import metrics\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn import feature_selection\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing\n\n# Gradient Boosting\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\nfrom sklearn.model_selection import KFold","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tf-Idf\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.pipeline import FeatureUnion\nfrom scipy.sparse import hstack, csr_matrix\nfrom nltk.corpus import stopwords ","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Viz\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport re\nimport string","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLDS = 5\nSEED = 42\nVALID = True\nclass SklearnWrapper(object):\n    def __init__(self, clf, seed=0, params=None, seed_bool = True):\n        if(seed_bool == True):\n            params['random_state'] = seed\n        self.clf = clf(**params)\n\n    def train(self, x_train, y_train):\n        self.clf.fit(x_train, y_train)\n\n    def predict(self, x):\n        return self.clf.predict(x)\n        \ndef get_oof(clf, x_train, y, x_test):\n    oof_train = np.zeros((ntrain,))\n    oof_test = np.zeros((ntest,))\n    oof_test_skf = np.empty((NFOLDS, ntest))\n\n    for i, (train_index, test_index) in enumerate(kf):\n        print('\\nFold {}'.format(i))\n        x_tr = x_train[train_index]\n        y_tr = y[train_index]\n        x_te = x_train[test_index]\n\n        clf.train(x_tr, y_tr)\n\n        oof_train[test_index] = clf.predict(x_te)\n        oof_test_skf[i, :] = clf.predict(x_test)\n\n    oof_test[:] = oof_test_skf.mean(axis=0)\n    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)\n    \ndef cleanName(text):\n    try:\n        textProc = text.lower()\n        # textProc = \" \".join(map(str.strip, re.split('(\\d+)',textProc)))\n        #regex = re.compile(u'[^[:alpha:]]')\n        #textProc = regex.sub(\" \", textProc)\n        textProc = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', textProc)\n        textProc = \" \".join(textProc.split())\n        return textProc\n    except: \n        return \"name error\"\n    \n    \ndef rmse(y, y0):\n    assert len(y) == len(y0)\n    return np.sqrt(np.mean(np.power((y - y0), 2)))\n","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"\\nData Load Stage\")\ntraining = pd.read_csv('../input/train.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\ntraindex = training.index\ntesting = pd.read_csv('../input/test.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\ntestdex = testing.index\n\nntrain = training.shape[0]\nntest = testing.shape[0]","execution_count":8,"outputs":[{"output_type":"stream","text":"\nData Load Stage\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLDS = 5\nSEED = 42\nVALID = True\nkf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)\n\ny = training.deal_probability.copy()\ntraining.drop(\"deal_probability\",axis=1, inplace=True)\nprint('Train shape: {} Rows, {} Columns'.format(*training.shape))\nprint('Test shape: {} Rows, {} Columns'.format(*testing.shape))\n\nprint(\"Combine Train and Test\")\ndf = pd.concat([training,testing],axis=0)\ndel training, testing\ngc.collect()\nprint('\\nAll Data shape: {} Rows, {} Columns'.format(*df.shape))\n\n\nprint(\"Feature Engineering\")\ndf[\"price\"] = np.log(df[\"price\"]+0.001)\ndf[\"price\"].fillna(df.price.mean(),inplace=True)\ndf[\"image_top_1\"].fillna(-999,inplace=True)\n\nprint(\"\\nCreate Time Variables\")\ndf[\"Weekday\"] = df['activation_date'].dt.weekday\n\n# Create Validation Index and Remove Dead Variables\ntraining_index = df.loc[df.activation_date<=pd.to_datetime('2017-04-07')].index\nvalidation_index = df.loc[df.activation_date>=pd.to_datetime('2017-04-08')].index\ndf.drop([\"activation_date\",\"image\"],axis=1,inplace=True)\n\nprint(\"\\nEncode Variables\")\ncategorical = [\"user_id\",\"region\",\"city\",\"parent_category_name\",\"category_name\",\"user_type\",\"image_top_1\",\"param_1\",\"param_2\",\"param_3\"]\nprint(\"Encoding :\",categorical)\n\n# Encoder:\nlbl = preprocessing.LabelEncoder()\nfor col in categorical:\n    df[col].fillna('Unknown')\n    df[col] = lbl.fit_transform(df[col].astype(str))\n    \nprint(\"\\nText Features\")\n\n# Feature Engineering \n\n# Meta Text Features\ntextfeats = [\"description\", \"title\"]\ndf['desc_punc'] = df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n\ndf['title'] = df['title'].apply(lambda x: cleanName(x))\ndf[\"description\"]   = df[\"description\"].apply(lambda x: cleanName(x))\n\nfor cols in textfeats:\n    df[cols] = df[cols].astype(str) \n    df[cols] = df[cols].astype(str).fillna('missing') # FILL NA\n    df[cols] = df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n    df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n    df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words\n    \n\nprint(\"\\n[TF-IDF] Term Frequency Inverse Document Frequency Stage\")\nrussian_stop = set(stopwords.words('russian'))\n\ntfidf_para = {\n    \"stop_words\": russian_stop,\n    \"analyzer\": 'word',\n    \"token_pattern\": r'\\w{1,}',\n    \"sublinear_tf\": True,\n    \"dtype\": np.float32,\n    \"norm\": 'l2',\n    #\"min_df\":5,\n    #\"max_df\":.9,\n    \"smooth_idf\":False\n}\n\n\ndef get_col(col_name): return lambda x: x[col_name]\n##I added to the max_features of the description. It did not change my score much but it may be worth investigating\nvectorizer = FeatureUnion([\n        ('description',TfidfVectorizer(\n            ngram_range=(1, 2),\n            max_features=17000,\n            **tfidf_para,\n            preprocessor=get_col('description'))),\n        ('title',CountVectorizer(\n            ngram_range=(1, 2),\n            stop_words = russian_stop,\n            #max_features=7000,\n            preprocessor=get_col('title')))\n    ])\n    \nstart_vect=time.time()\n\n#Fit my vectorizer on the entire dataset instead of the training rows\n#Score improved by .0001\nvectorizer.fit(df.to_dict('records'))\n\nready_df = vectorizer.transform(df.to_dict('records'))\ntfvocab = vectorizer.get_feature_names()\nprint(\"Vectorization Runtime: %0.2f Minutes\"%((time.time() - start_vect)/60))\n\n# Drop Text Cols\ntextfeats = [\"description\", \"title\"]\ndf.drop(textfeats, axis=1,inplace=True)\n\nfrom sklearn.metrics import mean_squared_error\nfrom math import sqrt\n\nridge_params = {'alpha':30.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,\n                'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}\n\n#Ridge oof method from Faron's kernel\n#I was using this to analyze my vectorization, but figured it would be interesting to add the results back into the dataset\n#It doesn't really add much to the score, but it does help lightgbm converge faster\nridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)\nridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])\n\nrms = sqrt(mean_squared_error(y, ridge_oof_train))\nprint('Ridge OOF RMSE: {}'.format(rms))\n\nprint(\"Modeling Stage\")\n\nridge_preds = np.concatenate([ridge_oof_train, ridge_oof_test])\n\ndf['ridge_preds'] = ridge_preds\n\n# Combine Dense Features with Sparse Text Bag of Words Features\nX = hstack([csr_matrix(df.loc[traindex,:].values),ready_df[0:traindex.shape[0]]]) # Sparse Matrix\ntesting = hstack([csr_matrix(df.loc[testdex,:].values),ready_df[traindex.shape[0]:]])\ntfvocab = df.columns.tolist() + tfvocab\nfor shape in [X,testing]:\n    print(\"{} Rows and {} Cols\".format(*shape.shape))\nprint(\"Feature Names Length: \",len(tfvocab))\ndel df\ngc.collect();\n\nprint(\"\\nModeling Stage\")\n\ndel ridge_preds,vectorizer,ready_df\ngc.collect();\n    \nprint(\"Light Gradient Boosting Regressor\")\nlgbm_params =  {\n    'task': 'train',\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': 'rmse',\n    # 'max_depth': 15,\n    'num_leaves': 270,\n    'feature_fraction': 0.5,\n    'bagging_fraction': 0.75,\n    'bagging_freq': 2,\n    'learning_rate': 0.016,\n    'verbose': 0\n}  \n\n\nif VALID == False:\n    X_train, X_valid, y_train, y_valid = train_test_split(\n        X, y, test_size=0.10, random_state=23)\n        \n    # LGBM Dataset Formatting \n    lgtrain = lgb.Dataset(X_train, y_train,\n                    feature_name=tfvocab,\n                    categorical_feature = categorical)\n    lgvalid = lgb.Dataset(X_valid, y_valid,\n                    feature_name=tfvocab,\n                    categorical_feature = categorical)\n    del X, X_train; gc.collect()\n    \n    # Go Go Go\n    lgb_clf = lgb.train(\n        lgbm_params,\n        lgtrain,\n        num_boost_round=20000,\n        valid_sets=[lgtrain, lgvalid],\n        valid_names=['train','valid'],\n        early_stopping_rounds=50,\n        verbose_eval=100\n    )\n    print(\"Model Evaluation Stage\")\n    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, lgb_clf.predict(X_valid))))\n    del X_valid ; gc.collect()\n\nelse:\n    # LGBM Dataset Formatting \n    lgtrain = lgb.Dataset(X, y,\n                    feature_name=tfvocab,\n                    categorical_feature = categorical)\n    del X; gc.collect()\n    # Go Go Go\n    lgb_clf = lgb.train(\n        lgbm_params,\n        lgtrain,\n        num_boost_round=1550,\n        verbose_eval=100\n    )\n\n\n\n# Feature Importance Plot\nf, ax = plt.subplots(figsize=[7,10])\nlgb.plot_importance(lgb_clf, max_num_features=50, ax=ax)\nplt.title(\"Light GBM Feature Importance\")\nplt.savefig('feature_import.png')\n\nprint(\"Model Evaluation Stage\")\nlgpred = lgb_clf.predict(testing) \n\n#Mixing lightgbm with ridge. I haven't really tested if this improves the score or not\n#blend = 0.95*lgpred + 0.05*ridge_oof_test[:,0]\nlgsub = pd.DataFrame(lgpred,columns=[\"deal_probability\"],index=testdex)\nlgsub['deal_probability'].clip(0.0, 1.0, inplace=True) # Between 0 and 1\nlgsub.to_csv(\"lgsub.csv\",index=True,header=True)\n#print(\"Model Runtime: %0.2f Minutes\"%((time.time() - modelstart)/60))\nprint(\"Notebook Runtime: %0.2f Minutes\"%((time.time() - notebookstart)/60))\n","execution_count":10,"outputs":[{"output_type":"error","ename":"TypeError","evalue":"__init__() got an unexpected keyword argument 'n_folds'","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-10-7ec2e7caf649>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mSEED\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m42\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mVALID\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mkf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mKFold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mntrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_folds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNFOLDS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSEED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeal_probability\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'n_folds'"]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}