{"cells":[{"metadata":{"_uuid":"6c15c7b0227a59c8d059269fdf62fb4e9bd606c1"},"cell_type":"markdown","source":"This script is a fork of [some great initial work by Samrat](https://www.kaggle.com/samratp/wordbatch-ridge-fm-frtl-target-encoding-lgbm), orginally based on [my solution that got 18th place in the Mercari competition](https://www.kaggle.com/peterhurford/lgb-and-fm-18th-place-0-40604/code).\n\nFrom his original solution, I...\n\n* Fixed the normalization to avoid removing Russian characters.\n* Removed some variables I thought were overfitting and not providing value.\n* Reduced the dimensions of the sparse matricies for easier fitting.\n* Added standard scaling for numeric data.\n* Simplified the logic for employing one hot encoding.\n* Tuned the individual models a little bit, but not much. (More tuning is left as an exercise for the reader.)\n\nOther things I'd suggest if improving this script:\n* Integrate data from train_active / test_active / periods_train / periods_test\n* Integrate image data\n* Test built-in LGB encoding for categoricals against TargetEncoding (or try both together)\n\nHope this helps. Good luck out there."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%%time\n# Based on this wonderful notebook by Peter - https://www.kaggle.com/peterhurford/lgb-and-fm-18th-place-0-40604\nimport time\nstart_time = time.time()\n\n# Use SUBMIT_MODE = False to tune your script!\n# Use SUBMIT_MODE = True to generate a submission for Kaggle.\nSUBMIT_MODE = True\n\nimport pandas as pd\nimport numpy as np\nimport time\nimport gc\nimport string\nimport re\n\nfrom nltk.corpus import stopwords\n\nfrom scipy.sparse import csr_matrix, hstack\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_selection.univariate_selection import SelectKBest, f_regression\nfrom sklearn.preprocessing import LabelBinarizer\n\nimport wordbatch\nfrom wordbatch.extractors import WordBag\nfrom wordbatch.models import FM_FTRL\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import Ridge\nfrom sklearn.naive_bayes import MultinomialNB\nimport lightgbm as lgb\n\n# Viz\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\ndef rmse(predicted, actual):\n    return np.sqrt(((predicted - actual) ** 2).mean())","execution_count":1,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e71d5b350e5588ffe8e808ce9f5ad3f9b3c6c031"},"cell_type":"code","source":"%%time\n\nclass TargetEncoder:\n    # Adapted from https://www.kaggle.com/ogrellier/python-target-encoding-for-categorical-features\n    def __repr__(self):\n        return 'TargetEncoder'\n\n    def __init__(self, cols, smoothing=1, min_samples_leaf=1, noise_level=0, keep_original=False):\n        self.cols = cols\n        self.smoothing = smoothing\n        self.min_samples_leaf = min_samples_leaf\n        self.noise_level = noise_level\n        self.keep_original = keep_original\n\n    @staticmethod\n    def add_noise(series, noise_level):\n        return series * (1 + noise_level * np.random.randn(len(series)))\n\n    def encode(self, train, test, target):\n        for col in self.cols:\n            if self.keep_original:\n                train[col + '_te'], test[col + '_te'] = self.encode_column(train[col], test[col], target)\n            else:\n                train[col], test[col] = self.encode_column(train[col], test[col], target)\n        return train, test\n\n    def encode_column(self, trn_series, tst_series, target):\n        temp = pd.concat([trn_series, target], axis=1)\n        # Compute target mean\n        averages = temp.groupby(by=trn_series.name)[target.name].agg([\"mean\", \"count\"])\n        # Compute smoothing\n        smoothing = 1 / (1 + np.exp(-(averages[\"count\"] - self.min_samples_leaf) / self.smoothing))\n        # Apply average function to all target data\n        prior = target.mean()\n        # The bigger the count the less full_avg is taken into account\n        averages[target.name] = prior * (1 - smoothing) + averages[\"mean\"] * smoothing\n        averages.drop(['mean', 'count'], axis=1, inplace=True)\n        # Apply averages to trn and tst series\n        ft_trn_series = pd.merge(\n            trn_series.to_frame(trn_series.name),\n            averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),\n            on=trn_series.name,\n            how='left')['average'].rename(trn_series.name + '_mean').fillna(prior)\n        # pd.merge does not keep the index so restore it\n        ft_trn_series.index = trn_series.index\n        ft_tst_series = pd.merge(\n            tst_series.to_frame(tst_series.name),\n            averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),\n            on=tst_series.name,\n            how='left')['average'].rename(trn_series.name + '_mean').fillna(prior)\n        # pd.merge does not keep the index so restore it\n        ft_tst_series.index = tst_series.index\n        return self.add_noise(ft_trn_series, self.noise_level), self.add_noise(ft_tst_series, self.noise_level)","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7c434d3aa7298a1997764b5b7e62cc210d2db9f"},"cell_type":"code","source":"%%time\n# Define helpers for text normalization\nstopwords = {x: 1 for x in stopwords.words('russian')}\nnon_alphanums = re.compile(u'[^A-Za-z0-9]+')\nnon_alphanumpunct = re.compile(u'[^A-Za-z0-9\\.?!,; \\(\\)\\[\\]\\'\\\"\\$]+')\nRE_PUNCTUATION = '|'.join([re.escape(x) for x in string.punctuation])","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e44441d000cc25d735993aa1f8820121086f6076"},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\ntest = pd.read_csv('../input/test.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\nprint('[{}] Finished load data'.format(time.time() - start_time))","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d697e7b99a93e4d2e20f0700496cadd735932b31"},"cell_type":"code","source":"%%time\nimport string\n\ndef normalize_text(text):\n    text = text.lower().strip()\n    for s in string.punctuation:\n        text = text.replace(s, ' ')\n    text = text.strip().split(' ')\n    return u' '.join(x for x in text if len(x) > 1 and x not in stopwords)\n\nprint(train.description[0])\nprint(normalize_text(train.description[0]))","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9cf6215b89d6f3cc45bf10b6763ee904512b3db"},"cell_type":"code","source":"%%time\ntrain['is_train'] = 1\ntest['is_train'] = 0\nprint('[{}] Compiled train / test'.format(time.time() - start_time))\nprint('Train shape: ', train.shape)\nprint('Test shape: ', test.shape)\n\ny = train.deal_probability.copy()\nnrow_train = train.shape[0]\n\nmerge = pd.concat([train, test])\nsubmission = pd.DataFrame(test.index)\nprint('[{}] Compiled merge'.format(time.time() - start_time))\nprint('Merge shape: ', merge.shape)\n\ndel train\ndel test\ngc.collect()\nprint('[{}] Garbage collection'.format(time.time() - start_time))","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cbd5e338fce59e69bd4adaabc6e56b12fa074fa"},"cell_type":"code","source":"%%time\nprint(\"Feature Engineering - Part 1\")\nmerge[\"price\"] = np.log(merge[\"price\"]+0.001)\nmerge[\"price\"].fillna(-999,inplace=True)\nmerge[\"image_top_1\"].fillna(-999,inplace=True)\n\nprint(\"\\nCreate Time Variables\")\nmerge[\"activation_weekday\"] = merge['activation_date'].dt.weekday\nprint(merge.head(5))\ngc.collect()","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8805b5699632edbc608c20237d815c0a41e397ae"},"cell_type":"code","source":"%%time\n# Create Validation Index and Remove Dead Variables\ntraining_index = merge.loc[merge.activation_date<=pd.to_datetime('2017-04-07')].index\nvalidation_index = merge.loc[merge.activation_date>=pd.to_datetime('2017-04-08')].index\nmerge.drop([\"activation_date\",\"image\"],axis=1,inplace=True)\n\n#Drop user_id\nmerge.drop([\"user_id\"], axis=1,inplace=True)","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"721f909402419df7f77280f8977eb5a8dfa82695"},"cell_type":"code","source":"%%time\n\n# Meta Text Features\nprint(\"\\nText Features\")\ntextfeats = [\"description\", \"title\"]\n\nfor cols in textfeats:\n    merge[cols] = merge[cols].astype(str) \n    merge[cols] = merge[cols].astype(str).fillna('missing') # FILL NA\n    merge[cols] = merge[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n    merge[cols + '_num_stopwords'] = merge[cols].apply(lambda x: len([w for w in x.split() if w in stopwords])) # Count number of Stopwords\n    merge[cols + '_num_punctuations'] = merge[cols].apply(lambda comment: (comment.count(RE_PUNCTUATION))) # Count number of Punctuations\n    merge[cols + '_num_alphabets'] = merge[cols].apply(lambda comment: len([c for c in comment if c.isupper()])) # Count number of Alphabets\n    merge[cols + '_num_digits'] = merge[cols].apply(lambda comment: (comment.count('[0-9]'))) # Count number of Digits\n    merge[cols + '_num_letters'] = merge[cols].apply(lambda comment: len(comment)) # Count number of Letters\n    merge[cols + '_num_words'] = merge[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n    merge[cols + '_num_unique_words'] = merge[cols].apply(lambda comment: len(set(w for w in comment.split())))\n    merge[cols + '_words_vs_unique'] = merge[cols+'_num_unique_words'] / merge[cols+'_num_words'] # Count Unique Words\n    merge[cols + '_letters_per_word'] = merge[cols+'_num_letters'] / merge[cols+'_num_words'] # Letters per Word\n    merge[cols + '_punctuations_by_letters'] = merge[cols+'_num_punctuations'] / merge[cols+'_num_letters'] # Punctuations by Letters\n    merge[cols + '_punctuations_by_words'] = merge[cols+'_num_punctuations'] / merge[cols+'_num_words'] # Punctuations by Words\n    merge[cols + '_digits_by_letters'] = merge[cols+'_num_digits'] / merge[cols+'_num_letters'] # Digits by Letters\n    merge[cols + '_alphabets_by_letters'] = merge[cols+'_num_alphabets'] / merge[cols+'_num_letters'] # Alphabets by Letters\n    merge[cols + '_stopwords_by_words'] = merge[cols+'_num_stopwords'] / merge[cols+'_num_words'] # Stopwords by Letters\n    merge[cols + '_mean'] = merge[cols].apply(lambda x: 0 if len(x) == 0 else float(len(x.split())) / len(x)) * 10 # Mean\n\n# Extra Feature Engineering\nmerge['title_desc_len_ratio'] = merge['title_num_letters']/merge['description_num_letters']","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a9e9a7cf048512afc25c2bd9f46b458802be2fd"},"cell_type":"code","source":"merge.head()","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"e576f4315a2f625cbf6c87d749323561bb1ac54f"},"cell_type":"code","source":"%%time\ndf_test = merge.loc[merge['is_train'] == 0]\ndf_train = merge.loc[merge['is_train'] == 1]\ndel merge\ngc.collect()\ndf_test = df_test.drop(['is_train'], axis=1)\ndf_train = df_train.drop(['is_train'], axis=1)\n\nprint(df_train.shape)\nprint(y.shape)\n\nif SUBMIT_MODE:\n    y_train = y\n    del y\n    gc.collect()\nelse:\n    df_train, df_test, y_train, y_test = train_test_split(df_train, y, test_size=0.2, random_state=144)\n\nprint('[{}] Splitting completed.'.format(time.time() - start_time))","execution_count":11,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d894074de64d50c8a7042aa45d92cf6b0cc13619","scrolled":true},"cell_type":"code","source":"%%time\nwb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {\"hash_ngrams\": 2,\n                                                              \"hash_ngrams_weights\": [1.5, 1.0],\n                                                              \"hash_size\": 2 ** 29,\n                                                              \"norm\": None,\n                                                              \"tf\": 'binary',\n                                                              \"idf\": None,\n                                                              }), procs=8)\nwb.dictionary_freeze = True\nX_name_train = wb.fit_transform(df_train['title'])\nprint(X_name_train.shape)\nX_name_test = wb.transform(df_test['title'])\nprint(X_name_test.shape)\ndel(wb)\ngc.collect()","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"f3416c91d19190e4ab0a87db827462cd5275172c"},"cell_type":"code","source":"%%time\nmask = np.where(X_name_train.getnnz(axis=0) > 3)[0]\nX_name_train = X_name_train[:, mask]\nprint(X_name_train.shape)\nX_name_test = X_name_test[:, mask]\nprint(X_name_test.shape)\nprint('[{}] Vectorize `title` completed.'.format(time.time() - start_time))","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb16fbee2f71049f09df4166b0951a6fd4271a33"},"cell_type":"code","source":"%%time\nX_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_name_train, y_train,\n                                                              test_size = 0.5,\n                                                              shuffle = False)\nprint('[{}] Finished splitting'.format(time.time() - start_time))\n\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=42, alpha=5)\nmodel.fit(X_train_1, y_train_1)\nprint('[{}] Finished to train name ridge (1)'.format(time.time() - start_time))\nname_ridge_preds1 = model.predict(X_train_2)\nname_ridge_preds1f = model.predict(X_name_test)\nprint('[{}] Finished to predict name ridge (1)'.format(time.time() - start_time))\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=42, alpha=5)\nmodel.fit(X_train_2, y_train_2)\nprint('[{}] Finished to train name ridge (2)'.format(time.time() - start_time))\nname_ridge_preds2 = model.predict(X_train_1)\nname_ridge_preds2f = model.predict(X_name_test)\nprint('[{}] Finished to predict name ridge (2)'.format(time.time() - start_time))\nname_ridge_preds_oof = np.concatenate((name_ridge_preds2, name_ridge_preds1), axis=0)\nname_ridge_preds_test = (name_ridge_preds1f + name_ridge_preds2f) / 2.0\nprint('RMSLE OOF: {}'.format(rmse(name_ridge_preds_oof, y_train)))\nif not SUBMIT_MODE:\n    print('RMSLE TEST: {}'.format(rmse(name_ridge_preds_test, y_test)))\ngc.collect()","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50bcac30668a45e205ac164093740e6e5fbc19da"},"cell_type":"code","source":"%%time\nwb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {\"hash_ngrams\": 2,\n                                                              \"hash_ngrams_weights\": [1.0, 1.0],\n                                                              \"hash_size\": 2 ** 28,\n                                                              \"norm\": \"l2\",\n                                                              \"tf\": 1.0,\n                                                              \"idf\": None}), procs=8)\nwb.dictionary_freeze = True\nX_description_train = wb.fit_transform(df_train['description'].fillna(''))\nprint(X_description_train.shape)\nX_description_test = wb.transform(df_test['description'].fillna(''))\nprint(X_description_test.shape)\nprint('-')\ndel(wb)\ngc.collect()","execution_count":15,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"866fcb2d55a46899176d687ba57c708f03eb9dd6"},"cell_type":"code","source":"%%time\nmask = np.where(X_description_train.getnnz(axis=0) > 8)[0]\nX_description_train = X_description_train[:, mask]\nprint(X_description_train.shape)\nX_description_test = X_description_test[:, mask]\nprint(X_description_test.shape)\nprint('[{}] Vectorize `description` completed.'.format(time.time() - start_time))","execution_count":16,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4245d356226cb4023f7e22a20ef8cf62da3d445"},"cell_type":"code","source":"X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_description_train, y_train,\n                                                              test_size = 0.5,\n                                                              shuffle = False)\nprint('[{}] Finished splitting'.format(time.time() - start_time))\n\n# Ridge adapted from https://www.kaggle.com/object/more-effective-ridge-script?scriptVersionId=1851819\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_1, y_train_1)\nprint('[{}] Finished to train desc ridge (1)'.format(time.time() - start_time))\ndesc_ridge_preds1 = model.predict(X_train_2)\ndesc_ridge_preds1f = model.predict(X_description_test)\nprint('[{}] Finished to predict desc ridge (1)'.format(time.time() - start_time))\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_2, y_train_2)\nprint('[{}] Finished to train desc ridge (2)'.format(time.time() - start_time))\ndesc_ridge_preds2 = model.predict(X_train_1)\ndesc_ridge_preds2f = model.predict(X_description_test)\nprint('[{}] Finished to predict desc ridge (2)'.format(time.time() - start_time))\ndesc_ridge_preds_oof = np.concatenate((desc_ridge_preds2, desc_ridge_preds1), axis=0)\ndesc_ridge_preds_test = (desc_ridge_preds1f + desc_ridge_preds2f) / 2.0\nprint('RMSLE OOF: {}'.format(rmse(desc_ridge_preds_oof, y_train)))\nif not SUBMIT_MODE:\n    print('RMSLE TEST: {}'.format(rmse(desc_ridge_preds_test, y_test)))\ngc.collect()","execution_count":17,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cce7e78cac51c4a160e8912fa29106424d9e5dd3"},"cell_type":"code","source":"del X_train_1\ndel X_train_2\ndel y_train_1\ndel y_train_2\ndel name_ridge_preds1\ndel name_ridge_preds1f\ndel name_ridge_preds2\ndel name_ridge_preds2f\ndel desc_ridge_preds1\ndel desc_ridge_preds1f\ndel desc_ridge_preds2\ndel desc_ridge_preds2f\ngc.collect()\nprint('[{}] Finished garbage collection'.format(time.time() - start_time))","execution_count":18,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4141b8ee155db386e58b6a21fe13b2e0f742edd"},"cell_type":"code","source":"%%time\ndf_train.drop(['deal_probability', 'title', 'description'], axis=1, inplace=True)\ndf_test.drop(['title', 'description'], axis=1, inplace=True)\nprint('Remerged')\n\ndummy_cols = ['parent_category_name', 'category_name', 'user_type', 'image_top_1',\n            'region', 'city', 'param_1', 'param_2', 'param_3', 'activation_weekday']\nnumeric_cols = list(set(df_train.columns.values) - set(dummy_cols))\nprint(numeric_cols)","execution_count":19,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9b8b4c0a1e13c29762edf875cc9e1081040c32e"},"cell_type":"code","source":"%%time\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\n# https://stackoverflow.com/questions/37685412/avoid-scaling-binary-columns-in-sci-kit-learn-standsardscaler\nclass Scaler(BaseEstimator, TransformerMixin):\n    def __init__(self, columns, copy=True, with_mean=True, with_std=True):\n        self.scaler = StandardScaler(copy, with_mean, with_std)\n        self.columns = columns\n\n    def fit(self, X, y=None):\n        self.scaler.fit(X[self.columns], y)\n        return self\n\n    def transform(self, X, y=None, copy=None):\n        init_col_order = X.columns\n        X_scaled = pd.DataFrame(self.scaler.transform(X[self.columns]), columns=self.columns, index=X.index)\n        X_not_scaled = X[list(set(init_col_order) - set(self.columns))]\n        return pd.concat([X_not_scaled, X_scaled], axis=1)[init_col_order]\n\nprint('Scaler')\nscaler = Scaler(columns=numeric_cols)\ndf_train = scaler.fit_transform(df_train)\ndf_test = scaler.transform(df_test)","execution_count":20,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cee9177155b8e9862cee0d8ae84fd1b1d0903dd3"},"cell_type":"code","source":"df_train.head()","execution_count":21,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75e057ed9ab0e83cc628afbcc4b8df78ae699248"},"cell_type":"code","source":"%%time\nfrom sklearn.preprocessing import LabelBinarizer\n\nsparse_merge_train = hstack((X_name_train, X_description_train)).tocsr()\nsparse_merge_test = hstack((X_name_test, X_description_test)).tocsr()\nprint(sparse_merge_train.shape)\nfor col in dummy_cols:\n    print(col)\n    lb = LabelBinarizer(sparse_output=True)\n    sparse_merge_train = hstack((sparse_merge_train, lb.fit_transform(df_train[[col]].fillna('')))).tocsr()\n    print(sparse_merge_train.shape)\n    sparse_merge_test = hstack((sparse_merge_test, lb.transform(df_test[[col]].fillna('')))).tocsr()","execution_count":22,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8013f379f0013fb042abbc4502e152759da9f85f","scrolled":false},"cell_type":"code","source":"del X_description_test, X_name_test\ndel X_description_train, X_name_train\ndel lb, mask\ngc.collect()","execution_count":26,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7227c87ae5baa643dc828cf61f95d6e5772942b7"},"cell_type":"code","source":"%%time\nprint(\"\\n FM_FTRL Starting...........\")\nif SUBMIT_MODE:\n    iters = 1\nelse:\n    iters = 1\n    rounds = 1\n\nmodel = FM_FTRL(alpha=0.035, beta=0.001, L1=0.00001, L2=0.15, D=sparse_merge_train.shape[1],\n                alpha_fm=0.05, L2_fm=0.0, init_fm=0.01,\n                D_fm=100, e_noise=0, iters=iters, inv_link=\"identity\", threads=4)\n\nif SUBMIT_MODE:\n    model.fit(sparse_merge_train, y_train)\n    print('[{}] Train FM completed'.format(time.time() - start_time))\n    predsFM = model.predict(sparse_merge_test)\n    print('[{}] Predict FM completed'.format(time.time() - start_time))\nelse:\n    for i in range(rounds):\n        model.fit(sparse_merge_train, y_train)\n        predsFM = model.predict(sparse_merge_test)\n        print('[{}] Iteration {}/{} -- RMSLE: {}'.format(time.time() - start_time, i + 1, rounds, rmse(predsFM, y_test)))\n\ndel model\ngc.collect()\nif not SUBMIT_MODE:\n    print(\"FM_FTRL dev RMSLE:\", rmse(predsFM, y_test))\n# 0.23046 in 1/3","execution_count":27,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a804f62965a9be8c0a255aea4e66a50f94a6217","scrolled":true},"cell_type":"code","source":"X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(sparse_merge_train, y_train,\n                                                              test_size = 0.5,\n                                                              shuffle = False)\nprint('[{}] Finished splitting'.format(time.time() - start_time))\n\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_1, y_train_1)\nprint('[{}] Finished to train ridge (1)'.format(time.time() - start_time))\nridge_preds1 = model.predict(X_train_2)\nridge_preds1f = model.predict(sparse_merge_test)\nprint('[{}] Finished to predict ridge (1)'.format(time.time() - start_time))\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_2, y_train_2)\nprint('[{}] Finished to train ridge (2)'.format(time.time() - start_time))\nridge_preds2 = model.predict(X_train_1)\nridge_preds2f = model.predict(sparse_merge_test)\nprint('[{}] Finished to predict ridge (2)'.format(time.time() - start_time))\nridge_preds_oof = np.concatenate((ridge_preds2, ridge_preds1), axis=0)\nridge_preds_test = (ridge_preds1f + ridge_preds2f) / 2.0\nprint('RMSLE OOF: {}'.format(rmse(ridge_preds_oof, y_train)))\nif not SUBMIT_MODE:\n    print('RMSLE TEST: {}'.format(rmse(ridge_preds_test, y_test)))","execution_count":28,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0b88ab5b8d429061b3a35473bb516ee14fcd217"},"cell_type":"code","source":"fselect = SelectKBest(f_regression, k=48000)\ntrain_features = fselect.fit_transform(sparse_merge_train, y_train)\ntest_features = fselect.transform(sparse_merge_test)\nprint('[{}] Select best completed'.format(time.time() - start_time))\n\n\ndel sparse_merge_train\ndel sparse_merge_test\ngc.collect()\nprint('[{}] Garbage collection'.format(time.time() - start_time))","execution_count":29,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"895cbd082a3a921b329058ed3946ab4faced1047"},"cell_type":"code","source":"del ridge_preds1\ndel ridge_preds1f\ndel ridge_preds2\ndel ridge_preds2f\ndel X_train_1\ndel X_train_2\ndel y_train_1\ndel y_train_2\ndel model\ngc.collect()\nprint('[{}] Finished garbage collection'.format(time.time() - start_time))","execution_count":30,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf1199c0c94ec61102fc00313bb440ae66ce5cac"},"cell_type":"code","source":"df_train['ridge'] = ridge_preds_oof\ndf_train['name_ridge'] = name_ridge_preds_oof\ndf_train['desc_ridge'] = desc_ridge_preds_oof\ndf_test['ridge'] = ridge_preds_test\ndf_test['name_ridge'] = name_ridge_preds_test\ndf_test['desc_ridge'] = desc_ridge_preds_test\nprint('[{}] Finished adding submodels'.format(time.time() - start_time))","execution_count":31,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5dc36a55389e291fe1e6cdfcd67036e8ddb43c97"},"cell_type":"code","source":"f_cats = [\"region\",\"city\",\"parent_category_name\",\"category_name\",\"user_type\",\"image_top_1\"]\ntarget_encode = TargetEncoder(min_samples_leaf=100, smoothing=10, noise_level=0.01,\n                              keep_original=True, cols=f_cats)\ndf_train, df_test = target_encode.encode(df_train, df_test, y_train)\nprint('[{}] Finished target encoding'.format(time.time() - start_time))","execution_count":32,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95cc6f83bf7bc23703f02a1d4f971d67bee9eac1"},"cell_type":"code","source":"df_train.head()","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"320be0fbed65236eb10333fe8f2dad9eb33858a0"},"cell_type":"code","source":"del ridge_preds_oof\ndel ridge_preds_test\ngc.collect()\nprint('[{}] Finished garbage collection'.format(time.time() - start_time))","execution_count":34,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f69d17ec198a5998f8367f6836e260fcb727b5e0"},"cell_type":"code","source":"cols = ['region_te', 'city_te', 'parent_category_name_te', 'category_name_te',\n        'user_type_te', 'image_top_1_te', 'desc_ridge', 'name_ridge', 'ridge']\ntrain_dummies = csr_matrix(df_train[cols].values)\nprint('[{}] Finished dummyizing model 1/5'.format(time.time() - start_time))\ntest_dummies = csr_matrix(df_test[cols].values)\nprint('[{}] Finished dummyizing model 2/5'.format(time.time() - start_time))\ndel df_train\ndel df_test\ngc.collect()\nprint('[{}] Finished dummyizing model 3/5'.format(time.time() - start_time))\ntrain_features = hstack((train_features, train_dummies)).tocsr()\nprint('[{}] Finished dummyizing model 4/5'.format(time.time() - start_time))\ntest_features = hstack((test_features, test_dummies)).tocsr()\nprint('[{}] Finished dummyizing model 5/5'.format(time.time() - start_time))","execution_count":35,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97230d3025631ee32ea9afc5d217e4eb513edb07"},"cell_type":"code","source":"d_train = lgb.Dataset(train_features, label=y_train)\ndel train_features\ngc.collect()\nif SUBMIT_MODE:\n    watchlist = [d_train]\nelse:\n    d_valid = lgb.Dataset(test_features, label=y_test)\n    watchlist = [d_train, d_valid]\n\nparams = {\n    'learning_rate': 0.15,\n    'application': 'regression',\n    'max_depth': 13,\n    'num_leaves': 400,\n    'verbosity': -1,\n    'metric': 'RMSE',\n    'data_random_seed': 1,\n    'bagging_fraction': 0.8,\n    'feature_fraction': 0.6,\n    'nthread': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 10\n}\nprint('[{}] Finished compiling LGB'.format(time.time() - start_time))","execution_count":36,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0d871bd3d8f7c878f129e933df22283a1f8063b"},"cell_type":"code","source":"%%time\nmodelL = lgb.train(params,\n                  train_set=d_train,\n                  num_boost_round=400,\n                  valid_sets=watchlist,\n                  verbose_eval=50)\n\npredsL = modelL.predict(test_features)\npredsL[predsL < 0] = 0\npredsL[predsL > 1] = 1\n\nif not SUBMIT_MODE:\n    print(\"LGB RMSLE:\", rmse(predsL, y_test))","execution_count":37,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df52cd5d4ce36b79957de388fa63513875b858dd"},"cell_type":"code","source":"del d_train\ndel modelL\nif not SUBMIT_MODE:\n    del d_valid\ngc.collect()","execution_count":38,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47caac2aedea8c7b064c5917600de9389d1da582"},"cell_type":"code","source":"preds_final = predsFM * 0.1 + predsL * 0.9\nif not SUBMIT_MODE:\n    print('Final RMSE: ', rmse(preds_final, y_test))","execution_count":44,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6f870cf491f063c20e093f42c627bcb726d751bd"},"cell_type":"code","source":"if SUBMIT_MODE:\n    submission['deal_probability'] = preds_final\n    submission['deal_probability'] = submission['deal_probability'].clip(0.0, 1.0) # Between 0 and 1\n    submission.to_csv('lgb_and_fm_separate_train_test.csv', index=False)\n    print('[{}] Writing submission done'.format(time.time() - start_time))\n    print(submission.head(5))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}