{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"# Based on this wonderful notebook by Peter - https://www.kaggle.com/peterhurford/lgb-and-fm-18th-place-0-40604\nimport time\nstart_time = time.time()\n\nSUBMIT_MODE = True\n#SUBMIT_MODE = False\n\nimport pandas as pd\nimport numpy as np\nimport time\nimport gc\nimport string\nimport re\nimport random\nrandom.seed(2018)\n\nfrom nltk.corpus import stopwords\n\nfrom scipy.sparse import csr_matrix, hstack\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_selection.univariate_selection import SelectKBest, f_regression\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\n\nimport wordbatch\nfrom wordbatch.extractors import WordBag\nfrom wordbatch.models import FM_FTRL\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import Ridge\nfrom sklearn.naive_bayes import MultinomialNB\nimport lightgbm as lgb\n\n# Viz\nimport seaborn as sns\nimport matplotlib.pyplot as plt","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def rmse(predicted, actual):\n    return np.sqrt(((predicted - actual) ** 2).mean())","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e71d5b350e5588ffe8e808ce9f5ad3f9b3c6c031"},"cell_type":"code","source":"class TargetEncoder:\n    # Adapted from https://www.kaggle.com/ogrellier/python-target-encoding-for-categorical-features\n    def __repr__(self):\n        return 'TargetEncoder'\n\n    def __init__(self, cols, smoothing=1, min_samples_leaf=1, noise_level=0, keep_original=False):\n        self.cols = cols\n        self.smoothing = smoothing\n        self.min_samples_leaf = min_samples_leaf\n        self.noise_level = noise_level\n        self.keep_original = keep_original\n\n    @staticmethod\n    def add_noise(series, noise_level):\n        return series * (1 + noise_level * np.random.randn(len(series)))\n\n    def encode(self, train, test, target):\n        for col in self.cols:\n            if self.keep_original:\n                train[col + '_te'], test[col + '_te'] = self.encode_column(train[col], test[col], target)\n            else:\n                train[col], test[col] = self.encode_column(train[col], test[col], target)\n        return train, test\n\n    def encode_column(self, trn_series, tst_series, target):\n        temp = pd.concat([trn_series, target], axis=1)\n        # Compute target mean\n        averages = temp.groupby(by=trn_series.name)[target.name].agg([\"mean\", \"count\"])\n        # Compute smoothing\n        smoothing = 1 / (1 + np.exp(-(averages[\"count\"] - self.min_samples_leaf) / self.smoothing))\n        # Apply average function to all target data\n        prior = target.mean()\n        # The bigger the count the less full_avg is taken into account\n        averages[target.name] = prior * (1 - smoothing) + averages[\"mean\"] * smoothing\n        averages.drop(['mean', 'count'], axis=1, inplace=True)\n        # Apply averages to trn and tst series\n        ft_trn_series = pd.merge(\n            trn_series.to_frame(trn_series.name),\n            averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),\n            on=trn_series.name,\n            how='left')['average'].rename(trn_series.name + '_mean').fillna(prior)\n        # pd.merge does not keep the index so restore it\n        ft_trn_series.index = trn_series.index\n        ft_tst_series = pd.merge(\n            tst_series.to_frame(tst_series.name),\n            averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),\n            on=tst_series.name,\n            how='left')['average'].rename(trn_series.name + '_mean').fillna(prior)\n        # pd.merge does not keep the index so restore it\n        ft_tst_series.index = tst_series.index\n        return self.add_noise(ft_trn_series, self.noise_level), self.add_noise(ft_tst_series, self.noise_level)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7dfde0c9f80d13088e3a2eb7b45118217161d7bc"},"cell_type":"code","source":"def to_number(x):\n    try:\n        if not x.isdigit():\n            return 0\n        x = int(x)\n        if x > 100:\n            return 100\n        else:\n            return x\n    except:\n        return 0\n\ndef sum_numbers(desc):\n    if not isinstance(desc, str):\n        return 0\n    try:\n        return sum([to_number(s) for s in desc.split()])\n    except:\n        return 0","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7c434d3aa7298a1997764b5b7e62cc210d2db9f","collapsed":true},"cell_type":"code","source":"# Define helpers for text normalization\nstopwords_en = {x: 1 for x in stopwords.words('english')}\nstopwords = {x: 1 for x in stopwords.words('russian')}\nnon_alphanums = re.compile(u'[^A-Za-z0-9]+')\nnon_alphanumpunct = re.compile(u'[^A-Za-z0-9\\.?!,; \\(\\)\\[\\]\\'\\\"\\$]+')\nRE_PUNCTUATION = '|'.join([re.escape(x) for x in string.punctuation])\n\ndef normalize_text(text):\n    return u\" \".join(\n        [x for x in [y for y in non_alphanums.sub(' ', text).lower().strip().split(\" \")] \\\n         if len(x) > 1 and x not in stopwords])\n\ndef clean_name(x):\n    if len(x):\n        x = non_alphanums.sub(' ', x).split()\n        if len(x):\n            return x[0].lower()\n    return ''\n\n    \nprint('[{}] Finished defining stuff'.format(time.time() - start_time))","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93e008503829120eace03d346dd52eb00e4da8cb","collapsed":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\ntest = pd.read_csv('../input/test.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])\nprint('[{}] Finished load data'.format(time.time() - start_time))","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9cf6215b89d6f3cc45bf10b6763ee904512b3db","collapsed":true},"cell_type":"code","source":"train['is_train'] = 1\ntest['is_train'] = 0\nprint('[{}] Compiled train / test'.format(time.time() - start_time))\nprint('Train shape: ', train.shape)\nprint('Test shape: ', test.shape)\n\ny = train.deal_probability.copy()\nnrow_train = train.shape[0]\n\nmerge = pd.concat([train, test])\nsubmission = pd.DataFrame(test.index)\nprint('[{}] Compiled merge'.format(time.time() - start_time))\nprint('Merge shape: ', merge.shape)\n\ndel train\ndel test\ngc.collect()\nprint('[{}] Garbage collection'.format(time.time() - start_time))","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cbd5e338fce59e69bd4adaabc6e56b12fa074fa","collapsed":true},"cell_type":"code","source":"print(\"Feature Engineering - Part 1\")\nmerge[\"price\"] = np.log(merge[\"price\"]+0.001)\nmerge[\"price\"].fillna(-999,inplace=True)\nmerge[\"image_top_1\"].fillna(-999,inplace=True)\n\nprint(\"\\nCreate Time Variables\")\nmerge[\"activation_weekday\"] = merge['activation_date'].dt.weekday\nmerge[\"Weekd_of_Year\"] = merge['activation_date'].dt.week\nmerge[\"Day_of_Month\"] = merge['activation_date'].dt.day\n\nprint(merge.head(5))\ngc.collect()","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8805b5699632edbc608c20237d815c0a41e397ae","collapsed":true},"cell_type":"code","source":"# Create Validation Index and Remove Dead Variables\ntraining_index = merge.loc[merge.activation_date<=pd.to_datetime('2017-04-07')].index\nvalidation_index = merge.loc[merge.activation_date>=pd.to_datetime('2017-04-08')].index\nmerge.drop([\"activation_date\",\"image\"],axis=1,inplace=True)\n\nmerge['param_1_copy'] = merge['param_1']\n\n#Drop user_id\nmerge.drop([\"user_id\"], axis=1,inplace=True)","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"721f909402419df7f77280f8977eb5a8dfa82695","collapsed":true},"cell_type":"code","source":"# Meta Text Features\nprint(\"\\nText Features\")\ntextfeats = [\"description\", \"title\", \"param_1_copy\"]\n\nfor cols in textfeats:\n    merge[cols] = merge[cols].astype(str) \n    merge[cols] = merge[cols].astype(str).fillna('missing') # FILL NA\n    merge[cols] = merge[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n    merge[cols + '_num_stopwords'] = merge[cols].apply(lambda x: len([w for w in x.split() if w in stopwords])) # Count number of Stopwords\n    merge[cols + '_num_stopwords_en'] = merge[cols].apply(lambda x: len([w for w in x.split() if w in stopwords_en])) # Count number of Stopwords\n    merge[cols + '_num_punctuations'] = merge[cols].apply(lambda comment: (comment.count(RE_PUNCTUATION))) # Count number of Punctuations\n    merge[cols + '_num_alphabets'] = merge[cols].apply(lambda comment: (comment.count(r'[a-zA-Z]'))) # Count number of Alphabets\n    merge[cols + '_num_alphanumeric'] = merge[cols].apply(lambda comment: (comment.count(r'[A-Za-z0-9]'))) # Count number of AlphaNumeric\n    merge[cols + '_num_digits'] = merge[cols].apply(lambda comment: (comment.count('[0-9]'))) # Count number of Digits\n    merge[cols + '_num_letters'] = merge[cols].apply(lambda comment: len(comment)) # Count number of Letters\n    merge[cols + '_num_words'] = merge[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n    merge[cols + '_num_unique_words'] = merge[cols].apply(lambda comment: len(set(w for w in comment.split())))\n    merge[cols + '_words_vs_unique'] = merge[cols+'_num_unique_words'] / merge[cols+'_num_words'] # Count Unique Words\n    merge[cols + '_letters_per_word'] = merge[cols+'_num_letters'] / merge[cols+'_num_words'] # Letters per Word\n    merge[cols + '_punctuations_by_letters'] = merge[cols+'_num_punctuations'] / merge[cols+'_num_letters'] # Punctuations by Letters\n    merge[cols + '_punctuations_by_words'] = merge[cols+'_num_punctuations'] / merge[cols+'_num_words'] # Punctuations by Words\n    merge[cols + '_digits_by_letters'] = merge[cols+'_num_digits'] / merge[cols+'_num_letters'] # Digits by Letters\n    merge[cols + '_alphanumeric_by_letters'] = merge[cols+'_num_alphanumeric'] / merge[cols+'_num_letters'] # AlphaNumeric by Letters\n    merge[cols + '_alphabets_by_letters'] = merge[cols+'_num_alphabets'] / merge[cols+'_num_letters'] # Alphabets by Letters\n    merge[cols + '_stopwords_by_letters'] = merge[cols+'_num_stopwords'] / merge[cols+'_num_letters'] # Stopwords by Letters\n    merge[cols + '_stopwords_by_words'] = merge[cols+'_num_stopwords'] / merge[cols+'_num_words'] # Stopwords by Letters\n    merge[cols + '_stopwords_by_letters_en'] = merge[cols+'_num_stopwords_en'] / merge[cols+'_num_letters'] # Stopwords by Letters\n    merge[cols + '_stopwords_by_words_en'] = merge[cols+'_num_stopwords_en'] / merge[cols+'_num_words'] # Stopwords by Letters    \n    merge[cols + '_mean'] = merge[cols].apply(lambda x: 0 if len(x) == 0 else float(len(x.split())) / len(x)) * 10 # Mean\n    merge[cols + '_num_sum'] = merge[cols].apply(sum_numbers) \n\n# Extra Feature Engineering\nmerge['title_desc_len_ratio'] = merge['title_num_letters']/(merge['description_num_letters']+1)\nmerge['title_param1_len_ratio'] = merge['title_num_letters']/(merge['param_1_copy_num_letters']+1)\nmerge['param_1_copy_desc_len_ratio'] = merge['param_1_copy_num_letters']/(merge['description_num_letters']+1)\n\ngc.collect()","execution_count":11,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4bb109350b3c6c63550347572418481cdb3b0389"},"cell_type":"code","source":"cols = set(merge.columns.values)\ncat_cols = {\"region\",\"city\",\"parent_category_name\",\"category_name\",\"user_type\",\"image_top_1\"}\nbasic_cols = {\"region\",\"city\",\"parent_category_name\",\"category_name\",\"user_type\",\"image_top_1\",\n               \"description\",\"title\",\"param_1_copy\",\"param_1\",\"param_2\",\"param_3\", \"price\", \"item_seq_number\"}","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"e576f4315a2f625cbf6c87d749323561bb1ac54f","collapsed":true},"cell_type":"code","source":"df_test = merge.loc[merge['is_train'] == 0]\ndf_train = merge.loc[merge['is_train'] == 1]\ndel merge\ngc.collect()\ndf_test = df_test.drop(['is_train'], axis=1)\ndf_train = df_train.drop(['is_train'], axis=1)\n\nprint(df_train.shape)\nprint(y.shape)\n\nif SUBMIT_MODE:\n    y_train = y\n    del y\n    gc.collect()\nelse:\n    df_train, df_test, y_train, y_test = train_test_split(df_train, y, test_size=0.2, random_state=144)\n\nprint('[{}] Splitting completed.'.format(time.time() - start_time))","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d894074de64d50c8a7042aa45d92cf6b0cc13619","collapsed":true},"cell_type":"code","source":"wb = wordbatch.WordBatch(None, extractor=(WordBag, {\"hash_ngrams\": 2,\n                                                              \"hash_ngrams_weights\": [1.5, 1.0],\n                                                              \"hash_size\": 2 ** 29,\n                                                              \"norm\": None,\n                                                              \"tf\": 'binary',\n                                                              \"idf\": None,\n                                                              }), procs=8)\nwb.dictionary_freeze = True\nX_name_train = wb.fit_transform(df_train['title'])\nX_name_test = wb.transform(df_test['title'])\ndel(wb)\nmask = np.where(X_name_train.getnnz(axis=0) > 2)[0]\nX_name_train = X_name_train[:, mask]\nX_name_test = X_name_test[:, mask]\nprint('[{}] Vectorize `title` completed.'.format(time.time() - start_time))\ngc.collect()","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb16fbee2f71049f09df4166b0951a6fd4271a33","collapsed":true},"cell_type":"code","source":"X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_name_train, y_train,\n                                                              test_size = 0.5,\n                                                              shuffle = False)\nprint('[{}] Finished splitting'.format(time.time() - start_time))\n\n# Ridge adapted from https://www.kaggle.com/object/more-effective-ridge-script?scriptVersionId=1851819\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=42, alpha=30)\nmodel.fit(X_train_1, y_train_1)\nprint('[{}] Finished to train name ridge (1)'.format(time.time() - start_time))\nname_ridge_preds1 = model.predict(X_train_2)\nname_ridge_preds1f = model.predict(X_name_test)\nprint('[{}] Finished to predict name ridge (1)'.format(time.time() - start_time))\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=42, alpha=30)\nmodel.fit(X_train_2, y_train_2)\nprint('[{}] Finished to train name ridge (2)'.format(time.time() - start_time))\nname_ridge_preds2 = model.predict(X_train_1)\nname_ridge_preds2f = model.predict(X_name_test)\nprint('[{}] Finished to predict name ridge (2)'.format(time.time() - start_time))\nname_ridge_preds_oof = np.concatenate((name_ridge_preds2, name_ridge_preds1), axis=0)\nname_ridge_preds_test = (name_ridge_preds1f + name_ridge_preds2f) / 2.0\nprint('RMSLE OOF: {}'.format(rmse(name_ridge_preds_oof, y_train)))\nif not SUBMIT_MODE:\n    print('RMSLE TEST: {}'.format(rmse(name_ridge_preds_test, y_test)))\ngc.collect()","execution_count":15,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50bcac30668a45e205ac164093740e6e5fbc19da","collapsed":true},"cell_type":"code","source":"wb = wordbatch.WordBatch(None, extractor=(WordBag, {\"hash_ngrams\": 2,\n                                                              \"hash_ngrams_weights\": [1.0, 1.0],\n                                                              \"hash_size\": 2 ** 28,\n                                                              \"norm\": \"l2\",\n                                                              \"tf\": 1.0,\n                                                              \"idf\": None}), procs=8)\nwb.dictionary_freeze = True\nX_description_train = wb.fit_transform(df_train['description'])\nX_description_test = wb.transform(df_test['description'])\ndel(wb)\nmask = np.where(X_description_train.getnnz(axis=0) > 3)[0]\nX_description_train = X_description_train[:, mask]\nX_description_test = X_description_test[:, mask]\nprint('[{}] Vectorize `description` completed.'.format(time.time() - start_time))\ngc.collect()","execution_count":16,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4245d356226cb4023f7e22a20ef8cf62da3d445","collapsed":true},"cell_type":"code","source":"X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_description_train, y_train,\n                                                              test_size = 0.5,\n                                                              shuffle = False)\nprint('[{}] Finished splitting'.format(time.time() - start_time))\n\n# Ridge adapted from https://www.kaggle.com/object/more-effective-ridge-script?scriptVersionId=1851819\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_1, y_train_1)\nprint('[{}] Finished to train desc ridge (1)'.format(time.time() - start_time))\ndesc_ridge_preds1 = model.predict(X_train_2)\ndesc_ridge_preds1f = model.predict(X_description_test)\nprint('[{}] Finished to predict desc ridge (1)'.format(time.time() - start_time))\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_2, y_train_2)\nprint('[{}] Finished to train desc ridge (2)'.format(time.time() - start_time))\ndesc_ridge_preds2 = model.predict(X_train_1)\ndesc_ridge_preds2f = model.predict(X_description_test)\nprint('[{}] Finished to predict desc ridge (2)'.format(time.time() - start_time))\ndesc_ridge_preds_oof = np.concatenate((desc_ridge_preds2, desc_ridge_preds1), axis=0)\ndesc_ridge_preds_test = (desc_ridge_preds1f + desc_ridge_preds2f) / 2.0\nprint('RMSLE OOF: {}'.format(rmse(desc_ridge_preds_oof, y_train)))\nif not SUBMIT_MODE:\n    print('RMSLE TEST: {}'.format(rmse(desc_ridge_preds_test, y_test)))\ngc.collect()","execution_count":17,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd03fad2e040b615df6c5b2dc433528d04eff783","collapsed":true},"cell_type":"code","source":"wb = wordbatch.WordBatch(None, extractor=(WordBag, {\"hash_ngrams\": 2,\n                                                              \"hash_ngrams_weights\": [1.0, 1.0],\n                                                              \"hash_size\": 2 ** 28,\n                                                              \"norm\": \"l2\",\n                                                              \"tf\": 1.0,\n                                                              \"idf\": None}), procs=8)\nwb.dictionary_freeze = True\nX_param1_train = wb.fit_transform(df_train['param_1_copy'])\nX_param1_test = wb.transform(df_test['param_1_copy'])\ndel(wb)\nmask = np.where(X_param1_train.getnnz(axis=0) > 3)[0]\nX_param1_train = X_param1_train[:, mask]\nX_param1_test = X_param1_test[:, mask]\nprint('[{}] Vectorize `param_1_copy` completed.'.format(time.time() - start_time))\ngc.collect()","execution_count":18,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b4a5d4b7cd956166ead76d7f7d129ebf6e2bbbf","collapsed":true},"cell_type":"code","source":"X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_param1_train, y_train,\n                                                              test_size = 0.5,\n                                                              shuffle = False)\nprint('[{}] Finished splitting'.format(time.time() - start_time))\n\n# Ridge adapted from https://www.kaggle.com/object/more-effective-ridge-script?scriptVersionId=1851819\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_1, y_train_1)\nprint('[{}] Finished to train param1 ridge (1)'.format(time.time() - start_time))\nparam1_ridge_preds1 = model.predict(X_train_2)\nparam1_ridge_preds1f = model.predict(X_param1_test)\nprint('[{}] Finished to predict param1 ridge (1)'.format(time.time() - start_time))\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_2, y_train_2)\nprint('[{}] Finished to train param1 ridge (2)'.format(time.time() - start_time))\nparam1_ridge_preds2 = model.predict(X_train_1)\nparam1_ridge_preds2f = model.predict(X_param1_test)\nprint('[{}] Finished to predict param1 ridge (2)'.format(time.time() - start_time))\nparam1_ridge_preds_oof = np.concatenate((param1_ridge_preds2, param1_ridge_preds1), axis=0)\nparam1_ridge_preds_test = (param1_ridge_preds1f + param1_ridge_preds2f) / 2.0\nprint('RMSLE OOF: {}'.format(rmse(param1_ridge_preds_oof, y_train)))\nif not SUBMIT_MODE:\n    print('RMSLE TEST: {}'.format(rmse(param1_ridge_preds_test, y_test)))\ngc.collect()","execution_count":19,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cce7e78cac51c4a160e8912fa29106424d9e5dd3","collapsed":true},"cell_type":"code","source":"del X_train_1\ndel X_train_2\ndel y_train_1\ndel y_train_2\ndel name_ridge_preds1\ndel name_ridge_preds1f\ndel name_ridge_preds2\ndel name_ridge_preds2f\ndel desc_ridge_preds1\ndel desc_ridge_preds1f\ndel desc_ridge_preds2\ndel desc_ridge_preds2f\ndel param1_ridge_preds1\ndel param1_ridge_preds1f\ndel param1_ridge_preds2\ndel param1_ridge_preds2f\ngc.collect()\nprint('[{}] Finished garbage collection'.format(time.time() - start_time))","execution_count":20,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75e057ed9ab0e83cc628afbcc4b8df78ae699248","collapsed":true},"cell_type":"code","source":"lb = LabelBinarizer(sparse_output=True)\nX_parent_category_train = lb.fit_transform(df_train['parent_category_name'])\nX_parent_category_test = lb.transform(df_test['parent_category_name'])\nprint('[{}] Finished label binarize `parent_category_name`'.format(time.time() - start_time))","execution_count":21,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b4fb365c81609a14663e787dbae2d7873d7668e","collapsed":true},"cell_type":"code","source":"X_category_train = lb.fit_transform(df_train['category_name'])\nX_category_test = lb.transform(df_test['category_name'])\nprint('[{}] Finished label binarize `category_name`'.format(time.time() - start_time))","execution_count":22,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ab311da783f5c52bc944c83072ca4bc14790b77","collapsed":true},"cell_type":"code","source":"X_region_train = lb.fit_transform(df_train['region'])\nX_region_test = lb.transform(df_test['region'])\nprint('[{}] Finished label binarize `region`'.format(time.time() - start_time))","execution_count":23,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53a4eea4ce0d0bddb5248add86add16465fe6f8a","collapsed":true},"cell_type":"code","source":"X_city_train = lb.fit_transform(df_train['city'])\nX_city_test = lb.transform(df_test['city'])\nprint('[{}] Finished label binarize `city`'.format(time.time() - start_time))","execution_count":24,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f00e7545cde32ab1cb5e6fd86232cb1071df216","collapsed":true},"cell_type":"code","source":"X_imagetop1_train = lb.fit_transform(df_train['image_top_1'])\nX_imagetop1_test = lb.transform(df_test['image_top_1'])\nprint('[{}] Finished label binarize `image_top_1`'.format(time.time() - start_time))","execution_count":25,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a053abaa134b7574343cb3714c0054a90dc0b53","collapsed":true},"cell_type":"code","source":"X_user_type_train = lb.fit_transform(df_train['user_type'])\nX_user_type_test = lb.transform(df_test['user_type'])\nprint('[{}] Finished label binarize `user_type`'.format(time.time() - start_time))","execution_count":26,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e59fa613d6ef42fb08ed77cfdc81cdd73778b11f","collapsed":true},"cell_type":"code","source":"num_features = list(cols - (basic_cols))\nnum_features.remove('is_train')\n\nscaler = StandardScaler()\n\ntrain_num_features = scaler.fit_transform(df_train[num_features].drop('deal_probability', axis=1))\ntest_num_features = scaler.fit_transform(df_test[num_features].drop('deal_probability', axis=1))\n\ntrain_num_features = csr_matrix(train_num_features)\ntest_num_features = csr_matrix(test_num_features)","execution_count":27,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96fbc067d5b3f9c32060238f33e7ff486eca0080","collapsed":true},"cell_type":"code","source":"sparse_merge_train = hstack((X_description_train, X_param1_train, X_name_train, X_region_train, X_city_train, X_imagetop1_train, X_user_type_train)).tocsr()\nsparse_merge_train = hstack([sparse_merge_train, train_num_features])\nprint('[{}] Create sparse merge train completed'.format(time.time() - start_time))\n\nsparse_merge_test = hstack((X_description_test, X_param1_test, X_name_test, X_region_test, X_city_test, X_imagetop1_test, X_user_type_test)).tocsr()\nsparse_merge_test = hstack([sparse_merge_test, test_num_features])\nprint('[{}] Create sparse merge test completed'.format(time.time() - start_time))\ngc.collect()","execution_count":28,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b3781c3e0d48ef6178ccddb1bca7894956b7b2e","collapsed":true},"cell_type":"code","source":"print(\"\\n FM_FTRL Starting...........\")\nif SUBMIT_MODE:\n    iters = 3\nelse:\n    iters = 1\n    rounds = 3\n\nmodel = FM_FTRL(alpha=0.035, beta=0.001, L1=0.00001, L2=0.15, D=sparse_merge_train.shape[1],\n                alpha_fm=0.05, L2_fm=0.0, init_fm=0.01,\n                D_fm=100, e_noise=0, iters=iters, inv_link=\"identity\", threads=4)","execution_count":29,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7227c87ae5baa643dc828cf61f95d6e5772942b7","collapsed":true},"cell_type":"code","source":"if SUBMIT_MODE:\n    model.fit(sparse_merge_train, y_train)\n    print('[{}] Train FM completed'.format(time.time() - start_time))\n    predsFM = model.predict(sparse_merge_test)\n    print('[{}] Predict FM completed'.format(time.time() - start_time))\nelse:\n    for i in range(rounds):\n        model.fit(sparse_merge_train, y_train)\n        predsFM = model.predict(sparse_merge_test)\n        print('[{}] Iteration {}/{} -- RMSLE: {}'.format(time.time() - start_time, i + 1, rounds, rmse(predsFM, y_test)))\n\ndel model\ngc.collect()\nif not SUBMIT_MODE:\n    print(\"FM_FTRL dev RMSLE:\", rmse(predsFM, y_test))","execution_count":30,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c286815b3ce90868fe47905a48336185f1ac19f9"},"cell_type":"code","source":"del X_description_train, lb, X_name_train, X_param1_train, X_region_train, X_city_train, X_imagetop1_train, X_user_type_train\ndel X_description_test, X_name_test, X_param1_test, X_region_test, X_city_test, X_imagetop1_test, X_user_type_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0b88ab5b8d429061b3a35473bb516ee14fcd217","collapsed":true},"cell_type":"code","source":"fselect = SelectKBest(f_regression, k=48000)\ntrain_features = fselect.fit_transform(sparse_merge_train, y_train)\ntest_features = fselect.transform(sparse_merge_test)\nprint('[{}] Select best completed'.format(time.time() - start_time))\n\n\ndel sparse_merge_train\ndel sparse_merge_test\ngc.collect()\nprint('[{}] Garbage collection'.format(time.time() - start_time))","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4d15d21b5e9f50ceefdb47d6e5cdffaa3b45b16","collapsed":true},"cell_type":"code","source":"tv = TfidfVectorizer(max_features=250000,\n                     ngram_range=(1, 3),\n                     stop_words=None)\nX_name_train = tv.fit_transform(df_train['title'])\nprint('[{}] Finished TFIDF vectorize `title` (1/2)'.format(time.time() - start_time))\nX_name_test = tv.transform(df_test['title'])\nprint('[{}] Finished TFIDF vectorize `title` (2/2)'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5746e38e406f8f4fc7b331a4c4de0afc27a796f3","collapsed":true},"cell_type":"code","source":"tv = TfidfVectorizer(max_features=100000,\n                     ngram_range=(1, 2),\n                     stop_words=None)\nX_description_train = tv.fit_transform(df_train['description'])\nprint('[{}] Finished TFIDF vectorize `description` (1/2)'.format(time.time() - start_time))\nX_description_test = tv.transform(df_test['description'])\nprint('[{}] Finished TFIDF vectorize `description` (2/2)'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0d5f5e13c9fb95c9e9b76e090cdf9ca7ab1e3cb","collapsed":true},"cell_type":"code","source":"tv = TfidfVectorizer(max_features=50000,\n                     ngram_range=(1, 2),\n                     stop_words=None)\nX_param1_train = tv.fit_transform(df_train['param_1_copy'])\nprint('[{}] Finished TFIDF vectorize `param_1_copy` (1/2)'.format(time.time() - start_time))\nX_param1_test = tv.transform(df_test['param_1_copy'])\nprint('[{}] Finished TFIDF vectorize `param_1_copy` (2/2)'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0499351e52d6c825efa115210c3e5f2114e9c8c","collapsed":true},"cell_type":"code","source":"sparse_merge_train = hstack((X_description_train, X_param1_train, X_name_train)).tocsr()\ndel X_description_train, X_param1_train, X_name_train\ngc.collect()\nprint('[{}] Create sparse merge train completed'.format(time.time() - start_time))\n\nsparse_merge_test = hstack((X_description_test, X_param1_test, X_name_test)).tocsr()\nX_description_test, X_param1_test, X_name_test\ngc.collect()\nprint('[{}] Create sparse merge test completed'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eabb537d9f943832f99f39449dada6f994eeaa11","collapsed":true},"cell_type":"code","source":"X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(sparse_merge_train, y_train,\n                                                              test_size = 0.5,\n                                                              shuffle = False)\nprint('[{}] Finished splitting'.format(time.time() - start_time))\n\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_1, y_train_1)\nprint('[{}] Finished to train ridge (1)'.format(time.time() - start_time))\nridge_preds1 = model.predict(X_train_2)\nridge_preds1f = model.predict(sparse_merge_test)\nprint('[{}] Finished to predict ridge (1)'.format(time.time() - start_time))\nmodel = Ridge(solver=\"sag\", fit_intercept=True, random_state=205, alpha=3.3)\nmodel.fit(X_train_2, y_train_2)\nprint('[{}] Finished to train ridge (2)'.format(time.time() - start_time))\nridge_preds2 = model.predict(X_train_1)\nridge_preds2f = model.predict(sparse_merge_test)\nprint('[{}] Finished to predict ridge (2)'.format(time.time() - start_time))\nridge_preds_oof = np.concatenate((ridge_preds2, ridge_preds1), axis=0)\nridge_preds_test = (ridge_preds1f + ridge_preds2f) / 2.0\nprint('RMSLE OOF: {}'.format(rmse(ridge_preds_oof, y_train)))\nif not SUBMIT_MODE:\n    print('RMSLE TEST: {}'.format(rmse(ridge_preds_test, y_test)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"895cbd082a3a921b329058ed3946ab4faced1047","collapsed":true},"cell_type":"code","source":"del ridge_preds1\ndel ridge_preds1f\ndel ridge_preds2\ndel ridge_preds2f\n#del mnb_preds1\n#del mnb_preds1f\n#del mnb_preds2\n#del mnb_preds2f\ndel X_train_1\ndel X_train_2\ndel y_train_1\ndel y_train_2\ndel sparse_merge_train\ndel sparse_merge_test\ndel model\ngc.collect()\nprint('[{}] Finished garbage collection'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf1199c0c94ec61102fc00313bb440ae66ce5cac","collapsed":true},"cell_type":"code","source":"df_train['ridge'] = ridge_preds_oof\ndf_train['name_ridge'] = name_ridge_preds_oof\ndf_train['desc_ridge'] = desc_ridge_preds_oof\ndf_train['param1_ridge'] = param1_ridge_preds_oof\n#df_train['mnb'] = mnb_preds_oof\ndf_test['ridge'] = ridge_preds_test\ndf_test['name_ridge'] = name_ridge_preds_test\ndf_test['desc_ridge'] = desc_ridge_preds_test\ndf_test['param1_ridge'] = param1_ridge_preds_test\n#df_test['mnb'] = mnb_preds_test\nprint('[{}] Finished adding submodels'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5dc36a55389e291fe1e6cdfcd67036e8ddb43c97","collapsed":true},"cell_type":"code","source":"f_cats = [\"region\",\"city\",\"parent_category_name\",\"category_name\",\"user_type\",\"image_top_1\"]\ntarget_encode = TargetEncoder(min_samples_leaf=100, smoothing=10, noise_level=0.01,\n                              keep_original=True, cols=f_cats)\ndf_train, df_test = target_encode.encode(df_train, df_test, y_train)\nprint('[{}] Finished target encoding'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"320be0fbed65236eb10333fe8f2dad9eb33858a0","collapsed":true},"cell_type":"code","source":"df_train.drop(f_cats, axis=1, inplace=True)\ndf_test.drop(f_cats, axis=1, inplace=True)\n#del mnb_preds_oof\n#del mnb_preds_test\ndel ridge_preds_oof\ndel ridge_preds_test\ngc.collect()\nprint('[{}] Finished garbage collection'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f69d17ec198a5998f8367f6836e260fcb727b5e0","collapsed":true},"cell_type":"code","source":"cols = ['region_te', 'city_te', 'parent_category_name_te', 'category_name_te',\n        'user_type_te', 'image_top_1_te', 'desc_ridge', 'name_ridge', 'ridge']\ntrain_dummies = csr_matrix(df_train[cols].values)\nprint('[{}] Finished dummyizing model 1/5'.format(time.time() - start_time))\ntest_dummies = csr_matrix(df_test[cols].values)\nprint('[{}] Finished dummyizing model 2/5'.format(time.time() - start_time))\ndel df_train\ndel df_test\ngc.collect()\nprint('[{}] Finished dummyizing model 3/5'.format(time.time() - start_time))\ntrain_features = hstack((train_features, train_dummies)).tocsr()\nprint('[{}] Finished dummyizing model 4/5'.format(time.time() - start_time))\ntest_features = hstack((test_features, test_dummies)).tocsr()\nprint('[{}] Finished dummyizing model 5/5'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97230d3025631ee32ea9afc5d217e4eb513edb07","collapsed":true},"cell_type":"code","source":"d_train = lgb.Dataset(train_features, label=y_train)\ndel train_features\ngc.collect()\nif SUBMIT_MODE:\n    watchlist = [d_train]\nelse:\n    d_valid = lgb.Dataset(test_features, label=y_test)\n    watchlist = [d_train, d_valid]\n\nparams = {\n    'learning_rate': 0.02,\n    'application': 'regression',\n    'max_depth': 13,\n    'num_leaves': 400,\n    'verbosity': -1,\n    'metric': 'RMSE',\n    'data_random_seed': 1,\n    'bagging_fraction': 0.8,\n    'feature_fraction': 0.6,\n    'nthread': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 10\n}\nprint('[{}] Finished compiling LGB'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0d871bd3d8f7c878f129e933df22283a1f8063b","collapsed":true},"cell_type":"code","source":"modelL = lgb.train(params,\n                  train_set=d_train,\n                  num_boost_round=1350,\n                  valid_sets=watchlist,\n                  verbose_eval=50)\n\npredsL = modelL.predict(test_features)\n\nif not SUBMIT_MODE:\n    print(\"LGB RMSLE:\", rmse(predsL, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df52cd5d4ce36b79957de388fa63513875b858dd","collapsed":true},"cell_type":"code","source":"del d_train\ndel modelL\nif not SUBMIT_MODE:\n    del d_valid\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47caac2aedea8c7b064c5917600de9389d1da582","collapsed":true},"cell_type":"code","source":"preds_final = predsFM * 0.30 + predsL * 0.70\nif not SUBMIT_MODE:\n    print('Final RMSE: ', rmse(preds_final, y_test))\n\nif SUBMIT_MODE:\n    submission['deal_probability'] = preds_final\n    submission['deal_probability'] = submission['deal_probability'].clip(0.0, 1.0) # Between 0 and 1\n    submission.to_csv('lgb_and_fm_separate_train_test.csv', index=False)\n    print('[{}] Writing submission done'.format(time.time() - start_time))\n\nprint(submission.head(5))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}