{"cells":[{"metadata":{"_uuid":"ab33115d7bad6b01a21e60538f066a5240dd84d4","_cell_guid":"5ac028e4-8414-406b-8425-896632f1337e"},"cell_type":"markdown","source":"This is an interesting idea based on https://www.kaggle.com/paulorzp/tfidf-tensor-starter-lb-0-234 and https://www.kaggle.com/lopuhin/mercari-golf-0-3875-cv-in-75-loc-1900-s, where we train a bunch of diverse, overfit simple MLP and then boost them together with a LightGBM as a stacking technique. Together, the hope is that all these bad models will jointly approximate something interesting.\n\nI'm not sure what to do with it, though. The final score is good but not amazing, and I suspect that other neural network techniques will be more competitive. I've thought about trying to include the individual MLP submodels into a more complex LGB or include this LGB in a more complex LGB via stacking. If you come up with an interesting application of this approach, please share!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","scrolled":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false,"collapsed":true},"cell_type":"code","source":"import gc\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL']='2'\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nfrom contextlib import contextmanager\nfrom operator import itemgetter\nimport time\nfrom typing import List, Dict\n\nimport keras as ks\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nfrom scipy.sparse import vstack\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n\nwith timer('reading data'):\n    train = pd.read_csv('../input/train.csv')\n    test = pd.read_csv('../input/test.csv')\n\nwith timer('imputation'):\n    train['param_1'].fillna('missing', inplace=True)\n    test['param_1'].fillna('missing', inplace=True)\n    train['param_2'].fillna('missing', inplace=True)\n    test['param_2'].fillna('missing', inplace=True)\n    train['param_3'].fillna('missing', inplace=True)\n    test['param_3'].fillna('missing', inplace=True)\n    train['image_top_1'].fillna(0, inplace=True)\n    test['image_top_1'].fillna(0, inplace=True)\n    train['price'].fillna(0, inplace=True)\n    test['price'].fillna(0, inplace=True)\n    train['price'] = np.log1p(train['price'])\n    test['price'] = np.log1p(test['price'])\n    price_mean = train['price'].mean()\n    price_std = train['price'].std()\n    train['price'] = (train['price'] - price_mean) / price_std\n    test['price'] = (test['price'] - price_mean) / price_std\n    train['description'].fillna('', inplace=True)\n    test['description'].fillna('', inplace=True)\n    # City names are duplicated across region, HT: Branden Murray https://www.kaggle.com/c/avito-demand-prediction/discussion/55630#321751\n    train['city'] = train['city'] + '_' + train['region']\n    test['city'] = test['city'] + '_' + test['region']\n\nwith timer('add new features'):\n    cat_cols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'user_type']\n    num_cols = ['price', 'deal_probability']\n    for c in cat_cols:\n        for c2 in num_cols:\n            enc = train.groupby(c)[c2].agg(['mean']).astype(np.float32).reset_index()\n            enc.columns = ['_'.join([str(c), str(c2), str(c3)]) if c3 != c else c for c3 in enc.columns]\n            train = pd.merge(train, enc, how='left', on=c)\n            test = pd.merge(test, enc, how='left', on=c)\n    del(enc)\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false,"collapsed":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ndef preprocess(df: pd.DataFrame) -> pd.DataFrame:\n    ex_col = ['item_id', 'user_id', 'deal_probability', 'title', 'param_1', 'param_2', 'param_3', 'activation_date']\n    df['description_len'] = df['description'].map(lambda x: len(str(x))).astype(np.float16) #Lenth\n    df['description_wc'] = df['description'].map(lambda x: len(str(x).split(' '))).astype(np.float16) #Word Count\n    df['description'] = (df['parent_category_name'] + ' ' + df['category_name'] + ' ' + df['param_1'] + ' ' + df['param_2'] + ' ' + df['param_3'] + ' ' +\n                        df['title'] + ' ' + df['description'].fillna(''))\n    df['description'] = df['description'].str.lower().replace(r\"[^[:alpha:]]\", \" \")\n    df['description'] = df['description'].str.replace(r\"\\\\s+\", \" \")\n    df['title_len'] = df['title'].map(lambda x: len(str(x))).astype(np.float16) #Lenth\n    df['title_wc'] = df['title'].map(lambda x: len(str(x).split(' '))).astype(np.float16) #Word Count\n    df['image'] = df['image'].map(lambda x: 1 if len(str(x))>0 else 0)\n    df['price'] = np.log1p(df['price'].fillna(0))\n    df['wday'] = pd.to_datetime(df['activation_date']).dt.dayofweek\n    col = [c for c in df.columns if c not in ex_col]\n    return df[col]\n\nwith timer('process train'):\n    train, valid = train_test_split(train, test_size=0.05, shuffle=True, random_state=37)\n    y_train = train['deal_probability'].values\n    X_train = preprocess(train)\n    print(f'X_train: {X_train.shape}')\n\nwith timer('process valid'):\n    X_valid = preprocess(valid)\n    print(f'X_valid: {X_valid.shape}')\n\nwith timer('process test'):\n    X_test = preprocess(test)\n    print(f'X_test: {X_test.shape}')\n\nX_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e304a0ddb9df673e096314486c2d64074790a8f","_cell_guid":"609a0184-8709-4726-a723-cb40275606c9","trusted":false,"collapsed":true},"cell_type":"code","source":"# Do some normalization\ndesc_len_mean = X_train['description_len'].mean()\ndesc_len_std = X_train['description_len'].std()\nX_train['description_len'] = (X_train['description_len'] - desc_len_mean) / desc_len_std\nX_valid['description_len'] = (X_valid['description_len'] - desc_len_mean) / desc_len_std\nX_test['description_len'] = (X_test['description_len'] - desc_len_mean) / desc_len_std\n\ndesc_wc_mean = X_train['description_wc'].mean()\ndesc_wc_std = X_train['description_wc'].std()\nX_train['description_wc'] = (X_train['description_wc'] - desc_wc_mean) / desc_wc_std\nX_valid['description_wc'] = (X_valid['description_wc'] - desc_wc_mean) / desc_wc_std\nX_test['description_wc'] = (X_test['description_wc'] - desc_wc_mean) / desc_wc_std\n\ntitle_len_mean = X_train['title_len'].mean()\ntitle_len_std = X_train['title_len'].std()\nX_train['title_len'] = (X_train['title_len'] - title_len_mean) / title_len_std\nX_valid['title_len'] = (X_valid['title_len'] - title_len_mean) / title_len_std\nX_test['title_len'] = (X_test['title_len'] - title_len_mean) / title_len_std\n\ntitle_wc_mean = X_train['title_wc'].mean()\ntitle_wc_std = X_train['title_wc'].std()\nX_train['title_wc'] = (X_train['title_wc'] - title_wc_mean) / title_wc_std\nX_valid['title_wc'] = (X_valid['title_wc'] - title_wc_mean) / title_wc_std\nX_test['title_wc'] = (X_test['title_wc'] - title_wc_mean) / title_wc_std\n\nimage_top_1_mean = X_train['image_top_1'].mean()\nimage_top_1_std = X_train['image_top_1'].std()\nX_train['image_top_1'] = (X_train['image_top_1'] - image_top_1_mean) / image_top_1_std\nX_valid['image_top_1'] = (X_valid['image_top_1'] - image_top_1_mean) / image_top_1_std\nX_test['image_top_1'] = (X_test['image_top_1'] - image_top_1_mean) / image_top_1_std","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1deb23b59985dde22e0805bb04f6e5b01e09b487","_cell_guid":"be640ce6-d4fa-4a6f-aafb-b7683584ffde","trusted":false,"collapsed":true},"cell_type":"code","source":"# I don't know why I need to fill NA a second time, but alas here we are...\nX_train.fillna(0, inplace=True)\nX_valid.fillna(0, inplace=True)\nX_test.fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f6323624c40039941c465cbeab6cd2003d8be8ca","_cell_guid":"56bc0919-f74a-401a-aa95-9f700d6f3132","trusted":false,"collapsed":true},"cell_type":"code","source":"X_train.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87683550aa3e01b9256719cea4bd31a2db1aa4b8","scrolled":true,"_cell_guid":"f4038bb3-fa7f-4bd4-a089-8197435cfc27","trusted":false,"collapsed":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder, StandardScaler\n\nwith timer('TFIDF'):\n    tfidf = TfidfVectorizer(ngram_range=(1, 2),\n                            max_features=100000,\n                             token_pattern='\\w+',\n                            encoding='KOI8-R')\n    tfidf_train = tfidf.fit_transform(X_train['description'])\n    tfidf_valid = tfidf.transform(X_valid['description'])\n    tfidf_test = tfidf.transform(X_test['description'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8cb2e899c4481c1883951fcf6b770144a569923c","scrolled":true,"_cell_guid":"c63807f5-8fab-4e5e-83f2-c3f59ca4f2a6","trusted":false,"collapsed":true},"cell_type":"code","source":"with timer('Dummy'):\n    dummy_cols = ['parent_category_name', 'category_name', 'user_type', 'image_top_1', 'wday', 'region', 'city']\n    for col in dummy_cols:\n        le = LabelEncoder()\n        le.fit(X_train[col] + X_valid[col] + X_test[col])\n        le.fit(list(X_train[col].values.astype('str')) + list(X_valid[col].values.astype('str')) + list(X_test[col].values.astype('str')))\n        X_train[col] = le.transform(list(X_train[col].values.astype('str')))\n        X_valid[col] = le.transform(list(X_valid[col].values.astype('str')))\n        X_test[col] = le.transform(list(X_test[col].values.astype('str')))\n\nwith timer('Dropping'):\n    X_train.drop('description', axis=1, inplace=True)\n    X_valid.drop('description', axis=1, inplace=True)\n    X_test.drop('description', axis=1, inplace=True)\n\nwith timer('OHE'):\n    ohe = OneHotEncoder(categorical_features=[X_train.columns.get_loc(c) for c in dummy_cols])\n    X_train = ohe.fit_transform(X_train)\n    print(f'X_train: {X_train.shape}')\n    X_valid = ohe.transform(X_valid)\n    print(f'X_valid: {X_valid.shape}')\n    X_test = ohe.transform(X_test)\n    print(f'X_test: {X_test.shape}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c418298959c4d8cf633ac37c83882de658d09404","_cell_guid":"677abb25-504b-4be0-8e6f-9491930571ca"},"cell_type":"markdown","source":"This is the key part. I train eight different MLP models -- four of them with huber loss and the rest optimizing mean squared error, and four of them with binarized data and the other eight using regular TFIDF variables, for a total of two copies each of four different model types."},{"metadata":{"_uuid":"199319896fb906d24bba59da3c1c6411749a5016","scrolled":false,"_cell_guid":"ba360959-07e2-4ddf-bc82-635c47281214","trusted":false,"collapsed":true},"cell_type":"code","source":"def huber_loss(y_true, y_pred, clip_delta=1.0):\n    error = y_true - y_pred\n    cond  = tf.keras.backend.abs(error) < clip_delta\n    squared_loss = 0.5 * tf.keras.backend.square(error)\n    linear_loss  = clip_delta * (tf.keras.backend.abs(error) - 0.5 * clip_delta)\n    return tf.where(cond, squared_loss, linear_loss)\n\ndef fit_predict(xs, y_train, loss_fn='mean_squared_error') -> np.ndarray:\n    X_train, X_test = xs\n    config = tf.ConfigProto(\n        intra_op_parallelism_threads=4, use_per_session_threads=4, inter_op_parallelism_threads=4)\n    with tf.Session(graph=tf.Graph(), config=config) as sess, timer('fit_predict'):\n        ks.backend.set_session(sess)\n        model_in = ks.Input(shape=(X_train.shape[1],), dtype='float32', sparse=True)\n        out = ks.layers.Dense(192, activation='relu')(model_in)\n        out = ks.layers.Dense(64, activation='relu')(out)\n        out = ks.layers.Dense(64, activation='relu')(out)\n        out = ks.layers.Dense(1)(out)\n        model = ks.Model(model_in, out)\n        model.compile(loss=loss_fn, optimizer=ks.optimizers.Adam(lr=2e-3))\n        for i in range(3):\n            with timer(f'epoch {i + 1}'):\n                model.fit(x=X_train, y=y_train, batch_size=2**(8 + i), epochs=1, verbose=0)\n        return model.predict(X_test, batch_size=2**(8 + i))[:, 0]\n\nX_train = X_train.tocsr()\nX_valid = X_valid.tocsr()\nX_test = X_test.tocsr()\nX_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_train, y_train, test_size = 0.5, shuffle = False)    \n\npreds_oofs = []\npreds_valids = []\npreds_tests = []\nfor r in range(8):\n    with timer('Round {}'.format(r)):\n        if r % 2 == 0:\n            loss_name = 'huber_loss'\n            loss = huber_loss\n        else:\n            loss_name = 'mean_squared_error'\n            loss = 'mean_squared_error'\n        if r >= 4:\n            print('Running loss = {}, binary = True'.format(loss_name))\n            xs = [x.astype(np.bool).astype(np.float32) for x in [X_train_1, X_train_2]]\n            y_pred1 = fit_predict(xs, y_train=y_train_1, loss_fn=loss)\n            xs = [x.astype(np.bool).astype(np.float32) for x in [X_train_2, X_train_1]]\n            y_pred2 = fit_predict(xs, y_train=y_train_2, loss_fn=loss)\n            xs = [x.astype(np.bool).astype(np.float32) for x in [X_train_1, X_valid]]\n            y_predf = fit_predict(xs, y_train=y_train_1, loss_fn=loss)\n            xs = [x.astype(np.bool).astype(np.float32) for x in [X_train_1, X_test]]\n            y_predt = fit_predict(xs, y_train=y_train_1, loss_fn=loss)\n        else:\n            print('Running loss = {}, binary = False'.format(loss_name))\n            xs = [X_train_1, X_train_2]\n            y_pred1 = fit_predict(xs, y_train=y_train_1, loss_fn=loss)\n            xs = [X_train_2, X_train_1]\n            y_pred2 = fit_predict(xs, y_train=y_train_2, loss_fn=loss)\n            xs = [X_train_1, X_valid]\n            y_predf = fit_predict(xs, y_train=y_train_1, loss_fn=loss)\n            xs = [X_train_1, X_test]\n            y_predt = fit_predict(xs, y_train=y_train_1, loss_fn=loss)\n        preds_oof = np.concatenate((y_pred2, y_pred1), axis=0)\n        preds_valid = y_predf\n        preds_test = y_predt\n        print('Round {} OOF RMSE: {:.4f}'.format(r, np.sqrt(mean_squared_error(train['deal_probability'], preds_oof))))\n        print('Round {} Valid RMSE: {:.4f}'.format(r, np.sqrt(mean_squared_error(valid['deal_probability'], preds_valid))))\n        preds_oofs.append(preds_oof)\n        preds_valids.append(preds_valid)\n        preds_tests.append(preds_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d05cd6a7067557bbe723a79be8671c7679c48c7","_cell_guid":"4a8d9042-989a-4208-8c6e-4ff95ee8e2a5","trusted":false,"collapsed":true},"cell_type":"code","source":"preds_oof = np.mean(preds_oofs, axis=0)\nprint('Overall OOF RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(train['deal_probability'], preds_oof))))\npreds_valid = np.mean(preds_valids, axis=0)\nprint('Overall Valid RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(valid['deal_probability'], preds_valid))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddff54d3c8c4c7e20b71032f354b75850ca1614d","scrolled":true,"_cell_guid":"74dc465d-6b5b-401d-976e-5b2583dac225","trusted":false,"collapsed":true},"cell_type":"code","source":"# As we can see, the individual submodels have very low correlation with each other!\nimport numpy as np\nnp.mean(np.corrcoef(preds_oofs), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e58b26b5ee218736b6eb1836ecfe31ce5d85e68","_cell_guid":"7765ef72-2c94-4d32-a3de-b772af94ad2b"},"cell_type":"markdown","source":"Now we build the LGB that will boost us to victory."},{"metadata":{"_uuid":"894a89f19e5458ab3fe8c44b4ef9105200913a3f","_cell_guid":"2a6a2cf8-b4a6-450c-b990-a753bd43857e","trusted":false,"collapsed":true},"cell_type":"code","source":"with timer('reading data'):\n    train = pd.read_csv('../input/train.csv')\n    test = pd.read_csv('../input/test.csv')\n    target = train['deal_probability']\n\nwith timer('imputation'):\n    train['param_1'].fillna('missing', inplace=True)\n    test['param_1'].fillna('missing', inplace=True)\n    train['param_2'].fillna('missing', inplace=True)\n    test['param_2'].fillna('missing', inplace=True)\n    train['param_3'].fillna('missing', inplace=True)\n    test['param_3'].fillna('missing', inplace=True)\n    train['price'].fillna(0, inplace=True)\n    test['price'].fillna(0, inplace=True)\n    # City names are duplicated across region, HT: Branden Murray https://www.kaggle.com/c/avito-demand-prediction/discussion/55630#321751\n    train['city'] = train['city'] + '_' + train['region']\n    test['city'] = test['city'] + '_' + test['region']\n\nwith timer('FE'):\n    trainp = preprocess(train)\n    testp = preprocess(test)\n    for col in ['param_1', 'param_2', 'param_3']:\n        trainp[col] = train[col]\n        testp[col] = test[col]\n\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed291422bcd2749ba4032f2b96a460b7fa79ed2e","scrolled":true,"_cell_guid":"d57638e2-6afd-4bca-954c-e1a2d37f7f91","trusted":false,"collapsed":true},"cell_type":"code","source":"with timer('drop'):\n    trainp.drop(['description', 'image'], axis=1, inplace=True)\n    testp.drop(['description', 'image'], axis=1, inplace=True)\nprint(trainp.shape)\nprint(testp.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a082f175bfc5cfb4b1e0ed4040467ef0781ed0df","_cell_guid":"4d6ca24d-28cf-4b83-9ed2-204d48068e35","trusted":false,"collapsed":true},"cell_type":"code","source":"with timer('To cat'):\n    trainp['image_top_1'] = trainp['image_top_1'].astype('str').fillna('missing')\n    testp['image_top_1'] = testp['image_top_1'].astype('str').fillna('missing') # My pet theory is that image_top_1 is categorical. Fight me.\n    cat_cols = ['region', 'city', 'parent_category_name', 'category_name',\n                'param_1', 'param_2', 'param_3', 'user_type', 'image_top_1', 'wday']\n    for col in trainp.columns:\n        print(col)\n        if col in cat_cols:\n            trainp[col] = trainp[col].astype('category')\n            testp[col] = testp[col].astype('category')\n        else:\n            trainp[col] = trainp[col].astype(np.float64)\n            testp[col] = testp[col].astype(np.float64)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c66f6215173e6eb4ec7b20a621e12efcda337dfc","_cell_guid":"9db6aa5a-ac9d-42a6-8669-ccb88e9ec4df","trusted":false,"collapsed":true},"cell_type":"code","source":"print(trainp.shape)\nprint(trainp.columns)\ntrainp.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c98bad08d7b9179094d7c3f6d2bd3b83f25f9165","_cell_guid":"34fbe8ab-d702-4ddf-9ecc-52eaa24d9a21","trusted":false,"collapsed":true},"cell_type":"code","source":"trainp.dtypes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bfca242287f416453f60556ee3720cddf230e12e","_cell_guid":"e000e258-8cec-470b-a41b-6d8279d6043a","trusted":false,"collapsed":true},"cell_type":"code","source":"with timer('Split'):\n    train, valid, y_train, y_valid = train_test_split(trainp, target, test_size=0.05, shuffle=True, random_state=37)\n    test = testp\n    print(train.shape)\n    print(valid.shape)\n    print(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"abe99a2c2d17bc60844c23d0264062c3b1510669","scrolled":false,"_cell_guid":"ae172bbf-628e-4ebe-8dbb-8a59a08baeb2","trusted":false,"collapsed":true},"cell_type":"code","source":"with timer('Submodels'):\n    train_models = pd.DataFrame(np.array(preds_oofs).transpose())\n    valid_models = pd.DataFrame(np.array(preds_valids).transpose())\n    test_models = pd.DataFrame(np.array(preds_tests).transpose())\n    train_models.columns = ['nn_' + str(i + 1) for i in range(train_models.shape[1])]\n    valid_models.columns = ['nn_' + str(i + 1) for i in range(train_models.shape[1])]\n    test_models.columns = ['nn_' + str(i + 1) for i in range(train_models.shape[1])]\n    print(train_models.shape)\n    print(valid_models.shape)\n    print(test_models.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6ac43ca057b169f3d60647837138c53898d6955","_cell_guid":"2ca209b6-4af6-4af4-b351-857b7196c78e","trusted":false,"collapsed":true},"cell_type":"code","source":"with timer('Concat'):\n    print(train.shape)\n    X_train = pd.concat([train.reset_index(), train_models.reset_index()], axis=1)\n    print(X_train.shape)\n    print('-')\n    print(valid.shape)\n    X_valid = pd.concat([valid.reset_index(), valid_models.reset_index()], axis=1)\n    print(X_valid.shape)\n    print('-')\n    print(test.shape)\n    X_test = pd.concat([test.reset_index(), test_models.reset_index()], axis=1)\n    print(X_test.shape)\n\nX_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c11faab84b6fd68e7be7b1b9a0a2d9325f0f553d","_cell_guid":"ec9448e5-b399-4257-bdef-01514a44f6a3","trusted":false,"collapsed":true},"cell_type":"code","source":"X_train.drop('index', axis=1, inplace=True)\nX_valid.drop('index', axis=1, inplace=True)\nX_test.drop('index', axis=1, inplace=True)\nprint(X_train.shape)\nprint(X_valid.shape)\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02402bd7720c751851f7377b78f9474f7a92e53e","scrolled":true,"_cell_guid":"762b6592-8b9e-4a6c-81ed-4230f923fe4f","trusted":false,"collapsed":true},"cell_type":"code","source":"#del X_train_1\n#del X_train_2\ndel trainp\ndel testp\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"038f77c6a86027eeb5a2ac5f0e1a0717e8ee49e2","_cell_guid":"9f1e6650-d823-42e9-9ea3-23d81ef64d9f","trusted":false,"collapsed":true},"cell_type":"code","source":"from pprint import pprint\nimport lightgbm as lgb\n\nd_train = lgb.Dataset(X_train, label=y_train)\nd_valid = lgb.Dataset(X_valid, label=y_valid)\nwatchlist = [d_train, d_valid]\nparams = {'application': 'regression',\n          'metric': 'rmse',\n          'nthread': 3,\n          'verbosity': -1,\n          'data_random_seed': 3,\n          'learning_rate': 0.05,\n          'num_leaves': 31,\n          'bagging_fraction': 0.8,\n          'feature_fraction': 0.2,\n          'lambda_l1': 3,\n          'lambda_l2': 3,\n          'min_data_in_leaf': 40}\nmodel = lgb.train(params,\n                  train_set=d_train,\n                  num_boost_round=1500,\n                  valid_sets=watchlist,\n                  verbose_eval=100)\npprint(sorted(list(zip(model.feature_importance(), X_train.columns)), reverse=True))\nprint('Done')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19a355b1ba7a18779bcf1a5b3c90e1ae79babdf4","_cell_guid":"711882ab-e46c-4632-8ffa-2e5d3e3498ce","trusted":false,"collapsed":true},"cell_type":"code","source":"valid_preds = model.predict(X_valid).clip(0, 1)\nprint('Overall Valid RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(y_valid, valid_preds))))\ntest_preds = model.predict(X_test).clip(0, 1)\nsubmission = pd.read_csv('../input/test.csv', usecols=[\"item_id\"])\nsubmission[\"deal_probability\"] = test_preds\nsubmission.to_csv(\"submit_boosting_mlp.csv\", index=False, float_format=\"%.2g\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c25111023a2a4dfbe8be7912d6d7e5ecec7b64f","_cell_guid":"740ae21b-1a0e-4cfb-9463-936f5b3fc1b7","trusted":false,"collapsed":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"file_extension":".py","pygments_lexer":"ipython3","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"name":"python","nbconvert_exporter":"python"}},"nbformat":4,"nbformat_minor":1}