{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport gc\nimport lightgbm as lgb","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"060f2673-7e56-4822-9208-0affd5a243df","_uuid":"6a1bc8dc5fccc86a43148ff709af0b23c2bc0c19","collapsed":true,"trusted":true},"cell_type":"code","source":"def readdf(filename, dataset, usecols=None, parse_dates=None):\n    print(f'Reading {filename}...', end=' ')\n    df = pd.read_csv(f'../input/{filename}', parse_dates=parse_dates, usecols=usecols)\n    df['dataset'] = dataset\n    gc.collect()\n    print('Done. Rows:', df.shape[0])\n    return df","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"usecols = ['category_name', 'city', \n       'description', 'image', 'image_top_1', 'item_id',\n       'item_seq_number', 'parent_category_name', 'price', 'region', 'title', \n       'user_id', 'user_type']\n\ntrain_df = readdf('train.csv', 0, usecols = usecols + ['deal_probability'])\ntest_df = readdf('test.csv', 1, usecols = usecols)","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"8e633358-d3cd-4a59-9d4a-07362ddcc33c","_uuid":"a25c77c7fc493960f01e7c2a8191c811d92f802a"},"cell_type":"markdown","source":"# Validation set preparation\n"},{"metadata":{"_cell_guid":"b44cfe9a-35a7-4da4-bc9e-e2460e8082f3","_uuid":"11f5501e8148d93baf6edf5bcef869b27e124ad3","collapsed":true,"trusted":true},"cell_type":"code","source":"train_df['dataset'] = 0\ntest_df['dataset'] = 1\n\ntrain_len = train_df.shape[0]\n# validation set = 5% of train\nval_len = train_len // 20","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"1b27ee93-32f0-432c-a62e-eae0f8e38ccb","_uuid":"5f948858d493ef4c3a41e57fccaf93acd840b3f9","collapsed":true,"trusted":true},"cell_type":"code","source":"train_df['train_probability'] = train_df.deal_probability\ntrain_df.loc[np.random.randint(0, train_len, train_len) < val_len, 'train_probability'] = np.nan\ntrain_df.loc[pd.isnull(train_df.train_probability), 'dataset'] = -1","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"3c3dd502-b6d2-4dfb-8b59-da982ddabded","_uuid":"43d2f6bcd8c5ecaffec4b6aa30426739cde839b7","collapsed":true,"trusted":true},"cell_type":"code","source":"df = train_df.append(test_df, ignore_index=True)","execution_count":6,"outputs":[]},{"metadata":{"_cell_guid":"0e0ccb4a-436b-4c55-8794-28fc3bc31c57","_uuid":"efdfd505c5597352b7bed07fcb5df4675c1b5ff0","trusted":true},"cell_type":"code","source":"del train_df, test_df\ngc.collect()","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"9a373bae-a129-40dd-bda5-34beaa2778ac","_uuid":"6b8157c67b941c357bef3766196f6083e879141e"},"cell_type":"markdown","source":"# Feature development"},{"metadata":{"_cell_guid":"710d11b3-3839-404f-867c-bd94a0c3e79e","_uuid":"d005d7f4819fb64cd65d7ec9dac03be87811c672","collapsed":true,"trusted":true},"cell_type":"code","source":"df.loc[pd.isnull(df.image), 'image_top_1'] = -1\ndf.image_top_1 = df.image_top_1.astype('int')","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"31c36cd3-5415-49be-8947-79701ab8e5a5","_uuid":"fcda845dca665e11038b128794ff4f5606261e42","collapsed":true,"trusted":true},"cell_type":"code","source":"df['has_image'] = pd.isnull(df.image).astype('int')","execution_count":9,"outputs":[]},{"metadata":{"_cell_guid":"907e0359-49aa-4202-8555-405e8df1429b","_uuid":"c02da94f333cbd5ce8c80285cc194e966765d196","collapsed":true,"trusted":true},"cell_type":"code","source":"df['desc_len'] = df.description.str.len()\ndf['title_len'] = df.title.str.len()\n\ndf['desc_words'] = df.description.str.split().str.len()\ndf['title_words'] = df.description.str.split().str.len()\n\ndf['desc_word_avg'] = (df.desc_len / df.desc_words)\ndf['title_word_avg'] = (df.title_len / df.title_words)","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"4b769d1b-676a-4aba-9961-acf2b37e7f9f","_uuid":"ee001ea4166c7b0390cb1593208208caaca6b8cc","collapsed":true,"trusted":true},"cell_type":"code","source":"def add_column(agg_name, column):\n    global df, predictors\n    df[agg_name] = column\n    predictors.append(agg_name)","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"b446d35a-0812-4326-a4b6-48f489aede78","_uuid":"c38e65df6246fb2111e6b8fdcd96aff0a418a8de","collapsed":true,"trusted":true},"cell_type":"code","source":"predictors = ['image_top_1', 'item_seq_number', 'price', 'has_image', \n       'desc_len', 'title_len', 'desc_words', 'title_words', 'desc_word_avg',\n       'title_word_avg']\nadd_column('user_id_a_has_image', df.groupby('user_id')['has_image'].transform('mean'))","execution_count":13,"outputs":[]},{"metadata":{"_cell_guid":"eb414055-4c00-41da-b9c0-ab9e7cdc78a3","_uuid":"e5f122e576ba7a7daf14519e321a35c7d8c51fe8","collapsed":true,"trusted":true},"cell_type":"code","source":"categorical = ['has_image', 'image_top_1']\n\nfor attr in ['city', 'category_name', 'user_type']:\n    df[attr] = df[attr].astype('category').cat.codes\n    predictors.append(attr)\n    categorical.append(attr)","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"60768707-7c8c-4c56-9ab7-3ee74ab5deea","_uuid":"2782137510e42560583cf88d8e6de1ccc8b3e3fe","trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"348c87ec-69b4-4c49-8eea-64feb08c4bf9","_uuid":"417300036880388811d95fb5d204633d7d88f487"},"cell_type":"markdown","source":"# Preparing data for modelling"},{"metadata":{"_cell_guid":"bd989d87-416d-4054-8c5a-0b334ef5dd96","_uuid":"7862717cc5b143100e0d1c39410c1adb4af4371e","trusted":true},"cell_type":"code","source":"test_df = df[df.dataset == 1]\nlen(test_df)","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"8951b07d-2100-409b-a626-02ad01acbb11","_uuid":"752764ae6ca74efd4c530fcabbec0a8b5333e164","trusted":true},"cell_type":"code","source":"train_df = df[df.dataset == 0]\nlen(train_df)","execution_count":17,"outputs":[]},{"metadata":{"_cell_guid":"117d99b4-e124-4b28-974f-9cdae97309ee","_uuid":"fe922e3985eba7aa6d7608c1b89bea6b0b53378c","trusted":true},"cell_type":"code","source":"val_df = df[df.dataset == -1]\nlen(val_df)","execution_count":18,"outputs":[]},{"metadata":{"_cell_guid":"5e169e57-d0fa-483f-be9c-0a391aed5d7e","_uuid":"7d794845758a6a6307212cb99b5e32305283c37c","collapsed":true,"trusted":true},"cell_type":"code","source":"train_target = 'train_probability'\nval_target = 'deal_probability'","execution_count":20,"outputs":[]},{"metadata":{"_cell_guid":"41012376-c937-44dc-9f05-a1e3cfd0ed1b","_uuid":"e0275d46734b2509762aeef9a7087517c409d076","trusted":true},"cell_type":"code","source":"xgtrain = lgb.Dataset(\n    train_df[predictors].values,\n    label=train_df[train_target].values,\n    feature_name=predictors,\n    categorical_feature=categorical,\n)\nxgvalid = lgb.Dataset(\n    val_df[predictors].values,\n    label=val_df[val_target].values,\n    feature_name=predictors,\n    categorical_feature=categorical,\n    reference = xgtrain\n)\ngc.collect()","execution_count":21,"outputs":[]},{"metadata":{"_cell_guid":"3c69889d-1093-4caf-ba25-5ada30ef085e","_uuid":"9e851e166afc08f28b11323e2c58956ce8e88672","trusted":true},"cell_type":"code","source":"evals_results = {}\n\nbst = lgb.train(\n    {\n        'task': 'train',\n        'boosting_type': 'gbdt',\n        'objective': 'regression',\n        'metric': 'rmse',\n        'num_leaves': 127,\n        'learning_rate': 0.02,\n        'verbose': 50,\n        'nthread': 4,\n        'seed': 1,\n        'data_random_seed': 1\n    },\n    xgtrain,\n    valid_sets=[xgvalid],\n    valid_names=['valid'],\n    evals_result=evals_results,\n    num_boost_round=2000,\n    early_stopping_rounds=100,\n    verbose_eval=50,\n    feval=None)\n\nprint(\"\\nModel Report\")\nprint(\"Best_iteration: \", bst.best_iteration)\nprint('rmse' + \":\", evals_results['valid']['rmse'][bst.best_iteration - 1])\n","execution_count":22,"outputs":[]},{"metadata":{"_cell_guid":"f433366e-dcbe-4a0f-8818-44bc7f81caa9","_uuid":"b3127b09fcd38236364a917c9e06b246f0421baa","trusted":true},"cell_type":"code","source":"lgb.plot_importance(bst, figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"63d80841-960f-4ff9-b9df-a9142ccfa055","_uuid":"b5a51f9f8aa70181e49a557b66982b2cf931fc7a","trusted":true},"cell_type":"code","source":"print('Preparing DataFrame...')\nsub = pd.DataFrame()\nsub['item_id'] = test_df['item_id'].values\nprint('Predicting...')\nsub['deal_probability'] = bst.predict(test_df[predictors].values, num_iteration=bst.best_iteration)\nsub.deal_probability.clip(0, 1, inplace=True)\nprint('Writing...')\nsub.to_csv('submission.csv', index=False)\nprint('done')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}