{"cells":[{"metadata":{},"cell_type":"markdown","source":"The feature engineering project and the method to check whether overfit happen was forked from Anton Enns and tunguz respectively, and thanks for their good work. This kernel is to prove that do not submit public kernels' output without considering. Please upvotes tunguz and Anton！ The following is copied from the words from Anton's kernel.\n\n“With a very small dataset, lots and lots of different engineered features, vastly different CV and LB scores, and high inconsisency of LB scores with just slight changes in seed, this competition is overripe for a major shakupe. In this kernel we'll take a look at adveserial validation, and what it may portend about the shakeup.”\n\nAfter running this kerenl，we can see that the auc score is up to 0.87 roughly. Accordingly, it was confirmed that there is a significant difference between test and train'feature created by tunguz."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nfrom sklearn import model_selection, preprocessing, metrics\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/lanl-master-s-features-creating-0\"))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_X_0 = pd.read_csv(\"../input/lanl-master-s-features-creating-0/train_X_features_865.csv\")\ntrain_X_1 = pd.read_csv(\"../input/lanl-master-s-features-creating-1/train_X_features_865.csv\")\ny_0 = pd.read_csv(\"../input/lanl-master-s-features-creating-0/train_y.csv\", index_col=False,  header=None)\ny_1 = pd.read_csv(\"../input/lanl-master-s-features-creating-1/train_y.csv\", index_col=False,  header=None)\ntrain_X = pd.concat([train_X_0, train_X_1], axis=0)\ntrain_X = train_X.reset_index(drop=True)\nprint(train_X.shape)\ntrain_X.head()\n\ny = pd.concat([y_0, y_1], axis=0)\ny = y.reset_index(drop=True)\ny[0].shape\n\ntrain_y = pd.Series(y[0].values)\n\n\ntest_X = pd.read_csv(\"../input/lanl-master-s-features-creating-0/test_X_features_10.csv\")\n# del X[\"seg_id\"], test_X[\"seg_id\"]\n\nscaler = StandardScaler()\ntrain_columns = train_X.columns\n\ntrain_X[train_columns] = scaler.fit_transform(train_X[train_columns])\ntest_X[train_columns] = scaler.transform(test_X[train_columns])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = train_X.columns\ntrain_X['target'] = 0\ntest_X['target'] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test = pd.concat([train_X, test_X], axis =0)\n\ntarget = train_test['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param = {'num_leaves': 50,\n         'min_data_in_leaf': 30, \n         'objective':'binary',\n         'max_depth': 5,\n         'learning_rate': 0.006,\n         \"min_child_samples\": 20,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 27,\n         \"metric\": 'auc',\n         \"verbosity\": -1}\n\nfolds = KFold(n_splits=5, shuffle=True, random_state=15)\noof = np.zeros(len(train_test))\n\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_test.values, target)):\n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(train_test.iloc[trn_idx][features], label=target[trn_idx])\n    val_data = lgb.Dataset(train_test.iloc[val_idx][features], label=target[val_idx])\n\n    num_round = 30000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 1400)\n    oof[val_idx] = clf.predict(train_test.iloc[val_idx][features], num_iteration=clf.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_imp = pd.DataFrame(sorted(zip(clf.feature_importance(),features)), columns=['Value','Feature'])\nplt.figure(figsize=(30, 30))\nsns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False).head(20))\nplt.title('GDBT Features')\nplt.tight_layout()\nplt.show()\n\nfeature_imp.sort_values(by=\"Value\", ascending=False).to_csv('feat.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}