{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 何かと使うやつ.\nimport os\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nimport time, gc, string, math\nfrom tqdm.notebook import tqdm\nimport warnings\nimport shutil\nfrom collections import defaultdict\nimport heapq\nimport datetime\nimport random\nfrom collections import OrderedDict\nimport glob\nimport copy\n\n# sklearn\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import StandardScaler\n\n# LGBM\nfrom lightgbm import LGBMClassifier\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\n\n# KdTree\nfrom scipy.spatial import KDTree","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-05T07:30:26.175849Z","iopub.execute_input":"2022-08-05T07:30:26.177012Z","iopub.status.idle":"2022-08-05T07:30:28.932081Z","shell.execute_reply.started":"2022-08-05T07:30:26.176909Z","shell.execute_reply":"2022-08-05T07:30:28.930746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', 200)\npd.set_option('display.max_rows', 200)\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:28.934442Z","iopub.execute_input":"2022-08-05T07:30:28.935675Z","iopub.status.idle":"2022-08-05T07:30:28.941504Z","shell.execute_reply.started":"2022-08-05T07:30:28.935603Z","shell.execute_reply":"2022-08-05T07:30:28.940069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv', index_col='id')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv', index_col='id')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:28.943419Z","iopub.execute_input":"2022-08-05T07:30:28.944610Z","iopub.status.idle":"2022-08-05T07:30:29.265161Z","shell.execute_reply.started":"2022-08-05T07:30:28.944551Z","shell.execute_reply":"2022-08-05T07:30:29.263996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:29.268062Z","iopub.execute_input":"2022-08-05T07:30:29.268521Z","iopub.status.idle":"2022-08-05T07:30:29.410211Z","shell.execute_reply.started":"2022-08-05T07:30:29.268472Z","shell.execute_reply":"2022-08-05T07:30:29.400789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:29.411515Z","iopub.execute_input":"2022-08-05T07:30:29.413024Z","iopub.status.idle":"2022-08-05T07:30:29.436515Z","shell.execute_reply.started":"2022-08-05T07:30:29.412964Z","shell.execute_reply":"2022-08-05T07:30:29.434190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## FE","metadata":{}},{"cell_type":"code","source":"all_df = pd.concat([train, test])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:29.438399Z","iopub.execute_input":"2022-08-05T07:30:29.439323Z","iopub.status.idle":"2022-08-05T07:30:29.459880Z","shell.execute_reply.started":"2022-08-05T07:30:29.439274Z","shell.execute_reply":"2022-08-05T07:30:29.458827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# string to int\nfor col in all_df.columns:\n    if all_df[col].dtype != 'O':\n        continue\n    le = LabelEncoder()\n    all_df[col] = le.fit_transform(all_df[col])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:29.463800Z","iopub.execute_input":"2022-08-05T07:30:29.466065Z","iopub.status.idle":"2022-08-05T07:30:29.514527Z","shell.execute_reply.started":"2022-08-05T07:30:29.466009Z","shell.execute_reply":"2022-08-05T07:30:29.513690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_col_list = []\nfor col in all_df.columns:\n    if col != 'failure':\n        train_col_list.append(col)\ntrain_col_list","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:29.516363Z","iopub.execute_input":"2022-08-05T07:30:29.517209Z","iopub.status.idle":"2022-08-05T07:30:29.525952Z","shell.execute_reply.started":"2022-08-05T07:30:29.517164Z","shell.execute_reply":"2022-08-05T07:30:29.524940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\nall_df[train_col_list] = scaler.fit_transform(all_df[train_col_list])\nall_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:29.527761Z","iopub.execute_input":"2022-08-05T07:30:29.529582Z","iopub.status.idle":"2022-08-05T07:30:29.777334Z","shell.execute_reply.started":"2022-08-05T07:30:29.529535Z","shell.execute_reply":"2022-08-05T07:30:29.775966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Impute","metadata":{}},{"cell_type":"code","source":"na_index_of = {}\nno_na_index_of = {}\nfor col in train_col_list:\n    na_index = all_df[all_df[col].isna() == True].index\n    na_index_of[col] = na_index\n    no_na_index = all_df[all_df[col].isna() == False].index\n    no_na_index_of[col] = no_na_index","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:29.778889Z","iopub.execute_input":"2022-08-05T07:30:29.779341Z","iopub.status.idle":"2022-08-05T07:30:29.971517Z","shell.execute_reply.started":"2022-08-05T07:30:29.779298Z","shell.execute_reply":"2022-08-05T07:30:29.969922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for cnt in range(1):\n    for col in train_col_list:\n        na_index = na_index_of[col]\n        no_na_index = no_na_index_of[col]\n        X_col_list = train_col_list[:]\n        X_col_list.remove(col)\n        print(\"=\"*10, col, \"=\"*10)\n\n        if len(na_index) == 0:\n            continue\n\n        # split data\n        imp_train = all_df.loc[no_na_index]\n        X_test = all_df.loc[na_index][X_col_list]\n        X = imp_train[X_col_list]\n        y = imp_train[col]\n        X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, random_state=42)\n\n        # train\n        lgb_train = lgb.Dataset(X_train, y_train)\n        lgb_valid = lgb.Dataset(X_valid, y_valid, reference=lgb_train)\n        param = {\n            \"objective\": \"regression\",\n            \"metric\": \"rmse\",\n            \"n_estimators\": 9999,\n        }\n        callbacks = [\n            lgb.log_evaluation(250),\n            lgb.early_stopping(100),\n        ]\n        model = lgb.train(param, lgb_train, valid_sets=[lgb_valid], callbacks=callbacks)\n        y_predict = model.predict(X_test, num_iteration=model.best_iteration)\n\n        if cnt == 0:\n            lgb.plot_importance(model, max_num_features=20, title=col)\n            plt.show()\n\n        # impute to data\n        all_df[col].iloc[na_index,] = y_predict","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:30:29.975448Z","iopub.execute_input":"2022-08-05T07:30:29.976252Z","iopub.status.idle":"2022-08-05T07:31:45.097030Z","shell.execute_reply.started":"2022-08-05T07:30:29.976198Z","shell.execute_reply":"2022-08-05T07:31:45.095884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop product and attribute\ncol_list = all_df.columns\nfor col in col_list:\n    if col == \"product_code\" or \"attribute\" in col:\n        all_df.drop(columns=col, inplace=True)\n        train_col_list.remove(col)\nall_df","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:31:45.099670Z","iopub.execute_input":"2022-08-05T07:31:45.100516Z","iopub.status.idle":"2022-08-05T07:31:45.148932Z","shell.execute_reply.started":"2022-08-05T07:31:45.100465Z","shell.execute_reply":"2022-08-05T07:31:45.147340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = all_df[:train.shape[0]]\ntest = all_df[train.shape[0]:]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T07:31:45.150729Z","iopub.execute_input":"2022-08-05T07:31:45.151977Z","iopub.status.idle":"2022-08-05T07:31:45.159105Z","shell.execute_reply.started":"2022-08-05T07:31:45.151926Z","shell.execute_reply":"2022-08-05T07:31:45.158128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CV","metadata":{}},{"cell_type":"code","source":"n_splits = 5\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\nfor n, (train_index, val_index) in enumerate(skf.split(train, train['failure'])):\n    train.loc[val_index, 'fold'] = int(n)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:01:19.472675Z","iopub.execute_input":"2022-08-05T08:01:19.473313Z","iopub.status.idle":"2022-08-05T08:01:19.495283Z","shell.execute_reply.started":"2022-08-05T08:01:19.473255Z","shell.execute_reply":"2022-08-05T08:01:19.493910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## data argumentation","metadata":{}},{"cell_type":"markdown","source":"Try two types of data argumentation\n* One is a weighted average of data with similar measurements\n* The other is to adjust only the loading","metadata":{}},{"cell_type":"code","source":"def data_arg(train, col_list, r_list, nbh_num):\n    # MixUp\n    print('start data_argumentation. before shape is', train.shape)\n    dummy_list = []\n    \n    # 1. make KdTree for search neighbor\n    train_0 = train[train['failure'] == 0]\n    train_1 = train[train['failure'] == 1]\n    kd_data = train[col_list]\n    kd_data_0 = train_0[col_list]\n    kd_data_1 = train_1[col_list]\n    kdtree_0 = KDTree(kd_data_0)\n    kdtree_1 = KDTree(kd_data_1)\n\n    for row_id in tqdm(range(len(train))):\n        row = train.iloc[row_id]\n        failure = row['failure']\n        # 2. search neighbor\n        if failure == 0:\n            kdtree = kdtree_0\n            _train = train_0\n        else:\n            kdtree = kdtree_1\n            _train = train_1\n        nbh_d_list, nbh_row_id_list = kdtree.query(kd_data.iloc[row_id], k=nbh_num)\n        nbh_row_id_list = nbh_row_id_list[1:]  # Remove the beginning because it always contains itself\n        nbh = copy.deepcopy(_train.iloc[nbh_row_id_list])\n\n        # mix up row and nbh\n        for r in r_list:\n            dummy = copy.deepcopy(nbh)\n            dummy[col_list] = nbh[col_list]*r + row[col_list]*(1-r)\n            dummy_list.append(dummy)\n\n    # loading adjust\n    print('start loading adjust')\n\n    def loading_adj(col_list, ratio):\n        loading, fail = col_list\n        if fail:\n            loading *= ratio\n        else:\n            loading /= ratio\n        return loading\n\n    for ratio in tqdm([1.01, 1.1, 1.2, 1.3]):\n        dummy = copy.deepcopy(train)\n        dummy['loading'] = dummy[['loading', 'failure']].apply(lambda x: loading_adj(x, ratio), axis=1)\n        dummy_list.append(dummy)\n    \n    # concate all data argmentation record and original train data\n    dummy_all = pd.concat([train]+dummy_list).reset_index(drop=True)\n    print('end data arg. after shape is', dummy_all.shape)\n    return dummy_all","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:01:23.153159Z","iopub.execute_input":"2022-08-05T08:01:23.153600Z","iopub.status.idle":"2022-08-05T08:01:23.169132Z","shell.execute_reply.started":"2022-08-05T08:01:23.153564Z","shell.execute_reply":"2022-08-05T08:01:23.167240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_preds = []\ntrain[\"stack\"] = 0\nfor fold in range(n_splits):\n    print(\"=\"*10, fold, \"=\"*10)\n    train_folds = train[train['fold']!=fold]\n    valid_folds = train[train['fold']==fold]\n    train_folds = data_arg(train_folds, train_col_list, [0.1, 0.2, 0.3, 0.4], 10)  # Mixup every time because of leakage when mixup with OOF included.\n    X_train = train_folds[train_col_list]\n    y_train = train_folds['failure']\n    X_valid = valid_folds[train_col_list]\n    y_valid = valid_folds['failure']\n    X_test = test[train_col_list]\n\n    # train\n    lgb_train = lgb.Dataset(X_train, y_train)\n    lgb_valid = lgb.Dataset(X_valid, y_valid, reference=lgb_train)\n    param = {\n        \"objective\": \"binary\",\n        # \"metric\": \"auc\",\n        \"n_estimators\": 9999,\n        \"random_state\": 42,\n    }\n    callbacks = [\n        lgb.log_evaluation(250),\n        lgb.early_stopping(100),\n    ]\n    model = lgb.train(param, lgb_train, valid_sets=[lgb_valid], callbacks=callbacks)\n    y_predict = model.predict(X_valid, num_iteration=model.best_iteration)\n    train[\"stack\"].loc[train['fold']==fold] = y_predict\n    \n    auc = roc_auc_score(y_valid, y_predict)\n    print(\"auc: \", auc)\n    if fold == 0:\n        lgb.plot_importance(model, max_num_features=20, title=fold)\n        plt.show()\n\n    y_test = model.predict(X_test, num_iteration=model.best_iteration)\n    all_preds.append(y_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:04:35.827776Z","iopub.execute_input":"2022-08-05T08:04:35.828217Z","iopub.status.idle":"2022-08-05T08:17:54.555570Z","shell.execute_reply.started":"2022-08-05T08:04:35.828181Z","shell.execute_reply":"2022-08-05T08:17:54.554527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_preds = np.mean(all_preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:33:49.841201Z","iopub.execute_input":"2022-08-05T08:33:49.841653Z","iopub.status.idle":"2022-08-05T08:33:49.848300Z","shell.execute_reply.started":"2022-08-05T08:33:49.841596Z","shell.execute_reply":"2022-08-05T08:33:49.847171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_preds","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:33:51.919108Z","iopub.execute_input":"2022-08-05T08:33:51.919528Z","iopub.status.idle":"2022-08-05T08:33:51.928509Z","shell.execute_reply.started":"2022-08-05T08:33:51.919492Z","shell.execute_reply":"2022-08-05T08:33:51.927429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_pickle(\"train.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:33:54.050569Z","iopub.execute_input":"2022-08-05T08:33:54.051133Z","iopub.status.idle":"2022-08-05T08:33:54.068132Z","shell.execute_reply.started":"2022-08-05T08:33:54.051082Z","shell.execute_reply":"2022-08-05T08:33:54.066957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')\nsub['failure'] = all_preds\nsub.to_csv(\"submission.csv\", index=False)\nsub","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:33:56.010556Z","iopub.execute_input":"2022-08-05T08:33:56.010946Z","iopub.status.idle":"2022-08-05T08:33:56.093217Z","shell.execute_reply.started":"2022-08-05T08:33:56.010915Z","shell.execute_reply":"2022-08-05T08:33:56.092024Z"},"trusted":true},"execution_count":null,"outputs":[]}]}