{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport joblib\nimport gc\nimport itertools\nfrom itertools import combinations\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\n\nimport lightgbm as lgb\nfrom sklearn.preprocessing import LabelEncoder\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\n\n\npd.set_option('display.width', 1000)\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\nimport warnings; warnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:40:59.819240Z","iopub.execute_input":"2022-08-02T09:40:59.819893Z","iopub.status.idle":"2022-08-02T09:41:01.795039Z","shell.execute_reply.started":"2022-08-02T09:40:59.819764Z","shell.execute_reply":"2022-08-02T09:41:01.794038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# trainデータのIDとデータ数とS_2のdf作成","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_parquet('../input/amex-parquet/train_data.parquet')\ntrain = train_df[[\"customer_ID\", \"S_2\"]]\ndisplay([\"customer_ID\", \"S_2\", \"target\"])\ndel train_df","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:41:01.797220Z","iopub.execute_input":"2022-08-02T09:41:01.797544Z","iopub.status.idle":"2022-08-02T09:41:44.145082Z","shell.execute_reply.started":"2022-08-02T09:41:01.797515Z","shell.execute_reply":"2022-08-02T09:41:44.143580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ID_unique = train[\"customer_ID\"].unique()\nprint(len(train_ID_unique))","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:41:44.147255Z","iopub.execute_input":"2022-08-02T09:41:44.147663Z","iopub.status.idle":"2022-08-02T09:41:45.220109Z","shell.execute_reply.started":"2022-08-02T09:41:44.147602Z","shell.execute_reply":"2022-08-02T09:41:45.218861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ID_value_counts = train[\"customer_ID\"].value_counts()\nID_value_counts_df = pd.DataFrame(ID_value_counts)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:41:45.224333Z","iopub.execute_input":"2022-08-02T09:41:45.224908Z","iopub.status.idle":"2022-08-02T09:41:45.816303Z","shell.execute_reply.started":"2022-08-02T09:41:45.224874Z","shell.execute_reply":"2022-08-02T09:41:45.815241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(ID_value_counts_df)\nID_value_counts_df.to_csv(\"train_ID_counts.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:41:45.818035Z","iopub.execute_input":"2022-08-02T09:41:45.818484Z","iopub.status.idle":"2022-08-02T09:41:46.061896Z","shell.execute_reply.started":"2022-08-02T09:41:45.818439Z","shell.execute_reply":"2022-08-02T09:41:46.060613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"S_2\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:41:46.063721Z","iopub.execute_input":"2022-08-02T09:41:46.064486Z","iopub.status.idle":"2022-08-02T09:41:46.315941Z","shell.execute_reply.started":"2022-08-02T09:41:46.064439Z","shell.execute_reply":"2022-08-02T09:41:46.314720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:41:46.317576Z","iopub.execute_input":"2022-08-02T09:41:46.318676Z","iopub.status.idle":"2022-08-02T09:41:46.593216Z","shell.execute_reply.started":"2022-08-02T09:41:46.318599Z","shell.execute_reply":"2022-08-02T09:41:46.590866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# testデータのIDとデータ数とS_2のdf作成","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_feather('../input/parquet-files-amexdefault-prediction/test_data.ftr')\ntest = test_df[[\"customer_ID\", \"S_2\"]]\ndel test_df","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:41:46.595239Z","iopub.execute_input":"2022-08-02T09:41:46.596234Z","iopub.status.idle":"2022-08-02T09:42:24.695767Z","shell.execute_reply.started":"2022-08-02T09:41:46.596186Z","shell.execute_reply":"2022-08-02T09:42:24.694142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ID_unique = test[\"customer_ID\"].unique()\nprint(len(test_ID_unique))","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:42:37.231361Z","iopub.execute_input":"2022-08-02T09:42:37.231819Z","iopub.status.idle":"2022-08-02T09:42:39.178516Z","shell.execute_reply.started":"2022-08-02T09:42:37.231780Z","shell.execute_reply":"2022-08-02T09:42:39.177052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ID_value_counts = test[\"customer_ID\"].value_counts()\nID_value_counts_df = pd.DataFrame(ID_value_counts)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:42:41.111012Z","iopub.execute_input":"2022-08-02T09:42:41.111446Z","iopub.status.idle":"2022-08-02T09:42:42.438148Z","shell.execute_reply.started":"2022-08-02T09:42:41.111412Z","shell.execute_reply":"2022-08-02T09:42:42.436913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(ID_value_counts_df)\nID_value_counts_df.to_csv(\"test_ID_counts.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:42:42.440886Z","iopub.execute_input":"2022-08-02T09:42:42.441286Z","iopub.status.idle":"2022-08-02T09:42:42.933545Z","shell.execute_reply.started":"2022-08-02T09:42:42.441232Z","shell.execute_reply":"2022-08-02T09:42:42.932243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"S_2\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:42:42.938085Z","iopub.execute_input":"2022-08-02T09:42:42.938470Z","iopub.status.idle":"2022-08-02T09:42:43.458344Z","shell.execute_reply.started":"2022-08-02T09:42:42.938428Z","shell.execute_reply":"2022-08-02T09:42:43.456963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:42:46.390225Z","iopub.execute_input":"2022-08-02T09:42:46.390913Z","iopub.status.idle":"2022-08-02T09:42:46.487197Z","shell.execute_reply.started":"2022-08-02T09:42:46.390879Z","shell.execute_reply":"2022-08-02T09:42:46.485831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# trainとtestのIDの違い確認","metadata":{}},{"cell_type":"code","source":"diff_train_test = set(train_ID_unique) - set(test_ID_unique)\ndiff_test_train = set(test_ID_unique) - set(train_ID_unique)\n\nprint(len(train_ID_unique))\nprint(len(diff_train_test))\nprint(\" \")\nprint(len(test_ID_unique))\nprint(len(diff_test_train))","metadata":{"execution":{"iopub.status.busy":"2022-08-02T09:44:04.455041Z","iopub.execute_input":"2022-08-02T09:44:04.455927Z","iopub.status.idle":"2022-08-02T09:44:05.189809Z","shell.execute_reply.started":"2022-08-02T09:44:04.455886Z","shell.execute_reply":"2022-08-02T09:44:05.188583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}