{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TPS0421 LGBM + Iterative Imputer","metadata":{"papermill":{"duration":0.01723,"end_time":"2021-10-26T07:43:51.454989","exception":false,"start_time":"2021-10-26T07:43:51.437759","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport random\nimport optuna\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":2.433958,"end_time":"2021-10-26T07:43:53.905228","exception":false,"start_time":"2021-10-26T07:43:51.471270","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:22:57.369900Z","iopub.execute_input":"2022-08-05T17:22:57.370317Z","iopub.status.idle":"2022-08-05T17:22:57.377772Z","shell.execute_reply.started":"2022-08-05T17:22:57.370277Z","shell.execute_reply":"2022-08-05T17:22:57.376225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer #after import enable_iterative_imputer","metadata":{"execution":{"iopub.status.busy":"2022-08-05T17:22:57.389485Z","iopub.execute_input":"2022-08-05T17:22:57.390819Z","iopub.status.idle":"2022-08-05T17:22:57.395342Z","shell.execute_reply.started":"2022-08-05T17:22:57.390758Z","shell.execute_reply":"2022-08-05T17:22:57.394338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train0 = pd.read_csv(\"../input/tabular-playground-series-apr-2021/train.csv\")\ntest0 = pd.read_csv(\"../input/tabular-playground-series-apr-2021/test.csv\")","metadata":{"papermill":{"duration":0.52749,"end_time":"2021-10-26T07:43:54.450958","exception":false,"start_time":"2021-10-26T07:43:53.923468","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:22:57.407957Z","iopub.execute_input":"2022-08-05T17:22:57.408358Z","iopub.status.idle":"2022-08-05T17:22:57.789760Z","shell.execute_reply.started":"2022-08-05T17:22:57.408328Z","shell.execute_reply":"2022-08-05T17:22:57.788414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols=train0.columns.tolist()\ndata0=pd.concat([train0,test0],axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T17:22:57.801580Z","iopub.execute_input":"2022-08-05T17:22:57.802339Z","iopub.status.idle":"2022-08-05T17:22:57.857147Z","shell.execute_reply.started":"2022-08-05T17:22:57.802303Z","shell.execute_reply":"2022-08-05T17:22:57.855678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef labelencoder(df):\n    for c in df.columns:\n        if df[c].dtype=='object':\n            df[c] = df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(df[c].values))\n            df[c] = lbl.transform(df[c].values)\n    return df","metadata":{"papermill":{"duration":0.786741,"end_time":"2021-10-26T07:43:55.485769","exception":false,"start_time":"2021-10-26T07:43:54.699028","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:22:57.860509Z","iopub.execute_input":"2022-08-05T17:22:57.861466Z","iopub.status.idle":"2022-08-05T17:22:57.871800Z","shell.execute_reply.started":"2022-08-05T17:22:57.861417Z","shell.execute_reply":"2022-08-05T17:22:57.870294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1=labelencoder(data0)\ndisplay(data1.info())","metadata":{"execution":{"iopub.status.busy":"2022-08-05T17:22:57.873963Z","iopub.execute_input":"2022-08-05T17:22:57.874446Z","iopub.status.idle":"2022-08-05T17:22:59.655151Z","shell.execute_reply.started":"2022-08-05T17:22:57.874403Z","shell.execute_reply":"2022-08-05T17:22:59.654164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_imp = IterativeImputer(\n    max_iter=9, random_state=42, verbose=0, \n    skip_complete=True, n_nearest_features=10, tol=0.001)\n\nmulti_imp.fit(data0)\ndata2 = multi_imp.transform(data1)\ndata3=pd.DataFrame(data=data2,columns=cols)\ndisplay(data3.info())","metadata":{"papermill":{"duration":0.702877,"end_time":"2021-10-26T07:43:56.207180","exception":false,"start_time":"2021-10-26T07:43:55.504303","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:22:59.656569Z","iopub.execute_input":"2022-08-05T17:22:59.658041Z","iopub.status.idle":"2022-08-05T17:23:00.657807Z","shell.execute_reply.started":"2022-08-05T17:22:59.658000Z","shell.execute_reply":"2022-08-05T17:23:00.656555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=data3.iloc[0:len(train0)]\ntest=data3.iloc[len(train0):]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T17:23:00.659575Z","iopub.execute_input":"2022-08-05T17:23:00.660009Z","iopub.status.idle":"2022-08-05T17:23:00.675091Z","shell.execute_reply.started":"2022-08-05T17:23:00.659969Z","shell.execute_reply":"2022-08-05T17:23:00.673410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainY = train['Survived']\ntrainX = train.drop(['Survived','PassengerId','Name','Ticket'],axis=1)\ntestX = test.drop(['Survived','PassengerId','Name','Ticket'],axis=1)","metadata":{"papermill":{"duration":0.036014,"end_time":"2021-10-26T07:43:56.262376","exception":false,"start_time":"2021-10-26T07:43:56.226362","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:00.677616Z","iopub.execute_input":"2022-08-05T17:23:00.683453Z","iopub.status.idle":"2022-08-05T17:23:00.707166Z","shell.execute_reply.started":"2022-08-05T17:23:00.683381Z","shell.execute_reply":"2022-08-05T17:23:00.705544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns=trainX.columns.to_list()\nprint(columns)","metadata":{"papermill":{"duration":0.025976,"end_time":"2021-10-26T07:43:56.306083","exception":false,"start_time":"2021-10-26T07:43:56.280107","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:00.715115Z","iopub.execute_input":"2022-08-05T17:23:00.719636Z","iopub.status.idle":"2022-08-05T17:23:00.731561Z","shell.execute_reply.started":"2022-08-05T17:23:00.719469Z","shell.execute_reply":"2022-08-05T17:23:00.729054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective(trial,data=trainX,target=trainY):\n    \n    train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)\n    param =   {\n        'num_leaves': trial.suggest_int('num_leaves', 2, 256),\n        'objective': trial.suggest_categorical('objective',['regression','rmse','binary']),  \n        'max_depth': trial.suggest_int('max_depth', -1, 10),\n        'learning_rate': trial.suggest_loguniform('learning_rate', 1e-2, 1.0),\n        \"boosting\": \"gbdt\",\n        'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),\n        'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),\n        \"bagging_freq\": trial.suggest_int('bagging_freq', 1, 10),\n        \"bagging_fraction\": trial.suggest_uniform('bagging_fraction', 0.1, 1.0),\n        \"feature_fraction\": trial.suggest_uniform('feature_fraction', 0.4, 1.0),\n        \"verbosity\": -1,\n    }\n    model = lgb.LGBMClassifier(**param)      \n    model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)\n    preds = model.predict(test_x)\n    rmse = mean_squared_error(test_y, preds,squared=False)\n    \n    return rmse","metadata":{"papermill":{"duration":0.029006,"end_time":"2021-10-26T07:43:56.353222","exception":false,"start_time":"2021-10-26T07:43:56.324216","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:00.738007Z","iopub.execute_input":"2022-08-05T17:23:00.738501Z","iopub.status.idle":"2022-08-05T17:23:00.751566Z","shell.execute_reply.started":"2022-08-05T17:23:00.738463Z","shell.execute_reply":"2022-08-05T17:23:00.750189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=32)\nprint('Number of finished trials:', len(study.trials))\nprint('Best trial:', study.best_trial.params)","metadata":{"papermill":{"duration":22.979688,"end_time":"2021-10-26T07:44:19.351150","exception":false,"start_time":"2021-10-26T07:43:56.371462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:00.753234Z","iopub.execute_input":"2022-08-05T17:23:00.753630Z","iopub.status.idle":"2022-08-05T17:23:29.363362Z","shell.execute_reply.started":"2022-08-05T17:23:00.753597Z","shell.execute_reply":"2022-08-05T17:23:29.361845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shows the scores from all trials\noptuna.visualization.plot_optimization_history(study)","metadata":{"papermill":{"duration":0.148545,"end_time":"2021-10-26T07:44:19.645821","exception":false,"start_time":"2021-10-26T07:44:19.497276","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:29.413770Z","iopub.execute_input":"2022-08-05T17:23:29.414169Z","iopub.status.idle":"2022-08-05T17:23:29.569433Z","shell.execute_reply.started":"2022-08-05T17:23:29.414133Z","shell.execute_reply":"2022-08-05T17:23:29.567782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# interactively visualizes the hyperparameters and scores\noptuna.visualization.plot_parallel_coordinate(study)","metadata":{"papermill":{"duration":0.096828,"end_time":"2021-10-26T07:44:19.779920","exception":false,"start_time":"2021-10-26T07:44:19.683092","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:29.571198Z","iopub.execute_input":"2022-08-05T17:23:29.572014Z","iopub.status.idle":"2022-08-05T17:23:29.664483Z","shell.execute_reply.started":"2022-08-05T17:23:29.571965Z","shell.execute_reply":"2022-08-05T17:23:29.663212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shows the evolution of the search\noptuna.visualization.plot_slice(study)","metadata":{"papermill":{"duration":0.350197,"end_time":"2021-10-26T07:44:20.167304","exception":false,"start_time":"2021-10-26T07:44:19.817107","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:29.666168Z","iopub.execute_input":"2022-08-05T17:23:29.666917Z","iopub.status.idle":"2022-08-05T17:23:30.059600Z","shell.execute_reply.started":"2022-08-05T17:23:29.666876Z","shell.execute_reply":"2022-08-05T17:23:30.057824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize parameter importances.\noptuna.visualization.plot_param_importances(study)","metadata":{"papermill":{"duration":0.871393,"end_time":"2021-10-26T07:44:21.403209","exception":false,"start_time":"2021-10-26T07:44:20.531816","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:30.380717Z","iopub.execute_input":"2022-08-05T17:23:30.381273Z","iopub.status.idle":"2022-08-05T17:23:31.165154Z","shell.execute_reply.started":"2022-08-05T17:23:30.381224Z","shell.execute_reply":"2022-08-05T17:23:31.163823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize empirical distribution function\noptuna.visualization.plot_edf(study)","metadata":{"papermill":{"duration":0.054343,"end_time":"2021-10-26T07:44:21.497457","exception":false,"start_time":"2021-10-26T07:44:21.443114","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:31.166950Z","iopub.execute_input":"2022-08-05T17:23:31.167480Z","iopub.status.idle":"2022-08-05T17:23:31.187225Z","shell.execute_reply.started":"2022-08-05T17:23:31.167439Z","shell.execute_reply":"2022-08-05T17:23:31.185971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Best_trial=study.best_trial.params\nprint(Best_trial)","metadata":{"papermill":{"duration":0.047827,"end_time":"2021-10-26T07:44:21.586253","exception":false,"start_time":"2021-10-26T07:44:21.538426","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:31.188872Z","iopub.execute_input":"2022-08-05T17:23:31.189206Z","iopub.status.idle":"2022-08-05T17:23:31.197126Z","shell.execute_reply.started":"2022-08-05T17:23:31.189175Z","shell.execute_reply":"2022-08-05T17:23:31.195629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv(\"../input/tabular-playground-series-apr-2021/sample_submission.csv\")\nprint(sample.shape)","metadata":{"papermill":{"duration":0.078189,"end_time":"2021-10-26T07:44:21.704950","exception":false,"start_time":"2021-10-26T07:44:21.626761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:31.199344Z","iopub.execute_input":"2022-08-05T17:23:31.199914Z","iopub.status.idle":"2022-08-05T17:23:31.246334Z","shell.execute_reply.started":"2022-08-05T17:23:31.199874Z","shell.execute_reply":"2022-08-05T17:23:31.244599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[columns]","metadata":{"papermill":{"duration":0.067797,"end_time":"2021-10-26T07:44:21.814665","exception":false,"start_time":"2021-10-26T07:44:21.746868","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:31.248722Z","iopub.execute_input":"2022-08-05T17:23:31.249287Z","iopub.status.idle":"2022-08-05T17:23:31.279235Z","shell.execute_reply.started":"2022-08-05T17:23:31.249241Z","shell.execute_reply":"2022-08-05T17:23:31.278173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.zeros((sample.shape[0]))\ntrain=trainX\ntarget=trainY\ntest=testX\nkf = KFold(n_splits=5,random_state=48,shuffle=True)\nfor trn_idx, test_idx in kf.split(train[columns],target):\n    X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]\n    y_tr,y_val=target.iloc[trn_idx],target.iloc[test_idx]\n    model = lgb.LGBMClassifier(**Best_trial)\n    model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)\n    preds+=model.predict(test[columns])/kf.n_splits   ###### predict_proba\n    rmse=mean_squared_error(y_val, model.predict(X_val),squared=False)\n    print(rmse)","metadata":{"papermill":{"duration":4.891369,"end_time":"2021-10-26T07:44:26.747003","exception":false,"start_time":"2021-10-26T07:44:21.855634","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:31.280844Z","iopub.execute_input":"2022-08-05T17:23:31.282561Z","iopub.status.idle":"2022-08-05T17:23:40.724742Z","shell.execute_reply.started":"2022-08-05T17:23:31.282513Z","shell.execute_reply":"2022-08-05T17:23:40.723801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(preds.shape)\nprint(preds[0])","metadata":{"papermill":{"duration":0.051683,"end_time":"2021-10-26T07:44:26.841532","exception":false,"start_time":"2021-10-26T07:44:26.789849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:40.729284Z","iopub.execute_input":"2022-08-05T17:23:40.731604Z","iopub.status.idle":"2022-08-05T17:23:40.739384Z","shell.execute_reply.started":"2022-08-05T17:23:40.731561Z","shell.execute_reply":"2022-08-05T17:23:40.737897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = sample\nsubm['Survived'] = preds.astype(int)\nsubm.to_csv('submission.csv',index=False)\ndisplay(subm)","metadata":{"papermill":{"duration":0.167149,"end_time":"2021-10-26T07:44:27.052059","exception":false,"start_time":"2021-10-26T07:44:26.884910","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:40.741447Z","iopub.execute_input":"2022-08-05T17:23:40.741906Z","iopub.status.idle":"2022-08-05T17:23:40.876824Z","shell.execute_reply.started":"2022-08-05T17:23:40.741872Z","shell.execute_reply":"2022-08-05T17:23:40.875127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm['Survived'].value_counts()","metadata":{"papermill":{"duration":0.054643,"end_time":"2021-10-26T07:44:27.150909","exception":false,"start_time":"2021-10-26T07:44:27.096266","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-05T17:23:40.878989Z","iopub.execute_input":"2022-08-05T17:23:40.879696Z","iopub.status.idle":"2022-08-05T17:23:40.891345Z","shell.execute_reply.started":"2022-08-05T17:23:40.879659Z","shell.execute_reply":"2022-08-05T17:23:40.889977Z"},"trusted":true},"execution_count":null,"outputs":[]}]}