{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import modules","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn import preprocessing\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis as lda\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.515047,"end_time":"2022-08-02T01:01:05.232854","exception":false,"start_time":"2022-08-02T01:01:02.717807","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-08T01:01:30.351161Z","iopub.execute_input":"2022-08-08T01:01:30.351708Z","iopub.status.idle":"2022-08-08T01:01:30.365466Z","shell.execute_reply.started":"2022-08-08T01:01:30.351670Z","shell.execute_reply":"2022-08-08T01:01:30.361565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install --ignore-installed --pre pycaret\n!pip install tune-sklearn ray[tune]\n!pip install hyperopt\nfrom pycaret.classification import *","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-08T01:01:30.368054Z","iopub.execute_input":"2022-08-08T01:01:30.368797Z","iopub.status.idle":"2022-08-08T01:02:34.176889Z","shell.execute_reply.started":"2022-08-08T01:01:30.368757Z","shell.execute_reply":"2022-08-08T01:02:34.166311Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/test.csv\")","metadata":{"papermill":{"duration":0.306332,"end_time":"2022-08-02T01:01:05.551187","exception":false,"start_time":"2022-08-02T01:01:05.244855","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-08T01:02:36.314747Z","iopub.execute_input":"2022-08-08T01:02:36.315351Z","iopub.status.idle":"2022-08-08T01:02:36.591309Z","shell.execute_reply.started":"2022-08-08T01:02:36.315298Z","shell.execute_reply":"2022-08-08T01:02:36.589574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# refer: https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342319\ndf_train['m_3_missing'] = df_train.measurement_3.isna()\ndf_train['m_5_missing'] = df_train.measurement_5.isna()\n\ndf_test['m_3_missing'] = df_test.measurement_3.isna()\ndf_test['m_5_missing'] = df_test.measurement_5.isna()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:02:42.512284Z","iopub.execute_input":"2022-08-08T01:02:42.514084Z","iopub.status.idle":"2022-08-08T01:02:42.531049Z","shell.execute_reply.started":"2022-08-08T01:02:42.514020Z","shell.execute_reply":"2022-08-08T01:02:42.529683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_numeric = df_train.select_dtypes(include=['int16', 'int32', 'int64', 'float16', 'float32', 'float64']).columns[:-1]\nimputer = IterativeImputer(max_iter=9, random_state=42, verbose = 0,\n                            skip_complete = True, n_nearest_features = 10,\n                            tol = 0.001)\n\nfor i in ['A', 'B', 'C', 'D', 'E']:\n    df_train.loc[df_train['product_code']==i, col_numeric] = imputer.fit_transform(df_train[df_train['product_code']==i][col_numeric])\nfor i in ['F', 'G', 'H', 'I']:\n    df_test.loc[df_test['product_code']==i, col_numeric] = imputer.fit_transform(df_test[df_test['product_code']==i][col_numeric])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:02:43.176656Z","iopub.execute_input":"2022-08-08T01:02:43.177768Z","iopub.status.idle":"2022-08-08T01:03:05.735490Z","shell.execute_reply.started":"2022-08-08T01:02:43.177708Z","shell.execute_reply":"2022-08-08T01:03:05.733913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# refer: https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342126\ndf_train['attribute_2*3'] = df_train['attribute_2'] * df_train['attribute_3']\ndf_test['attribute_2*3'] = df_test['attribute_2'] * df_test['attribute_3']","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:03:05.738251Z","iopub.execute_input":"2022-08-08T01:03:05.738794Z","iopub.status.idle":"2022-08-08T01:03:05.749437Z","shell.execute_reply.started":"2022-08-08T01:03:05.738754Z","shell.execute_reply":"2022-08-08T01:03:05.747568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# refer: https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342126\ndf_train['mean_3_to_16'] = df_train[[f'measurement_{i}' for i in range(3, 17)]].mean(axis=1)\ndf_train['std_3_to_16'] = df_train[[f'measurement_{i}' for i in range(3, 17)]].std(axis=1)\n\ndf_test['mean_3_to_16'] = df_test[[f'measurement_{i}' for i in range(3, 17)]].mean(axis=1)\ndf_test['std_3_to_16'] = df_test[[f'measurement_{i}' for i in range(3, 17)]].std(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:03:05.751882Z","iopub.execute_input":"2022-08-08T01:03:05.752281Z","iopub.status.idle":"2022-08-08T01:03:05.800715Z","shell.execute_reply.started":"2022-08-08T01:03:05.752239Z","shell.execute_reply":"2022-08-08T01:03:05.799651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['loading'] = np.log1p(df_train['loading'])\ndf_test['loading'] = np.log1p(df_test['loading'])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:50:54.047346Z","iopub.execute_input":"2022-08-08T01:50:54.048292Z","iopub.status.idle":"2022-08-08T01:50:54.057134Z","shell.execute_reply.started":"2022-08-08T01:50:54.048242Z","shell.execute_reply":"2022-08-08T01:50:54.055543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.concat((df_train, df_test), axis=0)\ndf_all = df_all.drop(['id', 'product_code'] + [f'measurement_{i}' for i in range(3, 17)], axis=1)\ndf_all[[f'attribute_{i}' for i in range(2)]] = df_all[[f'attribute_{i}' for i in range(2)]].astype('object')\ndf_all = pd.get_dummies(df_all)\ndf_train = df_all.iloc[:df_train.shape[0]]\ndf_test = df_all.iloc[df_train.shape[0]:]\n\ndf_train = df_train.drop(['attribute_1_material_7', 'measurement_17'], axis=1)\ndf_test = df_test.drop(['attribute_1_material_7', 'measurement_17', 'failure'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:03:05.802809Z","iopub.execute_input":"2022-08-08T01:03:05.803676Z","iopub.status.idle":"2022-08-08T01:03:05.886113Z","shell.execute_reply.started":"2022-08-08T01:03:05.803618Z","shell.execute_reply":"2022-08-08T01:03:05.884592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scaler = StandardScaler()\n# col_scaled = ['loading', 'attribute_2', 'attribute_3', 'measurement_0', 'measurement_1', 'measurement_2', 'measurement_17', 'attribute_2*3', 'mean_3_to_16', 'std_3_to_16']\n# df_train[col_scaled] = scaler.fit_transform(df_train[col_scaled])\n# df_test[col_scaled] = scaler.transform(df_test[col_scaled])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:02:34.203713Z","iopub.status.idle":"2022-08-08T01:02:34.205000Z","shell.execute_reply.started":"2022-08-08T01:02:34.204610Z","shell.execute_reply":"2022-08-08T01:02:34.204649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, y_train = df_train.drop(['failure'], axis=1), df_train['failure']","metadata":{"papermill":{"duration":1.030391,"end_time":"2022-08-02T01:25:04.934074","exception":false,"start_time":"2022-08-02T01:25:03.903683","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-08T01:03:05.887977Z","iopub.execute_input":"2022-08-08T01:03:05.888601Z","iopub.status.idle":"2022-08-08T01:03:05.899653Z","shell.execute_reply.started":"2022-08-08T01:03:05.888545Z","shell.execute_reply":"2022-08-08T01:03:05.897887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create and tune Model","metadata":{}},{"cell_type":"code","source":"# 6390\ns = setup(\n    session_id = 6390,\n    data=X_train,\n    target=y_train\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:03:05.901816Z","iopub.execute_input":"2022-08-08T01:03:05.902434Z","iopub.status.idle":"2022-08-08T01:03:07.180407Z","shell.execute_reply.started":"2022-08-08T01:03:05.902381Z","shell.execute_reply":"2022-08-08T01:03:07.179173Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lda = create_model('lda')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:36:27.209576Z","iopub.execute_input":"2022-08-08T01:36:27.210713Z","iopub.status.idle":"2022-08-08T01:39:35.346054Z","shell.execute_reply.started":"2022-08-08T01:36:27.210664Z","shell.execute_reply":"2022-08-08T01:39:35.344661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_score = roc_auc_score(y_train, 1 - predict_model(model_lda, X_train)['Score'])\n# test_score = roc_auc_score(y_test, 1 - predict_model(model_lda, X_test)['Score'])\ntrain_score","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:02:34.223295Z","iopub.status.idle":"2022-08-08T01:02:34.224429Z","shell.execute_reply.started":"2022-08-08T01:02:34.224087Z","shell.execute_reply":"2022-08-08T01:02:34.224121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tune_lda = tune_model(model_lda, n_iter=150, optimize='AUC', search_library='tune-sklearn', search_algorithm='hyperopt', choose_better=True)\n# tune_lr = tune_model(model_lr, n_iter=100, optimize='AUC', search_library='tune-sklearn', search_algorithm='hyperopt', choose_better=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:02:34.226587Z","iopub.status.idle":"2022-08-08T01:02:34.227716Z","shell.execute_reply.started":"2022-08-08T01:02:34.227347Z","shell.execute_reply":"2022-08-08T01:02:34.227383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_lda = lda(**tune_lda.get_params()).fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:02:34.229992Z","iopub.status.idle":"2022-08-08T01:02:34.231154Z","shell.execute_reply.started":"2022-08-08T01:02:34.230801Z","shell.execute_reply":"2022-08-08T01:02:34.230836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = final_lda.predict_proba(df_test)[:,1]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:02:34.233265Z","iopub.status.idle":"2022-08-08T01:02:34.234429Z","shell.execute_reply.started":"2022-08-08T01:02:34.234050Z","shell.execute_reply":"2022-08-08T01:02:34.234085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/sample_submission.csv\")\nsubmission.failure = y_pred\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:45:41.320429Z","iopub.execute_input":"2022-08-08T01:45:41.321737Z","iopub.status.idle":"2022-08-08T01:45:41.386584Z","shell.execute_reply.started":"2022-08-08T01:45:41.321696Z","shell.execute_reply":"2022-08-08T01:45:41.385224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:45:42.100647Z","iopub.execute_input":"2022-08-08T01:45:42.101103Z","iopub.status.idle":"2022-08-08T01:45:42.115987Z","shell.execute_reply.started":"2022-08-08T01:45:42.101063Z","shell.execute_reply":"2022-08-08T01:45:42.114817Z"},"trusted":true},"execution_count":null,"outputs":[]}]}