{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### We see we got better score when we use KNN Imputer instead of SimpleImputer. I used 5-Fold GroupKFold Cross validation technique to make groups based on\n1. Target i.e Failure\n2. Product_code. So in product_code we got 3 categories that are present in training set but not present in test set. So it is not use to use this feature. Instead we can use this feature so that model will learn to make decision based on unseen categories present in the product_code feature\n\nReference post --> https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/341070","metadata":{}},{"cell_type":"markdown","source":"# Importing necessary libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.metrics import roc_auc_score\nwarnings.filterwarnings(\"ignore\")\nplt.style.use(\"fivethirtyeight\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-03T03:12:57.477494Z","iopub.execute_input":"2022-08-03T03:12:57.477902Z","iopub.status.idle":"2022-08-03T03:13:00.498121Z","shell.execute_reply.started":"2022-08-03T03:12:57.477822Z","shell.execute_reply":"2022-08-03T03:13:00.497160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing Train, Test and Submission Dataset","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"../input/tabular-playground-series-aug-2022/train.csv\")\ntest_data = pd.read_csv(\"../input/tabular-playground-series-aug-2022/test.csv\")\nsubmission = pd.read_csv(\"../input/tabular-playground-series-aug-2022/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:13:00.500072Z","iopub.execute_input":"2022-08-03T03:13:00.500436Z","iopub.status.idle":"2022-08-03T03:13:00.732599Z","shell.execute_reply.started":"2022-08-03T03:13:00.500402Z","shell.execute_reply":"2022-08-03T03:13:00.731615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"useful_features = [feature for feature in train_data.columns if feature not in [\"id\", \"failure\",\"product_code\"]]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:13:00.736238Z","iopub.execute_input":"2022-08-03T03:13:00.736534Z","iopub.status.idle":"2022-08-03T03:13:00.742204Z","shell.execute_reply.started":"2022-08-03T03:13:00.736507Z","shell.execute_reply":"2022-08-03T03:13:00.741012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:13:00.744434Z","iopub.execute_input":"2022-08-03T03:13:00.745030Z","iopub.status.idle":"2022-08-03T03:13:00.791927Z","shell.execute_reply.started":"2022-08-03T03:13:00.744995Z","shell.execute_reply":"2022-08-03T03:13:00.790987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training LightGBM model with 5 Fold-GroupKFold Technique","metadata":{}},{"cell_type":"code","source":"gkf = GroupKFold(n_splits=5)\n\nfinal_predictions = []\nroc_score = []\n\nfor fold, (train_idx, valid_idx) in enumerate(gkf.split(train_data\n                                                        ,train_data.failure\n                                                        ,train_data.product_code\n                                                       )\n                                             ):\n    \n    params_lgb = {'learning_rate': 0.001,\n              'objective': 'binary',\n              'boosting': 'gbdt',\n              'verbosity': -1,\n              'n_jobs': -1, \n             \"device_type\":\"gpu\",\n             \"gpu_platform_id\" : 0,\n            \"gpu_device_id\" : 0} \n    \n    X_train = train_data.iloc[train_idx][useful_features]\n    X_valid = train_data.iloc[valid_idx][useful_features]\n    y_train = train_data.iloc[train_idx][\"failure\"]\n    y_valid = train_data.iloc[valid_idx][\"failure\"]\n\n    \n    X_test = test_data.copy()\n    X_test = X_test.drop([\"id\",\"product_code\"],axis=1)\n    \n    \n    # Code Reference :- https://www.kaggle.com/code/ambrosm/tpsaug22-eda-which-makes-sense\n    ohe_attributes = ['attribute_0', 'attribute_1']\n    ohe_output = ['ohe0_7', 'ohe1_6', 'ohe1_8']\n    ohe = OneHotEncoder(categories=[['material_5', 'material_7'],\n                                    ['material_5', 'material_6', 'material_8']],\n                        drop='first', sparse=False, handle_unknown='ignore')\n    ohe.fit(X_train[ohe_attributes])\n    for df in [X_train, X_valid, X_test]:\n        with warnings.catch_warnings(): # ignore \"Found unknown categories\"\n            warnings.filterwarnings('ignore', category=UserWarning)\n            df[ohe_output] = ohe.transform(df[ohe_attributes])\n        df.drop(columns=ohe_attributes, inplace=True)\n     \n    # Using KNNImputer to impute the missing values. This is not the mandatory method. We have lot of techniques \n    # to impute missing values. It all depends on the data the model which are going to use\n    imputer = KNNImputer()\n    \n    X_train = imputer.fit_transform(X_train)\n    X_valid = imputer.transform(X_valid)\n    X_test = imputer.transform(X_test)\n    \n    X_trn = lgb.Dataset(X_train, \n                        y_train)\n    \n    X_val = lgb.Dataset(X_valid, \n                        y_valid)\n    \n    model = lgb.train(params = params_lgb, \n                      train_set = X_trn, \n                      valid_sets =  X_val, \n                      num_boost_round = 5000, \n                      callbacks = [ lgb.early_stopping(stopping_rounds=100, verbose=True), \n                                 lgb.log_evaluation(period=200)])  \n    \n    preds_valid = model.predict(X_valid)\n    print(\"The ROC score after {} fold is {}\".format(fold,roc_auc_score(y_valid,preds_valid)))\n    roc_score.append(roc_auc_score(y_valid,preds_valid))\n    test_preds = model.predict(X_test)\n    final_predictions.append(test_preds)\n    \nprint(f\"Mean ROC_AUC Score is : {np.mean(roc_score)}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:13:09.977328Z","iopub.execute_input":"2022-08-03T03:13:09.977774Z","iopub.status.idle":"2022-08-03T03:17:56.223458Z","shell.execute_reply.started":"2022-08-03T03:13:09.977737Z","shell.execute_reply":"2022-08-03T03:17:56.222589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:27:21.636086Z","iopub.execute_input":"2022-08-03T03:27:21.636733Z","iopub.status.idle":"2022-08-03T03:27:21.644986Z","shell.execute_reply.started":"2022-08-03T03:27:21.636698Z","shell.execute_reply":"2022-08-03T03:27:21.644066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"failure = np.mean(np.column_stack(final_predictions), axis=1)\n\nids = test_data[\"id\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:27:27.753613Z","iopub.execute_input":"2022-08-03T03:27:27.753980Z","iopub.status.idle":"2022-08-03T03:27:27.761245Z","shell.execute_reply.started":"2022-08-03T03:27:27.753948Z","shell.execute_reply":"2022-08-03T03:27:27.760038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_data = pd.DataFrame({\"id\":ids,\"failure\":failure})\noutput_data","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:27:31.243466Z","iopub.execute_input":"2022-08-03T03:27:31.243824Z","iopub.status.idle":"2022-08-03T03:27:31.258173Z","shell.execute_reply.started":"2022-08-03T03:27:31.243792Z","shell.execute_reply":"2022-08-03T03:27:31.257162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_data.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:27:38.556501Z","iopub.execute_input":"2022-08-03T03:27:38.556853Z","iopub.status.idle":"2022-08-03T03:27:38.606186Z","shell.execute_reply.started":"2022-08-03T03:27:38.556823Z","shell.execute_reply":"2022-08-03T03:27:38.605295Z"},"trusted":true},"execution_count":null,"outputs":[]}]}