{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Submission 1 - Light GBM baseline","metadata":{}},{"cell_type":"code","source":"# Loading libraries\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"id":"1324dce0","execution":{"iopub.status.busy":"2022-07-20T09:57:45.778106Z","iopub.execute_input":"2022-07-20T09:57:45.778560Z","iopub.status.idle":"2022-07-20T09:57:45.788204Z","shell.execute_reply.started":"2022-07-20T09:57:45.778524Z","shell.execute_reply":"2022-07-20T09:57:45.786993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete submission.csv if it is already in Output folder\n\n# os.remove('submission2.csv')","metadata":{"id":"0d12d5ef","execution":{"iopub.status.busy":"2022-07-20T09:57:46.017438Z","iopub.execute_input":"2022-07-20T09:57:46.018065Z","iopub.status.idle":"2022-07-20T09:57:46.022864Z","shell.execute_reply.started":"2022-07-20T09:57:46.018019Z","shell.execute_reply":"2022-07-20T09:57:46.021617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading the datasets","metadata":{"id":"cO7i9w1bfP29"}},{"cell_type":"code","source":"# Loading the Train dataset (train.csv)\n\ntrain = pd.read_csv('/kaggle/input/tabular-playground-series-mar-2021/train.csv')\ntrain.head()","metadata":{"id":"45da6b04","execution":{"iopub.status.busy":"2022-07-20T09:57:46.196497Z","iopub.execute_input":"2022-07-20T09:57:46.196889Z","iopub.status.idle":"2022-07-20T09:57:47.502191Z","shell.execute_reply.started":"2022-07-20T09:57:46.196857Z","shell.execute_reply":"2022-07-20T09:57:47.500996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading the Test dataset (test.csv)\n\ntest = pd.read_csv('/kaggle/input/tabular-playground-series-mar-2021/test.csv')\ntest.head()","metadata":{"id":"3a2b4614","execution":{"iopub.status.busy":"2022-07-20T09:57:47.504568Z","iopub.execute_input":"2022-07-20T09:57:47.505501Z","iopub.status.idle":"2022-07-20T09:57:48.523085Z","shell.execute_reply.started":"2022-07-20T09:57:47.505450Z","shell.execute_reply":"2022-07-20T09:57:48.521931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Exploratory Data Analysis (EDA) to understand the dataset","metadata":{"id":"EiSCVOwoEGWy"}},{"cell_type":"code","source":"train.shape","metadata":{"id":"Z-Qx-aWqBeKJ","outputId":"ab2ddc25-d93b-43da-88f6-eee2a8ef0276","execution":{"iopub.status.busy":"2022-07-20T09:57:48.524743Z","iopub.execute_input":"2022-07-20T09:57:48.525789Z","iopub.status.idle":"2022-07-20T09:57:48.533192Z","shell.execute_reply.started":"2022-07-20T09:57:48.525734Z","shell.execute_reply":"2022-07-20T09:57:48.531961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape","metadata":{"id":"yTMiXabmBeZI","outputId":"7f8eb5e2-d87e-4031-d405-c72c8c3d28fc","execution":{"iopub.status.busy":"2022-07-20T09:57:48.536868Z","iopub.execute_input":"2022-07-20T09:57:48.537383Z","iopub.status.idle":"2022-07-20T09:57:48.545350Z","shell.execute_reply.started":"2022-07-20T09:57:48.537334Z","shell.execute_reply":"2022-07-20T09:57:48.544095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check the target feature","metadata":{"id":"OulUJZ-QELAY"}},{"cell_type":"code","source":"train['target'].value_counts()","metadata":{"id":"W-TrfSm4Dhf2","outputId":"0c915a8c-24a7-4b27-b6b2-ace9234c0436","execution":{"iopub.status.busy":"2022-07-20T09:57:48.546757Z","iopub.execute_input":"2022-07-20T09:57:48.547102Z","iopub.status.idle":"2022-07-20T09:57:48.559100Z","shell.execute_reply.started":"2022-07-20T09:57:48.547071Z","shell.execute_reply":"2022-07-20T09:57:48.558018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['target'].value_counts(normalize=True)","metadata":{"id":"aBhiW_VwFDJk","outputId":"a97f4d00-7c1b-4789-ac15-852273276f40","execution":{"iopub.status.busy":"2022-07-20T09:57:48.560965Z","iopub.execute_input":"2022-07-20T09:57:48.562320Z","iopub.status.idle":"2022-07-20T09:57:48.576607Z","shell.execute_reply.started":"2022-07-20T09:57:48.562273Z","shell.execute_reply":"2022-07-20T09:57:48.575578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='target', data=train).set(title='Target feature distribution')\nplt.show()","metadata":{"id":"-aOsMvByDhy7","outputId":"e7725650-ad8f-4560-b401-a36532386cd7","execution":{"iopub.status.busy":"2022-07-20T09:57:48.577951Z","iopub.execute_input":"2022-07-20T09:57:48.578874Z","iopub.status.idle":"2022-07-20T09:57:48.756492Z","shell.execute_reply.started":"2022-07-20T09:57:48.578837Z","shell.execute_reply":"2022-07-20T09:57:48.755394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Data Pre-processing - preparing the data for modelling\n","metadata":{"id":"NpLp3Hj3ttXS"}},{"cell_type":"markdown","source":"Combine Train and Test datasets","metadata":{"id":"bmH0tDFNGHpF"}},{"cell_type":"code","source":"# ignore_index parameter is not passed, so for test_data rows, index re-starts from the first index of test dataset\n\ndata_all = pd.concat([train, test], sort=False)\ndata_all.head()","metadata":{"id":"2EoO69bkEjAd","outputId":"4b0ade61-57b9-4ddb-d743-ee6969eb0249","execution":{"iopub.status.busy":"2022-07-20T09:57:48.759558Z","iopub.execute_input":"2022-07-20T09:57:48.760766Z","iopub.status.idle":"2022-07-20T09:57:48.979330Z","shell.execute_reply.started":"2022-07-20T09:57:48.760714Z","shell.execute_reply":"2022-07-20T09:57:48.978192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_all = data_all.drop(columns='id')","metadata":{"id":"9Ajkqg9DEjOs","execution":{"iopub.status.busy":"2022-07-20T09:57:48.980971Z","iopub.execute_input":"2022-07-20T09:57:48.981463Z","iopub.status.idle":"2022-07-20T09:57:49.146199Z","shell.execute_reply.started":"2022-07-20T09:57:48.981389Z","shell.execute_reply":"2022-07-20T09:57:49.144937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_all.shape","metadata":{"id":"835Z2jV3wXZW","outputId":"d8d88a77-4ff2-4d82-cc93-c8bcb2fed6c8","execution":{"iopub.status.busy":"2022-07-20T09:57:49.150007Z","iopub.execute_input":"2022-07-20T09:57:49.150348Z","iopub.status.idle":"2022-07-20T09:57:49.157512Z","shell.execute_reply.started":"2022-07-20T09:57:49.150315Z","shell.execute_reply":"2022-07-20T09:57:49.156218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_all.info()","metadata":{"id":"jmhIFO8xHJMP","outputId":"ecd79dbd-2edc-492a-f377-a5a6902d1b6f","execution":{"iopub.status.busy":"2022-07-20T09:57:49.158751Z","iopub.execute_input":"2022-07-20T09:57:49.159049Z","iopub.status.idle":"2022-07-20T09:57:50.189573Z","shell.execute_reply.started":"2022-07-20T09:57:49.159021Z","shell.execute_reply":"2022-07-20T09:57:50.188284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check for missing data","metadata":{"id":"aagQV0HnHHIg"}},{"cell_type":"code","source":"data_all.isnull().sum()","metadata":{"id":"zzX7xc7LEjWf","outputId":"a56a8070-fb0a-4c64-f801-701dc4f55de6","execution":{"iopub.status.busy":"2022-07-20T09:57:50.190891Z","iopub.execute_input":"2022-07-20T09:57:50.191191Z","iopub.status.idle":"2022-07-20T09:57:51.200184Z","shell.execute_reply.started":"2022-07-20T09:57:50.191163Z","shell.execute_reply":"2022-07-20T09:57:51.198948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_all.describe().T","metadata":{"id":"679fvIZyrVO2","outputId":"9d8735ca-4471-43f8-ea5a-d1f22380ad5f","execution":{"iopub.status.busy":"2022-07-20T09:57:51.201707Z","iopub.execute_input":"2022-07-20T09:57:51.202117Z","iopub.status.idle":"2022-07-20T09:57:51.526896Z","shell.execute_reply.started":"2022-07-20T09:57:51.202085Z","shell.execute_reply":"2022-07-20T09:57:51.525785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Group numerical columns","metadata":{"id":"pmZoF2UOhkns"}},{"cell_type":"code","source":"numerical_columns = data_all.drop(columns='target').select_dtypes(exclude=['object']).columns\nnumerical_columns","metadata":{"id":"YaED8m0cyPWV","outputId":"cfe5eddf-bb14-4ffe-be63-7b049c485f56","execution":{"iopub.status.busy":"2022-07-20T09:57:51.529675Z","iopub.execute_input":"2022-07-20T09:57:51.530534Z","iopub.status.idle":"2022-07-20T09:57:51.689928Z","shell.execute_reply.started":"2022-07-20T09:57:51.530486Z","shell.execute_reply":"2022-07-20T09:57:51.688740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Group categorial columns","metadata":{"id":"RfIciPD7hp6l"}},{"cell_type":"code","source":"categorical_columns = data_all.select_dtypes(include=['object']).columns\ncategorical_columns","metadata":{"id":"9E76n3peRomK","outputId":"d07c5cc0-6573-485d-d374-87e622d82377","execution":{"iopub.status.busy":"2022-07-20T09:57:51.693507Z","iopub.execute_input":"2022-07-20T09:57:51.693822Z","iopub.status.idle":"2022-07-20T09:57:51.802114Z","shell.execute_reply.started":"2022-07-20T09:57:51.693794Z","shell.execute_reply":"2022-07-20T09:57:51.800985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Label encode categorial columns","metadata":{"id":"sPnjMY2Qn78X"}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder","metadata":{"id":"zEFF9LBwqCVW","execution":{"iopub.status.busy":"2022-07-20T09:57:51.803869Z","iopub.execute_input":"2022-07-20T09:57:51.804570Z","iopub.status.idle":"2022-07-20T09:57:51.809881Z","shell.execute_reply.started":"2022-07-20T09:57:51.804524Z","shell.execute_reply":"2022-07-20T09:57:51.808779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = LabelEncoder()\nfor col in categorical_columns.tolist():\n  data_all[col] = le.fit_transform(data_all[col])","metadata":{"id":"apS3KRnJqCWC","execution":{"iopub.status.busy":"2022-07-20T09:57:51.810926Z","iopub.execute_input":"2022-07-20T09:57:51.811768Z","iopub.status.idle":"2022-07-20T09:57:55.563329Z","shell.execute_reply.started":"2022-07-20T09:57:51.811728Z","shell.execute_reply":"2022-07-20T09:57:55.562405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The label encoded columns are now all numerical\n","metadata":{"id":"B7weyPHqq9T-"}},{"cell_type":"code","source":"data_all.head()","metadata":{"id":"iBBf721HoC97","outputId":"8e5f569f-c431-4716-c7e4-caa5d4494b75","execution":{"iopub.status.busy":"2022-07-20T09:57:55.564784Z","iopub.execute_input":"2022-07-20T09:57:55.565092Z","iopub.status.idle":"2022-07-20T09:57:55.589768Z","shell.execute_reply.started":"2022-07-20T09:57:55.565064Z","shell.execute_reply":"2022-07-20T09:57:55.588524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Building the model\n","metadata":{"id":"HPxqRiB8uBpk"}},{"cell_type":"markdown","source":"Data Pre processing for baseline model is done\n\nand I will build a **Light GBM** baseline model to start with","metadata":{"id":"oSMY6XJduJHk"}},{"cell_type":"markdown","source":"Split back combined dataset into train and test datasets","metadata":{"id":"hwXi1p3zsHv_"}},{"cell_type":"code","source":"train_data = data_all.iloc[:len(train)]\ntest_data = data_all.iloc[len(train):].drop(columns='target')\n\n# separate train_data into X and y\n\nX = train_data.drop(columns='target')\ny = train_data['target']","metadata":{"id":"kwoKxJpyIhwT","execution":{"iopub.status.busy":"2022-07-20T09:57:55.591096Z","iopub.execute_input":"2022-07-20T09:57:55.591813Z","iopub.status.idle":"2022-07-20T09:57:55.679543Z","shell.execute_reply.started":"2022-07-20T09:57:55.591779Z","shell.execute_reply":"2022-07-20T09:57:55.678476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_data_train, X_data_valid, y_data_train, y_data_valid = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)","metadata":{"id":"0OT4tBcCI8g7","execution":{"iopub.status.busy":"2022-07-20T09:57:55.680920Z","iopub.execute_input":"2022-07-20T09:57:55.681778Z","iopub.status.idle":"2022-07-20T09:57:55.891208Z","shell.execute_reply.started":"2022-07-20T09:57:55.681745Z","shell.execute_reply":"2022-07-20T09:57:55.890226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\n\nmodel = lgb.LGBMClassifier()\nmodel.fit(X_data_train, y_data_train)\n\n# prediction for validation data\ny_pred = model.predict(X_data_valid)\n\n# predict the probability for validation data for class 0 and class 1\ny_pred_prob = model.predict_proba(X_data_valid)","metadata":{"id":"zfiiY29Biuml","execution":{"iopub.status.busy":"2022-07-20T09:57:55.893162Z","iopub.execute_input":"2022-07-20T09:57:55.893537Z","iopub.status.idle":"2022-07-20T09:57:58.909590Z","shell.execute_reply.started":"2022-07-20T09:57:55.893504Z","shell.execute_reply":"2022-07-20T09:57:58.908636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Actual value (of validation target data) and target prediction by the trained model","metadata":{"id":"AaoelWST1xW8"}},{"cell_type":"code","source":"df_pred = pd.DataFrame({'target': y_data_valid, 'target_pred': y_pred})\ndf_pred.head()","metadata":{"outputId":"e550ad52-10da-4aa0-a984-074e4d24e3da","id":"MoY59XdAkfIy","execution":{"iopub.status.busy":"2022-07-20T09:57:58.911066Z","iopub.execute_input":"2022-07-20T09:57:58.911731Z","iopub.status.idle":"2022-07-20T09:57:58.926642Z","shell.execute_reply.started":"2022-07-20T09:57:58.911694Z","shell.execute_reply":"2022-07-20T09:57:58.925591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Actual value (of validation target data) and target class probability prediction by the trained model","metadata":{"id":"W50ES3W-2FkF"}},{"cell_type":"code","source":"df_pred_prob = pd.DataFrame({'target': y_data_valid, 'class_0_prob': y_pred_prob[:, 0], 'class_1_prob': y_pred_prob[:, 1]})\ndf_pred_prob.head()","metadata":{"id":"q84OMO8Ekufk","outputId":"4b615894-6424-462a-db6c-c3c8ce846ee3","execution":{"iopub.status.busy":"2022-07-20T09:57:58.929627Z","iopub.execute_input":"2022-07-20T09:57:58.929962Z","iopub.status.idle":"2022-07-20T09:57:58.946075Z","shell.execute_reply.started":"2022-07-20T09:57:58.929933Z","shell.execute_reply":"2022-07-20T09:57:58.944967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Evaluation of the trained model with validation data prediction","metadata":{"id":"crRjwDQQ2brX"}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import log_loss\nfrom sklearn.metrics import roc_auc_score\n\nacc = accuracy_score(y_data_valid, y_pred)\nprint('accuracy: ', acc)\n\nlogloss = log_loss(y_data_valid, y_pred_prob)\nprint('logloss: ', logloss)\n\nauc = roc_auc_score(y_data_valid, y_pred_prob[:, 1])\nprint('AUC: ', auc)","metadata":{"outputId":"9b7f5019-5bfa-47b9-b022-565624bad33a","id":"_4yrcXlqkfI4","execution":{"iopub.status.busy":"2022-07-20T09:57:58.947889Z","iopub.execute_input":"2022-07-20T09:57:58.948678Z","iopub.status.idle":"2022-07-20T09:57:59.003122Z","shell.execute_reply.started":"2022-07-20T09:57:58.948643Z","shell.execute_reply":"2022-07-20T09:57:59.002000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Prediction on test dataset for submission","metadata":{"id":"iNIe7E5S3BrQ"}},{"cell_type":"code","source":"submission = model.predict_proba(test_data)\nsubmission = submission[:, 1]","metadata":{"id":"Xze5nKxI3Ptt","execution":{"iopub.status.busy":"2022-07-20T09:57:59.004484Z","iopub.execute_input":"2022-07-20T09:57:59.004791Z","iopub.status.idle":"2022-07-20T09:57:59.604725Z","shell.execute_reply.started":"2022-07-20T09:57:59.004763Z","shell.execute_reply":"2022-07-20T09:57:59.603769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[:10]","metadata":{"id":"R46yWvFJ59Fg","outputId":"9b86d6cf-1b62-4330-c9fb-f18a347cb822","execution":{"iopub.status.busy":"2022-07-20T09:57:59.609450Z","iopub.execute_input":"2022-07-20T09:57:59.610393Z","iopub.status.idle":"2022-07-20T09:57:59.616909Z","shell.execute_reply.started":"2022-07-20T09:57:59.610354Z","shell.execute_reply":"2022-07-20T09:57:59.616037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# id column for submission file\n\nid_column = test['id']","metadata":{"id":"tbdtn-UF3P_K","execution":{"iopub.status.busy":"2022-07-20T09:57:59.618573Z","iopub.execute_input":"2022-07-20T09:57:59.619306Z","iopub.status.idle":"2022-07-20T09:57:59.625929Z","shell.execute_reply.started":"2022-07-20T09:57:59.619264Z","shell.execute_reply":"2022-07-20T09:57:59.624914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(submission)","metadata":{"id":"Ro0iMn9u7ZtN","outputId":"3bdbee50-2744-4b2e-a3c8-ea6a387beb38","execution":{"iopub.status.busy":"2022-07-20T09:57:59.627759Z","iopub.execute_input":"2022-07-20T09:57:59.628097Z","iopub.status.idle":"2022-07-20T09:57:59.637407Z","shell.execute_reply.started":"2022-07-20T09:57:59.628069Z","shell.execute_reply":"2022-07-20T09:57:59.636317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(id_column)\nsubmission_df['target'] = submission\nsubmission_df.head()","metadata":{"id":"SXcR2sbF3QMx","outputId":"a5dfa7ab-c264-434a-eb91-8c4adf826067","execution":{"iopub.status.busy":"2022-07-20T09:57:59.638720Z","iopub.execute_input":"2022-07-20T09:57:59.640058Z","iopub.status.idle":"2022-07-20T09:57:59.653310Z","shell.execute_reply.started":"2022-07-20T09:57:59.640023Z","shell.execute_reply":"2022-07-20T09:57:59.652513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"id":"Cmaqw2vY3QaT","execution":{"iopub.status.busy":"2022-07-20T09:57:59.655058Z","iopub.execute_input":"2022-07-20T09:57:59.655790Z","iopub.status.idle":"2022-07-20T09:58:00.345721Z","shell.execute_reply.started":"2022-07-20T09:57:59.655754Z","shell.execute_reply":"2022-07-20T09:58:00.344638Z"},"trusted":true},"execution_count":null,"outputs":[]}]}