{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#  TPS AUG 22 - Lasso Regression","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import linear_model\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RepeatedKFold\nfrom sklearn.metrics import mean_squared_error\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-03T00:53:58.616180Z","iopub.execute_input":"2022-08-03T00:53:58.616600Z","iopub.status.idle":"2022-08-03T00:53:58.623840Z","shell.execute_reply.started":"2022-08-03T00:53:58.616567Z","shell.execute_reply":"2022-08-03T00:53:58.622898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T00:44:08.318913Z","iopub.execute_input":"2022-08-03T00:44:08.319321Z","iopub.status.idle":"2022-08-03T00:44:08.608616Z","shell.execute_reply.started":"2022-08-03T00:44:08.319284Z","shell.execute_reply":"2022-08-03T00:44:08.607368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-03T00:44:08.610467Z","iopub.execute_input":"2022-08-03T00:44:08.610916Z","iopub.status.idle":"2022-08-03T00:44:08.735327Z","shell.execute_reply.started":"2022-08-03T00:44:08.610871Z","shell.execute_reply":"2022-08-03T00:44:08.733970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train.drop(columns=['id'], inplace=True)\ntest.drop(columns=['id'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T00:44:09.217863Z","iopub.execute_input":"2022-08-03T00:44:09.218597Z","iopub.status.idle":"2022-08-03T00:44:09.231732Z","shell.execute_reply.started":"2022-08-03T00:44:09.218557Z","shell.execute_reply":"2022-08-03T00:44:09.230376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_columns = train.select_dtypes(include=[float]).columns\none_hot_columns = set(list(train.columns)) - set(list(float_columns)) - set(list(['failure']))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T00:44:09.838638Z","iopub.execute_input":"2022-08-03T00:44:09.839057Z","iopub.status.idle":"2022-08-03T00:44:09.849058Z","shell.execute_reply.started":"2022-08-03T00:44:09.839020Z","shell.execute_reply":"2022-08-03T00:44:09.847988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoders = {}\n\nfor col in one_hot_columns:\n    enc = OneHotEncoder(handle_unknown='ignore')\n    enc.fit(train[col].values.reshape(-1, 1))\n    encoders[col] = enc","metadata":{"execution":{"iopub.status.busy":"2022-08-03T00:44:10.209078Z","iopub.execute_input":"2022-08-03T00:44:10.210006Z","iopub.status.idle":"2022-08-03T00:44:10.223416Z","shell.execute_reply.started":"2022-08-03T00:44:10.209968Z","shell.execute_reply":"2022-08-03T00:44:10.222196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df_encoding(data, encoders):\n    \n    df = pd.DataFrame()\n\n    for col in one_hot_columns:\n        header_cols = encoders[col].get_feature_names_out([col])\n        arr = encoders[col].transform(data[col].values.reshape(-1, 1)).toarray()\n        df = pd.concat([df, pd.DataFrame(arr, columns=header_cols)], axis=1)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-03T00:44:11.752879Z","iopub.execute_input":"2022-08-03T00:44:11.753317Z","iopub.status.idle":"2022-08-03T00:44:11.760708Z","shell.execute_reply.started":"2022-08-03T00:44:11.753278Z","shell.execute_reply":"2022-08-03T00:44:11.759489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = pd.concat([train[float_columns], get_df_encoding(train, encoders)], axis=1)\nlabels = train['failure']\n\ntest_prep = pd.concat([test[float_columns], get_df_encoding(test, encoders)], axis=1)\n\nnew_one_hot_columns = set(features.columns) - set(float_columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T00:44:11.935659Z","iopub.execute_input":"2022-08-03T00:44:11.936498Z","iopub.status.idle":"2022-08-03T00:44:12.146480Z","shell.execute_reply.started":"2022-08-03T00:44:11.936448Z","shell.execute_reply":"2022-08-03T00:44:12.145275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Verify imbalance","metadata":{}},{"cell_type":"code","source":"x = [len(labels[labels==0]), len(labels[labels==1])]\nplt_labels = ['Negative', 'Positive']\n\nfig, ax = plt.subplots()\nax.pie(x, labels = plt_labels)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-03T00:44:14.081017Z","iopub.execute_input":"2022-08-03T00:44:14.081897Z","iopub.status.idle":"2022-08-03T00:44:14.221365Z","shell.execute_reply.started":"2022-08-03T00:44:14.081845Z","shell.execute_reply":"2022-08-03T00:44:14.219325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"n_splits = 5\nskf = StratifiedKFold(n_splits=n_splits)\noof_preds = np.zeros((n_splits, len(test)))\n\nlgb_best_params = []\nxgb_best_params = []\n\nfor i, (train_index, test_index) in enumerate(skf.split(features, labels)):\n    X_train, X_test = features.iloc[train_index], features.iloc[test_index]\n    y_train, y_test = labels.iloc[train_index], labels.iloc[test_index]\n    \n    imp = SimpleImputer(missing_values=np.nan, strategy='mean')\n    X_train = imp.fit_transform(X_train)\n    test_df = imp.transform(test_prep)\n    valid_df = imp.transform(X_test)\n    \n    X_train = pd.DataFrame(X_train, columns=features.columns)\n    test_df = pd.DataFrame(test_df, columns=features.columns)\n    valid_df = pd.DataFrame(valid_df, columns=features.columns)\n    \n    scaler = PowerTransformer().fit(X_train[float_columns])\n    temp_train_float = pd.DataFrame(scaler.transform(X_train[float_columns]), columns=float_columns)\n    temp_valid_float = pd.DataFrame(scaler.transform(valid_df[float_columns]), columns=float_columns)\n    temp_test_float = pd.DataFrame(scaler.transform(test_df[float_columns]), columns=float_columns)\n    \n    train_df = np.hstack((X_train[new_one_hot_columns].values, temp_train_float.values))\n    valid_df = np.hstack((X_test[new_one_hot_columns].values, temp_valid_float.values))\n    test_df = np.hstack((test_df[new_one_hot_columns].values, temp_test_float.values))\n    \n    X, y = train_df, y_train.values\n    X_v, y_v = valid_df, y_test.values\n    \n    # Grid Search Best Parameters\n    cv = RepeatedKFold(n_splits=5, n_repeats=2, random_state=1)\n    grid = dict()\n    grid['alpha'] = np.arange(0, 1, 0.01)\n    search = GridSearchCV(linear_model.Lasso(), grid, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1)\n    results = search.fit(X, y)\n\n    clf = linear_model.Lasso(**results.best_params_)\n    clf.fit(X, y)\n    \n    yhat = clf.predict(X_v.astype(np.float32))\n    rmse = mean_squared_error(y_v, yhat)\n    print('Fold:%2d, RMSE:%1.3f' % (i, rmse))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T00:57:26.583525Z","iopub.execute_input":"2022-08-03T00:57:26.583999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub['failure'] = np.mean(oof_preds, axis=0)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:25:05.20501Z","iopub.execute_input":"2022-08-02T19:25:05.205269Z","iopub.status.idle":"2022-08-02T19:25:05.258317Z","shell.execute_reply.started":"2022-08-02T19:25:05.205247Z","shell.execute_reply":"2022-08-02T19:25:05.256858Z"},"trusted":true},"execution_count":null,"outputs":[]}]}