{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TPS - AUG22 Logistic Regression + GA feature seleciton","metadata":{"papermill":{"duration":0.004862,"end_time":"2022-08-02T01:01:02.713911","exception":false,"start_time":"2022-08-02T01:01:02.709049","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom random import randint\nfrom sklearn import preprocessing\nfrom lightgbm import LGBMClassifier\nfrom sklearn.impute import KNNImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.linear_model import LogisticRegressionCV, LogisticRegression\nfrom sklearn.preprocessing import OneHotEncoder, RobustScaler, PowerTransformer, LabelEncoder, StandardScaler\nimport warnings; warnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.515047,"end_time":"2022-08-02T01:01:05.232854","exception":false,"start_time":"2022-08-02T01:01:02.717807","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-04T04:08:31.129351Z","iopub.execute_input":"2022-08-04T04:08:31.130454Z","iopub.status.idle":"2022-08-04T04:08:31.138529Z","shell.execute_reply.started":"2022-08-04T04:08:31.130393Z","shell.execute_reply":"2022-08-04T04:08:31.137372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Data Loading","metadata":{"papermill":{"duration":0.003939,"end_time":"2022-08-02T01:01:05.241028","exception":false,"start_time":"2022-08-02T01:01:05.237089","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/train.csv\", index_col='id')\ny = df_train['failure']\ngroups = df_train['product_code'].copy()\ndf_train = df_train.drop(['failure'], axis=1)\ndf_test = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/test.csv\", index_col='id')","metadata":{"papermill":{"duration":0.306332,"end_time":"2022-08-02T01:01:05.551187","exception":false,"start_time":"2022-08-02T01:01:05.244855","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-04T03:46:10.011633Z","iopub.execute_input":"2022-08-04T03:46:10.012016Z","iopub.status.idle":"2022-08-04T03:46:10.299249Z","shell.execute_reply.started":"2022-08-04T03:46:10.011988Z","shell.execute_reply":"2022-08-04T03:46:10.298329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Preprocessing","metadata":{"papermill":{"duration":0.003221,"end_time":"2022-08-02T01:01:05.558261","exception":false,"start_time":"2022-08-02T01:01:05.55504","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"**Dropping the target column**","metadata":{}},{"cell_type":"code","source":"train_df = df_train.copy()\ntest_df = df_test.copy()","metadata":{"papermill":{"duration":0.014304,"end_time":"2022-08-02T01:01:05.575946","exception":false,"start_time":"2022-08-02T01:01:05.561642","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-04T03:46:10.898542Z","iopub.execute_input":"2022-08-04T03:46:10.898931Z","iopub.status.idle":"2022-08-04T03:46:10.909979Z","shell.execute_reply.started":"2022-08-04T03:46:10.898902Z","shell.execute_reply":"2022-08-04T03:46:10.908922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Imputing Missing Values with KNN & Encoding categoricals with LabelEncoder**","metadata":{}},{"cell_type":"code","source":"train_df = train_df.reset_index()\ntest_df = test_df.reset_index()\nall_data = pd.concat([train_df, test_df],axis = 0).reset_index(drop=True)\nna_col = [col for col in train_df.columns if train_df[col].isnull().sum() !=0]\nimputer = KNNImputer(n_neighbors = 3)\nall_data[na_col] = imputer.fit_transform(all_data[na_col])\ncat_col = [col for col in train_df.columns[:-1] if train_df[col].dtypes == 'object']\nfor col in cat_col :\n    le = LabelEncoder()\n    all_data[col] = le.fit_transform(all_data[col])\ntrain_df = all_data[all_data['id'] < train_df.shape[0]].drop('id', axis = 1).copy()\ntest_df = all_data[all_data['id'] > train_df.shape[0] - 1].drop('id', axis = 1).copy()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T03:46:11.523853Z","iopub.execute_input":"2022-08-04T03:46:11.524257Z","iopub.status.idle":"2022-08-04T03:47:39.792357Z","shell.execute_reply.started":"2022-08-04T03:46:11.524228Z","shell.execute_reply":"2022-08-04T03:47:39.791480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Standard Scaling**","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler()\ntrain_df = scaler.fit_transform(train_df)\ntest_df = scaler.fit_transform(test_df)","metadata":{"papermill":{"duration":0.014304,"end_time":"2022-08-02T01:01:05.575946","exception":false,"start_time":"2022-08-02T01:01:05.561642","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-04T03:47:39.794201Z","iopub.execute_input":"2022-08-04T03:47:39.794514Z","iopub.status.idle":"2022-08-04T03:47:39.820657Z","shell.execute_reply.started":"2022-08-04T03:47:39.794486Z","shell.execute_reply":"2022-08-04T03:47:39.819736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_df\ndf_test = test_df","metadata":{"papermill":{"duration":0.014304,"end_time":"2022-08-02T01:01:05.575946","exception":false,"start_time":"2022-08-02T01:01:05.561642","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-04T03:47:39.821697Z","iopub.execute_input":"2022-08-04T03:47:39.821999Z","iopub.status.idle":"2022-08-04T03:47:39.828061Z","shell.execute_reply.started":"2022-08-04T03:47:39.821974Z","shell.execute_reply":"2022-08-04T03:47:39.826835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Genetic Algorithm\n> Credit for GA: [Kinnera Kiran GA Notebook](https://www.kaggle.com/code/kinnerakiran/feature-selection-by-genetic-algorithm)","metadata":{}},{"cell_type":"code","source":"def initilization_of_population(size,n_feat):\n    population = []\n    for i in range(size):\n        chromosome = np.ones(n_feat,dtype=np.bool)     \n        chromosome[:int(0.3*n_feat)]=False             \n        np.random.shuffle(chromosome)\n        population.append(chromosome)\n    return population\n\n\ndef fitness_score(population, X_train, Y_train, X_test, Y_test):\n    scores = []\n    for chromosome in population:\n        logmodel.fit(X_train.iloc[:,chromosome],Y_train)         \n        predictions = logmodel.predict(X_test.iloc[:,chromosome])\n        scores.append(accuracy_score(Y_test,predictions))\n    scores, population = np.array(scores), np.array(population) \n    inds = np.argsort(scores)                                    \n    return list(scores[inds][::-1]), list(population[inds,:][::-1]) \n\n\ndef selection(pop_after_fit,n_parents):\n    population_nextgen = []\n    for i in range(n_parents):\n        population_nextgen.append(pop_after_fit[i])\n    return population_nextgen\n\n\ndef crossover(pop_after_sel):\n    pop_nextgen = pop_after_sel\n    for i in range(0,len(pop_after_sel),2):\n        new_par = []\n        child_1 , child_2 = pop_nextgen[i] , pop_nextgen[i+1]\n        new_par = np.concatenate((child_1[:len(child_1)//2],child_2[len(child_1)//2:]))\n        pop_nextgen.append(new_par)\n    return pop_nextgen\n\n\ndef mutation(pop_after_cross,mutation_rate,n_feat):   \n    mutation_range = int(mutation_rate*n_feat)\n    pop_next_gen = []\n    for n in range(0,len(pop_after_cross)):\n        chromo = pop_after_cross[n]\n        rand_posi = [] \n        for i in range(0,mutation_range):\n            pos = randint(0,n_feat-1)\n            rand_posi.append(pos)\n        for j in rand_posi:\n            chromo[j] = not chromo[j]  \n        pop_next_gen.append(chromo)\n    return pop_next_gen\n\ndef generations(df,label,size,n_feat,n_parents,mutation_rate,n_gen,X_train,\n                                   X_test, Y_train, Y_test):\n    best_chromo= []\n    best_score= []\n    population_nextgen=initilization_of_population(size,n_feat)\n    for i in range(n_gen):\n        scores, pop_after_fit = fitness_score(population_nextgen, \n                                              X_train, Y_train,\n                                              X_test, Y_test)\n        print('Best score in generation',i+1,':',scores[:1])  #2\n        pop_after_sel = selection(pop_after_fit,n_parents)\n        pop_after_cross = crossover(pop_after_sel)\n        population_nextgen = mutation(pop_after_cross,mutation_rate,n_feat)\n        best_chromo.append(pop_after_fit[0])\n        best_score.append(scores[0])\n    return best_chromo,best_score","metadata":{"execution":{"iopub.status.busy":"2022-08-04T04:00:44.509978Z","iopub.execute_input":"2022-08-04T04:00:44.510387Z","iopub.status.idle":"2022-08-04T04:00:44.527091Z","shell.execute_reply.started":"2022-08-04T04:00:44.510355Z","shell.execute_reply":"2022-08-04T04:00:44.526268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Logistic Regression Prediction\n\n> **Grid search credit:** The great [notebook](https://www.kaggle.com/code/kostiantynlavronenko/tps-08-22-logistic-regression) by [Kostiantyn Lavronenko](https://www.kaggle.com/kostiantynlavronenko)\n\n**As it turns out:** Logistic regression works surprisingly well on this competition. For this reason we also train a logistic regression model that will be blended with our lightgbm model for final prediction.","metadata":{"papermill":{"duration":0.004944,"end_time":"2022-08-02T01:25:03.898832","exception":false,"start_time":"2022-08-02T01:25:03.893888","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"train_prep = pd.DataFrame(train_df, columns=all_data.drop(columns=['id']).columns)\ntrain_X, val_X, train_Y, val_Y = train_test_split(train_prep, y, test_size=0.05, random_state=42, stratify = y)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T03:53:36.592372Z","iopub.execute_input":"2022-08-04T03:53:36.592754Z","iopub.status.idle":"2022-08-04T03:53:36.601634Z","shell.execute_reply.started":"2022-08-04T03:53:36.592725Z","shell.execute_reply":"2022-08-04T03:53:36.600121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LogisticRegression GA","metadata":{}},{"cell_type":"code","source":"%%capture\nparam_grid = {'C': [0.0001, 0.001, 0.01, 0.1, 1], 'penalty':['l2','l1']}\nlogmodel = GridSearchCV(LogisticRegression(max_iter = 200), param_grid, cv = 5, verbose = 3)\n\nchromo_df_bc,score_bc=generations(train_prep,y,size=80,n_feat=train_prep.shape[1],n_parents=64,mutation_rate=0.20,n_gen=5,\n                         X_train=train_X,X_test=val_X,Y_train=train_Y,Y_test=val_Y)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T04:08:36.834936Z","iopub.execute_input":"2022-08-04T04:08:36.835348Z","iopub.status.idle":"2022-08-04T04:20:11.305445Z","shell.execute_reply.started":"2022-08-04T04:08:36.835314Z","shell.execute_reply":"2022-08-04T04:20:11.303858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Select top score combination","metadata":{}},{"cell_type":"code","source":"max_value = max(score_bc)\nindex = score_bc.index(max_value)\nprint('index:%2d, max_value: %1.4f' % (index, max_value))","metadata":{"execution":{"iopub.status.busy":"2022-08-04T04:37:08.104553Z","iopub.execute_input":"2022-08-04T04:37:08.104942Z","iopub.status.idle":"2022-08-04T04:37:08.110626Z","shell.execute_reply.started":"2022-08-04T04:37:08.104914Z","shell.execute_reply":"2022-08-04T04:37:08.109787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_cols = train_prep.columns[chromo_df_bc[index]]\nprint(selected_cols)\nX = train_prep[selected_cols].values","metadata":{"execution":{"iopub.status.busy":"2022-08-04T04:37:27.431950Z","iopub.execute_input":"2022-08-04T04:37:27.432325Z","iopub.status.idle":"2022-08-04T04:37:27.440157Z","shell.execute_reply.started":"2022-08-04T04:37:27.432292Z","shell.execute_reply":"2022-08-04T04:37:27.439124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LogisticRegression Training","metadata":{}},{"cell_type":"code","source":"train_X, val_X, train_Y, val_Y = train_test_split(X, y, test_size=0.05, random_state=42, stratify = y)\n\nlogmodel.fit(train_X, train_Y)\nprint(logmodel.best_params_)\n\nval_pred = logmodel.predict(val_X)\nprint(classification_report(val_Y, val_pred))\n\ndf_test_prep = pd.DataFrame(df_test, columns=all_data.drop(columns=['id']).columns)\npreds_test_lin = logmodel.predict_proba(df_test_prep[selected_cols])[:,1]","metadata":{"papermill":{"duration":1.030391,"end_time":"2022-08-02T01:25:04.934074","exception":false,"start_time":"2022-08-02T01:25:03.903683","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-04T05:14:07.261420Z","iopub.execute_input":"2022-08-04T05:14:07.262130Z","iopub.status.idle":"2022-08-04T05:14:08.769682Z","shell.execute_reply.started":"2022-08-04T05:14:07.262095Z","shell.execute_reply":"2022-08-04T05:14:08.768147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Training with the best hyperparams**","metadata":{}},{"cell_type":"code","source":"def inference(X, X_test, iterations):\n    pred_list = []\n    for i in range(iterations):\n        X_train = X.sample(int(0.8 * len(X)))\n        y_train = y.loc[X_train.index]\n        \n        model = LogisticRegression(C = 0.0001, penalty = 'l2', random_state=i, tol = 1e-2, max_iter = 1000)\n        model.fit(X_train, y_train)\n        y_pred = model.predict_proba(X_test)[:,1]\n\n        pred_list.append(y_pred)    \n    pred_df = pd.DataFrame(pred_list).T\n    pred_df = pred_df.rank()\n    pred_df[\"mean\"] = pred_df.mean(axis=1)    \n    return pred_df['mean']\n\npreds_test_lin = inference(train_prep[selected_cols], df_test_prep[selected_cols], iterations = 500)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T05:16:02.807740Z","iopub.execute_input":"2022-08-04T05:16:02.808171Z","iopub.status.idle":"2022-08-04T05:16:34.769458Z","shell.execute_reply.started":"2022-08-04T05:16:02.808139Z","shell.execute_reply":"2022-08-04T05:16:34.768187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Submission\n\n> The logistic regression itself scores: 0.5865 LB","metadata":{"papermill":{"duration":0.004695,"end_time":"2022-08-02T01:25:04.944035","exception":false,"start_time":"2022-08-02T01:25:04.93934","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/sample_submission.csv\")\nsubmission.failure = (preds_test_lin)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":0.071537,"end_time":"2022-08-02T01:25:05.020411","exception":false,"start_time":"2022-08-02T01:25:04.948874","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-08-03T23:56:18.714342Z","iopub.execute_input":"2022-08-03T23:56:18.71477Z","iopub.status.idle":"2022-08-03T23:56:18.77696Z","shell.execute_reply.started":"2022-08-03T23:56:18.714736Z","shell.execute_reply":"2022-08-03T23:56:18.775951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Blending","metadata":{}},{"cell_type":"code","source":"sub_1 = pd.read_csv('../input/tps-aug-top/submission_nn.csv') # Reading a submission file\nsub_2 = pd.read_csv('../input/tps-aug-top/submission_simple.csv') # Reading another submission file\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv') # Reading the sample submission file\nsub['failure'] = 0.2*sub_1['failure'] + 0.8*sub_2['failure']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['failure_2'] = sub['failure']\nsubmission['failure'] = submission['failure'].rank()\nsubmission['failure_2'] = submission['failure_2'].rank()\nsubmission['failure'] = (submission['failure_2'] + submission['failure']) / 2.0\nsubmission[['id', 'failure']].to_csv(\"submission_blend.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T00:35:47.840993Z","iopub.execute_input":"2022-08-04T00:35:47.841495Z","iopub.status.idle":"2022-08-04T00:35:47.933754Z","shell.execute_reply.started":"2022-08-04T00:35:47.841418Z","shell.execute_reply":"2022-08-04T00:35:47.932381Z"},"trusted":true},"execution_count":null,"outputs":[]}]}