{"cells":[{"metadata":{},"cell_type":"markdown","source":"xgb with these features seems good => 0.744 on LB\n\ncatboost => 0.5 (seems bad)\n\ntry lgbm\n\ntry adaboost\n\ntry Neural Nets\n\ntry more tuning ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Special Thanks to : @awsaf49 \n\nplease check and upvote his kernel : https://www.kaggle.com/awsaf49/xgboost-tabular-data-ml-cv-85-lb-787","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport random , os \nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split,cross_val_score,StratifiedKFold\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.svm import SVR, SVC\nfrom sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\nfrom xgboost import XGBClassifier, XGBRegressor\nfrom sklearn.linear_model import SGDRegressor, BayesianRidge\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\nfrom sklearn.ensemble import (RandomForestClassifier, AdaBoostClassifier, \n                              GradientBoostingClassifier, ExtraTreesClassifier)\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.ensemble import StackingClassifier\nfrom sklearn.neural_network import MLPClassifier\nimport warnings \nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install catboost\nfrom catboost import CatBoostClassifier","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsub = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_approx'].fillna(train['age_approx'].mean(),inplace = True)\ntrain['sex'].fillna('unknown_sex',inplace = True)\n\ntrain['anatom_site_general_challenge'].fillna('unknown_anatom',inplace = True)\ntest['anatom_site_general_challenge'].fillna('unknown_anatom',inplace = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# One Hot Encoding Anatom site challenge : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"one_hot_anatom = pd.get_dummies(train.anatom_site_general_challenge ,prefix = 'anatom')\ntrain = train.join(one_hot_anatom)\n\none_hot_anatom = pd.get_dummies(test.anatom_site_general_challenge ,prefix = 'anatom')\ntest = test.join(one_hot_anatom)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\none_hot_sex = pd.get_dummies(train.sex, prefix='sex')\n#one_hot_diagnosis = pd.get_dummies(train.diagnosis , prefix = 'disgnosis')\n\ntrain = train.join(one_hot_sex)\n#train = train.join(one_hot_diagnosis)\n\ntrain['id'] = train['patient_id'].map(lambda x : int(x[3:]))'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\none_hot_sex = pd.get_dummies(test.sex, prefix='sex')\ntest = test.join(one_hot_sex)\ntest['id'] = test['patient_id'].map(lambda x : int(x[3:]))'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''train.drop(['sex','diagnosis','anatom_site_general_challenge','benign_malignant','image_name','patient_id'],axis = 1,inplace = True)\ntest.drop(['sex','anatom_site_general_challenge','image_name','patient_id'],axis = 1,inplace = True)\ntrain.drop(['sex_unknown_sex','anatom_unknown_anatom'],axis=1, inplace = True)'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing / Feature Engineering :","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Label Encoding Sex , Anatom : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder \n\nle = LabelEncoder()\n\ntrain['sex_encoding'] = le.fit_transform(train['sex'].astype(str))\ntest['sex_encoding'] = le.transform(test['sex'].astype(str))\n\ntrain['anatom_site_general_challenge_encoding'] = le.fit_transform(train['anatom_site_general_challenge'].astype(str))\ntest['anatom_site_general_challenge_encoding'] = le.transform(test['anatom_site_general_challenge'].astype(str))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Images per Patient :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['n_images'] = train['patient_id'].map(train.groupby(['patient_id']).image_name.count())\ntest['n_images'] = test['patient_id'].map(test.groupby(['patient_id']).image_name.count())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Categorize number of images per patient :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import KBinsDiscretizer\n\ncategorize = KBinsDiscretizer(n_bins = 10, encode = 'ordinal', strategy = 'uniform')\ntrain['n_images_encoding'] = categorize.fit_transform(train['n_images'].values.reshape(-1,1)).astype(int).squeeze()\ntest['n_images_encoding'] = categorize.transform(test['n_images'].values.reshape(-1,1)).astype(int).squeeze()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Label Encoding Age : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder \n\nenc = LabelEncoder()\n\ntrain['age_enc'] = enc.fit_transform(train['age_approx'].astype('str'))\ntest['age_enc'] = enc.fit_transform(test['age_approx'].astype('str'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Size :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images = train['image_name'].values\ntrain_sizes = np.zeros(train.shape[0])\n\nfor i, img_path in enumerate(tqdm(train_images)) :\n    train_sizes[i] = os.path.getsize(os.path.join('../input/siim-isic-melanoma-classification/jpeg/train/',f'{img_path}.jpg'))\n    \ntrain['image_size'] = train_sizes\n\ntest_images = test['image_name'].values\ntest_sizes = np.zeros(test.shape[0])\n\nfor i, img_path in enumerate(tqdm(test_images)) :\n    test_sizes[i] = os.path.getsize(os.path.join('../input/siim-isic-melanoma-classification/jpeg/test/',f'{img_path}.jpg'))\n\ntest['image_size'] = test_sizes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Scaling Image sizes : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n\nscaler = MinMaxScaler()\n\ntrain['image_size_scaled'] = scaler.fit_transform(train['image_size'].values.reshape(-1,1))\ntest['image_size_scaled'] = scaler.transform(test['image_size'].values.reshape(-1,1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Categorize Image size :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import KBinsDiscretizer \n\ncategorize = KBinsDiscretizer(n_bins = 10,encode = 'ordinal' , strategy = 'uniform')\n\ntrain['image_size_encoding'] = categorize.fit_transform(train.image_size_scaled.values.reshape(-1,1)).astype(int).squeeze()\ntest['image_size_encoding'] = categorize.fit_transform(test.image_size_scaled.values.reshape(-1,1)).astype(int).squeeze()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adding mean color (using extra data) :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_color = pd.read_csv('../input/mean-color-isic2020/train_color.csv')\ntest_color = pd.read_csv('../input/mean-color-isic2020/test_color.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['mean_color'] = train_color.values\ntest['mean_color'] = test_color.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import KBinsDiscretizer \n\ncategorize  = KBinsDiscretizer(n_bins = 10 , encode = 'ordinal' , strategy = 'uniform')\n\ntrain['mean_color_encoding'] = categorize.fit_transform(train['mean_color'].values.reshape(-1,1)).astype(int).squeeze()\ntest['mean_color_encoding'] = categorize.transform(test['mean_color'].values.reshape(-1,1)).astype(int).squeeze()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Min-Max age of patient :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_id_min'] = train['patient_id'].map(train.groupby(['patient_id']).age_approx.min())\ntrain['age_id_max'] = train['patient_id'].map(train.groupby(['patient_id']).age_approx.max())\n\ntest['age_id_min'] = test['patient_id'].map(test.groupby(['patient_id']).age_approx.min())\ntest['age_id_max'] = test['patient_id'].map(test.groupby(['patient_id']).age_approx.max())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mean Encoding of age :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_approx_mean'] = train['age_approx'].map(train.groupby(['age_approx'])['target'].mean())\ntest['age_approx_mean'] = test['age_approx'].map(train.groupby(['age_approx'])['target'].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mean Encoding of sex :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['sex_encoding_mean'] = train['sex_encoding'].map(train.groupby(['sex_encoding'])['target'].mean())\ntest['sex_encoding_mean'] = test['sex_encoding'].map(train.groupby(['sex_encoding'])['target'].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mean Encoding of anatom :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['anatom_site_general_challenge_encoding_mean'] = train['anatom_site_general_challenge_encoding'].map(train.groupby(['anatom_site_general_challenge_encoding'])['target'].mean())\ntest['anatom_site_general_challenge_encoding_mean'] = test['anatom_site_general_challenge_encoding'].map(train.groupby(['anatom_site_general_challenge_encoding'])['target'].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mean Encoding of Number of Images :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['n_images_encoding_mean'] = train['n_images_encoding'].map(train.groupby(['n_images_encoding'])['target'].mean())\ntest['n_images_encoding_mean'] = test['n_images_encoding'].map(train.groupby(['n_images_encoding'])['target'].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mean Encoding of Image size :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['image_size_encoding_mean'] = train['image_size_encoding'].map(train.groupby(['image_size_encoding'])['target'].mean())\ntest['image_size_encoding_mean'] = test['image_size_encoding'].map(train.groupby(['image_size_encoding'])['target'].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Correlation Map :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"corr = train.corr(method = 'pearson')\ncorr = corr.abs()\ncorr.style.background_gradient(cmap='inferno')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test Correlation Map :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"corr = test.corr(method = 'pearson')\ncorr = corr.abs()\ncorr.style.background_gradient(cmap='inferno')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Play around with these feature ( you can get up to 0.8 LB)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [\n    \n    #'image_name',\n    #'patient_id',\n    #'sex', \n    'age_approx',\n    #'anatom_site_general_challenge', \n    'sex_encoding',\n    'anatom_site_general_challenge_encoding',\n    'n_images',\n    #'n_images_encoding',\n    # 'age_enc',\n   # 'image_size',\n    'image_size_scaled',\n   # 'image_size_encoding',\n    'mean_color',\n    #'mean_color_encoding',\n    'age_id_min',\n    'age_id_max',\n   # 'age_approx_mean',\n    #'sex_encoding_mean',\n    #'anatom_site_general_challenge_encoding_mean',\n   # 'n_images_encoding_mean',\n   # 'image_size_encoding_mean',\n       # 'anatom_head/neck',\n       # 'anatom_lower extremity',\n       #'anatom_oral/genital',\n       # 'anatom_palms/soles', \n      #  'anatom_torso',\n     #  'anatom_unknown_anatom', \n    #'anatom_upper extremity'\n    \n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train[features]\ny = train['target']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling : ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## XGBoost : (0.744 on LB) ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb = XGBRegressor()\nparameters = {'nthread':[4], #when use hyperthread, xgboost may become slower\n              'objective':['binary:logistic'],\n              'learning_rate': [0.04], #so called `eta` value\n              'max_depth': [9],\n              'min_child_weight': [5],\n              'silent': [1],\n              'subsample': [0.7],\n              'colsample_bytree': [0.7],\n           'n_estimators': [500]}\n\nxgb_grid = GridSearchCV(xgb,\n                        parameters,\n                        cv = 4,\n                        n_jobs = 5,\n                        verbose=True\n                       )\n\nxgb_grid.fit(X,\n         y)\n\nprint(xgb_grid.best_score_)\nprint(xgb_grid.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb = XGBRegressor(\n                nthread=4, #when use hyperthread, xgboost may become slower\n              objective='binary:logistic',\n              learning_rate= 0.04, #so called `eta` value\n              max_depth = 11,\n              min_child_weight = 5,\n              silent = 1,\n              subsample= 0.7,\n              colsample_bytree = 0.7,\n           n_estimators =  500\n\n)\n\nfolds = StratifiedKFold(n_splits = 5 , shuffle = True, random_state = 42 )\ncv_results = cross_val_score(xgb ,X , y, cv = folds, scoring = 'roc_auc', verbose = 3 ) \nprint(cv_results.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb.fit(X,y)\npredictions = xgb.predict(test[features])\nsub['target'] = predictions\nsub.to_csv('xgb_anatom_ohe.csv',index = False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CatBoost (0.5 on LB) :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CatBoostClassifier()\nparameters = {'depth'         : [6],\n                  'learning_rate' : [0.03],\n                  'iterations'    : [600]\n                 }\ngrid1 = GridSearchCV(estimator=model, param_grid = parameters, cv = 3, n_jobs=-1)\ngrid1.fit(X, y)    \n    # Results from Grid Search\nprint(\"\\n========================================================\")\nprint(\" Results from Grid Search \" )\nprint(\"========================================================\")       \nprint(\"\\n The best estimator across ALL searched params:\\n\",\n          grid1.best_estimator_)    \nprint(\"\\n The best score across ALL searched params:\\n\",\n          grid1.best_score_)\nprint(\"\\n The best parameters across ALL searched params:\\n\",\n          grid1.best_params_)  \nprint(\"\\n ========================================================\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat = CatBoostClassifier(\n    depth = 6,\n    iterations = 600,\n    learning_rate = 0.03,\n    verbose = 0\n)\n\ncv_results = cross_val_score(cat ,X , y, cv = folds, scoring = 'roc_auc', verbose = 3 ) \nprint(cv_results.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat.fit(X,y)\npredictions = cat.predict(test[features])\nsub['target'] = predictions\nsub.to_csv('cat_sub.csv',index = False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multinomial NB, Gaussian NB :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model  = GaussianNB()\n\ncv_results = cross_val_score(model,X , y, cv = folds, scoring = 'roc_auc', verbose = 3 ) \nprint(cv_results.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X,y)\npredictions = model.predict(test[features])\nsub['target'] = predictions\nsub.to_csv('gaussian_sub.csv',index = False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model  = MultinomialNB()\n\ncv_results = cross_val_score(model,X , y, cv = folds, scoring = 'roc_auc', verbose = 3 ) \nprint(cv_results.mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## AdaBoost : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = AdaBoostClassifier()\nparameters = {'n_estimators' : [25],\n                  'learning_rate' : [0.015],\n                 }\ngrid1 = GridSearchCV(estimator=model, param_grid = parameters, cv = 3, n_jobs=-1)\ngrid1.fit(X, y)    \n    # Results from Grid Search\nprint(\"\\n========================================================\")\nprint(\" Results from Grid Search \" )\nprint(\"========================================================\")       \nprint(\"\\n The best estimator across ALL searched params:\\n\",\n          grid1.best_estimator_)    \nprint(\"\\n The best score across ALL searched params:\\n\",\n          grid1.best_score_)\nprint(\"\\n The best parameters across ALL searched params:\\n\",\n          grid1.best_params_)  \nprint(\"\\n ========================================================\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model  = AdaBoostClassifier(\n)\n\ncv_results = cross_val_score(model,X , y, cv = folds, scoring = 'roc_auc', verbose = 3 ) \nprint(cv_results.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X,y)\npredictions = model.predict(test[features])\nsub['target'] = predictions\nsub.to_csv('ada_sub.csv',index = False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LightGBM :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = LGBMClassifier()\nparameters = {'n_estimators' : [50,100,150],\n                  'learning_rate' : [0.015,0.01,0.005],\n                 }\ngrid1 = GridSearchCV(estimator=model, param_grid = parameters, cv = 3, n_jobs=-1)\ngrid1.fit(X, y)    \n    # Results from Grid Search\nprint(\"\\n========================================================\")\nprint(\" Results from Grid Search \" )\nprint(\"========================================================\")       \nprint(\"\\n The best estimator across ALL searched params:\\n\",\n          grid1.best_estimator_)    \nprint(\"\\n The best score across ALL searched params:\\n\",\n          grid1.best_score_)\nprint(\"\\n The best parameters across ALL searched params:\\n\",\n          grid1.best_params_)  \nprint(\"\\n ========================================================\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from lightgbm import LGBMClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model  = LGBMClassifier()\n\ncv_results = cross_val_score(model,X , y, cv = folds, scoring = 'roc_auc', verbose = 3 ) \nprint(cv_results.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X,y)\npredictions = model.predict(test[features])\nsub['target'] = predictions\nsub.to_csv('lgbm_sub.csv',index = False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Neural Nets :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Activation, Dropout ,BatchNormalization\nfrom tensorflow.keras import utils\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import regularizers\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorflow-addons=='0.9.1'\nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight \nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(train.target),\n                                                 train.target)\n\nclass_weights = { 0 :  0.50897302 , 1 : 28.36130137 }\n\nclassweights =[item for k in class_weights for item in (k, class_weights[k])]\nprint(classweights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(512, activation='relu', input_dim=(8),\n                kernel_regularizer=regularizers.l2(0.01),\n                activity_regularizer=regularizers.l1(0.01)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.4))\nmodel.add(Dense(256, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.4))\nmodel.add(Dense(128, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dense(64, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.4))\nmodel.add(Dense(1, activation='sigmoid'))\n\nadam = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=10**-8, decay=0.0001, amsgrad=False)\nmodel.compile(optimizer= adam,\n              loss ='binary_crossentropy', # tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n              metrics=['accuracy',tf.keras.metrics.AUC()])\nhist = model.fit(X, y,\n                    batch_size=32,\n                    epochs=100,\n                    verbose=1,\n                    class_weight = class_weights\n                )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X,y)\npredictions = model.predict(test[features])\nsub['target'] = predictions\nsub.to_csv('nn_sub.csv',index = False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ensembling best results :","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"working on it ... stay tuned ! ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"If you like my kernel, don't forget to upvote, you can also check my work on image data :\n\nhttps://www.kaggle.com/aziz69/efficientnets-augs-0-925","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}