{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq #reading parquet files\nimport matplotlib.pyplot as plt\nimport os\nimport seaborn as sns\nimport scipy\nfrom statsmodels.robust import mad\nimport collections\nfrom scipy.signal import *\nimport random\nimport gc\n\n\n\n#used for plotting\nimport plotly.graph_objects as go\n#used for feature engineering(signal processing tools)\nfrom scipy.fftpack import fft\nfrom scipy.signal import welch\n#from siml.sk_utils import *\n#from siml.signal_analysis_utils import *\n\nfrom tqdm import tqdm\n\n\n## Sklearn Libraries\nfrom sklearn.utils import shuffle\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import label_binarize\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import make_scorer\nfrom sklearn.metrics import matthews_corrcoef\nfrom sklearn.metrics import f1_score, confusion_matrix, roc_curve, auc, classification_report, recall_score, precision_recall_curve\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import RandomizedSearchCV\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.linear_model import SGDRegressor\nfrom prettytable import PrettyTable\n\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-07-23T07:24:51.419602Z","iopub.execute_input":"2021-07-23T07:24:51.420158Z","iopub.status.idle":"2021-07-23T07:24:53.92883Z","shell.execute_reply.started":"2021-07-23T07:24:51.42005Z","shell.execute_reply":"2021-07-23T07:24:53.927835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_train = pd.read_csv('../input/vsb-power-line-fault-detection/metadata_train.csv')\nmetadata_train.head(6)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:22:29.87719Z","iopub.execute_input":"2021-07-21T11:22:29.877857Z","iopub.status.idle":"2021-07-21T11:22:29.902142Z","shell.execute_reply.started":"2021-07-21T11:22:29.877794Z","shell.execute_reply":"2021-07-21T11:22:29.901407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:22:34.082086Z","iopub.execute_input":"2021-07-21T11:22:34.082768Z","iopub.status.idle":"2021-07-21T11:22:34.09033Z","shell.execute_reply.started":"2021-07-21T11:22:34.082704Z","shell.execute_reply":"2021-07-21T11:22:34.089148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ckecking if each id measurement has 3 phases:\nfor i in range(0, len(metadata_train),3):\n    temp=[]\n    for i in range(3):\n        temp.append(metadata_train.loc[i]['phase'])\n    if (temp!= [0,1,2]):\n        print(\"error\")\n        break\n    \nprint('There are 3 phases 0, 1, 2 for each id_measurement')   ","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:22:35.836067Z","iopub.execute_input":"2021-07-21T11:22:35.836474Z","iopub.status.idle":"2021-07-21T11:22:36.831165Z","shell.execute_reply.started":"2021-07-21T11:22:35.836437Z","shell.execute_reply":"2021-07-21T11:22:36.829364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_train.corr()","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:22:40.60523Z","iopub.execute_input":"2021-07-21T11:22:40.605697Z","iopub.status.idle":"2021-07-21T11:22:40.623494Z","shell.execute_reply.started":"2021-07-21T11:22:40.605655Z","shell.execute_reply":"2021-07-21T11:22:40.622534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in metadata_train.columns:\n    print(\"Number of unique values in \", col, \"is\", metadata_train[col].unique().shape[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:22:45.820265Z","iopub.execute_input":"2021-07-21T11:22:45.821Z","iopub.status.idle":"2021-07-21T11:22:45.832813Z","shell.execute_reply.started":"2021-07-21T11:22:45.820952Z","shell.execute_reply":"2021-07-21T11:22:45.831164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nsplot = sns.countplot(x='target', data=metadata_train)\nfor ind, p in enumerate(splot.patches):\n  percent = np.round((metadata_train[metadata_train['target']==ind].shape[0]/metadata_train['target'].shape[0])*100, 2)\n  splot.annotate(str(metadata_train[metadata_train['target']==ind].shape[0]) + f\" ({percent}%)\", \n                 (p.get_x()+p.get_width()/3, p.get_height()))\nplt.title(\"Distribution of target classes\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:22:47.669077Z","iopub.execute_input":"2021-07-21T11:22:47.669542Z","iopub.status.idle":"2021-07-21T11:22:47.845508Z","shell.execute_reply.started":"2021-07-21T11:22:47.669498Z","shell.execute_reply":"2021-07-21T11:22:47.844431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nsplot = sns.countplot(x=\"phase\", data=metadata_train, hue=\"target\")\n# Get the total number of signals present in each phase\ntotal_phases = metadata_train[metadata_train['phase']==0].shape[0]\nnum_points = []\nfor ind, p in enumerate(splot.patches):\n    # Phase=[0,1,2] for indices [0,1,2] and indices [3,4,5] respectively\n    phase = ind%3\n    # target=[0,1] for indices [0,1], [2,3], [4,5] respectively\n    tar = ind//3\n    # Store the number of data points for the respective phase and target\n    num_points.append(metadata_train.loc[(metadata_train['target']==tar) & (metadata_train['phase']==phase)]['phase'].shape[0])\n    # Get the percentage of the number of data points\n    num_phase_percent = np.round((num_points[-1]/total_phases)*100, 2)\n    # Annotate the bar plot\n    splot.annotate(str(num_points[-1])+f\" ({num_phase_percent}%)\", (p.get_x(), p.get_height()))\nplt.title(\"Distribution of classes with respect to each phase\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:22:55.526977Z","iopub.execute_input":"2021-07-21T11:22:55.527449Z","iopub.status.idle":"2021-07-21T11:22:55.761791Z","shell.execute_reply.started":"2021-07-21T11:22:55.527403Z","shell.execute_reply":"2021-07-21T11:22:55.760522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check if id_measurement is same then is target label \nfor i in range(0, len(metadata_train), 3):\n    temp1= metadata_train.loc[i]['target']\n    #print(\"temp1= \",temp1)\n    f=0\n    for j in range(1,3):\n        temp2= metadata_train.loc[i+j]['target']\n        #print(\"temp2= \",temp2)\n        if temp1!=temp2:\n            print(metadata_train.loc[i:i+2])\n            f=1\n    if f==1:\n        print(\"The same id_measurement does not means the same target value!!! \")\n        break \n        ","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:23:02.008367Z","iopub.execute_input":"2021-07-21T11:23:02.008899Z","iopub.status.idle":"2021-07-21T11:23:02.041579Z","shell.execute_reply.started":"2021-07-21T11:23:02.008862Z","shell.execute_reply":"2021-07-21T11:23:02.04078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pq.read_pandas('../input/vsb-power-line-fault-detection/train.parquet', columns=[str(i) for i in range(100)]).to_pandas()","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:23:07.783893Z","iopub.execute_input":"2021-07-21T11:23:07.784413Z","iopub.status.idle":"2021-07-21T11:23:09.880374Z","shell.execute_reply.started":"2021-07-21T11:23:07.784378Z","shell.execute_reply":"2021-07-21T11:23:09.878953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sig_list = metadata_train.head(12).values\nplt.figure(figsize=(20,8))\nn_rows = 4\nn_cols = 1\nfor ind, val in enumerate(sig_list):\n  plt.subplot(n_rows, n_cols, (ind//3)+1)\n  plt.plot(df1[str(val[0])].values[::,], label=f\"signal_id-{val[0]}, phase-{val[2]}, id_measurement-{val[1]}\")\n  plt.legend()\n  plt.title(f'Power Line Signal with target={val[3]}')\n    \n  plt.xlabel('Samples')\n  plt.ylabel('Amplitude')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:23:13.420832Z","iopub.execute_input":"2021-07-21T11:23:13.421256Z","iopub.status.idle":"2021-07-21T11:23:43.695421Z","shell.execute_reply.started":"2021-07-21T11:23:13.421218Z","shell.execute_reply":"2021-07-21T11:23:43.690453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Feature Extraction**","metadata":{}},{"cell_type":"code","source":"def describe_freq(signal):\n    mean = np.mean(signal)\n    std = np.std(signal) \n    maxv = np.amax(signal) \n    minv = np.amin(signal) \n    median = np.median(signal)\n    skew = scipy.stats.skew(signal)\n    kurt = scipy.stats.kurtosis(signal)\n    q0 = np.quantile(signal, 0.10)\n    q1 = np.quantile(signal, 0.25)\n    q3 = np.quantile(signal, 0.75)\n    q4 = np.quantile(signal, 0.90)\n    mode = scipy.stats.mode(signal)[0][0]\n    iqr = scipy.stats.iqr(signal) \n    rmse= np.sqrt(np.mean(signal**2))\n    \n    return [mean, std, maxv, minv, median, skew, kurt, q0, q1, q3,q4, mode, iqr, rmse]\n\ndef describe_entropy(signal):\n    counter_values= collections.Counter(signal).most_common()\n    proba= [el[1]/len(signal) for el in counter_values]\n    entropy=scipy.stats.entropy(proba)\n    return entropy \n\ndef describe_crossings(signal):\n    zero_crossing_index= np.nonzero(np.diff(np.array(signal)> 0))[0]\n    len_zero_crossing= len(zero_crossing_index)\n    mean_crossing_index= np.nonzero(np.diff(np.array(signal)> np.nanmean(signal)))[0]\n    len_mean_crossing= len(mean_crossing_index)\n    return [len_zero_crossing, len_mean_crossing ]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:23:48.222926Z","iopub.execute_input":"2021-07-21T11:23:48.223428Z","iopub.status.idle":"2021-07-21T11:23:48.239083Z","shell.execute_reply.started":"2021-07-21T11:23:48.223381Z","shell.execute_reply":"2021-07-21T11:23:48.237938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def welch_max_power_and_frequency(signal):\n    f, Pxx = welch(signal)\n    ix = np.argmax(Pxx)\n    strong_count = np.sum(Pxx>2.5)\n    avg_amp = np.mean(Pxx)\n    sum_amp = np.sum(Pxx)\n    std_amp = np.std(Pxx)\n    median_amp = np.median(Pxx)\n    return [Pxx[ix], f[ix], strong_count, avg_amp, sum_amp, std_amp, median_amp]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f_values, psd_values = welch(df1['0'].values, fs=50)\nfig = go.Figure(data=go.Scatter(x = f_values, y = psd_values,mode = 'lines'))\nfig.update_layout(\n    title=\"frquency spectrum\",\n    xaxis_title=\"Frequency\",\n    yaxis_title=\"PSD\",\n    font=dict(\n        family=\"Courier New, monospace\",\n        size=18,\n        color=\"#7f7f7f\"\n    )\n)\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(signal):\n    entropy = describe_entropy(signal)\n    statistics = describe_freq(signal)\n    crossings = describe_crossings(signal)\n    powerfreq= welch_max_power_and_frequency(signal)\n    return [entropy] + statistics + crossings + powerfreq","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:24:38.119202Z","iopub.execute_input":"2021-07-21T11:24:38.120068Z","iopub.status.idle":"2021-07-21T11:24:38.127188Z","shell.execute_reply.started":"2021-07-21T11:24:38.120007Z","shell.execute_reply":"2021-07-21T11:24:38.125834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fourier_values = fft(df1['0'].values)\nfourier_values.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#N is number of points in our input signal\nN=800000\n#T is the time period which is inverseof frequency which is 50Hz\nT = 1/50\n#fft_values contains the filtered useful values from fft_values_ vector\nfourier_values_filtered = 2.0/N * np.abs(fourier_values[N//2:])\n#f_values contains the frquency values\nf_values = np.linspace(0.0, 1.0/(2.0*T), N//2)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure(data=go.Scatter(x = f_values, y = fourier_values_filtered,mode = 'lines'))\nfig.update_layout(\n    title=\"frquency spectrum\",\n    xaxis_title=\"Frequency\",\n    yaxis_title=\"Amplitude\",\n    font=dict(\n        family=\"Courier New, monospace\",\n        size=18,\n        color=\"#7f7f7f\"\n    )\n)\nfig.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_fourier_features(signal,N=800000,T=1/50):\n    '''\n    converts a signal from time spectrum to frequency spectrum\n    and returns only the features required as mentioned above\n    '''\n    fourier_values = fft(signal)\n    fourier_values_filtered = 2.0/N * np.abs(fourier_values[0:N//2])\n    return fourier_values_filtered\n\n\ndef filter_features_fourier(features,mph,no_features=8):\n    '''\n    returns fourier transformed features by extracting peaks and \n    considering only required number of peaks.\n    mph-detect peaks that are greater than minimum peak height\n    '''\n    indices_peaks = detect_peaks(features,mph = mph)\n    #print(indices_peaks)\n    values = features[indices_peaks]\n    if len(values)< no_features:\n        return np.append(values , [0]*(no_features-len(values)))\n    else:\n        return values[:no_features]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#total_features = []\n   \n#for i in tqdm(range(0,8712)):\n #   fourier_features = np.empty((3,8))\n #   features = []    \n #   signal =  pq.read_pandas('../input/vsb-power-line-fault-detection/train.parquet',columns=[str(i)]).to_pandas()[str(i)].values\n\n    #minimum peak height which can be used to filter fourier features\n  #  mph = signal.min() + (signal.max() - np.abs(signal.min()))/10\n    \n    #fourier features\n   # fourier_features_ = extract_fourier_features(signal)\n   # fourier_features = filter_features_fourier(fourier_features_,mph)\n    \n    #features.extend(fourier_features)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sig_features = np.array(total_features)\n\n#cols=[]\n#for i in range(0,8):\n # cols.append('feat'+str(i))\n#fourier_df = pd.DataFrame(sig_features, columns =cols)  \n#fourier_df.to_csv('../input/fourier-featcsv/'+\"fourier_feat.csv\", sep=\",\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#feature_columns= [ 'signal_id', 'entropy', 'mean', 'std', 'maxv', 'minv', 'median', 'skew', 'kurt', \n #                        'q0','q1', 'q3','q4', 'mode', 'iqr', 'rmse', 'len_zero_crossing', 'len_mean_crossing',\n  #                     'max_amp', 'max_freq', 'strong_amp_count', 'avg_amp', 'sum_amp', 'std_amp', 'median_amp', 'fault']\n\n#data_type = \"data\"\n#df_feature_matrix= pd.DataFrame([], columns= feature_columns)\n#for signal_id in metadata_train.signal_id:\n #   signal_0 = data[str(signal_id)]\n  #  signal_features= get_features(signal_0)\n   # df_features = pd.DataFrame([[signal_id] + signal_features + [metadata_train.target[metadata_train.signal_id == signal_id].values[0]]], columns=feature_columns)\n\n    #df_feature_matrix = df_feature_matrix.append(df_features, ignore_index=True)  # Append Feature Matrix Data Frame\n\n# Store feature matrix to CSV (commented out for this notebook, but left in for example.)\n#df_feature_matrix.to_csv('../input/data-get-feautures/'+data_type+\"_get_features.csv\", sep=\",\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_feature= pd.read_csv('../input/data-get-feautures/data_get_features.csv')\nfourier_df=pd.read_csv('../input/fourier-featcsv/fourier_feat.csv')\nfeatures= [ 'entropy', 'mean', 'std', 'maxv', 'minv', 'median', 'skew', 'kurt', \n                         'q0','q1', 'q3', 'q4','mode', 'iqr', 'rmse', 'len_zero_crossing', 'len_mean_crossing',\n                       'max_amp', 'max_freq', 'strong_amp_count', 'avg_amp', 'sum_amp', 'std_amp', 'median_amp']\nfourier_cols= ['feat0','feat1','feat2','feat3','feat4','feat5','feat6','feat7']                       \n\ntarget= [\"fault\"]\ndf= df_feature[features]\nfouri_df= fourier_df[fourier_cols]\ndata_features= pd.concat([df,fouri_df], axis=1)\ndata_features[\"fault\"]= df_feature[target]\ndata_features","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:26:21.898871Z","iopub.execute_input":"2021-07-21T11:26:21.89931Z","iopub.status.idle":"2021-07-21T11:26:22.096651Z","shell.execute_reply.started":"2021-07-21T11:26:21.899258Z","shell.execute_reply":"2021-07-21T11:26:22.095568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_features.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:26:40.61114Z","iopub.execute_input":"2021-07-21T11:26:40.611668Z","iopub.status.idle":"2021-07-21T11:26:40.620781Z","shell.execute_reply.started":"2021-07-21T11:26:40.611624Z","shell.execute_reply":"2021-07-21T11:26:40.619585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nimport seaborn as sns\nblues = [\"#66D7EB\", \"#51ACC5\", \"#3E849E\", \"#2C5F78\", \"#1C3D52\", \"#0E1E2B\"]\ncor = data_features.corr()\nf, ax = plt.subplots(figsize=(14, 8), dpi= 120, facecolor='w', edgecolor='k')\nsns.heatmap(cor, cmap=blues)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:26:50.102008Z","iopub.execute_input":"2021-07-21T11:26:50.10255Z","iopub.status.idle":"2021-07-21T11:26:51.211065Z","shell.execute_reply.started":"2021-07-21T11:26:50.102502Z","shell.execute_reply":"2021-07-21T11:26:51.209716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detail\nsns.pairplot(data_features[[\"std\", 'feat0','entropy',  \"fault\"]], hue=\"fault\",  height=3, diag_kind=\"kde\", diag_kws=dict(shade=True, bw=.05, vertical=False) )\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:27:04.05791Z","iopub.execute_input":"2021-07-21T11:27:04.058339Z","iopub.status.idle":"2021-07-21T11:27:11.353059Z","shell.execute_reply.started":"2021-07-21T11:27:04.058302Z","shell.execute_reply":"2021-07-21T11:27:11.351886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Machine Learning Algorithms**","metadata":{}},{"cell_type":"code","source":"random_state = 1\nclass Create_ensemble(object):\n    def __init__(self, n_splits, base_models):\n        self.n_splits = n_splits\n        self.base_models = base_models\n\n    def predict(self, X, y, T):\n        X = np.array(X)\n        y = np.array(y)\n        T = np.array(T)\n        no_class = len(np.unique(y))\n\n        folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True, \n                                     random_state = random_state).split(X, y))\n\n        train_proba = np.zeros((X.shape[0], no_class))\n        test_proba = np.zeros((T.shape[0], no_class))\n        \n        train_pred = np.zeros((X.shape[0], len(self.base_models)))\n        test_pred = np.zeros((T.shape[0], len(self.base_models)* self.n_splits))\n        f1_scores = np.zeros((len(self.base_models), self.n_splits))\n        recall_scores = np.zeros((len(self.base_models), self.n_splits))\n        clf_=[]\n        # PrettyTable\n        tb = PrettyTable()\n        tb.field_names = ['Model', 'CV-', 'recall', 'f1_score', 'mcc']\n        #DataFrame \n        Frame = pd.DataFrame()\n\n        test_col = 0\n        for i, clf in enumerate(self.base_models):\n            \n            for j, (train_idx, valid_idx) in enumerate(folds):\n                \n                X_train = X[train_idx]\n                Y_train = y[train_idx]\n                X_valid = X[valid_idx]\n                Y_valid = y[valid_idx]\n                \n                clf.fit(X_train, Y_train)\n                \n                valid_pred = clf.predict(X_valid)\n                recall  = recall_score(Y_valid, valid_pred, average='macro')\n                f1 = f1_score(Y_valid, valid_pred, average='macro')\n                mcc = matthews_corrcoef(Y_valid, valid_pred)\n                \n                recall_scores[i][j] = recall\n                f1_scores[i][j] = f1\n                \n                train_pred[valid_idx, i] = valid_pred\n                test_pred[:, test_col] = clf.predict(T)\n                test_col += 1\n                \n                ## Probabilities\n                valid_proba = clf.predict_proba(X_valid)\n                train_proba[valid_idx, :] = valid_proba\n                test_proba  += clf.predict_proba(T)\n                \n                #print( \"Model- {} and CV- {} recall: {}, f1_score: {}, mcc: {}\".format(i, j, recall, f1, mcc))\n                tb.add_row([i, j, recall, f1, mcc])\n                new_row= {'Model':i, 'CV':j, 'recall':recall, 'f1':f1, 'mcc':mcc}\n                Frame = Frame.append(new_row, ignore_index=True)\n                clf_.append(clf)\n            test_proba /= self.n_splits\n            \n        return train_proba, test_proba, train_pred, test_pred, clf_, Frame","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:27:34.89234Z","iopub.execute_input":"2021-07-21T11:27:34.892829Z","iopub.status.idle":"2021-07-21T11:27:34.912186Z","shell.execute_reply.started":"2021-07-21T11:27:34.89279Z","shell.execute_reply":"2021-07-21T11:27:34.91101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Unbalanced \n# LogisticRegression\nparams_log = {'C': [10**x for x in range(-4,5)],\n          'l1_ratio': [0, 0.1, 0.3, 0.5, 0.7, 0.9, 1]}\nlog_clf = LogisticRegression(penalty='elasticnet', solver='saga', random_state=42)\n\n# Perform stratified 5-fold cross validation\nrand_log_clf = RandomizedSearchCV(log_clf, param_distributions=params_log, random_state=42, return_train_score=True,\n                              n_jobs=-1)\n\n# Random Forest\nparams_rf = {'n_estimators': [10, 50, 100, 500, 1000], \n          'max_depth': [2, 3, 4, 5, 6],\n          'min_samples_split': [0.02, 0.04, 0.08, 0.16, 0.32, 0.50]}\nrf_clf = RandomForestClassifier(random_state=42)\nrand_rf_clf = RandomizedSearchCV(rf_clf, param_distributions=params_rf, random_state=42, return_train_score=True,\n                              n_jobs=-1)\n\n# LightGBM\nparams_lgb = {'n_estimators': [10, 50, 100, 500, 1000], \n          'max_depth': [2, 3, 4, 5, 6],\n          'learning_rate': [1e-2, 1e-1, 0.5, 0.9],\n          'reg_alpha': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2], \n          'reg_lambda': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2]}\nlgb_clf = lgb.LGBMClassifier(random_state=42,\n                             importance_type='gain')\nrand_lgb_clf = RandomizedSearchCV(lgb_clf, param_distributions=params_lgb, random_state=42, return_train_score=True,\n                              n_jobs=-1)\n\n# XGBoost\nparams_xgb = {'n_estimators': [10, 50, 100, 500, 1000], \n          'max_depth': [2, 3, 4, 5, 6],\n          'learning_rate': [1e-2, 1e-1, 0.5, 0.9],\n          'reg_alpha': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2], \n          'reg_lambda': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2]}\nxgb_clf = xgb.XGBClassifier(random_state=42)\nrand_xgb_clf = RandomizedSearchCV(xgb_clf, param_distributions=params_xgb, random_state=42, return_train_score=True,\n                              n_jobs=-1)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:27:39.71794Z","iopub.execute_input":"2021-07-21T11:27:39.718408Z","iopub.status.idle":"2021-07-21T11:27:39.733327Z","shell.execute_reply.started":"2021-07-21T11:27:39.718365Z","shell.execute_reply":"2021-07-21T11:27:39.732253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Balacend \n\n# LogisticRegression\nparams_log_b = {'C': [10**x for x in range(-4,5)],\n          'l1_ratio': [0, 0.1, 0.3, 0.5, 0.7, 0.9, 1]}\nlog_clf_b = LogisticRegression(penalty='elasticnet', solver='saga',class_weight='balanced', random_state=42)\n\n# Perform stratified 5-fold cross validation\nrand_log_clf_b = RandomizedSearchCV(log_clf_b, param_distributions=params_log, random_state=42, return_train_score=True,\n                              n_jobs=-1)\n\n# Random Forest\nparams_rf_b = {'n_estimators': [10, 50, 100, 500, 1000], \n          'max_depth': [2, 3, 4, 5, 6],\n          'min_samples_split': [0.02, 0.04, 0.08, 0.16, 0.32, 0.50]}\nrf_clf_b = RandomForestClassifier(random_state=42, class_weight='balanced')\nrand_rf_clf_b = RandomizedSearchCV(rf_clf_b, param_distributions=params_rf, random_state=42, return_train_score=True,\n                              n_jobs=-1)\n\n# LightGBM\nparams_lgb_b = {'n_estimators': [10, 50, 100, 500, 1000], \n          'max_depth': [2, 3, 4, 5, 6],\n          'learning_rate': [1e-2, 1e-1, 0.5, 0.9],\n          'reg_alpha': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2], \n          'reg_lambda': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2]}\nlgb_clf_b = lgb.LGBMClassifier(random_state=42, class_weight='balanced', \n                             importance_type='gain')\nrand_lgb_clf_b = RandomizedSearchCV(lgb_clf_b, param_distributions=params_lgb_b, random_state=42, return_train_score=True,\n                              n_jobs=-1)\n\n# XGBoost\nparams_xgb_b = {'n_estimators': [10, 50, 100, 500, 1000], \n          'max_depth': [2, 3, 4, 5, 6],\n          'learning_rate': [1e-2, 1e-1, 0.5, 0.9],\n          'reg_alpha': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2], \n          'reg_lambda': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2]}\nxgb_clf_b = xgb.XGBClassifier(random_state=42, class_weight='balanced')\nrand_xgb_clf_b = RandomizedSearchCV(xgb_clf_b, param_distributions=params_xgb_b, random_state=42, return_train_score=True,\n                              n_jobs=-1)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:27:44.046242Z","iopub.execute_input":"2021-07-21T11:27:44.046856Z","iopub.status.idle":"2021-07-21T11:27:44.061248Z","shell.execute_reply.started":"2021-07-21T11:27:44.046814Z","shell.execute_reply":"2021-07-21T11:27:44.060225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_models = [rand_log_clf, rand_rf_clf, rand_lgb_clf,rand_xgb_clf , rand_log_clf_b, rand_rf_clf_b, rand_lgb_clf_b,rand_xgb_clf_b ]\nn_splits=5\nlgb_stack = Create_ensemble(n_splits = n_splits, base_models = base_models) \n       \ndata_features.fillna(999, inplace=True)\n\nX = data_features.drop(['fault'], axis=1)\nY = data_features['fault'].values\nx_train, x_test, y_train, y_test = train_test_split(X, Y, test_size= 0.33, random_state=42)\nx_train.shape\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:28:12.396649Z","iopub.execute_input":"2021-07-21T11:28:12.397188Z","iopub.status.idle":"2021-07-21T11:28:12.418072Z","shell.execute_reply.started":"2021-07-21T11:28:12.397143Z","shell.execute_reply":"2021-07-21T11:28:12.416615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_proba, test_proba, train_pred, test_pred, clf , table= lgb_stack.predict(x_train, y_train, x_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:28:19.706569Z","iopub.execute_input":"2021-07-21T11:28:19.708966Z","iopub.status.idle":"2021-07-21T13:29:49.583334Z","shell.execute_reply.started":"2021-07-21T11:28:19.708903Z","shell.execute_reply":"2021-07-21T13:29:49.581905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table.loc[table['Model']==0.0, 'Model'] = 'Ub_LogisticRegression'\ntable.loc[table['Model']==1.0, 'Model'] = 'Ub_RandomForest'\ntable.loc[table['Model']==2.0, 'Model'] = 'Ub_LightGBM'\ntable.loc[table['Model']==3.0, 'Model'] = 'Ub_XGBoost'\ntable.loc[table['Model']==4.0, 'Model'] = 'B_LogisticRegression'\ntable.loc[table['Model']==5.0, 'Model'] = 'B_RandomForest'\ntable.loc[table['Model']==6.0, 'Model'] = 'B_LightGBM'\ntable.loc[table['Model']==7.0, 'Model'] = 'B_XGBoost'\ntable","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:41:36.693675Z","iopub.execute_input":"2021-07-21T13:41:36.694149Z","iopub.status.idle":"2021-07-21T13:41:36.737342Z","shell.execute_reply.started":"2021-07-21T13:41:36.694107Z","shell.execute_reply":"2021-07-21T13:41:36.735985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols=['Ub_LogisticRegression', 'Ub_RandomForest', 'Ub_LightGBM', 'Ub_XGBoost', 'B_LogisticRegression', 'B_RandomForest', 'B_LightGBM', 'B_XGBoost']\ntrain_pred_df = pd.DataFrame(train_pred, columns =cols)  \ntrain_pred_df","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:42:02.554511Z","iopub.execute_input":"2021-07-21T13:42:02.554932Z","iopub.status.idle":"2021-07-21T13:42:02.585454Z","shell.execute_reply.started":"2021-07-21T13:42:02.554893Z","shell.execute_reply":"2021-07-21T13:42:02.583821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nPerformance Metrics after Weighted XGBoost Cross Validation')\nprint('1. The F-1 score of the model {}\\n'.format(f1_score(y_train, train_pred_df['B_XGBoost'], average='macro')))\nprint('2. The recall score of the model {}\\n'.format(recall_score(y_train, train_pred_df['B_XGBoost'], average='macro')))\nprint('3. The Matthews Correlation Coefficient: {}\\n'.format(matthews_corrcoef(y_train, train_pred_df['B_XGBoost'])))\nprint('4. Classification report \\n {} \\n'.format(classification_report(y_train, train_pred_df['B_XGBoost'])))\nprint('5. Confusion matrix \\n {} \\n'.format(confusion_matrix(y_train, train_pred_df['B_XGBoost'])))","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:42:07.306675Z","iopub.execute_input":"2021-07-21T13:42:07.307103Z","iopub.status.idle":"2021-07-21T13:42:07.376795Z","shell.execute_reply.started":"2021-07-21T13:42:07.307067Z","shell.execute_reply":"2021-07-21T13:42:07.375948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# histogram of important features\nimp = clf[39].best_estimator_.feature_importances_\nimp, features = zip(*sorted(zip(imp, features)))\nblues = [\"#66D7EB\", \"#51ACC5\", \"#3E849E\", \"#2C5F78\", \"#1C3D52\", \"#0E1E2B\"]\nf, ax = plt.subplots(figsize=(14, 6), dpi= 120, facecolor='w', edgecolor='k')\nplt.barh(range(len(features)), imp, color=blues[1], align=\"center\")\nplt.yticks(range(len(features)), features)\nplt.xlabel(\"Importance of Features\")\nplt.ylabel(\"Features\")\nplt.title(\"Importance of Each Feature in Classifier Model\")","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:42:12.450073Z","iopub.execute_input":"2021-07-21T13:42:12.452196Z","iopub.status.idle":"2021-07-21T13:42:12.876338Z","shell.execute_reply.started":"2021-07-21T13:42:12.452147Z","shell.execute_reply":"2021-07-21T13:42:12.875558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# histogram of predicted probabilities\nblues = [\"#66D7EB\", \"#51ACC5\", \"#3E849E\", \"#2C5F78\", \"#1C3D52\", \"#0E1E2B\"]\nf, ax = plt.subplots(figsize=(14, 6), dpi= 120, facecolor='w', edgecolor='k')\nnclasses = 2\ntitles = [\"Probabilities for No Partial Discharge Fault Present\", \"Probabilities for Partial Discharge Fault Present\"]\nfor i in range(nclasses):\n    plt.subplot(1, nclasses, i+1)\n    plt.hist(train_proba[:, i], bins=50, histtype='bar', rwidth=0.95, color=blues[1])\n    plt.xlim(0,1)\n    plt.title(titles[i])\n    plt.xlabel('Probability')\n    plt.ylabel('Frequency')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:42:21.022537Z","iopub.execute_input":"2021-07-21T13:42:21.02327Z","iopub.status.idle":"2021-07-21T13:42:21.800319Z","shell.execute_reply.started":"2021-07-21T13:42:21.023153Z","shell.execute_reply":"2021-07-21T13:42:21.799187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = label_binarize(y_train, classes=[0, 1])\n_, _, th1 = roc_curve(y[:, 0], train_proba[:, 0])\n_, _, th2 = roc_curve(y[:, 0], train_proba[:, 1])\nprint('\\nMedian Detection Thresholds for Fault Detection')  # use for setting reprediction thresholds\nprint(np.median(th1))\nprint(np.median(th2))","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:42:27.308625Z","iopub.execute_input":"2021-07-21T13:42:27.309083Z","iopub.status.idle":"2021-07-21T13:42:27.325574Z","shell.execute_reply.started":"2021-07-21T13:42:27.309042Z","shell.execute_reply":"2021-07-21T13:42:27.324213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns=[] \nfor i in range(40):\n  col= 'y_pred'+str(i)\n  columns.append(col)\ne_dataframe = pd.DataFrame(test_pred, columns =columns)  \ne_dataframe['y_test'] =y_test.astype(float)\ne_dataframe","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:42:32.376309Z","iopub.execute_input":"2021-07-21T13:42:32.377009Z","iopub.status.idle":"2021-07-21T13:42:32.433694Z","shell.execute_reply.started":"2021-07-21T13:42:32.376969Z","shell.execute_reply":"2021-07-21T13:42:32.432931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint('\\nPerformance Metrics after Weighted Unbalanced XGBoost Cross Validation')\nprint('1. The F-1 score of the model {}\\n'.format(f1_score(e_dataframe.iloc[:,15], e_dataframe.iloc[:,-1], average='macro')))\nprint('2. The recall score of the model {}\\n'.format(recall_score(e_dataframe.iloc[:,15], e_dataframe.iloc[:,-1], average='macro')))\nprint('3. The Matthews Correlation Coefficient: {}\\n'.format(matthews_corrcoef(e_dataframe.iloc[:,15], e_dataframe.iloc[:,-1])))\nprint('4. Classification report \\n {} \\n'.format(classification_report(e_dataframe.iloc[:,15], e_dataframe.iloc[:,-1])))\nprint('5. Confusion matrix \\n {} \\n'.format(confusion_matrix(e_dataframe.iloc[:,15], e_dataframe.iloc[:,-1])))\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:42:37.53837Z","iopub.execute_input":"2021-07-21T13:42:37.539064Z","iopub.status.idle":"2021-07-21T13:42:37.578346Z","shell.execute_reply.started":"2021-07-21T13:42:37.539021Z","shell.execute_reply":"2021-07-21T13:42:37.577562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel = lgb.LGBMClassifier(boosting_type='gbdt', objective='binary', num_leaves=50,\n                                learning_rate=0.1, n_estimators=500, max_depth=6,\n                                bagging_fraction=0.9, feature_fraction=0.9, reg_lambda=0.2)\n\nbase_models = [model]\nn_splits = 5\nlgb_stack = Create_ensemble(n_splits = n_splits, base_models = base_models) ","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:44:49.299081Z","iopub.execute_input":"2021-07-21T13:44:49.29956Z","iopub.status.idle":"2021-07-21T13:44:49.306625Z","shell.execute_reply.started":"2021-07-21T13:44:49.299522Z","shell.execute_reply":"2021-07-21T13:44:49.305084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_features.fillna(999, inplace=True)\n\nX = data_features.drop(['fault'], axis=1)\nY = data_features['fault'].values\nx_train, x_test, y_train, y_test = train_test_split(X, Y, test_size= 0.33, random_state=42)\nx_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:45:01.966049Z","iopub.execute_input":"2021-07-21T13:45:01.96658Z","iopub.status.idle":"2021-07-21T13:45:01.983393Z","shell.execute_reply.started":"2021-07-21T13:45:01.966538Z","shell.execute_reply":"2021-07-21T13:45:01.982136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_proba, test_proba, train_pred, test_pred, clf , table= lgb_stack.predict(x_train, y_train, x_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:45:11.053043Z","iopub.execute_input":"2021-07-21T13:45:11.053445Z","iopub.status.idle":"2021-07-21T13:45:17.655127Z","shell.execute_reply.started":"2021-07-21T13:45:11.053402Z","shell.execute_reply":"2021-07-21T13:45:17.654298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:45:38.231681Z","iopub.execute_input":"2021-07-21T13:45:38.232047Z","iopub.status.idle":"2021-07-21T13:45:38.247641Z","shell.execute_reply.started":"2021-07-21T13:45:38.232015Z","shell.execute_reply":"2021-07-21T13:45:38.246377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nPerformance Metrics after Weighted Random Forest Cross Validation')\nprint('1. The F-1 score of the model {}\\n'.format(f1_score(y_train, train_pred, average='macro')))\nprint('2. The recall score of the model {}\\n'.format(recall_score(y_train, train_pred, average='macro')))\nprint('3. The Matthews Correlation Coefficient: {}\\n'.format(matthews_corrcoef(y_train, train_pred)))\nprint('4. Classification reporct \\n {} \\n'.format(classification_report(y_train, train_pred)))\nprint('5. Confusion matrix \\n {} \\n'.format(confusion_matrix(y_train, train_pred)))","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:46:22.602376Z","iopub.execute_input":"2021-07-21T13:46:22.602756Z","iopub.status.idle":"2021-07-21T13:46:22.67102Z","shell.execute_reply.started":"2021-07-21T13:46:22.602722Z","shell.execute_reply":"2021-07-21T13:46:22.6699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.DataFrame() \ntest_pred = np.median(test_pred).astype(int)\ndf_test['y_test'] =y_test.astype(int)\ndf_test[\"fault\"] = test_pred\n# Make Submission File\nsubmission_filename = \"../input/submission/prediction_submission_cv.csv\"\n\n# Commented out in this notebook but left in as an example to create a submission file\n#f_o = open(submission_filename, \"w+\")\n#f_o.write(\"signal_id,target\\n\")\n#for idx in range(len(df_test)):\n\n #   signal_id = df_feature[\"signal_id\"][idx]\n #   fault = df_test[\"fault\"][idx]\n #   f_o.write(str(signal_id)+\",\"+str(fault)+\"\\n\")\n#f_o.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:47:29.78204Z","iopub.execute_input":"2021-07-21T13:47:29.782487Z","iopub.status.idle":"2021-07-21T13:47:29.791879Z","shell.execute_reply.started":"2021-07-21T13:47:29.782449Z","shell.execute_reply":"2021-07-21T13:47:29.790636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test=pd.read_csv('../input/submission/prediction_submission_cv.csv')\ndf_test","metadata":{"execution":{"iopub.status.busy":"2021-07-21T13:47:57.54422Z","iopub.execute_input":"2021-07-21T13:47:57.544662Z","iopub.status.idle":"2021-07-21T13:47:57.574852Z","shell.execute_reply.started":"2021-07-21T13:47:57.544627Z","shell.execute_reply":"2021-07-21T13:47:57.573487Z"},"trusted":true},"execution_count":null,"outputs":[]}]}