{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Basic libraries of python for numeric and dataframe computations\n\nimport numpy as np                              \nimport pandas as pd\nfrom numpy import asarray\nfrom numpy import *\n#from sklearn.preprocessing import CategoricalEncoder\nfrom numba import njit\n\n# Basic library for data visualization\nimport matplotlib.pyplot as plt     \n\n# Slightly advanced library for data visualization            \nimport seaborn as sns                          \n\n# Featauretools for feature engineering\nimport featuretools as ft\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.impute import SimpleImputer\n\nimport statsmodels.formula.api as smf\n\n#Importing our ML toolkit\nfrom sklearn.metrics import r2_score\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve\nfrom sklearn.feature_selection import mutual_info_regression\n\n# Importing primitives\nfrom featuretools.primitives import (Minute, Hour, Day, Month,\n                                     Weekday, IsWeekend, Count, Sum, Mean, Median, Std, Min, Max)\n\n# Used to ignore the warning given as output of the code\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nprint(ft.__version__)\n%load_ext autoreload\n%autoreload 2","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating a function for comprehensive DEA, generic to any dataset\n#______________________________________________________________________\n#the input contains the dataset in data frame, and an optional column to drop, e.g. the target column\n#i.e. the input parameter 'target' is optional\n\ndef discover(df, target=''):\n    \n    #Creating Seris with Feature Types, removing the target feature\n    if target != '':\n        df1 = df.drop(target, axis = 1, inplace = False)\n    else:\n        df1 = df\n\n    #listing feature type\n    feature_type = []\n    for j in range(len(df1.columns)):\n        \n        if df1[df1.columns[j]].nunique()==2:\n            feature_type.append('Binary')\n            \n            \n        elif df1[df1.columns[j]].dtypes != 'O' and df1[df1.columns[j]].nunique()>10:\n            feature_type.append('Numerical')\n            \n        elif df1[df1.columns[j]].dtypes != 'O' and df1[df1.columns[j]].nunique()<10:\n            feature_type.append('Categorical')\n                    \n        elif df1[df1.columns[j]].nunique()<10:\n            feature_type.append('Categorical')\n        \n        else:\n            feature_type.append('Identifier')\n    \n    #listing feature content\n    features_content = []\n    for i in range(len(df1.columns)):\n        if df1[df1.columns[i]].nunique()<10: \n            features_content.append(df1[df1.columns[i]].unique())\n        else:\n            features_content.append('n = ' + str(df1[df1.columns[i]].nunique()))\n            \n    \n\n            \n    #listing empty cells feature type\n    num_of_empty = []\n    for k in df1.columns:\n        num_of_empty.append(df1[k].isnull().sum())\n\n    #listing empty cells ratio feature type\n    ratio_of_empty = []\n    for k in df1.columns:\n        ratio_of_empty.append(round(100*df1[k].isnull().sum()/df1[k].isnull().count(),2))\n        \n\n    discovered = pd.DataFrame({'Features': df1.columns, \n                               'Features Content': features_content, \n                               'Feature Type': feature_type, \n                               'Empty Cells': num_of_empty,\n                               '% Empty': ratio_of_empty\n                             })\n    discovered = discovered.sort_values(['Feature Type','Empty Cells'],ascending=False)\n    print(\"Shape of Dataset: \",df.shape)\n    return discovered\n#_______________________________________________________________________________________________________________\n#function to visualize the dataset, works best with classification datasets\n#to modify later to take also numerical target data\n#the function takes: df: the dataframe containing the dataset, and target: a string containing the column name of the target column (i.e. the one containing the classes)\n#_______________________________________________________________________________________________________________\n\ndef graph(df, target, spacing = 5):\n    \n        \n    if target != '':\n        df1 = df.drop(target, axis = 1, inplace = False)\n    else:\n        df1 = df\n\n#listing feature types\n    feature_type = []\n    for j in range(len(df1.columns)):\n        \n        if df1[df1.columns[j]].nunique()==2:\n            feature_type.append('Binary')\n            \n            \n        elif df1[df1.columns[j]].dtypes != 'O' and df1[df1.columns[j]].nunique()>10:\n            feature_type.append('Numerical')\n            \n        elif df1[df1.columns[j]].dtypes != 'O' and df1[df1.columns[j]].nunique()<10:\n            feature_type.append('Categorical')\n                    \n        elif df1[df1.columns[j]].nunique()<10:\n            feature_type.append('Categorical')\n        \n        else:\n            feature_type.append('Identifier')\n            \n#listing feature content\n    features_content = []\n    for i in range(len(df1.columns)):\n        if df1[df1.columns[i]].nunique()<10: \n            features_content.append(df1[df1.columns[i]].unique())\n        else:\n            features_content.append('n = ' + str(df1[df1.columns[i]].nunique()))\n\n\n\n\n#listing empty cells feature type\n    num_of_empty = []\n    for k in df1.columns:\n        num_of_empty.append(df1[k].isnull().sum())\n\n\n\n\n    discovered = pd.DataFrame({'Features': df1.columns, \n                                   'Features Content': features_content, \n                                   'Feature Type': feature_type, \n                                   'Empty Cells': num_of_empty\n                                 })\n\n#_________________________________________________________________________________________________    \n# Plotting:\n#_________________________________________________________________________________________________    \n\n#plotting the target column\n    plt.figure(figsize=(10,4))\n    plt.subplots_adjust(left=0.1,bottom=0.1, right=0.9,  top=0.9, wspace=0.4, hspace=0.4)\n    ax = df[target].value_counts().plot.barh()\n    plt.title(f'{target} Count Plot')\n\n\n\n#plotting Categorical & binary values:\n    \n    plt.subplots_adjust(left=0.1,bottom=0.1, right=0.9,  top=0.9, wspace=0.4, hspace=0.4)\n\n    plots = 0\n    legends = df[target].unique()\n    columns = int(min(spacing,len(discovered['Features'][discovered['Feature Type'] .isin(['Categorical', 'Binary'])])))\n    rows = int(np.ceil(len(legends)*len(discovered['Features'][discovered['Feature Type'] .isin(['Categorical', 'Binary'])])/spacing))\n    plt.figure(figsize=(20,5*rows))\n    \n    for i in discovered['Features'][discovered['Feature Type'] .isin(['Categorical', 'Binary'])]:\n        plots = plots + 1\n        plt.subplot(rows,columns,plots)\n        sns.countplot(x=i, hue=target, data=df, palette=\"Set1\")\n\n\n#plotting Numerical values:\n\n\n    plt.subplots_adjust(left=0.1,bottom=0.1, right=0.9,  top=0.9, wspace=0.4, hspace=0.4)\n\n    plots = 0\n    columns = int(min(spacing,len(discovered['Features'][discovered['Feature Type'] =='Numerical'])))\n    rows = int(np.ceil(len(discovered['Features'][discovered['Feature Type'] =='Numerical'])/spacing))\n    plt.figure(figsize=(20,5*rows))\n    for i in discovered['Features'][discovered['Feature Type'] =='Numerical']:\n        plots = plots + 1\n        plt.subplot(rows,columns,plots)\n        sns.histplot(data = df, x = df[i], hue = target, hue_order = legends, kde = True, element = 'step')\n        #plt.yscale('symlog')\n        if df[i].skew(axis = 0, skipna = True)>2:\n            plt.xscale('symlog')\n            plt.xlim(0,max(df[i]))\n            \n    plt.show()\n    \n#Plotting Overall Correlation Heatmap\n    \n    columns = int(min(spacing,len(discovered['Features'][discovered['Feature Type'] =='Numerical'])))\n    rows    = int(np.ceil(len(discovered['Features'][discovered['Feature Type'] =='Numerical'])/spacing))\n    plt.figure(figsize=(5*columns,5*columns))\n    cmap = sns.diverging_palette(230, 20, as_cmap = True)\n    sns.heatmap(df.corr().abs(), annot = True, fmt = '.2f', cmap = cmap, square=True )\n    plt.show()\n    \n    \n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#a function to test several models on several given datasets\n#x = dataset without target column\n#y = target column\n#i = name of the dataset or name of the trial user is doing, it will appear in the big title\n\nfrom sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve, cross_validate\n\ndef ModelTest(x,y, i = 'Title Here', kfold = StratifiedKFold(n_splits=10),n_jobs = -1):\n    \n    \n    random_state = 2\n    classifiers = []\n    classifiers.append(SVC(random_state=random_state))\n    classifiers.append(DecisionTreeClassifier(random_state=random_state))\n    classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state,learning_rate=0.1))\n    classifiers.append(RandomForestClassifier(random_state=random_state))\n    classifiers.append(ExtraTreesClassifier(random_state=random_state))\n    classifiers.append(GradientBoostingClassifier(random_state=random_state))\n    classifiers.append(MLPClassifier(random_state=random_state))\n    classifiers.append(KNeighborsClassifier())\n    classifiers.append(LogisticRegression(random_state = random_state))\n    classifiers.append(LinearDiscriminantAnalysis())\n\n    cv_results = []\n    for classifier in classifiers :\n        cv_results.append(cross_val_score(classifier, x, y = y, scoring = \"accuracy\", cv = kfold, n_jobs=n_jobs))\n\n    cv_means = []\n    cv_std = []\n    for cv_result in cv_results:\n        cv_means.append(cv_result.mean())\n        cv_std.append(cv_result.std())\n\n    cv_res = pd.DataFrame({\"CrossValMeans\":cv_means,\"CrossValerrors\": cv_std,\"Algorithm\":[\"SVC\",\"DecisionTree\",\"AdaBoost\",\n    \"RandomForest\",\"ExtraTrees\",\"GradientBoosting\",\"MultipleLayerPerceptron\",\"KNeighboors\",\"LogisticRegression\",\"LinearDiscriminantAnalysis\"]})\n\n    plt.figure(figsize=(10,4))\n    plt.suptitle(i,  y = 1.05, fontsize=12)\n    g = sns.barplot(\"Algorithm\",\"CrossValMeans\",data = cv_res, palette=\"Set3\",orient = \"v\",**{'xerr':cv_std})\n    for bar in g.patches: \n        g.annotate(format(bar.get_height(), '.4f'),\n                       (bar.get_x() + bar.get_width() / 2,\n                        bar.get_height()), ha='center', va='center',\n                       size=9, xytext=(0, 8),\n                       textcoords='offset points')\n\n\n    plt.xticks(rotation=70)\n    plt.ylim(cv_res[\"CrossValMeans\"].min()*0.95, cv_res[\"CrossValMeans\"].max()*1.05)\n    g.set_xlabel(\"Mean Accuracy\")\n    g = g.set_title(\"Cross validation scores\")","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#These two functions are used together to return the feature strength along with the VIF, for the feature selection step later\n\ndef make_mi_scores(X, y):\n    X = X.copy()\n    for colname in X.select_dtypes([\"object\", \"category\"]):\n        X[colname], _ = X[colname].factorize()\n    # All discrete features should now have integer dtypes\n    discrete_features = [pd.api.types.is_integer_dtype(t) for t in X.dtypes]\n    mi_scores = mutual_info_regression(X, y, random_state=0)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns)\n    mi_scores = mi_scores.sort_values(ascending=False)\n    df_mi_scores = pd.DataFrame({'Mutual Information Scores': mi_scores})\n    return df_mi_scores\n\n\ndef plot_mi_scores(scores):\n    scores = scores.sort_values(ascending=True)\n    width = np.arange(len(scores))\n    ticks = list(scores.index)\n    plt.barh(width, scores)\n    plt.yticks(width, ticks)\n    plt.title(\"Mutual Information Scores\")\n    \ndef get_vif(exogs, data):\n\n    # initialize dictionaries\n    vif_dict, tolerance_dict = {}, {}\n\n    # form input data for each exogenous variable\n    for exog in exogs:\n        not_exog = [i for i in exogs if i != exog]\n        X, y = data[not_exog], data[exog]\n\n        # extract r-squared from the fit\n        r_squared = LinearRegression().fit(X, y).score(X, y)\n\n        # calculate VIF\n        vif = 1/(1 - r_squared)\n        vif_dict[exog] = vif\n\n        # calculate tolerance\n        tolerance = 1 - r_squared\n        tolerance_dict[exog] = tolerance\n\n    # return VIF DataFrame\n    df_vif = pd.DataFrame({'VIF': vif_dict, 'Tolerance': tolerance_dict})\n    df_vif = df_vif.sort_values(['VIF'])\n    return df_vif\n\n\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv',encoding='utf-8')\ndf_test = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv', encoding='utf-8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discover(df_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discover(df_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph(df_train, 'failure')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating column names in groups to facilitate aggreagation\n\nattribute =   [f\"attribute_{i:d}\" for i in [0,1,2,3]]\nmeasure_A =   [f\"measurement_{i:d}\" for i in [0,1]]\nmeasure_B =   [f\"measurement_{i:d}\" for i in [3,4,5,6,7,8,9,10,11,12,13,14,15,16]]\nmaterial =    [f\"material_{i:d}\" for i in [5,6,7,8,9]]\nmeasure_all = [f\"measurement_{i:d}\" for i in [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Imputing:\n\n#in the imputing step, this code is incomplete: in fact I have repeated this chunck of code 3-4 times, with\n#different features in the groupby, the first step with 4 features, covered most of the null, the consecutive\n#steps I remove features one by one until reaching zero empty cells.\n\n\ncombine = [df_train, df_test]\nfor dataset in combine:\n    for i in measure_all:\n        dataset[i]     = dataset.groupby(['product_code', 'measurement_0','measurement_1','measurement_2'])[i]        .transform(lambda x: x.fillna(x.mean()))\n    dataset['loading'] = dataset.groupby(['product_code', 'measurement_0','measurement_1','measurement_2'])['loading'].transform(lambda x: x.fillna(x.mean()))\n    \nfor dataset in combine:\n    for i in measure_all:\n        dataset[i]     = dataset.groupby(['product_code', 'measurement_0','measurement_1'])[i]        .transform(lambda x: x.fillna(x.mean()))\n    dataset['loading'] = dataset.groupby(['product_code', 'measurement_0','measurement_1'])['loading'].transform(lambda x: x.fillna(x.mean()))\n\nfor dataset in combine:\n    for i in measure_all:\n        dataset[i]     = dataset.groupby(['product_code'])[i]        .transform(lambda x: x.fillna(x.mean()))\n    dataset['loading'] = dataset.groupby(['product_code'])['loading'].transform(lambda x: x.fillna(x.mean()))\n        \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating material columns (attribute):\n\n#first we need to remove the word material from attribute_0, attribute_1\n\ncombine = [df_train, df_test]\nfor dataset in combine:\n    dataset['attribute_0'] = dataset['attribute_0'].str.split(\"_\", 2, expand = True)[1].astype('int')\n    dataset['attribute_1'] = dataset['attribute_1'].str.split(\"_\", 2, expand = True)[1].astype('int')\n\n#then we create separate column per materials\n\ncombine = [df_train, df_test]\nfor dataset in combine:\n    for i in [5,6,7,8,9]:\n        dataset[material[i-5]]=0\n    for i in [5,6,7,8,9]:\n        for j in range (len(attribute)):\n            dataset.loc[dataset[attribute[j]] == i, material[i-5]] = 1\n            \n#creating new features, averaging measure_A, and measure_B\n\ncombine = [df_train, df_test]\nfor dataset in combine:\n    dataset['Avg_measure_A']=dataset[measure_A].mean(axis = 1)\n    dataset['Avg_measure_B']=dataset[measure_B].mean(axis = 1)\n    \n#creating dummy for product code:\n\ncombine = [df_train, df_test]\ndumFet = ['product_code']\ndummy_matrix   =pd.get_dummies(df_train[dumFet], drop_first=True)\ndf_train       = df_train      .drop(df_train[dumFet], axis = 1)\ndf_train       = pd.concat([df_train ,dummy_matrix], axis=1, ignore_index= False)\n\ndummy_matrix  =pd.get_dummies(df_test[dumFet], drop_first=True)\ndf_test       = df_test      .drop(df_test[dumFet], axis = 1)\ndf_test       = pd.concat([df_test ,dummy_matrix], axis=1, ignore_index= False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Attribute does not seem to correlate with distribution of any 'measurement'\nThough, attribute 1 impacts at measure 14, 15, 16\n\nProduct code is associated with its material content","metadata":{}},{"cell_type":"markdown","source":"all the available variables seems to be of low to no importance\neven in the private training set, the results are limited to 0.58\nit means we do not have any significant variable yet. In the coming codes, I will be hunting for a significant variable","metadata":{}},{"cell_type":"code","source":"#padding the columns of the two datasets with the dummy features that did not exist in either\n\nfor i in df_train.columns.difference(df_test.columns):\n    if i == 'failure':\n        continue\n    else:\n        df_test[i]=0\nfor i in df_test.columns.difference(df_train.columns):\n    df_train[i]=0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ymiscore = df_train['failure']\nxmiscore = df_train.drop(labels = ['failure'],axis = 1)\nmi_scores = make_mi_scores(xmiscore, ymiscore)\n\nxvif = df_train.drop(labels = ['failure'],axis = 1)\nFeature_Strength = get_vif(xvif,df_train).join(mi_scores)\nFeature_Strength.sort_values(['Mutual Information Scores'], ascending = False).style.background_gradient(cmap='coolwarm',axis=0, vmax = 5).set_precision(3)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping overly correlated features:\n\ncombine = [df_train, df_test]\nfor dataset in combine:\n    dataset = dataset.drop(columns = ['material_5', 'material_7', 'material_6']   , axis = 1, inplace = True)\n    \n#dropping the measure and attribute columns:\n#columns = measure_all + attribute\n#for dataset in combine:\n #   dataset = dataset.drop(columns = columns   , axis = 1, inplace = True)\n    \n#dropping the ID column:\ncombine = [df_train, df_test]\nfor dataset in combine:\n    dataset = dataset.drop(columns = ['id']   , axis = 1, inplace = True)\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#function to score the models:\ndef score(X, y, model, cv):\n    scoring = [\"roc_auc\"]\n    scores = cross_validate(\n        model, X, y, scoring=scoring, cv=cv, return_train_score=True,\n    )\n    scores = pd.DataFrame(scores).T\n    return scores.assign(\n        mean = lambda x: x.mean(axis=1),\n        std = lambda x: x.std(axis=1),\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fitting logistic regressor\n#copied from kaggle notebook TPS Aug 22 - VotingClassifier:\n\nfrom sklearn.preprocessing import StandardScaler\n\ncolumns = df_train.columns\nX = df_train.drop('failure', axis = 1)\nY = df_train['failure']\nTest = df_test[X.columns]\n\n\nstd_scaler = StandardScaler()\nX = std_scaler.fit_transform(X)\nTest = std_scaler.transform(Test)\n\nskf = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\nmodel_lr = LogisticRegression(max_iter = 500, C=0.0001, penalty='l2', solver='newton-cg')\nscores = score(X, Y, model_lr, cv=skf)\ndisplay(scores)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating submission:\n\nclf = LogisticRegression(max_iter = 500, C=0.0001, penalty='l2', solver='newton-cg')\nclf = clf.fit(X, Y)\n\n\na = np.array(pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv', encoding='utf-8').id).astype(int)\nb = clf.predict(Test)\nIbrahim_Seyam_Submission = pd.DataFrame([a,b], index =['id','failure']).T\nIbrahim_Seyam_Submission.to_csv(\"submission.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}