{"cells":[{"metadata":{"_uuid":"1f30ff6808444cecce422449067d07b072f2de8d"},"cell_type":"markdown","source":"# Reducing Commercial Aviation Fatalities"},{"metadata":{"_uuid":"e482a05ee3eca70dec7b58e691e75d6a9b3621f0"},"cell_type":"markdown","source":"### EDA for the competition Aviation Fatalities on Kaggle"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"46c743fd88d4ac0d2cb4e764e3ffb2da7a5b59e5"},"cell_type":"code","source":"# Imports for data manipulation and data vizualization\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import MinMaxScaler\nimport warnings\n\nwarnings.filterwarnings('ignore')\nscaler = MinMaxScaler()\nimport os\nprint(os.listdir(\"../input\"))\n\nprint (\"Hello World\")\n\ntrain = pd.read_csv(\"../input/train.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16af2ef59c968f1b6d0e0284f2b91963871a668a"},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"3d60a09e3376c6a3e3771efd1f0328ee40687857"},"cell_type":"code","source":"#Scaling the data\nimport gc \ntrainN = train.loc[:, train.dtypes == np.float64]\ntrainN['seat'] = train.seat\ntrainN['crew'] = train.crew\ntrainN[:] = scaler.fit_transform(trainN[:])\ntrainN['experiment'] = train['experiment'].map({'CA': -1, 'DA': 0,'SS':1})\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"442548eb87ab47b478abab6c19fba59b1c8b4a72"},"cell_type":"markdown","source":"#### Let's see how features are distributed by class"},{"metadata":{"trusted":true,"_uuid":"8977e35edc8bd5a9938fdcf08c287041e89f3226"},"cell_type":"code","source":"trainA = trainN[train.event=='A']\ntrainB = trainN[train.event=='B']\ntrainC = trainN[train.event=='C']\ntrainD = trainN[train.event=='D']\n\nfig = plt.figure(figsize=(65,65))\nfig.subplots_adjust(hspace=0.4, wspace=0.4)\nplt.grid()\n\nfor row,i in zip(trainN,range(0,len(trainN.columns))):\n    \n    plt.subplot(len(trainN.columns)/3, 4, i+1)\n    plt.hist(trainA[row],label='A',alpha=0.4)\n    plt.hist(trainB[row],label='B',alpha=0.4)\n    plt.hist(trainC[row],label='C',alpha=0.4)\n    plt.hist(trainD[row],label='D',alpha=0.4)\n    plt.xlabel(row,size=26)\n    plt.legend(fontsize=26)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"55b7dc70ff0124eeef08aa92248dbb3ac09ad5ce"},"cell_type":"markdown","source":"#### In this plot we can see that some features represents more embracing values,in this way, we also can notice that a lot of represents values with low variance. As this plot was divided by classes, in this group of histograms we can observe that some features could be useless \n\n#### Now lets see how are the correlation of the features "},{"metadata":{"trusted":true,"_uuid":"419ce6ad67bf0149c45d312b036077d9ff46168a"},"cell_type":"code","source":"fig = plt.figure(figsize=(55,25))\n#fig.subplots_adjust(hspace=0.4, wspace=0.4)\nplt.subplot(2, 2, 1)\ncorr = trainA.corr()\n\na = sns.heatmap(corr, \n            xticklabels=corr.columns.values,\n            yticklabels=corr.columns.values)\nplt.tick_params(axis='y', which='major', labelsize=26)\nplt.tick_params(axis='x', labelrotation = 90,which='major', labelsize=26)\n\nplt.subplot(2, 2, 2)\ncorr = trainB.corr()\nb = sns.heatmap(corr, \n            xticklabels=corr.columns.values,\n            yticklabels=corr.columns.values)\nplt.tick_params(axis='y', which='major', labelsize=26)\nplt.tick_params(axis='x', labelrotation = 90,which='major', labelsize=26)\nplt.subplot(2, 2, 3)\ncorr = trainC.corr()\nc = sns.heatmap(corr, \n            xticklabels=corr.columns.values,\n            yticklabels=corr.columns.values)\nplt.tick_params(axis='y', which='major', labelsize=26)\nplt.tick_params(axis='x', labelrotation = 90,which='major', labelsize=26)\nplt.subplot(2, 2, 4)\ncorr = trainD.corr()\nd = sns.heatmap(corr, \n            xticklabels=corr.columns.values,\n            yticklabels=corr.columns.values)\nplt.tick_params(axis='y', which='major', labelsize=26)\nplt.tick_params(axis='x', labelrotation = 90,which='major', labelsize=26)\n \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d196a89ea2a9e45d452a22a21481d66200ec187"},"cell_type":"markdown","source":"#### lets see now how every crew features are distributed "},{"metadata":{"trusted":true,"_uuid":"afa8b4f2d7275612d76d28e9bcbef58fdaebb224"},"cell_type":"code","source":"crews = np.unique(train.crew)\ngrCrews = []\nfor c in crews:\n    grCrews.append(trainN[train.crew==c])\nfig = plt.figure(figsize=(65,65))\nfig.subplots_adjust(hspace=0.4, wspace=0.4)\nplt.grid()\nfor row,i in zip(trainN,range(0,len(trainN.columns))):\n    \n    plt.subplot(len(trainN.columns)/3, 4, i+1)\n    for gr,l in zip(grCrews,np.unique(train.crew)):\n        plt.hist(gr[row].values,label=str(l),alpha=0.4)\n    plt.xlabel(row,size=26)\n    plt.legend(fontsize=26)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0dfbd3817bf66d44e390c57c3a08c0ac0b8b370"},"cell_type":"markdown","source":"#### it seems like every crew have your on characteristics, special in the features more relevant, by variance, showed in the first histogram\n\n#### lets see now the most important features by variance found by PCA technique"},{"metadata":{"trusted":true,"_uuid":"d51ae5924785df50297e0fda2ec4affcfabf1766"},"cell_type":"code","source":"cov_mat = np.cov(trainN.values.T)\neigen_vals, eigen_vecs = np.linalg.eig(cov_mat)\n\ntot = sum(eigen_vals)\nvar_exp = [(i / tot) for i in sorted(eigen_vals, reverse=True)]\ncum_var_exp = np.cumsum(var_exp)\n\nplt.bar(range(1,len(trainN.columns)+1), var_exp, alpha=0.5, align='center',label='individual explained variance')\nplt.step(range(1,len(trainN.columns)+1), cum_var_exp, where='mid',label='cumulative explained variance')\n\nplt.ylabel('Explained variance ratio')\nplt.xlabel('Principal components')\nplt.legend(loc='best')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de4e1955e18dcbbddd4441a55ad9bd187ea5d475"},"cell_type":"markdown","source":"#### only with 5\\4 features with explained more than 90% of the data!!!\n#### lets see how data looks like in 2D dimension"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"2284beaf281ea8e6fd9110e9dc13d00cf55579a2"},"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components=2)\nspace2D = pca.fit_transform(trainN.values)\nlabels = train.event.map({\"A\":0,\"B\":1,\"C\":2,\"D\":3}).values\nplt.scatter(space2D[:,0],space2D[:,1],c=labels)\nplt.grid()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1fc1f0f18bb99251491f2f215edc6e34293d5ef"},"cell_type":"markdown","source":"#### Now lets see if the data in test set seems like that train data , thereby, we can see if  the distribution in train data sounds equal to test data."},{"metadata":{"trusted":true,"_uuid":"ff15914df1ecc40e0283b1f2524829535da20b1f"},"cell_type":"code","source":"\n'''\ntest = pd.read_csv(\"../input/test.csv\")\n\ni = 0\neeg_features  = trainN.columns\nfor eeg in eeg_features:\n    i += 1\n    plt.subplot(len(test.columns)/4+1, 4, i)\n    sns.distplot(train[eeg], label='Test set', hist=False)\n    sns.distplot(test[eeg], label='Train set', hist=False)\n    #plt.xlim((-500, 500))\n    plt.legend()\n    plt.xlabel(eeg, fontsize=12)\n\nplt.show()\n'''\n## No space left in disk","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"783b1567ce66845ff20c39a235c83ad7ff823627"},"cell_type":"markdown","source":"#### Looks like ok! In this way, we can conclude that cluster analysis could be a good solution for this problem"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}