{"cells":[{"metadata":{"_uuid":"e1fea1602914cefd922b5b4e845fdcc531c601d2"},"cell_type":"markdown","source":"### This kernel uses output from [here](https://www.kaggle.com/jimpsull/explanation-of-the-current-training-set-ipynb), and multiple versions of [this kernel](https://www.kaggle.com/jimpsull/fast-test-set-reading-merged-with-fast-extractor).  There is also more background on the [SMOTE method here](https://www.kaggle.com/jimpsull/smote-training-set-test-feature-table)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#!pip install modin\nimport numpy as np # linear algebra\n#import modin.pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pandas as pd\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/explanation-of-the-current-training-set-ipynb\"))\nprint(os.listdir(\"../input/all-the-data-to-build-the-testfeaturetable\"))\nprint(os.listdir(\"../input/fastreadandextractl460-500\"))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bc7795fcd26f2a97048dabae44ad0616a2d62c6e"},"cell_type":"markdown","source":"### I just listed directory contents for easy copy and paste"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"trainDf=pd.read_csv(\"../input/explanation-of-the-current-training-set-ipynb/trainingSetToMatchCustomTestSet111518.csv\")\ntrainDf.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d42912c8ce31a2570a0d767fcb51929a79787440"},"cell_type":"markdown","source":"### The shape is off - I'll deal with that in a minute.  I must have written index as a column"},{"metadata":{"trusted":true,"_uuid":"03ba39d9fc89a5f4c5915d58400160ba2f98c259"},"cell_type":"code","source":"testDfBegin=pd.read_csv(\"../input/all-the-data-to-build-the-testfeaturetable/Objects0Through3219999.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64e9e93b71173a78d3ea91dc7e52cc12d6abc03e"},"cell_type":"markdown","source":"### My extractor got a little tripped up on the last dataFrame, so I had to rerun it"},{"metadata":{"trusted":true,"_uuid":"eb99119db9753796310c67e9703a621683834b13"},"cell_type":"code","source":"print(testDfBegin.shape)\ntestDfEnd=pd.read_csv(\"../input/fastreadandextractl460-500/testFeaturesFrom3220000TO3492890.csv\")\nprint(testDfEnd.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e200416a37b71a5fc960b6f3effce64ebadf553b"},"cell_type":"markdown","source":"### Getting rid of index columns"},{"metadata":{"trusted":true,"_uuid":"93d0f72390ab3a50007d4211369fea6b36cf5a3b"},"cell_type":"code","source":"testDfBegin=testDfBegin.drop(['Unnamed: 0', 'Unnamed: 0.1'], axis=1)\nprint(testDfBegin.shape)\ntestDfBegin.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4e99a4b698c6a6edb123f02c41338e2ebf72058"},"cell_type":"markdown","source":"### More of the same"},{"metadata":{"trusted":true,"_uuid":"7eb7573b47fc235d8f1e2647b10ba331cf48783f"},"cell_type":"code","source":"trainDf=trainDf.drop('Unnamed: 0', axis=1)\n#print(testDfEnd.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f97fd6fee0328fc9343ab128d2d731e7433f050"},"cell_type":"code","source":"\ntestDfEnd=testDfEnd.drop(['Unnamed: 0'], axis=1)\nprint(testDfEnd.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0dc843942fb7987dae15cbd6df8a52f99f2c953a"},"cell_type":"markdown","source":"### Merge them for a complete test dataFrame"},{"metadata":{"trusted":true,"_uuid":"58b6e8fe8dc9df524ac586029ce5d5e1923ce961"},"cell_type":"code","source":"testDf=testDfBegin.append(testDfEnd, sort=False)\ntestDf.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffdac64173d6344a6ac02b0027f2f3b177dbf397"},"cell_type":"markdown","source":"### Look to see if there are any nans that need to be dealt with"},{"metadata":{"trusted":true,"_uuid":"de3d146e1dfda40c1d8caf8aa14a3585d91b9edb"},"cell_type":"code","source":"\n#missing values in:\n#distmod\n#hostgal_specz - \n#deltaDetect - NaN would mean nothing was detected treat as zero\ntrainDf['deltaDetect'].fillna(0,inplace=True)\ntestDf['deltaDetect'].fillna(0, inplace=True)\ntestDf.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b092ccc3b10c1e62eb2ca25678bec115c93dce20"},"cell_type":"markdown","source":"## What do the missing distmod look like?"},{"metadata":{"trusted":true,"_uuid":"9b881c6a486dd2c58482536003ff58c95c28ebf6"},"cell_type":"code","source":"nullFilter=testDf.loc[:,'distmod'].isnull()\nmissingDm=testDf.loc[nullFilter]\nmissingDm.head()\nmissingDm.describe()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c716176aebc939373218bdfaf36cff5b187fe765"},"cell_type":"markdown","source":"## Same as the training data.  Missing distmod=intragalactic\n### Get tigdf and tegdf (train intergalactic data frame, train extragalactic data frame)"},{"metadata":{"trusted":true,"_uuid":"d66fdfd24a3e83490ae7747025da5d610917ae04"},"cell_type":"code","source":"#tig is training intergalactic.  For test we'll just use ig\nigdf=missingDm\negdf=testDf[nullFilter==False]\n\nprint(igdf.shape)\nprint(egdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"401aa90e7c21ed025a5c88f7f2c6ce47ff10c7d3"},"cell_type":"markdown","source":"## We're going to want object_id out of the data when we train the model\n- but we'll want it back when we put together our submission file for the 'real / test' set"},{"metadata":{"trusted":true,"_uuid":"2b011b677208b6195fde8b00e56a9959636d3a14"},"cell_type":"code","source":"eobjdf=pd.DataFrame()\neobjdf.loc[:,'object_id']=egdf.loc[:,'object_id']\neobjdf.head()\n\niobjdf=pd.DataFrame()\niobjdf.loc[:,'object_id']=igdf.loc[:,'object_id']\niobjdf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3cf12d8e6eab384db356d7a4d92014825097d9ba"},"cell_type":"markdown","source":"# We'll check again for NaNs\n- we'll be eliminating hostgal_specz since its missing for most of the test data"},{"metadata":{"trusted":true,"_uuid":"fbc4de325e649ebba17a1a08ea8aaaee2f3ce7e2"},"cell_type":"code","source":"igdf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6aa8b172bb36e4e97f7ed1e850ec819d74f0fc86"},"cell_type":"code","source":"egdf.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"286d1259121d02956b39049a282d83027f247bf3"},"cell_type":"markdown","source":"### [Adapted from this kernel](https://www.kaggle.com/qianchao/smote-with-imbalance-data)**\n"},{"metadata":{"trusted":true,"_uuid":"2bbc66ba68387f7a57dab8ced2d0cdf0b3829c3b"},"cell_type":"code","source":"igdf.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5a8f74f3a37b205abea2e49af6bbad99463cfa4"},"cell_type":"markdown","source":"### As I was adding markup I noticed a couple of oversights\n- the transitory TF were supposed to be gone as they were rolled into outlierScore\n- the object_id shouldn't be in the model\n- I meant to replace the pbSectionValues with some aggregates on them"},{"metadata":{"trusted":true,"_uuid":"2f27dbc9ee63cb79855667a7eb0ce2bb78035099"},"cell_type":"code","source":"#!pip install -U imbalanced-learn\n#traindf=pd.read_csv('../input/trainingSetToMatchCustomTestSet111518.csv')\ntraindf=trainDf\nprint(traindf.shape)\nprint(traindf.head())\n\n#hostgal_specz isn't in most rows of the test data\ndef dropUselessFeatures(df):\n    print(df.shape)\n    df=df.drop(['hephs','hepos','hepts','lephs','lepos','lepts','hostgal_specz',\n                  'hmphs','hmpos','hmpts','lmphs','lmpos','lmpts',\n                  'hlphs','hlpos','hlpts','llphs','llpos','llpts',\n                'highEnergy_transitory_1.0_TF', \n                'highEnergy_transitory_1.5_TF',\n                'lowEnergy_transitory_1.0_TF', \n                'lowEnergy_transitory_1.5_TF', \n               'object_id'], axis=1)\n    \n    #df.loc[:,'hMin']=np.min([df.loc[:,'heavg'], df.loc[:,'hmavg'],df.loc[:,'hlavg']])\n    #df.loc[:,'hMax']=np.max([df.loc[:,'heavg'], df.loc[:,'hmavg'],df.loc[:,'hlavg']])\n    #df.loc[:,'hVar']=np.average([df.loc[:,'hestd'], df.loc[:,'hmstd'],df.loc[:,'hlstd']])\n    #df.loc[:,'hSpread']=(df.loc[:,'hMax']-df.loc[:,'hMin']) / df.loc[:,'hVar']\n    \n    #df.loc[:,'lMin']=np.min([df.loc[:,'leavg'], df.loc[:,'lmavg'],df.loc[:,'llavg']])\n    #df.loc[:,'lMax']=np.max([df.loc[:,'leavg'], df.loc[:,'lmavg'],df.loc[:,'llavg']])\n    #df.loc[:,'lVar']=np.average([df.loc[:,'lestd'], df.loc[:,'lmstd'],df.loc[:,'llstd']])\n    #df.loc[:,'lSpread']=(df.loc[:,'lMax']-df.loc[:,'lMin']) / df.loc[:,'lVar']\n    \n    #df=df.drop(['hMin','hMax','hVar','lMin','lMax','lVar'],axis=1)\n    \n    print(df.shape)\n    return df\n\ntraindf=dropUselessFeatures(traindf)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"584e8b0ed82c80e3dca89756a5e8cfd7ea9c8b49"},"cell_type":"code","source":"traindf.head()\ntraindf.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d5c508bac7acf1c0b34bef92924af3399dc506b"},"cell_type":"code","source":"traindf.loc[:,'target']=traindf.loc[:,'target'].astype(str)\n\n#from stacy's code\n# move target to end\ntraindf = traindf[[c for c in traindf if c not in ['target']] + ['target']]\ntraindf.head()\n\n#df[1].fillna(0, inplace=True)\ntraindf['deltaDetect'].fillna(0,inplace=True)\ntigdf=traindf[traindf['hostgal_photoz']==0]\ntegdf=traindf[traindf['hostgal_photoz']!=0]\n\nprint(tigdf.shape)\nprint(tegdf.shape)\n\n\ntigdf=tigdf.drop(['hostgal_photoz', 'distmod', 'hostgal_photoz_err'], axis=1)\nprint(tigdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0c318063d52fc33fe0c241c354a3693147e1c59"},"cell_type":"markdown","source":"## Let's pare down the test dataFrames as well"},{"metadata":{"trusted":true,"_uuid":"49660f0ba6072bece0d0a7e760fbf32f8cc64c2b"},"cell_type":"code","source":"igdf=dropUselessFeatures(igdf)\nprint(igdf.shape)\negdf=dropUselessFeatures(egdf)\nprint(egdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96f75501f5cac3a24dfa0c811c21b60d56caf09a"},"cell_type":"code","source":"igdf=igdf.drop(['hostgal_photoz', 'distmod', 'hostgal_photoz_err'], axis=1)\nprint(igdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0814b351a2e0544fa64d80ef02ebc9327d8866b1"},"cell_type":"markdown","source":"## How badly is the data imbalanced?  Intergalactic:"},{"metadata":{"trusted":true,"_uuid":"1308f60d589cbe0ce8d24ad22fe5097df9304807"},"cell_type":"code","source":"print('inter-galactic')\nfor theClass in tigdf.loc[:,'target'].unique():\n    print('class ' + str(theClass) + ':')\n    trueFilter=tigdf['target']==theClass\n    print(trueFilter.sum())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b07a9bf2473527cf902830c3353e4ca1c7723118"},"cell_type":"markdown","source":"## Extra-galactic"},{"metadata":{"trusted":true,"_uuid":"4ad0852c4e3c77f8cbc9941bdefff9d7b22a63d2"},"cell_type":"code","source":"print('extra-galactic')\nfor theClass in tegdf.loc[:,'target'].unique():\n    print('class ' + str(theClass) + ':')\n    trueFilter=tegdf['target']==theClass\n    print(trueFilter.sum())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66015635e76e13df9bf56225a26ea458d5eb9c4b"},"cell_type":"markdown","source":"### I'm using [qianchao's](https://www.kaggle.com/qianchao) kernel as a starting point"},{"metadata":{"trusted":true,"_uuid":"ff72c0115430e3e9ebf2e2a3e9db870995466b50"},"cell_type":"code","source":"#https://www.kaggle.com/qianchao/smote-with-imbalance-data\n#from sklearn.preprocessing import StandardScaler\nXig = np.array(tigdf.iloc[:, tigdf.columns != 'target'])\nyig = np.array(tigdf.iloc[:, tigdf.columns == 'target'])\nprint('Shape of X: {}'.format(Xig.shape))\nprint('Shape of y: {}'.format(yig.shape))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c86b62dd613ed820826f55066736755c162beb9"},"cell_type":"markdown","source":"## Repeat for Extra galactic"},{"metadata":{"trusted":true,"_uuid":"f1b29ce01ce18ec294b0899dee980264ede8abd0"},"cell_type":"code","source":"\nXeg = np.array(tegdf.iloc[:, tegdf.columns != 'target'])\nyeg = np.array(tegdf.iloc[:, tegdf.columns == 'target'])\nprint('Shape of X: {}'.format(Xeg.shape))\nprint('Shape of y: {}'.format(yeg.shape))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e246c4a5f2c07d2a3a00b64524b9db695daa48a"},"cell_type":"markdown","source":"## The first time I did it manually\n- I copied [qianchao's](https://www.kaggle.com/qianchao) [kernel](https://www.kaggle.com/qianchao/smote-with-imbalance-data) and then modified the classes"},{"metadata":{"trusted":true,"_uuid":"e392cc492af160cbfffd78e6f1af5046467bd2c7"},"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\n\nfrom sklearn.model_selection import train_test_split\n\nXig_train, Xig_test, yig_train, yig_test = train_test_split(Xig, yig, test_size=0.3, random_state=0)\n\nprint(\"Number transactions X_train dataset: \", Xig_train.shape)\nprint(\"Number transactions y_train dataset: \", yig_train.shape)\nprint(\"Number transactions X_test dataset: \", Xig_test.shape)\nprint(\"Number transactions y_test dataset: \", yig_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ace324096572d2e169bf4102e1c20e47b2ec7eae"},"cell_type":"markdown","source":"## Make sure you have no NaNs at this point"},{"metadata":{"trusted":true,"_uuid":"6a8b979a748884199d750bd181fa7ca2901dfcf2"},"cell_type":"code","source":"print(\"Before OverSampling, counts of label '92': {}\".format(sum(yig_train=='92')))\nprint(\"Before OverSampling, counts of label '65': {} \\n\".format(sum(yig_train=='65')))\nprint(\"Before OverSampling, counts of label '16': {}\".format(sum(yig_train=='16')))\nprint(\"Before OverSampling, counts of label '6': {} \\n\".format(sum(yig_train=='6')))\nprint(\"Before OverSampling, counts of label '53': {}\".format(sum(yig_train=='53')))\n\nsm = SMOTE(random_state=2)\nXig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())\n\nprint('After OverSampling, the shape of train_X: {}'.format(Xig_train_res.shape))\nprint('After OverSampling, the shape of train_y: {} \\n'.format(yig_train_res.shape))\n\nprint(\"After OverSampling, counts of label '92': {}\".format(sum(yig_train_res=='92')))\nprint(\"After OverSampling, counts of label '65': {}\".format(sum(yig_train_res=='65')))\nprint(\"After OverSampling, counts of label '16': {}\".format(sum(yig_train_res=='16')))\nprint(\"After OverSampling, counts of label '6': {}\".format(sum(yig_train_res=='6')))\nprint(\"After OverSampling, counts of label '53': {}\".format(sum(yig_train_res=='53')))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"691805e5638af07a066fb7abcfeb352c106e2a57"},"cell_type":"markdown","source":"## We really need to make this a method\n- We're going to have to do it over and over again because SMOTE should be done AFTER cross validation splitting\n- and there's no guarantee every class will be in every cross validated sample"},{"metadata":{"trusted":true,"_uuid":"db7ad8acb60c6a59e4528fabdfcf06ea79e1f7a2"},"cell_type":"code","source":"def smoteAdataset(Xig, yig, test_size=0.2, random_state=0):\n    \n    Xig_train, Xig_test, yig_train, yig_test = train_test_split(Xig, yig, test_size=test_size, random_state=random_state)\n    print(\"Number transactions X_train dataset: \", Xig_train.shape)\n    print(\"Number transactions y_train dataset: \", yig_train.shape)\n    print(\"Number transactions X_test dataset: \", Xig_test.shape)\n    print(\"Number transactions y_test dataset: \", yig_test.shape)\n\n    classes=[]\n    for i in np.unique(yig):\n        classes.append(i)\n        print(\"Before OverSampling, counts of label \" + str(i) + \": {}\".format(sum(yig_train==i)))\n        \n    sm=SMOTE(random_state=2)\n    Xig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())\n\n    print('After OverSampling, the shape of train_X: {}'.format(Xig_train_res.shape))\n    print('After OverSampling, the shape of train_y: {} \\n'.format(yig_train_res.shape))\n    \n    for eachClass in classes:\n        print(\"After OverSampling, counts of label \" + str(eachClass) + \": {}\".format(sum(yig_train_res==eachClass)))\n        \n    return Xig_train_res, yig_train_res, Xig_test, yig_test\n\nXeg_train_res, yeg_train_res, Xeg_test, yeg_test=smoteAdataset(Xeg, yeg)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7782a6de883f2de4ecadf2b6fda38a13b59b8e6"},"cell_type":"markdown","source":"## We're ready to start training models\n- We have oversampling training sets for both inter-galactic and extra-galactic objects"},{"metadata":{"_uuid":"4643f8deac8601b50e5e9fac32cd4427c7b17eb3"},"cell_type":"markdown","source":"# Let's recount what we have\n- igdf and egdf are the 'real' dataFrames (we won't call them test anymore) for inter and extra galactic\n- Xeg_train_res and yeg_train_res are the SMOTE training sets for extragalactic\n- Xeg_test and yeg_test are the test split of the training data for extragalactic\n- Xig_train_res and yig_train_res are the SMOTE training sets for intragalactic\n- Xig_test and yig_test are the test split of the training data for intragalactic\n"},{"metadata":{"trusted":true,"_uuid":"caadbe93de42691fded2f7f2d13dd8855906ad12"},"cell_type":"code","source":"#from HW5\nfrom sklearn.ensemble import GradientBoostingClassifier\n\ndef getGbm(X_encoded_train, Y1, X_encoded_test, Y2,\n           nTrees=100, max_depth=5, min_node_size=5, verbose=0, learning_rate=0.05):\n\n    gbm_clf = GradientBoostingClassifier(n_estimators=nTrees, loss='deviance', learning_rate=learning_rate, max_depth=max_depth, \\\n                                        min_samples_leaf=min_node_size)\n    gbm_clf.fit(X_encoded_train, Y1)\n    \n    \n    Y_test_hat = gbm_clf.predict_proba(X_encoded_test)\n    #Accuracy = [1 for i in range(len(Y_test_hat)) if Y2.iloc[i] == Y_test_hat[i]]\n    #Accuracy = round(float(np.sum(Accuracy))/len(Y_test_hat)*100,2)\n    #rocAuc=roc_auc_score(Y2Vals, Y_test_hat)\n    \n    #Y1Vals=np.array(Y1)\n    \n    #Y_train_hat = clf.predict(X_encoded_train)\n    #trainAcc = [1 for i in range(len(Y_train_hat)) if Y1.iloc[i] == Y_train_hat[i]]\n    #trainAcc = round(float(np.sum(trainAcc))/len(Y_train_hat)*100,2)\n    #trainAuc=roc_auc_score(Y1Vals, Y_train_hat)\n    \n    \n    return gbm_clf, Y_test_hat\ntiggbm_clf, tigY_test_hat=getGbm(Xig_train_res, yig_train_res, Xig_test, yig_test,\n                                 nTrees=100, max_depth=5, min_node_size=5, verbose=0, learning_rate=0.05)\n\nprint(tiggbm_clf.feature_importances_)\n#print(\"Accuracy on Testing Data = %.2f%%\"%Accuracy)\n#print(\"AUC for ROC curve on Testing Data = %.2f\"%rocAuc)\n#print(\"Accuracy on Training Data = %.2f%%\"%trainAcc)\n#print(\"AUC for ROC curve on Training Data = %.2f\"%trainAuc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"570036c61c8d6d11421fcf3dc28aff8a3e3dccda"},"cell_type":"markdown","source":"### The method returned the actual model (which we'll use on the 'real / test' data\n- it also returned predicted probabilities from which we could compute the loss function"},{"metadata":{"trusted":true,"_uuid":"8de9c0ad251f8c0b75d7bbf159a26a43b64f2538"},"cell_type":"code","source":"print(tigY_test_hat)\nprint(tiggbm_clf.classes_)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8010a1aeacef60e46c4eeed29f93545c5b9c4184"},"cell_type":"markdown","source":"### Repeat for the extra-galactic data and model"},{"metadata":{"trusted":true,"_uuid":"ce7d9ea2c4e7a5a336779cb9cdf6480644060933"},"cell_type":"code","source":"teggbm_clf, tegY_test_hat=getGbm(Xeg_train_res, yeg_train_res, Xeg_test, yeg_test,\n                                 nTrees=100, max_depth=5, min_node_size=5, verbose=0, learning_rate=0.05)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3dcd790aa6a1e055c799b012917b6de75829e7a"},"cell_type":"code","source":"print(teggbm_clf.feature_importances_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"431e9920a6e2f1bea21073c99fc41bbc3d06ab7e"},"cell_type":"code","source":"print(tegY_test_hat)\nprint(teggbm_clf.classes_)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ca31a22de3114b3457e3abfe8a5d3274feb802a"},"cell_type":"markdown","source":"### Get the predictions based on the feature values for intra and extra galactic"},{"metadata":{"trusted":true,"_uuid":"e7bf230b8ec950ebab14a2812351b8e2d6c83d2a"},"cell_type":"code","source":"actualIgX=igdf.values\nactualIgPredictions = tiggbm_clf.predict_proba(actualIgX)\n#igPredictions = tiggbm_clf.predict_proba(X_encoded_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d1f96b5fad39537a080bffe4ac6a5d4ca3be21d"},"cell_type":"code","source":"actualEgX=egdf.values\nactualEgPredictions = teggbm_clf.predict_proba(actualEgX)\n#igPredictions = tiggbm_clf.predict_proba(X_encoded_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da815541083ec2604bc37d5d47705c2ff4db5d5b"},"cell_type":"code","source":"actualIgPredictions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dac1bc7d05f037a9251e08d563a74fd20281cd88"},"cell_type":"code","source":"actualEgPredictions.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2379eb7ef7e5638cbe9257036ca6a3a97ca59402"},"cell_type":"markdown","source":"### We miss the object_id - I wanted it out of the model but I need it for the submission file\n- we'll need to index them to merge them with the results arrays"},{"metadata":{"trusted":true,"_uuid":"88f224b7c549954f68c3383eac558a3e6e3aadbe"},"cell_type":"code","source":"iobjdf.index=range(iobjdf.shape[0])\niobjdf.head()\n\neobjdf.index=range(eobjdf.shape[0])\neobjdf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d21c80a7d3bfd4ae4068849878438f0d8b94be7"},"cell_type":"markdown","source":"### We have to get everything into the submission file format"},{"metadata":{"trusted":true,"_uuid":"7d893908c833c0c639b944f0385c56e4a4928052"},"cell_type":"code","source":"igResultDf=pd.DataFrame(data=actualIgPredictions, columns=tiggbm_clf.classes_)\n#igResultDf.loc[:,'object_id']=igdf.loc[:,'object_id']\nigResultDf.loc[:,'object_id']=iobjdf.loc[:,'object_id']\nigResultDf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3866e81f919f063060e50690621a0f404c474c4"},"cell_type":"code","source":"egResultDf=pd.DataFrame(data=actualEgPredictions, columns=teggbm_clf.classes_)\n#igResultDf.loc[:,'object_id']=igdf.loc[:,'object_id']\negResultDf.loc[:,'object_id']=eobjdf.loc[:,'object_id']\negResultDf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c26f82f9c0c8484cb08511126acd02d3e99c063f"},"cell_type":"markdown","source":"## Merge them into one results set"},{"metadata":{"trusted":true,"_uuid":"206dafa05bee46066f8a08ac6f6e102c3a8ea568"},"cell_type":"code","source":"fulldf=igResultDf.append(egResultDf, sort=False)\nprint(fulldf.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"046c1ee825b691394b9a02ca123d5d4ab9f9c293"},"cell_type":"markdown","source":"### To use the max_value I had to get object_id out\n- in hindsight I should've waited to put it back in"},{"metadata":{"trusted":true,"_uuid":"ccb5165a2e7841581a7db327ef4f9cbf839821bc"},"cell_type":"code","source":"import copy\nnobjdf=copy.deepcopy(fulldf)\nnobjdf=nobjdf.drop('object_id', axis=1)\nnobjdf['max_value'] = nobjdf.max(axis=1)\nprint(nobjdf.head())\n#I chose 7000 arbitrarily because its about 1/5 of 1%\n\n#arbGuess99=7000\n#arb99Proba=1.00\n#mysteryThresh=np.max(nobjdf.nsmallest(arbGuess99, 'max_value').loc[:,'max_value'])\n#print(mysteryThresh)\n#nobjdf['99']=0\n#nobjdf.loc[nobjdf['max_value']<]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e47d624b916653afd7911795acc4662a92236521"},"cell_type":"markdown","source":"## Would've been more efficient to do it other ways but I'm just commenting\n- and trying to change what I did after the fact"},{"metadata":{"_uuid":"6e88cbcf1463cb806f298f8b77aba4ce947a1ad3","trusted":true},"cell_type":"code","source":"averages=[]\nfor columns in nobjdf.columns:\n    nobjdf[columns].fillna(0, inplace=True)\n    #print(columns)\n    averages.append(np.average(nobjdf.loc[:,columns]))\n    #print(np.average(nobjdf.loc[:,columns]))\n    \nnobjdf.loc[:,'g99']=1-nobjdf.loc[:,'max_value']**2\n#print(np.min(averages))\n#print(nobjdf.head())\nnormalizingConstant = np.average(nobjdf.loc[:,'g99'])/np.min(averages)\nprint(normalizingConstant)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc7cb27e2be838e8e783eca541e3594e0dbf2830"},"cell_type":"markdown","source":"## This is arbitrary.  I'm saying class 99 is about as common as the most rare of the known classes"},{"metadata":{"trusted":true,"_uuid":"e8f53332ba53df5617e399fbeade7b4bf4ac341e"},"cell_type":"code","source":"fulldf.loc[:,'99']=nobjdf.loc[:,'g99']/normalizingConstant\nfulldf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc2b938ccdedc2d02322332b92ac1c16826ed9c5"},"cell_type":"code","source":"for cindex in fulldf.columns:\n    fulldf[cindex].fillna(0, inplace=True)\n    \nfulldf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"802ba7ba4741f5f78c010907edd1c0aae9ec4c8f"},"cell_type":"code","source":"def submitOrder(fulldf):\n    \n    deleteCols=[]\n    for cindex in fulldf.columns:\n        fulldf = fulldf.rename(columns={cindex: 'was'+str(cindex)})\n        deleteCols.append('was'+str(cindex))\n    \n    newNames=[6,15, 16, 42, 52,53,62,\n             64,65,67,88,90,92,95,99]\n    \n    #string column names seemed to cause a problem\n    fulldf.loc[:,'object_id']=fulldf.loc[:,'wasobject_id']\n    for name in newNames:\n        fulldf.loc[:,'class_'+str(name)]=fulldf.loc[:,'was'+str(name)]\n    \n    fulldf=fulldf.drop(deleteCols, axis=1)\n    return fulldf\n\n\nsubmitdf=submitOrder(fulldf)\n#for cindex in submitdf.columns:\n#    if cindex != 'object_id':\n#        submitdf.loc[:,cindex]=submitdf.loc[:,cindex].astype(float)\nprint(submitdf.shape)\nprint(submitdf.columns)\nsubmitdf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d887de2fee7ea115f2a6bd55345651cbe43b62b"},"cell_type":"markdown","source":"### We can submit to the competition directly from the output of this kernel"},{"metadata":{"trusted":true,"_uuid":"fa0b9c6f5099ab6934c4eeb7d6e57774fea9dd07"},"cell_type":"code","source":"submitdf.to_csv('fastFeatsOnly111718.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54b909c9dea857213a717b5e7c450aa44d03aca7"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}