{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir('../input/blendmodels'))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"trmeta=pd.read_csv('../input/PLAsTiCC-2018/training_set_metadata.csv')\ntemeta=pd.read_csv('../input/PLAsTiCC-2018/test_set_metadata.csv')\n\npdf=pd.read_csv('../input/blendmodels/blend_submission.csv')\n\nprint(trmeta.shape)\nprint(temeta.shape)\nprint(pdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4b0c4a7e40a8dbc31259f01667adff611072896"},"cell_type":"code","source":"trmeta=trmeta.fillna(0)\ntemeta=temeta.fillna(0)\n\ntrIgFilt=trmeta.loc[:,'distmod']==0\ntrEgFilt=trmeta.loc[:,'distmod']!=0\nteIgFilt=temeta.loc[:,'distmod']==0\n\nigClasses=trmeta.loc[trIgFilt,'target'].unique()\negClasses=trmeta.loc[trEgFilt,'target'].unique()\n\n\ntrIgFrac = trIgFilt.sum()/trmeta.shape[0]\nteIgFrac = teIgFilt.sum()/temeta.shape[0]\n\n\ntrEgFrac = 1.0 - trIgFrac\nteEgFrac = 1.0 - teIgFrac\n\n\nprint(trIgFrac)\nprint(teIgFrac)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34c53f084ff732d1365ad28a87594effd4176255"},"cell_type":"code","source":"import copy\ndef setZeroProbas(opdf, temeta, igClasses, egClasses):\n    pdf=copy.deepcopy(opdf)\n    pdf=pdf.merge(temeta, on='object_id')\n    rdf=pd.DataFrame()\n    rdf['object_id']=pdf['object_id']\n\n    igFilter=pdf.loc[:,'distmod']==0\n    egFilter=pdf.loc[:,'distmod']!=0\n    \n    for eg in egClasses:\n        pdf.loc[igFilter,'class_' + str(eg)]=0\n        rdf.loc[:,'class_' + str(eg)]=pdf.loc[:,'class_' + str(eg)]\n        \n    for ig in igClasses:\n        pdf.loc[egFilter,'class_' + str(ig)]=0\n        rdf.loc[:,'class_' + str(ig)]=pdf.loc[:,'class_' + str(ig)]\n    \n    rdf['class_99']=pdf['class_99']\n    return rdf\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e299a11feebd283398c4bd9926547151f836b3e"},"cell_type":"code","source":"predictions=setZeroProbas(pdf, temeta, igClasses, egClasses)\npredictions.to_csv('justSetZeroProbas.csv', index=False)\n\n#predictions=rdf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45ae0755481ce047bdf3a65479c421c9925854e4"},"cell_type":"code","source":"predictions.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c81737bcfc880106ff8cc7eac5a80f491063810"},"cell_type":"code","source":"fil={}\ndistmod={}\nsigma={}\n\ntrFracs={}\nteFracs={}\nteMult={}\n\n\nfor i in trmeta.loc[:,'target'].unique():\n    fil[i] = trmeta['target']==i\n    distmod[i] = np.average(trmeta.loc[fil[i],'distmod'])\n    sigma[i] = np.std(trmeta.loc[fil[i],'distmod'])\n    \n    print('class ' + str(i) + ': ' + str(distmod[i]) +' +/- ' + str(sigma[i]))\n    print(fil[i].sum())\n    \nfor ig in igClasses:\n    trFracs[ig]=fil[ig].sum() / trmeta.shape[0]\n    teFracs[ig]=trFracs[ig] * teIgFrac / trIgFrac\n    print('class ' + str(ig) + 'tr : ' + str(trFracs[ig]) +', te : ' + str(teFracs[ig]))\n    #teMult[ig]=teFracs[ig] / np.average(rdf.loc[:,'class_' + str(ig)])\n    #print(teMult[ig])\n    \nfor eg in egClasses:\n    trFracs[eg]=fil[eg].sum() / trmeta.shape[0]\n    teFracs[eg]=trFracs[eg] * teEgFrac / trEgFrac\n    print('class ' + str(eg) + 'tr : ' + str(trFracs[eg]) +', te : ' + str(teFracs[eg]))\n    #teMult[eg]=teFracs[eg] / np.average(rdf.loc[:,'class_' + str(eg)])\n    #print(teMult[eg])\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d4ef109a0a1ab821f870ce8bbd767d2266139e6"},"cell_type":"code","source":"def applyRebalance(ordf, teMult):\n    \n    rdf=copy.deepcopy(ordf)\n    for cindex in rdf.columns:\n        if cindex != 'object_id':\n            theClass = int(cindex[6:])\n            print(theClass)\n            rdf[cindex]*=teMult[theClass]\n            \n    return rdf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c525532ff9eb9836e0eaad7650722d1d3a2e7d7"},"cell_type":"code","source":"#fadf=applyRebalance(rdf, teMult)\n#fadf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d30378a07f747b066c3d0ba2779df1bf47daf7b"},"cell_type":"code","source":"#from Scirpus discussion:\n\ndef GenUnknown(data):\n    return ((((((data[\"mymedian\"]) + (((data[\"mymean\"]) / 2.0)))/2.0)) + (((((1.0) - (((data[\"mymax\"]) * (((data[\"mymax\"]) * (data[\"mymax\"]))))))) / 2.0)))/2.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab1daf92fde0022369ebc1877f6cc66862e85885"},"cell_type":"code","source":"#predictions['class_99'] = 1 - predictions.max(axis=1)\n#predictions['object_id'] = object_ids\n\n#pdf=predictions\nfeats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',\n         'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',\n         'class_92', 'class_95']\n\ny = pd.DataFrame()\ny['mymean'] = predictions[feats].mean(axis=1)\ny['mymedian'] = predictions[feats].median(axis=1)\ny['mymax'] = predictions[feats].max(axis=1)\n\npredictions['class_99'] = GenUnknown(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73c874041fb3691f5d9b0ee2b54a592e1bcd3356"},"cell_type":"code","source":"#meta=pd.read_csv('../input/PLAsTiCC-2018/test_set_metadata.csv')\n#import copy\ndef modUnknown(opdf, meta, ddfMult=0.5, mwMult=0.5, preserveMed=False):\n    pdf=copy.deepcopy(opdf)\n    mdf=pdf.merge(meta,on='object_id')\n    ddfilter=mdf.loc[:,'ddf']==1\n    mwfilter=mdf.loc[:,'hostgal_photoz']==0\n    print(ddfilter.sum())\n    print(mwfilter.sum())\n    \n    mdf.loc[mwfilter,'class_99']=mwMult*mdf.loc[mwfilter,'class_99']\n    mdf.loc[ddfilter,'class_99']=ddfMult*mdf.loc[ddfilter,'class_99']\n    pdf.loc[:,'class_99']=mdf.loc[:,'class_99']\n    \n    return pdf\n\nnpdf=modUnknown(predictions, temeta)\nnpdf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81656bbc7d745e37bcfaf9993b3849353cf88c93"},"cell_type":"code","source":"npdf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6eef553ec6de4a8de726cf84f250a141c3a58222"},"cell_type":"code","source":"npdf.to_csv('probaZeroAndRe99.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"237fabdb6015aa0ecfba0ff5d0d46df09a168ec0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}