{"cells":[{"metadata":{"_uuid":"4e489fa0-66ea-42d5-8725-a31208a15930","_cell_guid":"1d307167-5a6d-45fc-b8a6-6e56c8b20d14","trusted":true},"cell_type":"code","source":"# Data Preprocessing\n\n\nimport numpy as np \nimport pandas as pd \nimport csv\n\n\ndef addrow(full_band_dict, object_id, mjd, wfile, label):\n\tnew_row = [object_id, mjd]\n\tfor band in ['u','g','r','i','z','y']:\n\t\tif band not in full_band_dict:\n\t\t\tnew_row += [None, None]\n\t\telse:\n\t\t\tnew_row += full_band_dict[band]\n\tnew_row.append(label)\n\twfile.writerow(new_row)\n\ndata = pd.read_csv('../input/divided-plasticc-testset/plasticc_test_set_batch2_2.csv') #csv file need to be converted\nmeta = pd.read_csv('../input/plasticcunblindeddatasets/plasticc_test_metadata.csv') # metadata\n\nmeta = meta[(meta['ddf_bool']<1)]\nWDF_obj = meta.groupby('object_id')['object_id'].apply(int)\n# WDF_label = meta.groupby('object_id')['target'].apply(list)\n\nprint(WDF_obj)\n\nfile = open('test_set_converted_batch2_2.csv','w')\n\nwfile = csv.writer(file)\nwfile.writerow(['id','mjd','u','u_err','g','g_err','r','r_err','i','i_err','z','z_err','y','y_err','label'])\nband_dict = {0:'u',1:'g',2:'r',3:'i',4:'z',5:'y'}\nlast_id = None\nlast_mjd = None\nlast_label = None\nfull_band_dict = {}\nn = 0\ncheck = 0\nwhile n < len(data):\n\t\n\tif data['object_id'][n] in WDF_obj:\n\t\tif data['object_id'][n]==last_id:\n\t\t\tdelta_mjd = data['mjd'][n] - last_mjd\n\t\t\tif delta_mjd >=0.5:\n\t\t\t\taddrow(full_band_dict, last_id, last_mjd, wfile, last_label)\n\t\t\t\tfull_band_dict = {}\n\t\t\t\tlast_mjd = data['mjd'][n]\n\t\t\tif band_dict[data['passband'][n]] not in full_band_dict:\n\t\t\t\tfull_band_dict[band_dict[data['passband'][n]]] = [data['flux'][n],data['flux_err'][n]]\n\t\telse:\n\t\t\tif last_id != None and last_mjd != None:\n\t\t\t\taddrow(full_band_dict, last_id, last_mjd, wfile, last_label)\n\t\t\tlast_id = data['object_id'][n]\n\t\t\tlast_mjd = data['mjd'][n]\n\t\t\tlabel = int(meta[meta.object_id == last_id]['target'])\n\t\t\tif label == 88:\n\t\t\t\tlast_label = 1\n\t\t\telse:\n\t\t\t\tlast_label = 0\n\t\t\tprint(last_id)\n\t\t\tcheck +=1\n\t\t\tn -=1\n\tn +=1\n\nprint('check:', check)\nprint('test num:', len(WDF_obj))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# divide origninal files\n\nimport pandas as pd \nimport numpy as np \n\n\ninput_path = '../input/plasticcunblindeddatasets/'\nfile = 'plasticc_test_set_batch2'\nfile_path = input_path+file\n# data = pd.read_csv(file_path+'.csv')\n\n\nid_set = set()\nn = 0\nfor chunk in pd.read_csv(file_path+'.csv', chunksize = 2000000):\n    print(n)\n    chunk_id_list = []\n    groups = chunk.groupby(chunk['object_id'])\n    num_group = groups.ngroups\n    for _id, group in groups:\n        chunk_id_list.append(_id)\n    if chunk_id_list[0] in id_set:\n        add_part = chunk[chunk.object_id == chunk_id_list[0]]\n        add_part.to_csv(file+'_'+str(n-1)+'.csv', mode='a', header=False)\n        chunk = chunk[~(chunk.object_id == chunk_id_list[0])]\n    chunk.to_csv(file+'_'+str(n)+'.csv')\n    id_set.update(chunk_id_list)  \n    n +=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test convert dataframe using  pivot\nimport pandas as pd \nimport numpy as np \n\ndf  = pd.read_csv('../input/PLAsTiCC-2018/test_set_batch2.csv')\ndf[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.pivot_table(index = ['object_id','mjd'], columns = ['passband'],values = ['flux','flux_err']).reset_index()\ndf[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.columns = ['object_id','mjd','u_flux','g_flux','r_flux','i_flux','z_flux','y_flux','u_flux_err','g_flux_err','r_flux_err','i_flux_err','z_flux_err','y_flux_err']\ndf[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use .rename first, then use map() to combine multiIndex header\n# df = df.rename(columns = {0:'u',1:'g',2:'r',3:'i',4:'z',5:'y'})\n# df.columns = list(map('_'.join, df3.columns))\ndf.to_csv('converted_test_batch2.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df1 = pd.pivot(df, columns=\"passband\", values=[\"flux\",\"flux_err\"])\n# df['flux,flux_err'] = df['flux'].map(str)+','+df['flux_err'].map(str)\n# df.drop(labels = ['flux','flux_err'], axis = 1)\n# # df[:10]\n# df1 = df.set_index(['object_id','mjd','detected','passband'])['flux,flux_err'].unstack().rename_axis(columns=None).reset_index()\n\n# df1[:10]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}