{"cells":[{"metadata":{"_uuid":"7bb2cb906bf41eeec8927068220a62e12f0b7d75"},"cell_type":"markdown","source":"**EXPLORATION ET TRAITEMENT DES DONNEES:**\n1. objets galactiques et extragalactiques.(chque classe est soit galactique(redshift=0) soit extragalactique)\n1.  affichage en histagram\n1. valeur nan dans les attributs (distmod, et hostgal_spectz pour les donnees de test)\n1.  les attributs correles sont: \n                                          * hostgal_photoz et distmod.\n                                          * flux et distmod.\n                                          * hostgal_photoz et hostgal_spectz.\n                                          * passband et flux\n                                          * ra et gal_l\n                                          * dec et gal_b\n                                              \n                                              \n\n\n"},{"metadata":{"_uuid":"410bd90db0dfa35c046d26f6c97dd7c08b27cd8b"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"7d93b0f69b90cfe528f974b5f5009de8b7c49719"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom scipy import optimize","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07771416a764cb921cf9fca1ae5d38053714a6f3"},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"training_set = pd.read_csv('../input/training_set.csv')\nmeta_data = pd.read_csv('../input/training_set_metadata.csv')\ntest_meta_data = pd.read_csv('../input/test_set_metadata.csv')\ntraining_set.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0370a63f801e40b6abc0b81e24bcea00ee431a7a"},"cell_type":"code","source":"targets = np.hstack([np.unique(meta_data['target']), [99]])\ntarget_map = {j:i for i, j in enumerate(targets)}\ntarget_ids = [target_map[i] for i in meta_data['target']]\nmeta_data['target_id'] = target_ids\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a112b08530ad43013c08783198981461398d0e2b"},"cell_type":"code","source":"galactic_cut = meta_data['hostgal_specz'] == 0\nplt.figure(figsize=(10, 8))\nplt.hist(meta_data[galactic_cut]['target_id'], 15, (0, 15), label='Galactique')\nplt.hist(meta_data[~galactic_cut]['target_id'], 15, (0, 15), label='Extragalactique')\nplt.xticks(np.arange(15)+0.5,   targets)\nplt.gca().set_yscale(\"log\")\nplt.xlabel('Classe')\nplt.ylabel('nombre')\nplt.xlim(0, 15)\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3714915a09d1aab61d0bc1a15a7ecd654a09886e"},"cell_type":"markdown","source":"attribuant a chque classe son nom tel que definie par les astronautes ."},{"metadata":{"trusted":true,"_uuid":"eff5e8fe2122841c00fa001df70d305d4d4d2bec"},"cell_type":"code","source":"\ntarget_types={6:'Microlensing', 15:'Explosive Type V', 16:'Transits', 42:'Explosive type W', 52:'Explosive Type X', \n                  53:'Long periodic', 62:'Explosive Type Y', 64:'Near Burst', 65:'Flare', 67:'Explosive Type Z',\n                  88:'AGN', 90:'SN Type U', 92:'Periodic', 95:'SN Type T'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e289853a5b570824331a49dd7ec0070a3ff6e8a8"},"cell_type":"code","source":"object_list=times.groupby('object_id').apply(lambda x: x['object_id'].unique()[0]).tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30e8ad50508d3c91675ebf7997123c4594081b4c"},"cell_type":"code","source":"colors = ['purple', 'blue', 'green', 'orange', 'red', 'black']\n\ndef plot_one_object(obj_id):\n        \n    for band in range(len(colors)):\n        sample = train_series[(train_series['object_id'] == obj_id) & (train_series['passband']==band)]\n        plt.errorbar(x=sample['mjd'],y=sample['flux'],yerr=sample['flux_err'],c = colors[band],fmt='o',alpha=0.7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d72e37b1f7111c11647072dac3f2de59f94dfcd"},"cell_type":"code","source":"#le nombre d'objet dans chaque classe.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c272d35946e16b80bfe5078c5b21c3c8cde6f3c4"},"cell_type":"code","source":"for t in sorted(meta_data['target'].unique()):\n    print (t,meta_data[meta_data['target']== t]['target'].count(),target_types[t],meta_data[meta_data['target']== t]['hostgal_specz'].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6af7566de6ae0862ba27b6da9c7573508f24d347"},"cell_type":"code","source":"print(meta_data[meta_data.isnull().any(axis=1)][null_columns].head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31cd4ab6c85fd912bb067fc0887d1893c51f23be"},"cell_type":"code","source":"null_columns=test_meta_data.columns[test_meta_data.isnull().any()]\ntest_meta_data[null_columns].isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57f1024ebb7bc8244ea85144473e1dc265553f19"},"cell_type":"code","source":"\ngroups = training_set.groupby(['object_id', 'passband'])\ngroups","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46997882f24f1603026f8e15e1ab5c268bdfdbe9"},"cell_type":"code","source":"times = groups.apply(\n    lambda block: block['mjd'].values).reset_index().rename(columns={0: 'seq'})\nflux = groups.apply(\n    lambda block: block['flux'].values\n).reset_index().rename(columns={0: 'seq'})\nerr = groups.apply(\n    lambda block: block['flux_err'].values\n).reset_index().rename(columns={0: 'seq'})\ndet = groups.apply(\n    lambda block: block['detected'].astype(bool).values\n).reset_index().rename(columns={0: 'seq'})\ntimes_list = times.groupby('object_id').apply(lambda x: x['seq'].tolist()).tolist()\nflux_list = flux.groupby('object_id').apply(lambda x: x['seq'].tolist()).tolist()\nerr_list = err.groupby('object_id').apply(lambda x: x['seq'].tolist()).tolist()\ndet_list = det.groupby('object_id').apply(lambda x: x['seq'].tolist()).tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c12d766aa3d6aed5076bef0b0aae40c78743c397"},"cell_type":"code","source":"flux","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00007b598f7d70898a07ebd4e8fe924117224948"},"cell_type":"code","source":"object_list=times.groupby('object_id').apply(lambda x: x['object_id'].unique()[0]).tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44151c669605fc8b8daacc27f1b4637676bf4451"},"cell_type":"code","source":"colors = ['purple', 'blue', 'green', 'orange', 'red', 'black']\n\ndef plot_one_object(obj_id):\n        \n    for band in range(len(colors)):\n        sample = training_set[(training_set['object_id'] == obj_id) & (training_set['passband']==band)]\n        plt.errorbar(x=sample['mjd'],y=sample['flux'],yerr=sample['flux_err'],c = colors[band],fmt='o',alpha=0.7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c0f72b1ea56d31076c3fce9bd283f46240ab6a9"},"cell_type":"code","source":"for t in sorted(meta_data['target'].unique()):\n    print (t,meta_data[meta_data['target']== t]['target'].count(),target_types[t],meta_data[meta_data['target']== t]['hostgal_specz'].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7736efa5be431c765785f4f5c176988fb92f581f"},"cell_type":"code","source":"def fit_kernel_length_only(times_band,flux_band,err_band):\n    \n    def _kernel_likelihood(length):\n        sigma=siguess\n        #length=params\n        kernel=np.exp(-(np.reshape(times_band,(-1,1)) - times_band)**2/2/length**2)\n        np.fill_diagonal(kernel,0)\n        sumw=kernel.dot(1./err_band**2) + 1./sigma**2\n        pred=kernel.dot(flux_band/err_band**2) / sumw\n        chi2 = (pred - flux_band)**2 / ( err_band**2 + 1./sumw )\n        # -2 ln likelihood\n        logl=np.sum(chi2 + np.log(err_band**2 + 1./sumw))\n        return logl\n    \n    lguess=(np.max(times_band)-np.min(times_band))/len(times_band)\n    siguess=np.std(flux_band)\n    output=optimize.fmin(_kernel_likelihood,lguess,disp=False,xtol=0.01,full_output=1)\n    return (siguess,output[0][0]), output[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc978c8ec41e7710b714372d9dfeb08152e19ea7"},"cell_type":"code","source":"def kernel_predict(params,times_band,flux_band,err_band):\n    sigma=params[0]\n    length=params[1]\n    kernel=np.exp(-(np.reshape(time_grid,(-1,1)) - times_band)**2/2/length**2)\n    sumw=kernel.dot(1./err_band**2) + 1./sigma**2\n    pred=kernel.dot(flux_band/err_band**2) / sumw\n    return pred, np.sqrt(1./sumw)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8f3c231a0cee9b67f1d44e89fe5c0ffc40bf77a"},"cell_type":"code","source":"def make_kernel(tlist,flist,elist,fit_kernel_function=fit_kernel_length_only):\n    flux_grid = []\n    err_grid = []\n    kernel_sigma = []\n    kernel_length = []\n    kernel_logl=[]\n    for iobj,(times_obj,flux_obj,err_obj) in enumerate(zip(tlist,flist,elist)):\n        flux_grid_obj=[]\n        err_grid_obj=[]\n        kernel_sigma_obj = []\n        kernel_length_obj = []\n        kernel_logl_obj=[]\n        if iobj in meta_data[meta_data['hostgal_photoz']!=0.0].index:\n            for times_band,flux_band,err_band in zip(times_obj,flux_obj,err_obj):\n                (sigma,length),logl = fit_kernel_function(times_band,flux_band,err_band)\n                k_flux,k_err=kernel_predict((sigma,length),times_band,flux_band,err_band)\n                flux_grid_obj.append(k_flux)\n                err_grid_obj.append(k_err)\n                kernel_sigma_obj.append(sigma)\n                kernel_length_obj.append(length)\n                kernel_logl_obj.append(logl)\n        else:\n            kernel_sigma_obj=[0]*6\n            kernel_length_obj=[0]*6\n            kernel_logl_obj=[0]*6\n        flux_grid.append(flux_grid_obj)\n        err_grid.append(err_grid_obj)\n        kernel_sigma.append(kernel_sigma_obj)\n        kernel_length.append(kernel_length_obj)\n        kernel_logl.append(kernel_logl_obj)\n    return flux_grid,err_grid, kernel_sigma, kernel_length,kernel_logl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22ee1cdeff34e02edade57c5ae202d7e563c91aa"},"cell_type":"code","source":"iobj=1\nband=3\ntime_grid=(np.arange(59550,60705,5.))\n(sigma,length),logl = fit_kernel_length_only(times_list[iobj][band],flux_list[iobj][band],err_list[iobj][band])\n#length=4.0\nk_flux,k_err=kernel_predict((sigma,length),times_list[iobj][band],flux_list[iobj][band],err_list[iobj][band])\nplt.errorbar(times_list[iobj][band],flux_list[iobj][band],yerr=err_list[iobj][band],color=colors[band],fmt='o')\nplt.plot(time_grid,k_flux)\nplt.fill_between(time_grid,k_flux-k_err,k_flux+k_err,alpha=0.3)\nplt.ylim(np.min(flux_list[iobj][band]*1.5,0),np.max(flux_list[iobj][band]*1.5,0))\n#plt.xlim(60100,60300)\nprint (sigma,length,logl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"466da5d3f3aaab673530908f4abd5ad60faf5709"},"cell_type":"code","source":"klonly_flux_grid,klonly_err_grid,klonly_sigma,klonly_length,klonly_logl = make_kernel(\n    times_list,flux_list,err_list,fit_kernel_function=fit_kernel_length_only)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b120fa5c38a0c9a9e69eb81038aa1ac5745fea68"},"cell_type":"code","source":"def plot_interpolations(iobj,times_list,flux_list,err_list,flux_grid,err_grid):\n    fig, axes = plt.subplots(2, 3, sharex=True, sharey=True, figsize=(12, 8))\n    plt.title(target_types[meta_data.loc[iobj,'target']]) \n    for band in range(6):\n        ax = axes[band // 3, band % 3]\n        ax.errorbar(times_list[iobj][band],flux_list[iobj][band],yerr=err_list[iobj][band],color=colors[band],fmt='o')\n        ax.plot(time_grid,flux_grid[iobj][band],color=colors[band])\n        ax.fill_between(time_grid,flux_grid[iobj][band]-err_grid[iobj][band],\n                        flux_grid[iobj][band]+err_grid[iobj][band],alpha=0.3,color=colors[band])\n        ax.set_xlabel('MJD')\n        ax.set_ylabel('Flux')\n    plt.title(target_types[meta_data.loc[iobj,'target']])\nplot_interpolations(300,times_list,flux_list,err_list,klonly_flux_grid,klonly_err_grid)\n#plt.ylim(-50,200)\nplt.xlim(60000,60250)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98bf1f52fb9bef2b25387f0dec6aa29b2d1038dd"},"cell_type":"code","source":"for iobj in meta_data[(meta_data['ddf']==0)]['object_id'][:25]:\n   plt.figure()\n   plot_one_object(iobj)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f0c38be4213b6f04803d0aef5ba1553cf53a2f1","scrolled":false},"cell_type":"code","source":"x1=meta_data[\"hostgal_specz\"].tolist()\nx2=meta_data[\"hostgal_photoz\"].tolist()\nplt.scatter(x1, meta_data[\"distmod\"],color = 'red');\nplt.scatter(x2, meta_data[\"distmod\"],color = 'blue');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"25f60e37549e8abf79c29fc9a0d0935f7976c44d"},"cell_type":"code","source":"true_photoz= meta_data[~meta_data.hostgal_specz.isna()==~meta_data.hostgal_photoz.isna()]\ntrue_photoz.plot.scatter(x=\"hostgal_specz\", y=\"hostgal_photoz\",color = 'green');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4dd96985934cf1fe824d0a74cebd51143686bd3","scrolled":true},"cell_type":"code","source":"\ntraining_set.plot.scatter(y=\"passband\", x=\"flux\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6178b5fef438db69cdf2bda33290addcd1f4863f"},"cell_type":"code","source":"meta_data.plot.scatter(x=\"gal_l\", y=\"ra\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07d643885c24171e924a4f50d56c0b370a9369ba"},"cell_type":"code","source":"meta_data.plot.scatter(x=\"decl\", y=\"gal_b\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c16fb07e701ce1777979f44de5340fb6ebe9b1a8"},"cell_type":"code","source":"l=len(meta_data['target_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a31858a8120b374378ddc36fd2120cab4e3c0409"},"cell_type":"code","source":"l=len(unique(training_set['object_id']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7dbcafe8ce5a8d5c78b8ae9ba21dc48461e9fd8"},"cell_type":"code","source":"l","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}