{"cells":[{"metadata":{"_uuid":"9333ca2a02be5ce116b6fa3b677e7d8b9e19ed63"},"cell_type":"markdown","source":"**Import libraries:**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport math\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport statsmodels.api as sm\nimport statsmodels.formula.api as smf\nsns.set(style=\"darkgrid\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4858935279005ae050d6798ccf3660f2b55c0e02"},"cell_type":"markdown","source":"## **Training data**"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"**Import light curve data:**"},{"metadata":{"trusted":true,"_uuid":"6e3a21f4ccddd8d6a43bc46dec094f801ffa5ae0"},"cell_type":"code","source":"train_df = pd.read_csv('../input/training_set.csv')\ntrain_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00cfc65ae7b7ed2012d7f0edbee032369b7d6925"},"cell_type":"markdown","source":"**Import meta data:**"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"1089393591579618aa841a4b123d91dbd31d2091"},"cell_type":"code","source":"train_meta = pd.read_csv('../input/training_set_metadata.csv')\ntrain_meta.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21b69a4a08216d79a24b4b2fa73c50c5d4cde62a"},"cell_type":"code","source":"train_meta.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d09778460cc7cfad86978f74e1dbf4a823a574d"},"cell_type":"code","source":"help_dict = {'object_id' : 'Object ID',\n                'ra' : 'Right Ascension, corresponding to latitude', \n                'decl' : 'Declination (corresponding to longitude)',\n                'gal_l': 'Galactic coordinate 1',\n                'gal_b': 'Galactic coordinate 2',\n                'ddf': 'Object is observed usning Deep Drilling Field (DDFs have more observations but cover a smaller chunk of sky)',\n                'hostgal_specz': 'The spectroscopic redshift of the source8. This is an extremely accurate measure of redshift, provided for the training set and a small fraction of the test set (given as float32 numbers).',\n                'hostgal_photoz': 'The photometric redshift of the host galaxy of the astronomical source. While this is meant to be a proxy for hostgal specz, there can be large differences between the two and hostgal photoz should be regarded as a far less accurate version of hostgal specz',\n                'hostgal_photoz_err': 'The uncertainty on the hostgal photoz based on LSST survey projections.',\n                'distmod': 'The distance (modulus) calculated from the hostgal photoz since this redshift is given for all objects (given as float32 numbers). Computing the distance modulus requires knowledge of General Relativity, and assumed values of the dark energy and dark matter content of the Universe, as mentioned in the introduction section.',\n                'mwebv' : 'MW E(B-V): this ‘extinction’ of light is a property of the Milky Way (MW) dust along the line of sight to the astronomical source, and is thus a function of the sky coordinates of the source ra, decl. This is used to determine a passband dependent dimming and reddening of light from astronomical sources as described in subsection 2.1',\n                'target': 'The class of the astronomical source. This is provided in the training data.'\n               }\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3567a7ae0a6093bc3b3f7cb903ddcada69577a8a"},"cell_type":"markdown","source":"**Position in the sky:**"},{"metadata":{"trusted":true,"_uuid":"7f82d5f658f495ce8b35fc3dbdd338e39064a711"},"cell_type":"code","source":"plt.figure(figsize = (10,6))\nplt.subplot(111,projection ='aitoff')\nplt.scatter(x = (train_meta['ra']-180)*math.pi/180, y = (train_meta['decl'])*math.pi/180);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5541984dbc70112b04bda2d2dce2f315b2cfb80e"},"cell_type":"markdown","source":"**DDF vs non-DDF Count:**[](http://)"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"bc3c08a83a681730359122c101a9c0484cd9e8a2"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (10,6))\nsns.countplot(train_meta['ddf']);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ea999434e61201377ff42f36e01553ef02fc62c"},"cell_type":"markdown","source":"**Count of each astronomical object:**"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":true,"_uuid":"e06fcc8f872fd658a986fc0264a62246b3fc51d3"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (10,6))\nsns.countplot(train_meta['target']);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"406195ca91de69c612317f73f1c61e8b1e10cfa2"},"cell_type":"markdown","source":"**Percentage of each astronomical object (DDf vs non DDF):**"},{"metadata":{"trusted":true,"_uuid":"0078219560afd9843a5ad4b03c8905d2c5a6c6b9","_kg_hide-input":true},"cell_type":"code","source":"df_0 = train_meta.loc[train_meta['ddf']==0]\ndf_1 = train_meta.loc[train_meta['ddf']==1]\n\n\nymax = 1.1* max(list(100*df_1['target'].value_counts()/len(df_1))+list(100*df_0['target'].value_counts()/len(df_0)))\n\nplt.figure(figsize = (20,6))\nplt.subplot(121)\nax = sns.barplot(x=\"target\", y=\"target\", data=df_1, estimator=lambda x: len(x) / len(df_1) * 100)\nax.set(ylabel=\"%\", xlabel ='Target')\nax.set_ylim(0,ymax)\nplt.title('DDF')\nplt.subplot(122)\nax = sns.barplot(x=\"target\", y=\"target\", data=df_0, estimator=lambda x: len(x) / len(df_0) * 100)\nax.set(ylabel=\"%\", xlabel ='Target')\nax.set_ylim(0,ymax)\nplt.title('Wide Angle');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bef58f112310a616ac65d935aa821e5dacec1032"},"cell_type":"markdown","source":"Things which immediately jump out:\n* The DDF group has a much higher proportion of object class 90.\n* The DDF group very little objects of class 6 or 15 compared to Wide Angle.\n* The DDF group has (almost) no objects of class 53 or 64 wheras Wide Angle does."},{"metadata":{"_uuid":"6f5982c2280638f8e36970a591ec1acd75440e94"},"cell_type":"markdown","source":"If we actually look at the numbers for DDF:"},{"metadata":{"trusted":true,"_uuid":"06f9f6b8c5268c54052e480a722f805a12a81ce8"},"cell_type":"code","source":"df_1['target'].loc[df_1['target'].isin([53,64])].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7faa30fa1282f6868ea54ddc4a0d0aa2a7d19385"},"cell_type":"markdown","source":"We can see that infact there are entries for classes 53 and 64, but not many."},{"metadata":{"_uuid":"b60e8fd6865d7ba54b8b89cd43e2619e21b5db4c"},"cell_type":"markdown","source":"**Redshift measurment differences:**"},{"metadata":{"trusted":true,"_uuid":"530f6863935c0eca71accf0adf1c943ab6de3875"},"cell_type":"code","source":"plt.figure(figsize = (10,8))\nax = plt.subplot(111)\nax.scatter(train_meta['hostgal_photoz'],train_meta['hostgal_specz'], marker = '.')\nax.errorbar(train_meta['hostgal_photoz'],train_meta['hostgal_specz'], xerr= None, yerr= train_meta['hostgal_photoz_err'],fmt='none');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbb424238f229e2b1eac7ed1a5c546162374a64d"},"cell_type":"markdown","source":"Run a linear regression on this data:"},{"metadata":{"trusted":true,"_uuid":"f8d1b75070c279dd88ad38f3ab33ea5a3d3dcbd2"},"cell_type":"code","source":"model = smf.ols(formula='hostgal_specz ~ hostgal_photoz', data=train_meta)\nresults = model.fit()\nprint(results.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00d1da5c7780bc37b1b8e802aa5d777ac976339c"},"cell_type":"code","source":"plt.figure(figsize = (10,8))\nax = plt.subplot(111)\nfig = sm.graphics.plot_fit(results, 1, ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ee79fd5d5694bf65c741961ea81bab3063541941"},"cell_type":"markdown","source":"There is a cluster of points which in reality have very low redshift but are measured at a a quite high photometric redshift. We will see if the error is correlated with any other variables:"},{"metadata":{"trusted":true,"_uuid":"7e542e0886ab1f1447d210b0b2966388a8563aca"},"cell_type":"code","source":"train_meta['hostgal_sq_diff'] = (train_meta['hostgal_photoz'] - train_meta['hostgal_specz'])**2\ntrain_meta['hostgal_diff'] = train_meta['hostgal_photoz'] - train_meta['hostgal_specz']\ncorr = train_meta.corr()\n\n# create a heatmap plot\nsns.set(style=\"white\")\n# Generate a mask for the upper triangle\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(11, 9))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(150, 275, s=80, l=55, n=9,as_cmap = True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5});","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ef0c00b6bef7ad0ab03462ba0d4dd7cb0bdae3e"},"cell_type":"markdown","source":"The difference in the redshift measurements is basically not correlated with anything except the measurements themselves. By visual inspection of the first plot it looks as if a predictive model using the photometric measurment and known error may predict spectroscopic measurement fairly well:"},{"metadata":{"trusted":true,"_uuid":"cdc669e36d9ed1be6fc8b09f28a5810c1a8ea083"},"cell_type":"code","source":"model = smf.ols(formula='hostgal_specz ~ hostgal_photoz + hostgal_photoz_err', data=train_meta)\nresults = model.fit()\nprint(results.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52886b3d78f5de4a49dafd225f205f0838261ced"},"cell_type":"code","source":"plt.figure(figsize = (20,8))\nax = plt.subplot(121)\nfig = sm.graphics.plot_fit(results, 1, ax=ax)\nax = plt.subplot(122)\nfig = sm.graphics.plot_fit(results, 2, ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39b37caf62824b16e786cd03766ad6d8906af5e1"},"cell_type":"markdown","source":"We now use an interaction effect as well:"},{"metadata":{"trusted":true,"_uuid":"46557b5684376e11d9a70a36fcd310f4724ccc62"},"cell_type":"code","source":"model = smf.ols(formula='hostgal_specz ~ hostgal_photoz * hostgal_photoz_err', data=train_meta)\nresults = model.fit()\nprint(results.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d1b59bbc852159be636f12f007fc4b41b7520ee"},"cell_type":"code","source":"plt.figure(figsize = (21,8))\nax = plt.subplot(131)\nfig = sm.graphics.plot_fit(results, 1, ax=ax)\nax = plt.subplot(132)\nfig = sm.graphics.plot_fit(results, 2, ax=ax)\nax = plt.subplot(133)\nfig = sm.graphics.plot_fit(results, 3, ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27455293a26d8ef2c504b0c051618853a4998e5b"},"cell_type":"markdown","source":"Much better results."},{"metadata":{"_uuid":"c318636222d48ff94f58f359a4ccf1987f2dcef5"},"cell_type":"markdown","source":"**Plot histograms for each variable in metadata:**"},{"metadata":{"trusted":true,"_uuid":"93d51f27235b99fa5c121c652f2bcf3fb4f815b9"},"cell_type":"code","source":"cols = list(train_meta)\nplt.subplots_adjust(hspace = 2)\nplt.figure(figsize = (20,30))\nplt.title('Histograms');\nfor counter, col in enumerate(cols):\n    index = counter+1\n    ax = plt.subplot(7,2,index)\n    ax.hist(train_meta[col].dropna(), bins=10)\n    plt.title(col);\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a11e6716e6f92d7dd5be9980e5d8a0fb5ff3f6b"},"cell_type":"markdown","source":"**Find missing values in all datasets**"},{"metadata":{"trusted":true,"_uuid":"bce9ca0e85622c9df2ee9501ac231db6ca3ecc5f"},"cell_type":"code","source":"train_meta.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02ab5589a1e5ee6c5a81af2688126fedc436ca02","scrolled":true},"cell_type":"code","source":"train_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6ee25aea05acc1783f4b1e3cb695edca05b93ce"},"cell_type":"markdown","source":"Only missing values are in distance variable. See if the missingness is related to anything else:"},{"metadata":{"trusted":true,"_uuid":"a6c38c915d9826303cd2de4a745a84e6da1a049b"},"cell_type":"code","source":"train_meta['dist_mod_missing'] = train_meta['distmod'].isna()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18b0246855131728bff9be3b9f555ef40db2b962"},"cell_type":"code","source":"corr = train_meta.corr()\n\n# create a heatmap plot\nsns.set(style=\"white\")\n# Generate a mask for the upper triangle\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(11, 9))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(150, 275, s=80, l=55, n=9,as_cmap = True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5});","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4cbc4e455cf92a30e2026ea11b7fef6b6886e44c"},"cell_type":"markdown","source":"Missingness strongly negatively correlated with redshift measures (and error) and the target.  Looking closer at the target correlation:"},{"metadata":{"trusted":true,"_uuid":"a4869fc65f8f2b9258223d7f2a3f008168e7658b"},"cell_type":"code","source":"plt.figure(figsize = (10,8))\nsns.barplot(train_meta.groupby(['target']).sum().index,train_meta.groupby(['target']).sum()['dist_mod_missing']);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d95de8fd248e51efd8d250466b42d51102e445b6"},"cell_type":"markdown","source":"So to fill these distances in we will build a classification system for these five classes and then fill in within the class."},{"metadata":{"_uuid":"659f518f34ef188cb1ef30e61fb2e1f444301402"},"cell_type":"markdown","source":"## **Test data**"},{"metadata":{"_uuid":"37a645cda51bc790a9937192fa7d8e6e5bbf57ff"},"cell_type":"markdown","source":"**Import light curve data:****"},{"metadata":{"trusted":true,"_uuid":"e50858fe0f20fdfd3fd19c1bdcbbc3f29267e8ff"},"cell_type":"code","source":"test_df_reader = pd.read_csv('../input/test_set.csv',chunksize = 1000) # use an iterator as otherwise file too large\ntest_df_reader.get_chunk(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"abf6bc37111b8f7908eec7988713bf6b9ad08662"},"cell_type":"markdown","source":"Look for missing data:"},{"metadata":{"trusted":true,"_uuid":"d4988db4395c9fe5a6246ef6ac06e970353a735e"},"cell_type":"code","source":"# arrays_of_sums = [chunk.isna().sum() for chunk in test_df_reader]\n# sum(arrays_of_sums)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7e1ec6b9d1095ccd6a5c68fc4256d703892b275"},"cell_type":"markdown","source":"**Import meta data:**"},{"metadata":{"trusted":true,"_uuid":"34e44d0d5788ff58722fa588875b9fa5a569e612"},"cell_type":"code","source":"test_meta = pd.read_csv('../input/test_set_metadata.csv')\ntest_meta.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}