{"cells":[{"metadata":{"_uuid":"057792ece78e10037d4e014a4f83a17659878cc3"},"cell_type":"markdown","source":"## Import libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport math\nfrom astropy.coordinates import SkyCoord\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"## Load the data"},{"metadata":{"trusted":true,"_uuid":"750855ccb0adcf873f6c4e58fcf2e124ce8a9646"},"cell_type":"code","source":"train_data = pd.read_csv('../input/training_set.csv')\ntrain_metadata = pd.read_csv('../input/training_set_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"387b0fd669d8f7f53b1ff0650791b7b20ba59b3c"},"cell_type":"markdown","source":"## Quick look at the tables"},{"metadata":{"trusted":true,"_uuid":"fe1026dbfa9f1bc77ef04629cedc249e5778b776"},"cell_type":"code","source":"train_metadata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c63f7cb2e460ae47b282e0d1392ae93abd44d3c"},"cell_type":"code","source":"train_metadata.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40dd64d84accf3706e1e496a4c341061e483848c"},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cbd3451ed5dd5e939b4c651f65bfd23412793a4d"},"cell_type":"markdown","source":"## Investigate target"},{"metadata":{"trusted":true,"_uuid":"900ef80a1dfea9560ea0bb5a0737a3f91544e282","_kg_hide-input":true},"cell_type":"code","source":"target_counts = train_metadata.groupby(by='target')['object_id'].count().sort_values(ascending=False)\nplt.figure(figsize=(15, 8))\nplt.bar(target_counts.index.astype(str), target_counts.values)\nplt.xlabel('astro class')\nplt.ylabel('occurences in training data')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10b308d0fab8c0a248fe7b51de49406c76512e14"},"cell_type":"markdown","source":"## Plot the sources location in both coordinate systems"},{"metadata":{"trusted":true,"_uuid":"2984e4cfd077bab7c9c1966e5b7e68c3a682d5d6","_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(20, 8))\nplt.subplot(121, projection='aitoff')\nplt.scatter(x=(train_metadata['ra']-180)*math.pi/180, y=train_metadata['decl']*math.pi/180, alpha=0.1)\nplt.title('Equatorial coordinates')\nplt.subplot(122, projection='aitoff')\nplt.scatter(x=(train_metadata['gal_l']-180)*math.pi/180, y=train_metadata['gal_b']*math.pi/180, alpha=0.1)\nplt.title('Galactic coordinates')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbeb2b423f69807aab66b4a5c101d14c123bcd9d"},"cell_type":"markdown","source":"## Show just DDFs (Deep Drilling Fields)"},{"metadata":{"trusted":true,"_uuid":"881e01c4b561dd6f6c950d2dcb5717f9cdfbcbc5","_kg_hide-input":true},"cell_type":"code","source":"ddf_metadata = train_metadata[train_metadata['ddf'] == 1].copy()\nwfd_metadata = train_metadata[train_metadata['ddf'] != 1].copy()\nplt.figure(figsize=(20, 8))\nplt.subplot(121, projection='aitoff')\nplt.scatter(x=(ddf_metadata['ra']-180)*math.pi/180, y=ddf_metadata['decl']*math.pi/180, alpha=0.1)\nplt.title('Equatorial coordinates')\nplt.subplot(122, projection='aitoff')\nplt.scatter(x=(ddf_metadata['gal_l']-180)*math.pi/180, y=ddf_metadata['gal_b']*math.pi/180, alpha=0.1)\nplt.title('Galactic coordinates')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"637be19d65c000303a22eaa8544be8f5466f6415"},"cell_type":"markdown","source":"## DDFs vs WFDs (Wide Fast Deep)"},{"metadata":{"trusted":true,"_uuid":"0d80e40021e4beb27453f143d205453bf27fe70d"},"cell_type":"code","source":"plt.figure(figsize=(20, 5))\nplt.barh(y=['DDF', 'WFD'], \n        width=[ddf_metadata['object_id'].count(), wfd_metadata['object_id'].count()])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bff8f0d3a77686c09b59500b7db4efc3298e9464"},"cell_type":"markdown","source":"## Examine distance to sources"},{"metadata":{"trusted":true,"_uuid":"91ad3d959f82108765781f8f10ecbdedcb6816f9"},"cell_type":"code","source":"train_metadata['distmod'].isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcd0461988655990b50e17c70c2d13ee7e7902ac"},"cell_type":"code","source":"train_metadata[train_metadata['distmod'].isna()].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"183eb89adb6dfeb065d274db13ac2df37bb255c8"},"cell_type":"code","source":"train_metadata[train_metadata['distmod'].isna()]['hostgal_photoz'].describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c78d50c36765c90c57f6142088115c15abf82b1c"},"cell_type":"markdown","source":"## Sources with redshift values of zero belong to our own galaxy - Milky Way\nLets create new feature with flag for that"},{"metadata":{"trusted":true,"_uuid":"951858872af5d6574da35a4922b945a056957c3c"},"cell_type":"code","source":"train_metadata['is_milky_way'] = train_metadata['hostgal_photoz'] == 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f219a06bb5106650f3ea856b2aafb9fdad15e46"},"cell_type":"markdown","source":"## Plot distance to sources from outside Milky Way\nThe distance is going to be relative to others so that we can use distance as colour"},{"metadata":{"trusted":true,"_uuid":"a6d02688212b4d49c5c7b38a392288117d6a8df7"},"cell_type":"code","source":"outside_sources = train_metadata.loc[~train_metadata['is_milky_way']].copy()\nmax_distance = outside_sources['distmod'].max()\noutside_sources['scaled_distance'] = outside_sources['distmod'] / max_distance\n\nplt.figure(figsize=(20, 8))\nplt.gray()\nplt.subplot(121, projection='aitoff')\nplt.scatter(x=(outside_sources['ra']-180)*math.pi/180, y=outside_sources['decl']*math.pi/180, c=outside_sources['scaled_distance'])\nplt.title('Equatorial coordinates')\nplt.subplot(122, projection='aitoff')\nplt.scatter(x=(outside_sources['gal_l']-180)*math.pi/180, y=outside_sources['gal_b']*math.pi/180, c=outside_sources['scaled_distance'])\nplt.title('Galactic coordinates')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c6cff7f9a495e7bf1fd5ffd783ce98c61d92d57"},"cell_type":"markdown","source":"## Light extinction of Milky Way Dust"},{"metadata":{"trusted":true,"_uuid":"6c7327a90f8773a70b2a5a63255794d8591b0fd8"},"cell_type":"code","source":"outside_sources['mwebv_scaled'] = outside_sources['mwebv'] / outside_sources['mwebv'].max()\ns = [4 for i in range(len(outside_sources['mwebv']))]\n\nplt.figure(figsize=(20, 8))\nplt.gray()\nplt.subplot(121, projection='aitoff')\nplt.scatter(x=(outside_sources['ra']-180)*math.pi/180, y=outside_sources['decl']*math.pi/180, c=outside_sources['mwebv_scaled'], s=s)\nplt.title('Equatorial coordinates')\nplt.subplot(122, projection='aitoff')\nplt.scatter(x=(outside_sources['gal_l']-180)*math.pi/180, y=outside_sources['gal_b']*math.pi/180, c=outside_sources['mwebv_scaled'], s=s)\nplt.title('Galactic coordinates')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fed70d077cb358387c084ac8355bce0736d6331"},"cell_type":"markdown","source":"## Plot sources just in our galaxy"},{"metadata":{"trusted":true,"_uuid":"b11444cf1d226403c11d96671dedf769b7fad8d1"},"cell_type":"code","source":"milky_way_sources = train_metadata.loc[train_metadata['is_milky_way']].copy()\n\nplt.figure(figsize=(20, 8))\nplt.subplot(121, projection='aitoff')\nplt.scatter(x=(milky_way_sources['ra']-180)*math.pi/180, y=milky_way_sources['decl']*math.pi/180, alpha=0.1)\nplt.title('Equatorial coordinates')\nplt.subplot(122, projection='aitoff')\nplt.scatter(x=(milky_way_sources['gal_l']-180)*math.pi/180, y=milky_way_sources['gal_b']*math.pi/180, alpha=0.1)\nplt.title('Galactic coordinates')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed3bac5404864fbc42706cea091940bb929c17ac"},"cell_type":"code","source":"milky_way_ddf_count = len(train_metadata[(train_metadata['ddf']==1) & (train_metadata['is_milky_way'])])\noutside_ddf_count = len(train_metadata[(train_metadata['ddf']==1) & (~train_metadata['is_milky_way'])])\n\nplt.figure(figsize=(15, 5))\nplt.barh(y=['Milky Way', 'Outside'], width=[milky_way_ddf_count, outside_ddf_count])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"720280dac20a0ff01a123cdf8f6bfea6458f3e11"},"cell_type":"markdown","source":"## Quick look at one light curve"},{"metadata":{"trusted":true,"_uuid":"e5e94372259011c8796109913b6beb7d6a5fec63"},"cell_type":"code","source":"# Convert from Modified Julian Date to normal date\n# Unix = (MJD−40587)×86400\ntrain_data['date'] = pd.to_datetime((train_data['mjd'] - 40587) * 86400, unit='s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f79d0e51adb8179fce7804f7c7455028b4062909","scrolled":true},"cell_type":"code","source":"light_curve_615 = train_data[train_data['object_id']==615]\nlight_curve_615['passband'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85037f9ffc94bb453487452d1bb3fbf02f10ee4a"},"cell_type":"code","source":"bands = {0: 'b', 1: 'g', 2: 'r', 3: 'm', 4: 'y', 5: 'k'}\nplt.figure(figsize=(15, 8))\nfor band, color in bands.items():\n    plt.errorbar(x=light_curve_615[light_curve_615['passband']==band]['mjd'], \n                 y=light_curve_615[light_curve_615['passband']==band]['flux'], \n                 yerr=light_curve_615[light_curve_615['passband']==band]['flux_err'], \n                 fmt='o', color=color)\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5d8a89920d4f9f532ffb0c180330e7c16bfaf05"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}