{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sb\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# We can try to guess the physical object types be studying their properties.\n# This information will be useful, in case we want to fit the light curves with some of the proposed algorithms (see 'The PLAsTiCC Astronomy Classification Demo' by michaelapers)\n# Fine tuning these algorithms will potentially speed up and improve the machine learning predictions, because the training data set is not representative of test data set.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5288274061919315b1ce7bc6eb2f152d9dcbb755"},"cell_type":"code","source":"# Loading the data\ntrain = pd.read_csv('../input/training_set.csv')\ntrain.name = 'Training Set'\ntrain_meta = pd.read_csv('../input/training_set_metadata.csv')\ntrain_meta.name = 'Training Metadata Set'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"909c8d39e6042deec30ddd4b130a1a3fff14b912"},"cell_type":"code","source":"train_comp = pd.merge(train_meta, train, on='object_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a54e5e67dc25ab7bccf9c438e6d3892de9ffef05"},"cell_type":"code","source":"order = [53, 64, 6, 95, 52, 67, 92, 88, 62, 15, 16, 65, 42, 90]\nlegend_class = train_meta['target'].value_counts().to_frame().sort_values(['target'])\nlegend_class.index.name = 'Class'\nlegend_class.columns = ['Number count']\n# 14 out of 15 object classes are populated.\n# The distribution of objects is certainly irregular.\nsb.set(rc={'figure.figsize':(15,10)})\nsb.barplot(x=train_meta['target'].value_counts().index.ravel(), y=train_meta['target'].value_counts().ravel(), label=legend_class, order=order)\nplt.legend(loc='upper left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2500b8861b76f69884db592151ca4d1264868bd2"},"cell_type":"code","source":"# Let us have a look at the redshift statistics.\n# We want to know the minimum, median and maximum redshifts for each of the classes.\n\norder = [6, 16, 53, 65, 92, 64, 67, 62, 52, 15, 90, 42, 95, 88] # order originates from the maximum redshift measured\nsb.set_color_codes(\"pastel\")\nmax_z = train_meta.groupby('target')['hostgal_specz'].max().sort_values()\nsb.barplot(x=max_z.index,\n           y=max_z.values,  label=\"Max\", color='b', order=order)\nmed_z = train_meta.groupby('target')['hostgal_specz'].median().sort_values()\nsb.barplot(x=med_z.index,\n           y=med_z.values, label=\"Mean\", color='r', order=order)\nmin_z = train_meta.groupby('target')['hostgal_specz'].min().sort_values()\nsb.set_color_codes(\"muted\")\nsb.barplot(x=min_z.index, \n           y=min_z.values, label=\"Min\", color='y', order=order)\nplt.legend(loc='upper left')\nplt.ylabel('hostgal_specz')\nplt.axis([-0.5,13.5,0,1.5])\n\n# We learn that\n# 6, 16, 53, 65, 92 have to be innergalactic objects: stars or star-like objects\n# 64, 67, 62, 52, 15, 90, 42, 95, 88 are extragalactic objects (Active Galactic Nuclei, Supernovae, etc.)\n# furthermore, we can identify at least two subcategories:\n# 95 and 88 seem to be very distant objects (scale factor of the Universe ~ 1 / (1 + z) , for z <= 1000), these objects must be bright and present at earlier times of the Universe\n# the other objects have a median redshift of 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"638ef73f0cf3a5265cb27d394e0ad1b7b0362240"},"cell_type":"code","source":"# Now we go through each of the classes (a very traditional approach ... )\n# Physically correct reasoning will follow...\n#comment: the sb.scatterplot(hue=...) does not seem to work properly. any ideas?\n\nfrom random import randint\n\ndef plt_cls(df, cls, obj=None, hue=False, ln_plt=False):\n    plt.figure(figsize=(15,5))\n    obj_class = df[(df['target']==cls)]\n    unique_ids = obj_class['object_id'].unique()\n    if obj == None:\n        obj = unique_ids[randint(0,len(unique_ids)-1)]\n    label = 'class: ' + str(cls) + ', obj_id: '  + str(obj) + ', z: ' + str(list(df[df['object_id']==obj]['hostgal_photoz'])[0])\n    print(label)\n    mjd = obj_class[obj_class['object_id']==obj]['mjd']\n    flx = obj_class[obj_class['object_id']==obj]['flux']\n    if ln_plt == False:\n        if hue == False:\n            sb.scatterplot(mjd, flx, label=label)\n        else:\n            sb.scatterplot(mjd, flx, hue=df['passband'])\n    else:\n        sb.lineplot(mjd, flx, hue=df['passband'])\n    plt.legend(loc='upper left')\n    plt.plot()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e27c7ca6507af30fc434fefa75aa316a8cb12fa"},"cell_type":"markdown","source":"## Innergalatic objects"},{"metadata":{"trusted":true,"_uuid":"18e3968ffe63e6213a590f66ae425f0f9c5f4476"},"cell_type":"code","source":"# random behavior\n# guess: variable star\nplt_cls(train_comp, 16, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4084327d809dee570c435cc975870d8c7884040f"},"cell_type":"code","source":"# periodicity of about 100 days\n# achromatic flux change\n# guess: pulsating star\nplt_cls(train_comp, 53, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f06f3d511f013640497931753aecb47d388a7b9"},"cell_type":"code","source":"# single event\n# short timescale of few days\n# chromatic flux change\n# guess: microlensing event\nplt_cls(train_comp, 6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20750cf78f11b746343018a4342cd7b3c05b9f03"},"cell_type":"code","source":"# single event\n# random behavior\n# guess: variable star/eruptive\nplt_cls(train_comp, 65, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53d81295239adf69a8a5827ad7c05cb3d7d45aae"},"cell_type":"code","source":"# random behavior & huge fluctuations\n# guess:  variable star\nplt_cls(train_comp, 92, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"739d53bb7c9da6a3b79622514ed02b0bae1715f5"},"cell_type":"markdown","source":"## Extragalactic objects"},{"metadata":{"trusted":true,"_uuid":"a20211f8afbb76dfce2a76b2889809511f4ce119"},"cell_type":"code","source":"# single event\n# achromatic\n# long timescale\n# guess: supernova 1a\nplt_cls(train_comp, 95, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"183bce02e944efff7aaf2aa8623b05417bcaf117"},"cell_type":"code","source":"# high redshift\n# variability\n# long timescale\n# guess: AGN\nplt_cls(train_comp, 88, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"030ce6d20341f10e90e79fad68d914c4ef7dcc80"},"cell_type":"code","source":"# single event\n# short timescale\n# guess: merging event\nplt_cls(train_comp, 64, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14b0f2c22a468003641137a2e7bcd34404a35010"},"cell_type":"code","source":"# single event\n# symmetric\n# short timescale\n# guess: merging event\nplt_cls(train_comp, 67, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16436d94401739ff8ca21e84c1f0f68ddca19ae8"},"cell_type":"code","source":"# single event\n# long timescale\n# guess: supernova type 1\nplt_cls(train_comp, 62, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"508b0470a92b8729196e6fdb2dcf0d29357d61e6"},"cell_type":"code","source":"# single event\n# asymmetric\n# guess: supernova type 2\nplt_cls(train_comp, 52, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"701e37a9731fcc3151cd25fd6b8ec67fc294471c"},"cell_type":"code","source":"# single event\n# asymmetric\n# guess: supernova type 2\nplt_cls(train_comp, 15, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38f67cd8b1542da0ffa1964505a7fe386c78ec47"},"cell_type":"code","source":"# single event\n# guess: supernova type 1\nplt_cls(train_comp, 90, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f44e72781be250c6b85a1a4e8f208d97df5bf6e2"},"cell_type":"code","source":"# single event\n# short timescale\n# guess: supernova type 1\nplt_cls(train_comp, 42, hue=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19a7bf6663ab921380bbf36eed427af6727640ac"},"cell_type":"markdown","source":"## Studying the populations representativeness"},{"metadata":{"trusted":true,"_uuid":"b615727db1f274545e506fb446c526c1763055a4"},"cell_type":"code","source":"import pandas as pd\n# One quick addition:\n#test = pd.read_csv('../input/test_set.csv', low_memory=True, chunk)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cc2358728ce08f01548b772d15df232392bcf5c"},"cell_type":"code","source":"test_meta = pd.read_csv('../input/test_set_metadata.csv', low_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d870c2d94a0b46fbab81c4905c5199584dabc69"},"cell_type":"code","source":"# From the redshift distribution of the training and test set follows, that\n# zero redshift, with redshift between 0.05 and 0.3 and around 2.5 are overrepresented in the training set\n# objects with redshift between 0.3 and 1.0 are underrepresented as well -> data augmentation can be useful to avoid misclassification\n\nplt.figure(figsize=(20,5))\nsb.distplot(test_meta['hostgal_photoz'])\nsb.distplot(train_meta['hostgal_photoz'])\nplt.axis([0.0,1.0,0.0,2.0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3663191770fde76ac49d5bdafe3eaf74642681c"},"cell_type":"code","source":"plt.figure(figsize=(20,5))\nsb.distplot(test_meta['hostgal_photoz'])\nsb.distplot(train_meta['hostgal_photoz'])\nplt.axis([1.0,3.0,0.0,0.1])\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1611c7b22f05df0bb450c607a70399de141b0ee0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}