{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport time\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm._tqdm_notebook import tqdm_notebook as tqdm\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7fe3b2446b39f63bf30e2a0e4130792c31d8abb"},"cell_type":"code","source":"pb = [0, 1, 2, 3, 4, 5]\n# choice = 'extragalactic'\n# choice = 'galactic'\nchoice = 'all'","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/training_set.csv')\ntrain_meta = pd.read_csv('../input/training_set_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76ccbdcd54656babb7ad1427fba33e9bf0fbd8da"},"cell_type":"code","source":"train_meta.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"addd8f17c2bae4be38d152ed2aa0acec2060b3f4"},"cell_type":"code","source":"extra_cols = ['hostgal_specz', 'hostgal_photoz', 'hostgal_photoz_err', 'distmod']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bff790a77429905ca6ab2186304a75513dff31b8"},"cell_type":"code","source":"gal_mask = train_meta['distmod'].isnull().values #galactic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01498ce111ad2b9fd77b37b413224d5b8b9f4fc6"},"cell_type":"code","source":"print(train_meta.shape, gal_mask.sum())\nprint(train_meta['target'].unique())\n\nprint(f'Select {choice}')\nif choice=='galactic':\n    train_meta = train_meta[gal_mask]\n    train_meta.drop(extra_cols, axis=1, inplace=True)\nelif choice=='extragalactic':\n    train_meta = train_meta[~gal_mask]\nelse:\n    pass\n\nprint(train_meta.shape, gal_mask.sum())\nprint(train_meta['target'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e934bad7749ec6b8e57d4a423bba9c230e2e0886"},"cell_type":"code","source":"df = train.merge(train_meta, on='object_id', how='inner').reset_index(drop=True)\ntarget = train_meta[['object_id', 'target']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f20ef3bd90f0ba413c77bb61b528462792f32ad"},"cell_type":"code","source":"print(df.shape)\ndisplay(df.head(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b94c260eac9c032000152ad69b35590c18907d0"},"cell_type":"code","source":"grps = df.groupby('object_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79fab0090d448b9cd6fb2c43d39c6f830993b8af"},"cell_type":"code","source":"tmp = target.merge(grps.size().rename('obj_size').reset_index(), on='object_id')\nsns.barplot(x='target', y='obj_size', data=tmp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9402431d81f91ee6c4094105b0933ccacf43f3e"},"cell_type":"code","source":"def plot_agg(col, func, grps=grps, target=target):\n    tmp = grps.agg({col:func})\n    tmp.columns = [f'{col}_{func}']\n    tmp = target.merge(tmp, on='object_id')\n    sns.boxplot(x='target', y=f'{col}_{func}', data=tmp)\n    plt.grid()\n    if tmp[f'{col}_{func}'].max()>1000:\n        plt.yscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c9f8dbf607fc53987b97e34433b2029285c7a45"},"cell_type":"code","source":"func_li = ['mean', 'std']\ncols = df.columns.tolist()\ncols.remove('object_id')\ncols.remove('target')\nprint(cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4aab20845cf25a91c3b1683e5a1efb8d6f420763"},"cell_type":"code","source":"from itertools import product\npairs = list(product(cols, func_li))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43b42e9d8232908f77ee3a3b7375db149e73667b"},"cell_type":"code","source":"plt.figure(figsize=[24, 28])\ncnt = 0\nfor i,(col,func) in enumerate(pairs):\n    if col in train_meta and func=='std':\n        continue\n    else:\n        plt.subplot(7, 3, cnt+1)\n        plot_agg(col, func)\n        cnt += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dee2e7de7ddfad40d52829d80b8f2b44e5517f0"},"cell_type":"code","source":"plt.figure(figsize=[18, 4])\nsns.distplot(df['mjd'], bins=100)\nplt.grid();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2316c11ce967f59dafcfa2f5ac08b91bc0cf820"},"cell_type":"code","source":"plt.figure(figsize=[16, 5])\nplt.subplot(1,2,1)\ntmp = grps['mjd'].apply(lambda x: x.max()-x.min())\ntmp = tmp.rename('mjd_length').reset_index()\ntmp = target.merge(tmp, on='object_id')\nsns.boxplot(x='target', y=f'mjd_length', data=tmp)\nplt.grid()\n\nplt.subplot(1,2,2)\ntmp = grps['mjd'].apply(lambda x: (x%1).mean())\ntmp = tmp.rename('day_mean').reset_index()\ntmp = target.merge(tmp, on='object_id')\nsns.boxplot(x='target', y=f'day_mean', data=tmp)\nplt.grid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2129c9e4589c80b6cffa822d23468246bb0a839"},"cell_type":"code","source":"tmp = df['mjd']%1\ntmp.to_frame().describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb805b6d23575ccf2fcbdddb10b581ee95a8a1fc"},"cell_type":"code","source":"tmp[tmp<=0.5].min(), tmp[tmp<=0.5].max(), tmp[tmp>0.5].min(), tmp[tmp>0.5].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ef6c475f79ed3c6ff109811b7b62c87b9f2a142"},"cell_type":"code","source":"(86400*tmp[tmp<0.5].max())/3600, (86400*tmp[tmp>0.5].min())/3600","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"781fa5ddad9eebbab6b235b7e1f2b5edf7d81899"},"cell_type":"code","source":"plt.figure(figsize=[18, 3])\nsns.distplot(df['mjd']%1, bins=100)\nplt.grid();\nplt.title('Hide during the day, come out at night')\nplt.xlabel('Time within a day');","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}