{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"Am I the only one to see two distinct distributions in train?  I am fast coming to the conclusion that this competition should be to focus on the data as just a **simulation** rather than modelling nature.  As an indicator I used the columns that have relatively few NaNs and then performed a TSne."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import log_loss\nfrom scipy.stats import skew, kurtosis\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.manifold import TSNE\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82ce30f1187e1952b181ebfe108ad604511e771d"},"cell_type":"code","source":"def get_inputs(data, metadata):\n    metadata.drop(['ra','decl','gal_l','gal_b','mwebv','hostgal_photoz','ddf','distmod'],inplace=True,axis=1)\n    data['flux_ratio_sq'] = np.power(data['flux'] / data['flux_err'], 2.0)\n    data['flux_by_flux_ratio_sq'] = data['flux'] * data['flux_ratio_sq']\n    aggdata = data.groupby(['object_id','passband']).agg({'mjd': ['min', 'max', 'size'],\n                                             'flux': ['min', 'max', 'mean', 'median', 'std','skew'],\n                                             'flux_err': ['min', 'max', 'mean', 'median', 'std','skew'],\n                                             'flux_by_flux_ratio_sq': ['sum'],    \n                                             'flux_ratio_sq': ['sum'],                      \n                                             'detected': ['mean','std']}).reset_index(drop=False)\n    \n    cols = ['_'.join(str(s).strip() for s in col if s) if len(col)==2 else col for col in aggdata.columns ]\n    aggdata.columns = cols\n    aggdata = aggdata.merge(metadata,on='object_id',how='left')\n    aggdata.insert(1,'delta_passband', aggdata.mjd_max-aggdata.mjd_min)\n    aggdata.drop(['mjd_min','mjd_max'],inplace=True,axis=1)\n    aggdata['flux_diff'] = aggdata['flux_max'] - aggdata['flux_min']\n    aggdata['flux_dif2'] = (aggdata['flux_max'] - aggdata['flux_min']) / aggdata['flux_mean']\n    aggdata['flux_w_mean'] = aggdata['flux_by_flux_ratio_sq_sum'] / aggdata['flux_ratio_sq_sum']\n    aggdata['flux_dif3'] = (aggdata['flux_max'] - aggdata['flux_min']) / aggdata['flux_w_mean']\n    return aggdata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23a03351e7168e0dca2cbfe99f515618e061f7bb"},"cell_type":"code","source":"meta_train = pd.read_csv('../input/training_set_metadata.csv')\ntrain = pd.read_csv('../input/training_set.csv')\ntraindata = get_inputs(train,meta_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f7748ab6177945def43b3e1f7e587c34e69641d"},"cell_type":"code","source":"features = list(set(traindata.columns).difference(set(['target','object_id','hostgal_photoz_err','hostgal_specz'])))\nallfeatures = ['object_id']+features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8692e2ca54d9dd11a9c9764c53b24415fd3b208a"},"cell_type":"code","source":"alldata = traindata.loc[:,allfeatures].copy()\nfor c in features:\n    print(c)\n    if(alldata[c].min()<0):\n        alldata.loc[~alldata[c].isnull(),c] = np.sign(alldata.loc[~alldata[c].isnull(),c])*np.log1p(np.abs(alldata.loc[~alldata[c].isnull(),c]))\n    elif((alldata[c].max()-alldata[c].min())>10):\n        alldata.loc[~alldata[c].isnull(),c] = np.log1p(alldata.loc[~alldata[c].isnull(),c])\nalldata.fillna(alldata.mean(),inplace=True)\nss = None\nss = StandardScaler()\nalldata.loc[:,features] = ss.fit_transform(alldata.loc[:,features])\nprint(ss.mean_)\nprint(ss.scale_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14d2eac3747d19516167a014c58d8b890d9e9005"},"cell_type":"code","source":"model = TSNE(n_components=2, perplexity=30,random_state=0)\ntsnedata = model.fit_transform(alldata[features])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"352867f75f5c49a243e62518ca6447aa48299ac3"},"cell_type":"code","source":"plt.plot(tsnedata[:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7ee0386bff33388c90129f4e4ad0237e334c258"},"cell_type":"code","source":"plt.plot(tsnedata[:,1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e36a1ee185e2c60c9d201836c8a3ce875fbedd75"},"cell_type":"markdown","source":"I could certainly be missing something but one can see a disting partition in the training data in terms of TSne"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}