{"cells":[{"metadata":{"_uuid":"89583ca5bcd56754dcb34df33863b2cd758b5a5a"},"cell_type":"markdown","source":"The Lomb-Scargle Periodogram is a commonly-used statistical tool designed to detect periodic signals in unevenly-spaced observations. The AstropyLombScargle class is a unified interface to several implementations of the Lomb-Scargle periodogram, including a fast O[NlogN] implementation following the algorithm presented by Press & Rybicki]."},{"metadata":{"trusted":true,"_uuid":"869a1335e3363a787dc48ce58f25f52a8394224c"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport logging\nfrom astropy.stats import LombScargle\nfrom astropy.time import Time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"548e942f702d816687e46bd5b5fc00bb914f1d4e"},"cell_type":"code","source":"train = pd.read_csv('../input/training_set.csv')\nprint(train.shape)\nmeta_train = pd.read_csv('../input/training_set_metadata.csv')\nprint(meta_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48ae36b99a7eab24ea1fe488e06a686a45795ba2"},"cell_type":"code","source":"train['mjd'] = Time(train.mjd.values, format='mjd').iso\ntrain['mjd'] = pd.to_datetime(train['mjd'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f96889848c547c50cd21b130569521645b11443"},"cell_type":"code","source":"x = train.copy()\nx.head()\nx = x.sort_values(by=['object_id','passband','mjd'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2f2904032b8c28a462d67115e73402176317874"},"cell_type":"code","source":"x['cc'] = x.groupby(['object_id','passband'])['mjd'].cumcount()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"623e87881a8c36060501ac1b026c34bf1c0593fe"},"cell_type":"code","source":"x.mjd = x.groupby(['object_id','passband'])['mjd'].diff()\nx.loc[~x.mjd.isnull(),'mjd'] = x.loc[~x.mjd.isnull(),'mjd'].apply(lambda a: a.total_seconds())\nx.loc[x.mjd.isnull(),'mjd'] = 0\nx.mjd = x.mjd.astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6490b4cafa28eb6ae0455290aa0268a61881a00c"},"cell_type":"code","source":"x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eafc611a596a934f30d7e27392d50ff06430a3cf"},"cell_type":"code","source":"x = x.set_index(['object_id','passband','cc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4804d4f8cbf74d8cdfe46a2566369c1def023320"},"cell_type":"code","source":"x = x.unstack()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3d4a6ab8ecfe45531e8371074a60f7f8c24e393"},"cell_type":"code","source":"x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1394da6a9e4a84ab5ab62fe310149e05573ace1b"},"cell_type":"code","source":"cols = ['_'.join(str(s).strip() for s in col if s) if len(col)==2 else col for col in x.columns ]\ncols\nx.columns = cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9e9074d6b3aca209fd608e9e5a49b60957e0493"},"cell_type":"code","source":"x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a4a3d8c36c7d33d968d00753580b486658e4a95"},"cell_type":"code","source":"mjdcolumns = [a  for a in x.columns if a.startswith('mjd')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88527811672696032d5720657af28dcce942795d"},"cell_type":"code","source":"fluxcolumns = [a  for a in x.columns if a.startswith('flux') and not a.startswith('flux_err')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8365a853f4be662c65f3eabc15ae2b2d7ee20cd"},"cell_type":"code","source":"def GenerateLS(r):\n    t = r[mjdcolumns].dropna().apply(lambda a: a/(3600)).cumsum().astype('float32')\n    p = r[fluxcolumns].dropna().astype('float32')\n    frequency, power = LombScargle(t.values,p.values).autopower(nyquist_factor=1)\n    w = pd.DataFrame()\n    w['f'] = (frequency*100000).astype(int)\n    w['p'] = power\n    w = w.groupby('f').sum().reset_index(drop=False)\n    return  {'f'+str(int(i)):j for i,j in zip(w.f,w.p)}\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bab248d1d8d51de21ddf3371861cc1eb4a7cb110"},"cell_type":"code","source":"d = x.apply(lambda r: GenerateLS(r),axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28561787c69ac543ab384ed93c70b96aa494f06e"},"cell_type":"code","source":"trn_all_predictions = pd.DataFrame(list(d))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c6734bb30014c9fbe769b6498b4d9ca2c945368"},"cell_type":"code","source":"trn_all_predictions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"993cfcfb37ebaa5297d9618ed3ba183ab4a8f002"},"cell_type":"code","source":"meta_train = meta_train.set_index('object_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af554ee31ad61fde8e97ea7bea86aa9ae823faca"},"cell_type":"code","source":"x = x.join(meta_train,on='object_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbc715c16beff784985249d7f99ba2113b62cbef"},"cell_type":"code","source":"x = x.reset_index(drop=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f354fa2ed97121d34bab3b1528151b05f31ca78"},"cell_type":"code","source":"x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b085feff8d0725e05c1ab33ed8cf081c6369c13"},"cell_type":"code","source":"trn_all_predictions.insert(0,'passband',x.passband.ravel())\ntrn_all_predictions.insert(0,'object_id',x.object_id.ravel())\ntrn_all_predictions = trn_all_predictions.join(meta_train,on='object_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b02160a6e21f5b8d178059534f6501b8a69dbc1"},"cell_type":"code","source":"trn_all_predictions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"089c2f2968069d805a41e59b8aed7d25b33beede"},"cell_type":"code","source":"cols = []\nfor i in range(186):\n    cols.append('f'+str(i))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03e160f63bd8750f97e9ccad64e80001b0c09e9d"},"cell_type":"code","source":"f, ax = plt.subplots(6)\nf.set_figheight(15)\nf.set_figwidth(15)\nfor i in range(6):\n    ax[i].xaxis.set_ticks(np.arange(0, 186, 20))   \nax[0].plot(trn_all_predictions[(trn_all_predictions.passband==0)][cols].mean())\nax[1].plot(trn_all_predictions[(trn_all_predictions.passband==1)][cols].mean())\nax[2].plot(trn_all_predictions[(trn_all_predictions.passband==2)][cols].mean())\nax[3].plot(trn_all_predictions[(trn_all_predictions.passband==3)][cols].mean())\nax[4].plot(trn_all_predictions[(trn_all_predictions.passband==4)][cols].mean())\nax[5].plot(trn_all_predictions[(trn_all_predictions.passband==5)][cols].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"972a0b4bb5ad14a1a4bb8cccb7a41495e36e1693"},"cell_type":"code","source":"uniques = sorted(trn_all_predictions.target.unique())\nf, ax = plt.subplots(len(uniques),6)\nf.set_figheight(30)\nf.set_figwidth(15)\nfor a in range(len(uniques)):\n    for b in range(6):\n        ax[a,b].xaxis.set_ticks(np.arange(0, 186, 50))   \n   \nfor i in range(len(uniques)):\n    ax[i][0].plot(trn_all_predictions[(trn_all_predictions.target==uniques[i])&(trn_all_predictions.passband==0)][cols].mean()-trn_all_predictions[(trn_all_predictions.passband==0)][cols].mean())\n    ax[i][1].plot(trn_all_predictions[(trn_all_predictions.target==uniques[i])&(trn_all_predictions.passband==1)][cols].mean()-trn_all_predictions[(trn_all_predictions.passband==1)][cols].mean())\n    ax[i][2].plot(trn_all_predictions[(trn_all_predictions.target==uniques[i])&(trn_all_predictions.passband==2)][cols].mean()-trn_all_predictions[(trn_all_predictions.passband==2)][cols].mean())\n    ax[i][3].plot(trn_all_predictions[(trn_all_predictions.target==uniques[i])&(trn_all_predictions.passband==3)][cols].mean()-trn_all_predictions[(trn_all_predictions.passband==3)][cols].mean())\n    ax[i][4].plot(trn_all_predictions[(trn_all_predictions.target==uniques[i])&(trn_all_predictions.passband==4)][cols].mean()-trn_all_predictions[(trn_all_predictions.passband==4)][cols].mean())\n    ax[i][5].plot(trn_all_predictions[(trn_all_predictions.target==uniques[i])&(trn_all_predictions.passband==5)][cols].mean()-trn_all_predictions[(trn_all_predictions.passband==5)][cols].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"da19033b513be5b073405ffcf0dbe92d41144ae1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}