{"cells":[{"metadata":{"trusted":true,"_uuid":"38cefbe458cdbdb2ded6cf5737c49f2ff46339e1"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.preprocessing import sequence\nfrom keras.utils import to_categorical\nimport tensorflow as tf\nfrom keras import backend as K\nfrom keras.models import Sequential,Model\nfrom keras.layers import Dense,LSTM,Dropout,Masking\nfrom keras.callbacks import ModelCheckpoint\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport matplotlib.pyplot as plt\nimport gc\nimport os\nimport time\nfrom collections import Counter\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.preprocessing import sequence\nfrom keras.utils import to_categorical\nimport tensorflow as tf\nfrom keras import backend as K\nfrom keras.models import Sequential\nfrom keras.layers import Dense,LSTM,Dropout,Masking,concatenate,Input\nfrom keras.callbacks import ModelCheckpoint\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport matplotlib.pyplot as plt\nimport gc\nimport os\nimport time\nfrom collections import Counter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0fbd6527c62e305841cb83bd67542f637315411"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.preprocessing import sequence\nfrom keras.utils import to_categorical\nimport tensorflow as tf\nfrom keras import backend as K\nfrom keras.models import Sequential,Model\nfrom keras.layers import Dense,LSTM,Dropout,Masking\nfrom keras.callbacks import ModelCheckpoint\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport matplotlib.pyplot as plt\nimport gc\nimport os\nimport time\nfrom collections import Counter\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.preprocessing import sequence\nfrom keras.utils import to_categorical\nimport tensorflow as tf\nfrom keras import backend as K\nfrom keras.models import Sequential\nfrom keras.layers import Dense,LSTM,Dropout,Masking,concatenate,Input\nfrom keras.callbacks import ModelCheckpoint\n\nimport tsfresh\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport matplotlib.pyplot as plt\nimport gc\nimport os\nimport time\nfrom collections import Counter\n\ndef mywloss(y_true,y_pred):\n    yc=tf.clip_by_value(y_pred,1e-15,1-1e-15)\n    loss=-(tf.reduce_mean(tf.reduce_mean(y_true*tf.log(yc),axis=0)/wtable))\n    return loss\n\ndef multi_weighted_logloss(y_ohe, y_p):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set\n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss\n\ndef normalize(df):\n    df_mean = df.mean(axis=0)\n    df.fillna(df_mean, inplace=True)\n\n    df_new = df.copy()\n    ss = StandardScaler()\n    return ss.fit_transform(df_new)\n\ndef plot_loss_acc(history):\n    plt.plot(history.history['loss'][1:])\n    plt.plot(history.history['val_loss'][1:])\n    plt.title('model loss')\n    plt.ylabel('val_loss')\n    plt.xlabel('epoch')\n    plt.legend(['train','Validation'], loc='upper left')\n    plt.show()\n\n    plt.plot(history.history['acc'][1:])\n    plt.plot(history.history['val_acc'][1:])\n    plt.title('model Accuracy')\n    plt.ylabel('val_acc')\n    plt.xlabel('epoch')\n    plt.legend(['train','Validation'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n\ngc.enable()\ntrain = pd.read_csv('../input/training_set.csv')\nmeta_train = pd.read_csv('../input/training_set_metadata.csv')\nmeta_train.head()\n\nfull_train = train.reset_index().merge(\n    right=meta_train,\n    how='outer',\n    on='object_id'\n)\n\ndel train\ngc.collect()\n\nsplits = full_train[['object_id']].diff()[\"object_id\"].nonzero()[0]\noof_df = full_train[['object_id']].iloc[splits[1:]-1]\npassband = full_train[[\"passband\"]]\npassband = to_categorical(passband)\nmeta_train = full_train[meta_train.keys()].iloc[splits[1:]-1]\ndel meta_train['object_id'], meta_train['target']\n\nif 'target' in full_train:\n    y = full_train['target'][splits[1:] - 1]\n    del full_train['target']\nclasses = sorted(y.unique())\nclass_weight = {\n    c: 1 for c in classes\n}\nfor c in [64, 15]:\n    class_weight[c] = 2\n\nif 'object_id' in full_train:\n    del full_train['passband'], full_train['object_id'], full_train['distmod'], full_train['hostgal_specz'], full_train['hostgal_photoz'], full_train['hostgal_photoz_err'], full_train['mwebv']\n    del full_train['ra'], full_train['decl'], full_train['gal_l'],full_train['gal_b'],full_train['ddf']\n\nfull_train_ss = normalize(full_train)\ntrain_data2 = normalize(meta_train)\nsequence_length = max(np.diff(splits))\nfull_train_ss = np.concatenate([full_train_ss, passband], axis=1)\ntrain_data = np.split(full_train_ss, splits[1:-1])\ntrain_data = sequence.pad_sequences(train_data, maxlen=sequence_length, dtype=train_data[0].dtype)\n\nunique_y = np.unique(y)\nclass_map = dict()\nfor i,val in enumerate(unique_y):\n    class_map[val] = i\n\ny_map = np.zeros((y.shape[0],))\ny_map = np.array([class_map[val] for val in y])\ny_categorical = to_categorical(y_map)\n\ny_count = Counter(y_map)\nwtable = np.zeros((len(unique_y),))\nfor i in range(len(unique_y)):\n    wtable[i] = y_count[i]/y_map.shape[0]\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"K.clear_session()\n\ndef build_model(dropout_rate=.2, activation=\"relu\"):\n    input1 = Input(shape=(train_data.shape[-2], train_data.shape[-1]))\n    x1 = Masking(mask_value=0.)(input1)\n    x1 = LSTM(50, return_sequences=True)(x1)\n    x1 = LSTM(25)(x1)\n    x1 = Dense(30, activation=activation)(x1)\n    x1 = Dropout(dropout_rate)(x1)\n\n    input2 = Input(shape=(train_data2.shape[-1],))\n    x2 = Dense(30, activation=activation)(input2)\n    x2 = Dropout(dropout_rate)(x2)\n    x2 = Dense(30, activation=activation)(x2)\n    x2 = Dropout(dropout_rate)(x2)\n\n    x3 = concatenate([x1, x2])\n    x3 = Dense(30, activation=activation)(x3)\n    x3 = Dropout(dropout_rate)(x3)\n    x3 = Dense(30, activation=activation)(x3)\n    x3 = Dropout(dropout_rate)(x3)\n    x3 = Dense(len(classes), activation='softmax')(x3)\n\n    return Model(inputs=[input1, input2], output=x3)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"8798a82b126d921c84e72b7cd8ed4e2f6702400d"},"cell_type":"code","source":"K.clear_session()\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)\nclfs = []\noof_preds = np.zeros((len(oof_df), len(classes)))\nepochs = 600\nbatch_size = 10000\nstart_time = time.time()\nfor fold_, (trn_, val_) in enumerate(folds.split(y_map, y_map)):\n    checkPoint = ModelCheckpoint(\"./keras.model\",monitor='val_loss',mode = 'min', save_best_only=True, verbose=0)\n    x_train, x_train2, y_train = train_data[trn_], train_data2[trn_], y_categorical[trn_]\n    x_valid, x_valid2, y_valid = train_data[val_], train_data2[val_], y_categorical[val_]\n\n    model = build_model(dropout_rate=0.2,activation='relu')\n    model.compile(loss=mywloss, optimizer='adam', metrics=['accuracy'])\n    history = model.fit([x_train, x_train2], y_train,\n                    validation_data=[[x_valid, x_valid2], y_valid],\n                    epochs=epochs,\n                    batch_size=batch_size,shuffle=True,verbose=0,callbacks=[checkPoint])\n\n    plot_loss_acc(history)\n\n    print('Loading Best Model')\n    model.load_weights('./keras.model')\n    # # Get predicted probabilities for each class\n    oof_preds[val_, :] = model.predict([x_valid,x_valid2],batch_size=batch_size)\n    print(multi_weighted_logloss(y_valid, model.predict([x_valid,x_valid2],batch_size=batch_size)))\n    clfs.append(model)\n    break\n\nend_time = time.time()\nprint(\"Took %.5f minutes\" % ((end_time - start_time) / 60.0))\nprint('MULTI WEIGHTED LOG LOSS : %.5f ' % multi_weighted_logloss(y_categorical,oof_preds))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d91829829f5f7389d710f53c8f703bf16464299"},"cell_type":"code","source":"import tsfresh\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.ensemble import ExtraTreesClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9f57070ffcab1a3ba0f10eae044c36a54ac714e"},"cell_type":"code","source":"#gc.enable()\ntrain = pd.read_csv('../input/training_set.csv')\nmeta_train = pd.read_csv('../input/training_set_metadata.csv')\nmeta_train.head()\n\nfull_train = train.reset_index().merge(\n    right=meta_train,\n    how='outer',\n    on='object_id'\n)\n\n#del train\n#gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59a08310ef7df014175021c27d00748bc2b524b3","scrolled":true},"cell_type":"code","source":"enc = OneHotEncoder()\nmeta_train['target'] = meta_train['target'].astype(\"object\")\nenc.fit(meta_train[[\"target\"]])\ntarget = meta_train[[\"object_id\", \"target\"]]\ntarget.index = target['object_id']\nnew_features = None\nfor i in [0,1,2,3,4,5]:\n    passband_train = train[train['passband'] == i]\n    features = tsfresh.extract_relevant_features(\n        passband_train,\n        target.loc[passband_train[\"object_id\"].unique()][\"target\"],\n        column_id=\"object_id\", \n        column_sort=\"mjd\",\n        column_value=\"flux\"\n    )\n\n    clf = ExtraTreesClassifier(n_estimators=50)\n    clf = clf.fit(features, target.loc[features.index][[\"target\"]].astype(\"str\"))\n\n    model = SelectFromModel(clf, prefit=True, max_features=20)\n    features = features[features.columns[model.get_support()]]\n    if new_features is not None:\n        new_features = pd.concat([new_features, features], axis=1)\n    else:\n        new_features = features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18fa1545b4f48f48641222fef4f3dd6300f9acf0"},"cell_type":"code","source":"new_features.to_csv(\"new_features.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"978f2244908b62f67b905b7f3a276f53ef096cb9"},"cell_type":"code","source":"new_features.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3955294eeae15efa07d75578ee8315946fa862b0"},"cell_type":"code","source":"def multi_weighted_logloss(y_true, y_preds, classes, class_weights):\n    \"\"\"\n    refactor from\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')\n    # Trasform y_true in dummies\n    y_ohe = pd.get_dummies(y_true)\n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)\n    # Transform to log\n    y_p_log = np.log(y_p)\n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set\n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)\n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).values.astype(float)\n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])\n    y_w = y_log_ones * class_arr / nb_pos\n\n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss\n\n\ndef lgbm_multi_weighted_logloss(y_true, y_preds):\n    \"\"\"\n    refactor from\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"  \n    # Taken from Giba's topic : https://www.kaggle.com/titericz\n    # https://www.kaggle.com/c/PLAsTiCC-2018/discussion/67194\n    # with Kyle Boone's post https://www.kaggle.com/kyleboone\n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n    class_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n\n    loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)\n    return 'wloss', loss, False\n\n\ndef xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):\n    loss = multi_weighted_logloss(y_true.get_label(), y_predicted, \n                                  classes, class_weights)\n    return 'wloss', loss\n\n\ndef save_importances(importances_):\n    mean_gain = importances_[['gain', 'feature']].groupby('feature').mean()\n    importances_['mean_gain'] = importances_['feature'].map(mean_gain['gain'])\n    return importances_\n\n\ndef xgb_modeling_cross_validation(params,\n                                  full_train, \n                                  y, \n                                  classes, \n                                  class_weights, \n                                  nr_fold=5, \n                                  random_state=1):\n    # Compute weights\n    w = y.value_counts()\n    weights = {i : np.sum(w) / w[i] for i in w.index}\n\n    # loss function\n    func_loss = partial(xgb_multi_weighted_logloss, \n                        classes=classes, \n                        class_weights=class_weights)\n\n    clfs = []\n    importances = pd.DataFrame()\n    folds = StratifiedKFold(n_splits=nr_fold, \n                            shuffle=True, \n                            random_state=random_state)\n    \n    oof_preds = np.zeros((len(full_train), np.unique(y).shape[0]))\n    for fold_, (trn_, val_) in enumerate(folds.split(y, y)):\n        trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]\n        val_x, val_y = full_train.iloc[val_], y.iloc[val_]\n    \n        clf = XGBClassifier(**params)\n        clf.fit(\n            trn_x, trn_y,\n            eval_set=[(trn_x, trn_y), (val_x, val_y)],\n            eval_metric=func_loss,\n            verbose=100,\n            early_stopping_rounds=50,\n            sample_weight=trn_y.map(weights)\n        )\n        clfs.append(clf)\n\n        oof_preds[val_, :] = clf.predict_proba(val_x, ntree_limit=clf.best_ntree_limit)\n        print('no {}-fold loss: {}'.format(fold_ + 1, \n              multi_weighted_logloss(val_y, oof_preds[val_, :], \n                                     classes, class_weights)))\n    \n        imp_df = pd.DataFrame({\n                'feature': full_train.columns,\n                'gain': clf.feature_importances_,\n                'fold': [fold_ + 1] * len(full_train.columns),\n                })\n        importances = pd.concat([importances, imp_df], axis=0, sort=False)\n\n    score = multi_weighted_logloss(y_true=y, y_preds=oof_preds, \n                                   classes=classes, class_weights=class_weights)\n    print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))\n    df_importances = save_importances(importances_=importances)\n    df_importances.to_csv('xgb_importances.csv', index=False)\n    \n    return clfs, score\n\n\ndef lgbm_modeling_cross_validation(params,\n                                   full_train, \n                                   y, \n                                   classes, \n                                   class_weights, \n                                   nr_fold=5, \n                                   random_state=1):\n\n    # Compute weights\n    w = y.value_counts()\n    weights = {i : np.sum(w) / w[i] for i in w.index}\n\n    clfs = []\n    importances = pd.DataFrame()\n    folds = StratifiedKFold(n_splits=nr_fold, \n                            shuffle=True, \n                            random_state=random_state)\n    \n    oof_preds = np.zeros((len(full_train), np.unique(y).shape[0]))\n    for fold_, (trn_, val_) in enumerate(folds.split(y, y)):\n        trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]\n        val_x, val_y = full_train.iloc[val_], y.iloc[val_]\n    \n        clf = LGBMClassifier(**params)\n        clf.fit(\n            trn_x, trn_y,\n            eval_set=[(trn_x, trn_y), (val_x, val_y)],\n            eval_metric=lgbm_multi_weighted_logloss,\n            verbose=100,\n            early_stopping_rounds=50,\n            sample_weight=trn_y.map(weights)\n        )\n        clfs.append(clf)\n\n        oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)\n        print('no {}-fold loss: {}'.format(fold_ + 1, \n              multi_weighted_logloss(val_y, oof_preds[val_, :], \n                                     classes, class_weights)))\n    \n        imp_df = pd.DataFrame({\n                'feature': full_train.columns,\n                'gain': clf.feature_importances_,\n                'fold': [fold_ + 1] * len(full_train.columns),\n                })\n        importances = pd.concat([importances, imp_df], axis=0, sort=False)\n\n    score = multi_weighted_logloss(y_true=y, y_preds=oof_preds, \n                                   classes=classes, class_weights=class_weights)\n    print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))\n    df_importances = save_importances(importances_=importances)\n    df_importances.to_csv('lgbm_importances.csv', index=False)\n    \n    return clfs, score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45cd100b34857599c11b1c333e7d0d8965df3b5b"},"cell_type":"code","source":"   best_params = {\n            'device': 'cpu', \n            'objective': 'multiclass', \n            'num_class': 14, \n            'boosting_type': 'gbdt', \n            'n_jobs': -1, \n            'max_depth': 7, \n            'n_estimators': 500, \n            'subsample_freq': 2, \n            'subsample_for_bin': 5000, \n            'min_data_per_group': 100, \n            'max_cat_to_onehot': 4, \n            'cat_l2': 1.0, \n            'cat_smooth': 59.5, \n            'max_cat_threshold': 32, \n            'metric_freq': 10, \n            'verbosity': -1, \n            'metric': 'multi_logloss', \n            'xgboost_dart_mode': False, \n            'uniform_drop': False, \n            'colsample_bytree': 0.5, \n            'drop_rate': 0.173, \n            'learning_rate': 0.0267, \n            'max_drop': 5, \n            'min_child_samples': 10, \n            'min_child_weight': 100.0, \n            'min_split_gain': 0.1, \n            'num_leaves': 7, \n            'reg_alpha': 0.1, \n            'reg_lambda': 0.00023, \n            'skip_drop': 0.44, \n            'subsample': 0.75}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38b03ec473dcb1fd75abe96ae079643f3006ee30","scrolled":true},"cell_type":"code","source":"from lightgbm import LGBMClassifier\nclasses = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\nclass_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n\nfull_features = pd.merge(new_features, meta_train, left_on=new_features.index, right_on=\"object_id\")\nfull_features = full_features.drop(\"target\", axis=1)\nlgbm_modeling_cross_validation(best_params,\n                                   full_features, \n                                   target[\"target\"].astype(\"str\"), \n                                   classes, \n                                   class_weights, \n                                   nr_fold=5, \n                                   random_state=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"21a76311af2c632f06e6575df393e9d1ea6c6f94"},"cell_type":"code","source":"y = target[\"target\"].astype(\"str\")\nw = y.value_counts()\nweights = {i : np.sum(w) / w[i] for i in w.index}\n\nclf = LGBMClassifier(**best_params)\nclf.fit(\n    full_features, y,\n    #eval_set=[(trn_x, trn_y), (val_x, val_y)],\n    eval_metric=lgbm_multi_weighted_logloss,\n    verbose=100,\n    #early_stopping_rounds=50,\n    sample_weight=y.map(weights)\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf830a1d106f840ffb2744c5020be7ee5fb2dd57"},"cell_type":"code","source":"meta_test = pd.read_csv(\"../input/PLAsTiCC-2018/test_set_metadata.csv\")\ntry:\n    os.remove('predictions.csv')\nexcept FileNotFoundError:\n    pass\nchunks = 5000000\nfor i_c, test in enumerate(pd.read_csv(\"../input/PLAsTiCC-2018/test_set.csv\", chunksize=chunks, iterator=True)):\n    test_features = None\n    for i in [0,1,2,3,4,5]:\n        x = tsfresh.feature_extraction.settings.from_columns(new_features.columns[i*20+1:(i+1)*20+1])\n        passband_test = test[test[\"passband\"] == i]\n        t_features = tsfresh.extract_features(\n                passband_test,\n                #test_target.loc[passband_train[\"object_id\"].unique()][\"target\"],\n                column_id=\"object_id\", \n                column_sort=\"mjd\",\n                column_value=\"flux\",\n                kind_to_fc_parameters=x\n            )\n\n        if test_features is not None:\n            test_features = pd.concat([test_features, t_features], axis=1)\n        else:\n            test_features = t_features\n\n    test_full_features = pd.merge(test_features, meta_test, left_on=new_features.index, right_on=\"object_id\")\n    preds_df = clf.predict(test_full_features)\n    preds_df.to_csv('predictions.csv',  header=True, mode='a', index=False)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17628f2ba6d57ed76d77759b2a6d6d36a92f6cdc"},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01ed54035b2a5b3a17f8aa52fd841b86f63e9489"},"cell_type":"code","source":"#test = pd.read_csv(\"../input/test_set.csv\")\nmeta_test = pd.read_csv(\"../input/PLAsTiCC-2018/test_set_metadata.csv\")\nchunks = 5000000\nfor i_c, test in enumerate(pd.read_csv('../input/PLAsTiCC-2018/test_set.csv', chunksize=chunks, iterator=True)):\n    \n    test_features = None\n    for i in [0,1,2,3,4,5]:\n        x = tsfresh.feature_extraction.settings.from_columns(new_features.columns[i*20+1:(i+1)*20+1])\n        passband_test = test[test[\"passband\"] == i]\n        t_features = tsfresh.extract_features(\n                passband_test,\n                test_target.loc[passband_train[\"object_id\"].unique()][\"target\"],\n                column_id=\"object_id\", \n                column_sort=\"mjd\",\n                column_value=\"flux\",\n                kind_to_fc_parameters=x\n            )\n\n        if test_features is not None:\n            test_features = pd.concat([test_features, t_features], axis=1)\n        else:\n            test_features = t_features\n        \n        test_full_features = pd.merge(test_features, meta_test, left_on=new_features.index, right_on=\"object_id\")\n        test_full_features.to_csv('test_features.csv',  header=True, mode='a', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67e7cc5c2289bef29b7d9f2b19f2fc91e78e66da"},"cell_type":"code","source":"import tsfresh\nimport os\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"093109c96ef47a886cfdc387997226be358e7477"},"cell_type":"code","source":"new_features = pd.read_csv('../input/new-features/new_features.csv')\nmeta_train = pd.read_csv('../input/PLAsTiCC-2018/training_set_metadata.csv')\nenc = OneHotEncoder()\nmeta_train['target'] = meta_train['target'].astype(\"object\")\nenc.fit(meta_train[[\"target\"]])\ntarget = meta_train[[\"object_id\", \"target\"]]\ntarget.index = target['object_id']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e1034d16e31fb515775064200c0d5fed315b713"},"cell_type":"code","source":"x = tsfresh.feature_extraction.settings.from_columns(new_features.columns[i*20+1:(i+1)*20+1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"949aeaab3ffc844ad1246a215ddbd666efea908a"},"cell_type":"code","source":"new_features.columns[i*20+1:(i+1)*20+1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0460e3396d6466097d5119738d664544439464ba"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}