{"cells":[{"metadata":{"_uuid":"609006646970a83c89ff8a332459ddceb2e02e4c"},"cell_type":"markdown","source":"# PyTorch Net with Competition Loss"},{"metadata":{"_uuid":"7f9c2d449672e9f7fc453c2cc20be3fa2f4dfc2a"},"cell_type":"markdown","source":"### Sources\n- https://www.kaggle.com/meaninglesslives/simple-neural-net-for-time-series-classification\n- https://www.kaggle.com/cttsai/forked-lgbm-w-ideas-from-kernels-and-discuss\n- https://www.kaggle.com/mithrillion/know-your-objective\n- https://www.kaggle.com/ogrellier/plasticc-in-a-kernel-meta-and-data/code\n\nLet me know if I have missed anyone."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport gc\nimport os\nfrom collections import Counter\nfrom datetime import datetime as dt\nimport time\nfrom numba import jit\n\n# Plots\nimport matplotlib.pyplot as plt\nimport seaborn as sns \n\n# Sklearn\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import confusion_matrix\n\n# PyTorch\nimport torch\nfrom torch import autograd, nn, optim\nfrom torch.autograd import grad\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\n# TSFresh\nfrom tsfresh.feature_extraction import extract_features\n\n# Path to data directory\ndata_dir = '../input'\n\ntrain = pd.read_csv(data_dir + '/training_set.csv')\ntrain.head()\n\n@jit\ndef haversine_plus(lon1, lat1, lon2, lat2):\n    \"\"\"\n    Calculate the great circle distance between two points \n    on the earth (specified in decimal degrees) from \n    #https://stackoverflow.com/questions/4913349/haversine-formula-in-python-bearing-and-distance-between-two-gps-points\n    \"\"\"\n    # Convert decimal degrees to Radians:\n    lon1 = np.radians(lon1)\n    lat1 = np.radians(lat1)\n    lon2 = np.radians(lon2)\n    lat2 = np.radians(lat2)\n\n    # Implementing Haversine Formula: \n    dlon = np.subtract(lon2, lon1)\n    dlat = np.subtract(lat2, lat1)\n\n    a = np.add(np.power(np.sin(np.divide(dlat, 2)), 2),  \n                          np.multiply(np.cos(lat1), \n                                      np.multiply(np.cos(lat2), \n                                                  np.power(np.sin(np.divide(dlon, 2)), 2))))\n    \n    haversine = np.multiply(2, np.arcsin(np.sqrt(a)))\n    return {\n        'haversine': haversine, \n        'latlon1': np.subtract(np.multiply(lon1, lat1), np.multiply(lon2, lat2)), \n   }\n\n# TSFresh features\nfcp = {\n    'flux': {\n        'longest_strike_above_mean': None,\n        'longest_strike_below_mean': None,\n        'mean_change': None,\n        'mean_abs_change': None,\n        'length': None\n    },\n\n    'flux_by_flux_ratio_sq': {\n        'longest_strike_above_mean': None,\n        'longest_strike_below_mean': None,      \n    },\n\n    'flux_passband': {\n        'kurtosis' : None,\n        'skewness' : None,\n        'standard_deviation': None\n    },\n    'fluxerr_passband': {\n        'kurtosis' : None,\n        'skewness' : None,\n        'standard_deviation': None\n    },\n    'mjd': {\n        'maximum': None, \n        'minimum': None,\n        'mean_change': None,\n        'mean_abs_change': None,\n    },\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7dbccfa992f29644a47a31341b2c68c6b42b835d"},"cell_type":"markdown","source":"## Extracting Features Meta and flux"},{"metadata":{"trusted":true,"_uuid":"e0e09c98ad0ede4b6ca094466a28928f78a3d5e2"},"cell_type":"code","source":"def process_meta(filename):\n    meta_df = pd.read_csv(filename)\n    \n    meta_dict = dict()\n    \n    # Distance\n    meta_dict.update(haversine_plus(meta_df['ra'].values, meta_df['decl'].values, \n                   meta_df['gal_l'].values, meta_df['gal_b'].values))\n    \n    meta_dict['hostgal_photoz_certain'] = np.multiply(\n            meta_df['hostgal_photoz'].values, \n             np.exp(meta_df['hostgal_photoz_err'].values))\n    \n    meta_df = pd.concat([meta_df, pd.DataFrame(meta_dict, index=meta_df.index)], axis=1)\n    return meta_df\n\ndef featurize(df, df_meta, fcp, n_jobs=4):\n    \"\"\"\n    Extracting Features from train or test set\n    Features from olivier's kernel\n    very smart and powerful feature that is generously given here https://www.kaggle.com/c/PLAsTiCC-2018/discussion/69696#410538\n    per passband features with tsfresh library. fft features added to capture periodicity https://www.kaggle.com/c/PLAsTiCC-2018/discussion/70346#415506\n    \"\"\"\n    \n    # Add more features with\n    agg_df_ts_flux_passband = extract_features(df, \n                                               column_id='object_id', \n                                               column_sort='mjd', \n                                               column_kind='passband', \n                                               column_value='flux', \n                                               default_fc_parameters=fcp['flux_passband'], n_jobs=n_jobs)\n\n    agg_df_ts_fluxerr_passband = extract_features(df, \n                                               column_id='object_id', \n                                               column_sort='mjd', \n                                               column_kind='passband', \n                                               column_value='flux_err', \n                                               default_fc_parameters=fcp['flux_passband'], n_jobs=n_jobs)\n\n    # Add smart feature that is suggested here https://www.kaggle.com/c/PLAsTiCC-2018/discussion/69696#410538\n    # dt[detected==1, mjd_diff:=max(mjd)-min(mjd), by=object_id]\n    df_det = df[df['detected']==1].copy()\n    agg_df_mjd = extract_features(df_det, \n                                  column_id='object_id', \n                                  column_value='mjd',\n                                  default_fc_parameters=fcp['mjd'], n_jobs=n_jobs)\n    agg_df_mjd['mjd_diff_det'] = agg_df_mjd['mjd__maximum'].values - agg_df_mjd['mjd__minimum'].values\n    del agg_df_mjd['mjd__maximum'], agg_df_mjd['mjd__minimum']\n    \n    # Rename the index for merging with metadata (tsfresh renames the index)\n    agg_df_ts_flux_passband.index.rename('object_id', inplace=True) \n    agg_df_ts_fluxerr_passband.index.rename('object_id', inplace=True)\n    agg_df_mjd.index.rename('object_id', inplace=True)\n\n      \n    agg_df_ts = pd.concat([\n                           agg_df_ts_flux_passband, \n                           agg_df_ts_fluxerr_passband,\n                            agg_df_mjd\n                          ], axis=1).reset_index()\n\n    result = agg_df_ts.merge(right=df_meta, how='left', on='object_id')\n    return result","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ed21e25fb5678c78bb19a8297bf83db00ccae01"},"cell_type":"markdown","source":"## Merging extracted features with meta data"},{"metadata":{"_uuid":"696ee870837c23375bc7f0388de1b07510351123","trusted":true},"cell_type":"code","source":"meta_train = process_meta(data_dir + '/training_set_metadata.csv')\nfull_train = featurize(train, meta_train, fcp, n_jobs=6)\n\nif 'target' in full_train:\n    y = full_train['target']\n    del full_train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d98a75e19b7e702b0c8848ff3538516e21fd14a7"},"cell_type":"code","source":"full_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b6f79b71d0e9b62266a52d9e7ac209c56e14aff","trusted":true},"cell_type":"code","source":"if 'object_id' in full_train:\n    oof_df = full_train[['object_id']]\n    del full_train['object_id'], full_train['hostgal_specz'],  full_train['hostgal_photoz']\n    del full_train['ra'], full_train['decl'], full_train['gal_l'],full_train['gal_b'],full_train['ddf']\n    \ntrain_mean = full_train.mean(axis=0)\npd.set_option('display.max_rows', 500)\n\nfull_train.fillna(train_mean, inplace=True)\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7aaadd7fb68f26be1aa78e0721332689365f2322"},"cell_type":"markdown","source":"### Standard Scaling the input"},{"metadata":{"trusted":true,"_uuid":"6f78d821b79955c58780e27de4c621b895d6276c"},"cell_type":"code","source":"ss = StandardScaler()\n\n# @todo figure out something to replace this\nfull_train_values = np.nan_to_num(full_train.values)\nfull_train_ss = ss.fit_transform(full_train_values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb9ca16c7a5acea826e03e9a0a9fc0f1651660e4"},"cell_type":"code","source":"del full_train_values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba205f31590b109b05bcccfb0a1719ab370332b2"},"cell_type":"code","source":"def to_categorical(y, num_classes):\n    \"\"\" 1-hot encodes a tensor \"\"\"\n    return np.eye(num_classes, dtype='uint8')[y]\n\n# Get unique target classes\nunique_y = np.unique(y)\nclass_map = dict()\nfor i,val in enumerate(unique_y):\n    class_map[val] = i\n\ny_map = np.zeros((y.shape[0],))\ny_map = np.array([class_map[val] for val in y])\ny_categorical = to_categorical(y_map, len(unique_y))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e482d999e397eb13d5407dbe8f4601021b3cdb95"},"cell_type":"markdown","source":"# Model and Dataset\nGPU kernel's do not support custom packages so need to train on cpu on here. The model is not complex and training on cpu is not terribly painful.\n"},{"metadata":{"trusted":true,"_uuid":"188c1d69ea964d522ccd88bde57186aad72c0cff"},"cell_type":"code","source":"#device = 'cuda'\ndevice = 'cpu'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd813cf6349490ba472b82017747b716f28079be"},"cell_type":"code","source":"class PlasticcNet(nn.Module):\n    def __init__(self, input_size, hidden_size, num_classes):\n        \"\"\"\n        Initialize the class and call the parent init method.\n        Create two hidden layers. Derived classes from nn.Module\n        will assign learnable params to self.parameters\n        \"\"\"\n        super(PlasticcNet, self).__init__()\n        \n        dropout_proba = 0.6\n        bias = False\n        \n        \n        # define hidden linear layers, with optional batch norm on their outputs\n        # layers with batch_norm applied have no bias term\n        self.dense1 = nn.Linear(input_size, hidden_size, bias=bias)\n        \n        # It seems that even Christian Szegedy now likes to perform BatchNorm after the ReLU (not before it). \n        # Quote by F. Chollet, the author of Keras: \"I haven't gone back to check what they are suggesting \n        # in their original paper, but I can guarantee that recent code written by Christian \n        # applies relu before BN. It is still occasionally a topic of debate, though.\"\n        self.batch1 = nn.BatchNorm1d(hidden_size)\n        \n        self.dense2 = nn.Linear(hidden_size, hidden_size, bias=bias)\n        self.batch2 = nn.BatchNorm1d(hidden_size)\n        \n        self.dense3 = nn.Linear(hidden_size, hidden_size, bias=bias)\n        self.batch3 = nn.BatchNorm1d(hidden_size)\n        \n        self.dense4 = nn.Linear(hidden_size, hidden_size, bias=bias)\n        self.batch4 = nn.BatchNorm1d(hidden_size)\n        \n        self.dropout = nn.Dropout(p=dropout_proba)\n        \n        self.dense5 = nn.Linear(hidden_size, num_classes)\n        \n    def forward(self, x):\n        \"\"\"\n        Feed data forward through the layers.\n        \"\"\"\n        x = self.dense1(x)\n        x = self.batch1(x)\n        x = self.dropout(torch.relu(x))\n        \n        x = self.dense2(x)\n        x = self.batch2(x)\n        x = self.dropout(torch.relu(x))\n        \n        x = self.dense3(x)\n        x = self.batch3(x)\n        x = self.dropout(torch.relu(x))\n        \n        x = self.dense4(x)\n        x = self.batch4(x)\n        x = self.dropout(torch.relu(x))\n        \n        x = self.dense5(x)\n        x = F.log_softmax(x, dim=1)\n        return x\n    \nclass PlasticcDataset(Dataset):\n    def __init__(self, x, y_categorical, mode='train'):\n        self.x = x\n        self.y_categorical = y_categorical\n        self.mode = mode\n    def __len__(self):\n        return len(self.x)\n    \n    def __getitem__(self, index):\n        x, y = self.x[index], self.y_categorical[index]\n        if self.mode == 'validate':\n            x = torch.tensor(x, dtype=torch.float32, requires_grad=True).to(device)\n            y = torch.tensor(y, dtype=torch.int64).to(device)\n            return x, y\n        else:\n            x = torch.tensor(x, dtype=torch.float32).to(device)\n            y = torch.tensor(y, dtype=torch.int64).to(device)\n            return x, y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b915ef7ac26164d0b83e265a277ea50e8557eac"},"cell_type":"markdown","source":"### The Weights"},{"metadata":{"trusted":true,"_uuid":"b3d9e33f20f3874ec01db24b51638b7ea28986c7"},"cell_type":"code","source":"num_classes = len(unique_y)\nclasses = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\nclass_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1,\n                64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\nweight_tensor = torch.tensor(list(class_weight.values()),\n                             requires_grad=False, dtype=torch.float32).to(device)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3e1ab5e28ff7a3d3922c4a8f3bb926ef4362df5"},"cell_type":"markdown","source":"### Olivier's implementation of multi weighted log loss\nThis works well as an evalation metric but can't be used as a PyTorch loss function"},{"metadata":{"trusted":true,"_uuid":"524ea9d7b20aaffbb1e540f5347cdf9402fe54ac"},"cell_type":"code","source":"def multi_weighted_logloss(y_ohe, y_p, class_weight):\n    \"\"\"\n    @author olivier https://www.kaggle.com/ogrellier\n    multi logloss for PLAsTiCC challenge\n    \"\"\"\n    #class_weight = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}\n    \n    # Normalize rows and limit y_preds to 1e-15, 1-1e-15\n    y_p = np.clip(a=y_p, a_min=1e-15, a_max=1-1e-15)\n    \n    # Transform to log\n    y_p_log = np.log(y_p)\n    \n    # Get the log for ones, .values is used to drop the index of DataFrames\n    # Exclude class 99 for now, since there is no class99 in the training set \n    # we gave a special process for that class\n    y_log_ones = np.sum(y_ohe * y_p_log, axis=0)\n    \n    # Get the number of positives for each class\n    nb_pos = y_ohe.sum(axis=0).astype(float)\n    \n    # Weight average and divide by the number of positives\n    class_arr = np.array([class_weight[k] for k in sorted(class_weight.keys())])\n    y_w = y_log_ones * class_arr / nb_pos    \n    loss = - np.sum(y_w) / np.sum(class_arr)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7018d396fa01fcb3663cc60c73a41662141d87f1"},"cell_type":"markdown","source":"## PyTorch competition loss function\nI'm not 100% sure if this correct but it works and gives the same output as Olivier's function above"},{"metadata":{"trusted":true,"_uuid":"1e58d5899488d8932cd4dd553f93e9f91a664c4f"},"cell_type":"code","source":"def criterion(ln_p, y_h, wt):\n    \"\"\"\n    Modified implementation of wloss_objective competiton weighted log loss\n    from @mithrillion Know Your Objective kernel https://www.kaggle.com/mithrillion/know-your-objective\n    Main changes:\n        - Moved softmax back to net and take log probs as first arg\n        - Second arg is one hot matrix of target classes so we don't need to call scatter to expand the labels\n        back into a one hot matrix\n        - Divide everything by the sum of the weights as is shown in competition evaluation equation. Olivier's\n        function also does this\n        - Removed gradient computations as PyTorch loss function does not need them\n    \"\"\"\n    y_h = y_h.float()\n    ys = y_h.sum(dim=0, keepdim=True)\n    \n    # Replace 0's with 1's so we don't get nans from division by 0\n    ys[ys==0] = 1\n    y_h = y_h / ys\n    wll = torch.sum(y_h * ln_p, dim=0)\n    wsum = torch.sum(wt, dim=0)\n    loss = -(torch.dot(wt, wll) / wsum)\n    return loss\n\ndef batch_accuracy(probs, labels):\n    \"\"\"\n    Batch accuracy from https://github.com/udacity/deep-learning-v2-pytorch/blob/master/intro-to-pytorch\n    part 5\n    \"\"\"\n    top_p_validate, top_class_validate = probs.topk(1, dim=1)\n    equals_validate = top_class_validate == labels.view(*top_class_validate.shape)\n    return torch.mean(equals_validate.type(torch.FloatTensor))\n\ndef check_probs(probs):\n    \"\"\"\n    Check that predictions sum to 1\n    \"\"\"\n    preds_batch_sum = torch.sum(torch.sum(probs, dim=1), dim=0).item()\n    sum_diff = (np.absolute(batch_size - preds_batch_sum))\n    if (sum_diff > 0.1) and (preds_batch_sum != probs.shape[0]):\n\n        # Should be the same size as the number of samples\n        print(\"preds_sum: \", preds_batch_sum)\n\ndef fold_validation(model, xv, yv):\n    \"\"\"\n    Validate on all samples in current fold validate split\n    \"\"\"\n    x_valid_mwll = torch.tensor(xv, dtype=torch.float32, requires_grad=False).to(device)\n    y_valid_mwll = torch.tensor(yv, dtype=torch.int64).to(device)\n    olivier_loss = multi_weighted_logloss(yv, torch.exp(model(x_valid_mwll)).cpu().numpy(), class_weight)\n    pytorch_loss = criterion(model(x_valid_mwll), y_valid_mwll, weight_tensor).cpu().item()\n    print('Olivier loss: {:.6f}'.format(olivier_loss))\n    print('PyTorch criterion loss: {:.6f}'.format(pytorch_loss))\n    return pytorch_loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bea0a03720fea2ef4395056708c7c50879afa578"},"cell_type":"markdown","source":"## Train the model"},{"metadata":{"trusted":true,"_uuid":"c9d7fb069d1f8b45ff7c11f9d2ba5093ff09e4de"},"cell_type":"code","source":"hidden_size = 256\nbatch_size = 100\nepochs = 100\n\nbest_model = {\n    'epoch': 0,\n    'loss': np.Inf,\n    'fold': 0\n}\n\nfor fold_, (trn_, val_) in enumerate(folds.split(y_map, y_map)):\n    x_train, y_train = full_train_ss[trn_], y_categorical[trn_]\n    x_valid, y_valid = full_train_ss[val_], y_categorical[val_]\n    \n    # Create datasets (Make sure trn_ and val_ indices are correct)\n    ds_train = PlasticcDataset(full_train_ss[trn_], y_categorical[trn_], mode='train')\n    ds_val = PlasticcDataset(full_train_ss[val_], y_categorical[val_], mode='validate')\n    \n    # Create dataloaders\n    train_loader = DataLoader(ds_train, batch_size=batch_size, shuffle=False)\n    validate_loader = DataLoader(ds_val, batch_size=batch_size, shuffle=False)\n    \n    # Create a new model instance\n    model = PlasticcNet(full_train_ss.shape[1], hidden_size, num_classes)\n    \n    # Run model on gpu\n    model.to(device)\n    \n    # Create optimizer \n    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n    \n    train_losses = []\n    validate_losses = []\n    validate_accuracy = []\n    \n    # Initialize tracker for minimum validation loss\n    # Set initial min to infinity because any loss we get will be less than infinity\n    validate_loss_min = np.Inf\n    \n    # Epochs loop\n    for e in range(epochs):\n        running_train_loss = 0\n        running_validate_accuracy = 0\n        running_validate_loss = 0\n        all_validate_probs = []\n        \n        # Switch to train mode\n        model.train()\n        for train_bx, train_by in train_loader:\n            log_probs = model(train_bx)\n            loss = criterion(log_probs, train_by, weight_tensor)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            running_train_loss += loss.item()\n            \n        ##### Validation (After each epoch) #####\n        # Turn off gradients for validation, saves memory and computations\n        with torch.no_grad():\n            \n            # Switch to eval mode\n            model.eval()\n            \n            for validate_bx, validate_by in validate_loader:\n                validate_log_probs = model(validate_bx)\n                validate_probs = torch.exp(validate_log_probs)\n                    \n                # Check the sum of the predicted class probabilities\n                check_probs(validate_probs)\n                    \n                loss = criterion(validate_log_probs, validate_by, weight_tensor)\n                \n                # Get target class index from one hot vector\n                validate_labels = torch.argmax(validate_by, 1)\n                \n                # Sum batch loss and accuracy\n                running_validate_loss += loss.item()\n                running_validate_accuracy += batch_accuracy(validate_probs, validate_labels)\n        \n            # Validate on all samples in current fold validate split\n            pytorch_loss = fold_validation(model, x_valid, y_valid)\n            \n        vln = len(validate_loader)\n        tln = len(train_loader)\n        train_loss = (running_train_loss / tln)\n        validate_loss = (running_validate_loss / vln)\n        validate_accuracy = (running_validate_accuracy / vln)\n        \n        train_losses.append(train_loss)\n        validate_losses.append(validate_loss)\n\n        print(\"Epoch: {}/{}.. \".format(e+1, epochs),\n              \"Train Avg Batch Loss: {:.4f}.. \".format(train_loss),\n              \"Val Avg Batch Loss: {:.4f}.. \".format(validate_loss),\n              \"Val Avg Batch Accuracy: {:.4f}\".format(validate_accuracy))\n        \n        # save model if validation loss has decreased\n        if pytorch_loss <= validate_loss_min:\n            print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n            validate_loss_min,\n            validate_loss))\n            torch.save(model.state_dict(), \"pt_fold{}_model.pt\".format(fold_))\n            validate_loss_min = pytorch_loss\n            \n        # Record best model across all folds\n        if pytorch_loss <= best_model['loss']:\n            best_model['loss'] = pytorch_loss\n            best_model['epoch'] = e\n            best_model['fold'] = fold_\n            print(\"best_model updated: \", best_model)\n            \n    plt.plot(train_losses, label='Training loss')\n    plt.plot(validate_losses, label='Validation loss')\n    plt.title('model loss')\n    plt.xlabel('epoch')\n    plt.ylabel('loss')\n    plt.legend(['train','Validation'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f65492856178f1692de22989481d7dde8afd1e84"},"cell_type":"markdown","source":"## Process test set"},{"metadata":{"trusted":true,"_uuid":"8b726fec9a6ecfa9343e82745c908000e82f2664"},"cell_type":"code","source":"# Get the column names from the sample submission\nsample_sub = pd.read_csv(data_dir + '/sample_submission.csv')\nclass_names = list(sample_sub.columns[1:-1])\ndel sample_sub;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5689da7a2175f2cbeb15c950758916cba3c07ec2"},"cell_type":"code","source":"# meta_test is 285M so we can load that into memory in one go\nmeta_test = pd.read_csv(data_dir + '/test_set_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a3b84b5aef94dd9e0dbb0a983b466c89ff44a48"},"cell_type":"code","source":"def predict_chunk(df_, model_eval_, ss_, meta_, features, featurize_configs, train_mean):\n    \n    # Process all features\n    full_test = featurize(df_, meta_, featurize_configs['fcp'])\n    full_test.fillna(train_mean, inplace=True)\n    \n    partial_test = full_test.copy()\n    \n    # Delete unused features\n    if 'object_id' in full_test:\n        del full_test['object_id'], full_test['hostgal_specz'],  full_test['hostgal_photoz']\n        del full_test['ra'], full_test['decl'], full_test['gal_l'],full_test['gal_b'],full_test['ddf']\n\n    test_chunk_values = np.nan_to_num(full_test.values)\n    \n    # Only do transform on test set, scaler paramters have been learned\n    # on the train set\n    test_chunk_ss = ss.transform(test_chunk_values)\n    x_test = torch.tensor(test_chunk_ss, dtype=torch.float32).to('cuda')\n\n    test_log_probs = model_eval(x_test)\n    test_probs = torch.exp(test_log_probs)\n    preds_ = test_probs.cpu().numpy()\n\n    check_probs(test_probs)\n    \n    # Compute preds_99 as the proba of class not being any of the others\n    # preds_99 = 0.1 gives 1.769\n    # Create an empty vector the size of the number of samples\n    preds_99 = np.ones(preds_.shape[0])\n    \n    # Loop over the number of classes (should be 14)\n    for i in range(preds_.shape[1]):\n        preds_99 *= (1 - preds_[:, i])\n        \n    classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]\n\n    # Create DataFrame from predictions\n    preds_df_ = pd.DataFrame(preds_, \n                             columns=['class_{}'.format(s) for s in classes])\n    \n    # Add the object ids back (this is why we select instead of delete cols above)\n    preds_df_['object_id'] = partial_test['object_id']\n    \n    # Add probability for class 99\n    mean_preds_99 = 1\n    if np.mean(preds_99) > 0:\n        mean_preds_99 = np.mean(preds_99)\n        \n    preds_df_['class_99'] = 0.14 * preds_99 / mean_preds_99\n    return preds_df_\n\n\ndef process_test(model_eval_,\n                 ss_,\n                 features, \n                 featurize_configs,\n                 train_mean,\n                 filename='predictions.csv',\n                 chunks=5000000):\n    start = time.time()\n\n    meta_test = process_meta(data_dir + '/test_set_metadata.csv')\n    \n    remain_df = None\n    for i_c, df in enumerate(pd.read_csv(data_dir + '/test_set.csv', chunksize=chunks, iterator=True)):\n        \n        # Check object_ids\n        # I believe np.unique keeps the order of group_ids as they appear in the file\n        unique_ids = np.unique(df['object_id'])\n        \n        new_remain_df = df.loc[df['object_id'] == unique_ids[-1]].copy()\n        if remain_df is None:\n            df = df.loc[df['object_id'].isin(unique_ids[:-1])]\n        else:\n            df = pd.concat([remain_df, df.loc[df['object_id'].isin(unique_ids[:-1])]], axis=0)\n        # Create remaining samples df\n        remain_df = new_remain_df\n        \n        preds_df = predict_chunk(df_=df,\n                                 model_eval_=model_eval_,\n                                 ss_=ss,\n                                 meta_=meta_test,\n                                 features=features,\n                                 featurize_configs=featurize_configs,\n                                 train_mean=train_mean)\n    \n        if i_c == 0:\n            preds_df.to_csv(filename, header=True, mode='a', index=False)\n        else:\n            preds_df.to_csv(filename, header=False, mode='a', index=False)\n    \n        del preds_df\n        gc.collect()\n        print('{:15d} done in {:5.1f} minutes' .format(\n                chunks * (i_c + 1), (time.time() - start) / 60), flush=True)\n        \n    # Compute last object in remain_df\n    preds_df = predict_chunk(df_=remain_df,\n                             model_eval_=model_eval_,\n                             ss_=ss,\n                             meta_=meta_test,\n                             features=features,\n                             featurize_configs=featurize_configs,\n                             train_mean=train_mean)\n        \n    preds_df.to_csv(filename, header=False, mode='a', index=False)\n    return","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66d59fbaf1b985b418535be0a331a4a2db5061d7"},"cell_type":"markdown","source":"### Prediction on test set and create submission"},{"metadata":{"trusted":false,"_uuid":"62cbe73d5852d0f216c2057e305fdb9cb9d55b90"},"cell_type":"code","source":"# with torch.no_grad():\n    \n#     model_eval = PlasticcNet(full_train_ss.shape[1], hidden_size, num_classes)\n#     mpt = 'pt_fold{}_model.pt'.format(best_model['fold'])\n#     print('Loading best model state: {}...'.format(mpt))\n#     model_eval.load_state_dict(torch.load(mpt))\n#     model_eval.eval().to(device)\n\n#     filename = 'pt_wll_subm_{:.6f}_{}.csv'.format(best_model['loss'], dt.now().strftime('%Y-%m-%d-%H-%M'))\n#     print('save to {}'.format(filename))\n#     process_test(model_eval_=model_eval,\n#                  ss_=ss,\n#                  features=full_train.columns, \n#                  featurize_configs={'fcp': fcp}, \n#                  train_mean=train_mean, \n#                  filename=filename,\n#                  chunks=5000000)\n\n#     z = pd.read_csv(filename)\n#     print(\"Shape BEFORE grouping: {}\".format(z.shape))\n#     z = z.groupby('object_id').mean()\n#     print(\"Shape AFTER grouping: {}\".format(z.shape))\n#     z.to_csv('single_{}'.format(filename), index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5d91503f976168b6aead4d64b53e808bd689ab79"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}