{"cells":[{"metadata":{"_uuid":"322a25f6bfff15207d9534acd7eb8104e68833aa"},"cell_type":"markdown","source":"## Convert an existing kernel from Keras to Pytorch\nhttps://www.kaggle.com/fernandoramacciotti/cnn-with-class-weights "},{"metadata":{"trusted":true,"_uuid":"edd8378e3ba039ce6c3d84652958574ccb32aa44"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy.stats as stats\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\nimport pyarrow.parquet as pq\nimport math\nimport os\nimport gc\nfrom tqdm import tqdm_notebook as tqdm, tnrange\n\nprint(os.listdir(\"../input\"))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc8fb69953478ae997fb3439370aec8620729898"},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e40e3c4c1be7416fe393d58987815f34dfec4b4"},"cell_type":"code","source":"train = pq.read_pandas('../input/train.parquet').to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72efeb828b682df0a4c813dc0b774290d774b76d"},"cell_type":"code","source":"mdtrain = pd.read_csv('../input/metadata_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30cbfed294ddcd16cacf75563b064793a1f6e4f5"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c67673357173d5c7e85128bfb0bec70fe4af707c"},"cell_type":"code","source":"mdtrain.head()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b913f7153ad59d43837d5b1c119eac3dc190b8d7"},"cell_type":"markdown","source":"## Extract features"},{"metadata":{"trusted":true,"_uuid":"546387a7ffd8a90610734fb6a4a8031421a0f193"},"cell_type":"code","source":"def feature_extractor(x, n_part=1000):\n    length = len(x)\n    n_feat = 7\n    pool = np.int32(np.ceil(length/n_part))\n    output = np.zeros((n_part, n_feat))\n    for j, i in enumerate(range(0,length, pool)):\n        if i+pool < length:\n            k = x[i:i+pool]\n        else:\n            k = x[i:]\n        output[j, 0] = np.mean(k, axis=0) #mean\n        output[j, 1] = np.min(k, axis=0) #min\n        output[j, 2] = np.max(k, axis=0) #max\n        output[j, 3] = np.std(k, axis=0) #std\n        output[j, 4] = np.median(k, axis=0) #median\n        output[j, 5] = stats.skew(k, axis=0) #skew\n        output[j, 6] = stats.kurtosis(k, axis=0) # kurtosis\n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"393f0db1e183f91510201a7840e35c5931a7f1fa"},"cell_type":"code","source":"X = []\ny = []\nfor i in tqdm(mdtrain.signal_id):\n    idx = mdtrain.loc[mdtrain.signal_id==i, 'signal_id'].values.tolist()\n    y.append(mdtrain.loc[mdtrain.signal_id==i, 'target'].values)\n    X.append(feature_extractor(train.iloc[:, idx].values, n_part=400))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12f80a2aefd5da5d23786eb12b25c829f6bd6eb2"},"cell_type":"code","source":"X = np.array(X).reshape(-1, X[0].shape[0], X[0].shape[1])\nX = np.transpose(X, [0,2,1]) # Make X shape (batch size, channels, time steps) \n# because channels/time-steps are different in Keras\n\ny = np.array(y).reshape(-1,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d77c0d7cba100436239910115a0720fe56fbcd08"},"cell_type":"code","source":"X.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9cd768d177e51ff4337d6db4e87d5adbd43c10e"},"cell_type":"code","source":"del train; gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a68b87cd99dc96202c9b88553af4f592a270cc8a"},"cell_type":"markdown","source":"## Split into test/val sets and normalize"},{"metadata":{"trusted":true,"_uuid":"58d6caa007692fb87272eb1be44dba885c1b163e"},"cell_type":"code","source":"sss = StratifiedShuffleSplit(n_splits=1, test_size=0.05, random_state=0)\n(train_idx, val_idx) = next(sss.split(X, y))\n\nX_train, X_val = X[train_idx], X[val_idx]\ny_train, y_val = y[train_idx], y[val_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be17e4fd31d57c4893810ac29de7b97bad1eda40"},"cell_type":"code","source":"X_train.shape, X_val.shape, y_train.shape, y_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b41995c46f8dd60f82b33447085269ed5cb956b4"},"cell_type":"code","source":"scalers = {} # This code actually looked wrong for Keras but right for Pytorch already!\nfor i in range(X_train.shape[1]):\n    scalers[i] = MinMaxScaler(feature_range=(-1, 1))\n    X_train[:, i, :] = scalers[i].fit_transform(X_train[:, i, :]) \n\nfor i in range(X_val.shape[1]):\n    X_val[:, i, :] = scalers[i].transform(X_val[:, i, :]) \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"47b3675836dfd6e44324cf20da4efec0979fb4b4"},"cell_type":"markdown","source":"## Evaluation Metrics"},{"metadata":{"trusted":true,"_uuid":"be7c7f7497b8d1df99dd2b2516910cee73c26157"},"cell_type":"code","source":"def confusion(prediction, truth):\n    \"\"\" Returns the confusion matrix for the values in the `prediction` and `truth`\n    tensors, i.e. the amount of positions where the values of `prediction`\n    and `truth` are\n    - 1 and 1 (True Positive)\n    - 1 and 0 (False Positive)\n    - 0 and 0 (True Negative)\n    - 0 and 1 (False Negative)\n    \"\"\"\n\n    confusion_vector = torch.as_tensor(prediction, dtype=torch.float32) / torch.as_tensor(truth, dtype=torch.float32)\n    # Element-wise division of the 2 tensors returns a new tensor which holds a\n    # unique value for each case:\n    #   1     where prediction and truth are 1 (True Positive)\n    #   inf   where prediction is 1 and truth is 0 (False Positive)\n    #   nan   where prediction and truth are 0 (True Negative)\n    #   0     where prediction is 0 and truth is 1 (False Negative)\n\n    true_positives = torch.sum(confusion_vector == 1).item()\n    false_positives = torch.sum(confusion_vector == float('inf')).item()\n    true_negatives = torch.sum(torch.isnan(confusion_vector)).item()\n    false_negatives = torch.sum(confusion_vector == 0).item()\n\n    return true_positives, false_positives, true_negatives, false_negatives","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be458397993a9b4f15717fd9872d8b89701069b7"},"cell_type":"code","source":"def matthews(TP, FP, TN, FN):\n    nom = TP*TN - FP*FN\n    denom = math.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN))\n    return nom/denom\n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9c1b97055fb6370f3dbd6903dffd77a85a12e1c"},"cell_type":"markdown","source":"## Neural Network in Pytorch"},{"metadata":{"trusted":true,"_uuid":"924ed516779ba1f1649726a551b1a4cf72411d4b"},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0de1d180896e2346f1fa4d0d904b94592f4cb58b"},"cell_type":"code","source":"class VSBNet1(nn.Module):\n    \n    def __init__(self):\n        super(VSBNet1, self).__init__()\n        \n        self.conv1 = nn.Conv1d(in_channels=7, out_channels=64, kernel_size=4)\n        self.conv2 = nn.Conv1d(in_channels=64, out_channels=64, kernel_size=4)\n        \n        self.mp1 = nn.MaxPool1d(2, padding=1)\n        \n        self.conv3 = nn.Conv1d(in_channels=64, out_channels=20, kernel_size=4)\n        self.conv4 = nn.Conv1d(in_channels=20, out_channels=20, kernel_size=4)\n        \n        #self.gap1 = nn.AdaptiveMaxPool1d(20)\n        self.gap1 = nn.AvgPool1d(192)\n        \n        self.do1 = nn.Dropout(0.2)\n        \n        # Flatten\n        \n        self.lin1 = nn.Linear(20,32)\n        self.do2 = nn.Dropout(0.5)\n        \n        self.lin2 = nn.Linear(32,8)\n        self.do3 = nn.Dropout(0.5)\n\n        self.lin3 = nn.Linear(8,1)\n        \n        def init_weights(m):\n            # Conv1d defaults to kaiming just like Keras does\n            if type(m) == nn.Linear:\n                # Keras defaults Dense to glorot_uniform (which is also called xavier uniform)\n                # Whereas Pytorch default for Linear is kaiming\n                torch.nn.init.xavier_uniform_(m.weight)\n                \n        self.apply(init_weights)\n        \n        \n    def forward(self,x):\n        x = F.relu(self.conv1(x))\n        x = F.relu(self.conv2(x))\n        \n        x = self.mp1(x)\n        \n        x = F.relu(self.conv3(x))\n        x = F.relu(self.conv4(x))\n\n        x = self.gap1(x)\n        \n        x = self.do1(x)\n        \n        x = x.view(x.shape[0],-1)\n        \n        x = torch.tanh(self.lin1(x))\n        x = self.do2(x)\n        \n        x = torch.tanh(self.lin2(x))\n        x = self.do3(x)\n        \n        x = self.lin3(x) # Leave sigmoid for the loss function\n            \n        return x\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05eba98359a1ce011e342cc5dc752540068f6347"},"cell_type":"code","source":"net = VSBNet1()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdb0521a030d64ac2a5411322c35efa233c5cd70"},"cell_type":"code","source":"print(net)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dda56d01f322e274de7d4396d9fb4614cc5dc76e"},"cell_type":"markdown","source":"## Train the model"},{"metadata":{"trusted":true,"_uuid":"248e749501dd08a6c55729a38f68dbbca056cd58"},"cell_type":"code","source":"NUM_EPOCHS = 30\nBS = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12427374e18f62ef57baa94917a1011f1d12abda"},"cell_type":"code","source":"X_train.shape, y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7227b6e3ba7ab35fff0060a344e583d4967d9a0"},"cell_type":"code","source":"trainloader = DataLoader(list(zip(X_train,y_train)), batch_size=BS, shuffle=False, num_workers=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c09d9ca7c48769409b53803ed3d725161e2a4f05"},"cell_type":"code","source":"criterion = torch.nn.BCEWithLogitsLoss() #pos_weight=torch.Tensor([1.0,1.2])) - pos_weight seems to work differently on latest pytorch\noptimizer = torch.optim.Adam(net.parameters(), lr=1e-3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dbd605ca04c12d9468ae83be11e3d6f65d03bb7f"},"cell_type":"code","source":"net.train()\n    \nfor t in tnrange(NUM_EPOCHS, desc='Epochs'):\n    \n    #running_loss = 0.0\n    for i, (X_batch, y_target_batch) in tqdm(enumerate(trainloader), total=len(trainloader)):\n              \n        # Forward pass: Compute predicted y by passing x to the model\n        y_pred = net(X_batch.float())\n        # y_pred = net(X.view(X.shape[0], 1, X.shape[1]))\n\n        # Compute and print loss\n        loss = criterion(y_pred, y_target_batch.float())\n\n\n        # Zero gradients, perform a backward pass, and update the weights.\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17313a6c941ee2bfe54fdeed865dfcc151ed171e"},"cell_type":"markdown","source":"## Evaluate"},{"metadata":{"trusted":true,"_uuid":"08ea6433430b6c2c094565cec46bae2d21189400"},"cell_type":"code","source":"net.eval()\ny_preds = net(torch.Tensor(X_train).float()).detach()\nloss = criterion(y_preds, torch.Tensor(y_train).float()).detach(); loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddad6cdc70dd2a574854eee7497bca0bdaa8dfda"},"cell_type":"code","source":"# Number of target==0, no of target==1\n(y_preds <= 0).sum().item(), (y_preds > 0).sum().item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f7f639293a92ecb81c54dd1aff9e30cf49af14d"},"cell_type":"code","source":"net.eval()\ny_val_preds = net(torch.Tensor(X_val).float()).detach()\nloss_val = criterion(y_val_preds, torch.Tensor(y_val).float()).detach(); loss_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24afd62efee709fd210d1cbbe726a056f43cc0e8"},"cell_type":"code","source":"(y_val_preds <= 0).sum().item(), (y_val_preds > 0).sum().item()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2e69e7e3f38534fdd47e30e346685dd0a70ae05"},"cell_type":"markdown","source":"### Confusion and Matthews"},{"metadata":{"trusted":true,"_uuid":"64c1c53c20c2a1acbff62f64f279c5c373ec82e7"},"cell_type":"code","source":"c_train = confusion(y_preds > 0, y_train); c_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bfef3d627c23b6a90342e742dd8a63207d26aa8"},"cell_type":"code","source":"matthews(*c_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da9dc5344cbc0312ae10618433c26f4a81811259"},"cell_type":"code","source":"c_val = confusion(y_val_preds > 0, y_val); c_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"328466d787aa44943f67b20c3e547273f6d6c70c"},"cell_type":"code","source":"matthews(*c_val)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02af63cdd75a1433fefe6ae171e689aa568da2b0"},"cell_type":"markdown","source":"## Now apply to test dataset"},{"metadata":{"trusted":true,"_uuid":"1481d37bb6e78f46c4614774233708d65fa581a7"},"cell_type":"code","source":"mdtest = pd.read_csv('../input/metadata_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d3a1217861cc0beafe30c380fa78559d8fdc5a5"},"cell_type":"code","source":"mdtest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b876ccfc0b2a5a246fe36cc43965e7e259e670d"},"cell_type":"code","source":"start_test = mdtest.signal_id.min()\nend_test = mdtest.signal_id.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c41c479be58677e85ce4381fc83064927b5f0382"},"cell_type":"code","source":"X_test = []\n\npool_test = 2000\n\nfor start_col in tqdm(range(start_test, end_test + 1, pool_test)):\n    end_col = min(start_col + pool_test, end_test + 1)\n    test = pq.read_pandas('../input/test.parquet',\n                          columns=[str(c) for c in range(start_col, end_col)]).to_pandas()\n\n    for i in tqdm(test.columns, desc=str(start_test)):\n        X_test.append(feature_extractor(test[i].values, n_part=400))\n        #test.drop([i], axis=1, inplace=True); gc.collect()\n    del test; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fee84533cf15be2254641cfb812e22f49ccd32c"},"cell_type":"code","source":"X_test = np.array(X_test).reshape(-1, X_test[0].shape[0], X_test[0].shape[1])\nX_test = np.transpose(X_test, [0,2,1]) # Make X shape (batch size, channels, time steps) \n# because channels/time-steps are different in Keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21a4a92530fffaa105d0ece5770b49ac99cf40db"},"cell_type":"code","source":"for i in range(X_test.shape[1]):\n    X_test[:, i, :] = scalers[i].transform(X_test[:, i, :]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9e7f20d064d00919d4e26e29b3aa925474264f8"},"cell_type":"code","source":"testloader = DataLoader(X_test, batch_size=100, shuffle=False, num_workers=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4ed6d1127c973bc65125586994166a3fad6991b"},"cell_type":"code","source":"net.eval()\n\ny_test_preds = None\nfor i, X_test_batch in tqdm(enumerate(testloader), total=len(testloader)):\n    y_test_preds_batch = net(torch.as_tensor(X_test_batch, dtype=torch.float32)).detach()\n    if y_test_preds is None:\n        y_test_preds = y_test_preds_batch\n    else:\n        y_test_preds = torch.cat([y_test_preds, y_test_preds_batch])\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad28d423f47fc511e7bf6e46576a65cc5d404bd6"},"cell_type":"code","source":"y_test_classes = y_test_preds > 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0da9d6aaadec5ffa47df3f0de1070878f1dbf86"},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv')\nsubmission['signal_id'] = mdtest.signal_id.values\nsubmission['target'] = y_test_classes.data.numpy().astype(int)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8796a5a425bef9f9b41cfefa1018e7de63127938"},"cell_type":"code","source":"(y_test_classes <= 0).sum().item(), (y_test_classes > 0).sum().item()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"notify_time":"30"},"nbformat":4,"nbformat_minor":1}