{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import numpy as np\n\n# print('Loading data ...')\n\n# data_root='./timit_11/'\n# train = np.load(data_root + 'train_11.npy')\n# train_label = np.load(data_root + 'train_label_11.npy')\n# test = np.load(data_root + 'test_11.npy')\n\n# print('Size of training data: {}'.format(train.shape))\n# print('Size of testing data: {}'.format(test.shape))\n\n# PyTorch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\n# For data preprocess\nimport numpy as np\nimport csv\nimport os\n\n# For plotting\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\n\nmyseed = 42069  # set a random seed for reproducibility\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\nnp.random.seed(myseed)\ntorch.manual_seed(myseed)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed_all(myseed)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:12:55.730909Z","iopub.execute_input":"2022-08-10T03:12:55.731738Z","iopub.status.idle":"2022-08-10T03:12:56.380902Z","shell.execute_reply.started":"2022-08-10T03:12:55.731630Z","shell.execute_reply":"2022-08-10T03:12:56.379848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Loading data ...')\n\ntrain = np.load('../input/ml2021spring-hw2/timit_11/timit_11/train_11.npy')#得到function  obj\ntrain_label = np.load('../input/ml2021spring-hw2/timit_11/timit_11/train_label_11.npy')#得到ndarray\ntest = np.load('../input/ml2021spring-hw2/timit_11/timit_11/test_11.npy')#得到function  obj\n\nprint('Size of training data: {}'.format(train.shape))\nprint('Size of testing data: {}'.format(test.shape))\n\ntrain_label","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:12:56.383434Z","iopub.execute_input":"2022-08-10T03:12:56.384531Z","iopub.status.idle":"2022-08-10T03:13:43.988054Z","shell.execute_reply.started":"2022-08-10T03:12:56.384484Z","shell.execute_reply":"2022-08-10T03:13:43.986958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](http://)","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\n\nclass TIMITDataset(Dataset):\n    def __init__(self, X, y=None):\n        self.data = torch.from_numpy(X).float()\n        if y is not None:\n            y = y.astype(np.int)\n            self.label = torch.LongTensor(y)\n        else:\n            self.label = None\n\n    def __getitem__(self, idx):\n        if self.label is not None:\n            return self.data[idx], self.label[idx]\n        else:\n            return self.data[idx]\n\n    def __len__(self):\n        return len(self.data)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:13:43.989734Z","iopub.execute_input":"2022-08-10T03:13:43.990375Z","iopub.status.idle":"2022-08-10T03:13:43.999575Z","shell.execute_reply.started":"2022-08-10T03:13:43.990326Z","shell.execute_reply":"2022-08-10T03:13:43.998370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VAL_RATIO = 0.2\n\npercent = int(train.shape[0] * (1 - VAL_RATIO))\ntrain_x, train_y, val_x, val_y = train[:percent], train_label[:percent], train[percent:], train_label[percent:]\nprint('Size of training set: {}'.format(train_x.shape))\nprint('Size of validation set: {}'.format(val_x.shape))","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:13:44.002633Z","iopub.execute_input":"2022-08-10T03:13:44.003605Z","iopub.status.idle":"2022-08-10T03:13:44.011844Z","shell.execute_reply.started":"2022-08-10T03:13:44.003574Z","shell.execute_reply":"2022-08-10T03:13:44.010723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\n\nfrom torch.utils.data import DataLoader\n\ntrain_set = TIMITDataset(train_x, train_y)\nval_set = TIMITDataset(val_x, val_y)\ntrain_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True) #only shuffle the training data\nval_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:13:44.013760Z","iopub.execute_input":"2022-08-10T03:13:44.014124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ndel train, train_label, train_x, train_y, val_x, val_y\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass Classifier(nn.Module):\n    def __init__(self):\n        super(Classifier, self).__init__()\n        self.layer0 = nn.Linear(429,2048)\n        self.layer1 = nn.Linear(2048, 1024)\n        self.layer2 = nn.Linear(1024, 512)\n        self.layer3 = nn.Linear(512, 128)\n        self.out = nn.Linear(128, 39) \n\n        self.act_fn = nn.ReLU()\n        self.dropout = nn.Dropout(0.25)\n        self.batchnorm0 = nn.BatchNorm1d(2048)\n        self.batchnorm1 = nn.BatchNorm1d(1024)\n        self.batchnorm2 = nn.BatchNorm1d(512)\n        self.batchnorm3 = nn.BatchNorm1d(128)\n\n\n    def forward(self, x):\n        x = self.layer0(x)\n        x = self.batchnorm0(x)\n        x = self.act_fn(x)\n        x = self.dropout(x)\n        x = self.layer1(x)\n        x = self.batchnorm1(x)\n        x = self.act_fn(x)\n        x = self.dropout(x)\n\n        x = self.layer2(x)\n        x = self.batchnorm2(x)\n        x = self.act_fn(x)\n        x = self.dropout(x)\n\n        x = self.layer3(x)\n        x = self.batchnorm3(x)\n        x = self.act_fn(x)\n\n        x = self.out(x)\n        \n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check device\ndef get_device():\n  return 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fix random seed\ndef same_seeds(seed):\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)  \n    np.random.seed(seed)  \n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 模型改进\n1\n- 使用dropout 过后 acc提升了3%，并且模型再缓慢提升\n- val_acc 和 train_acc 之间的差距变小，初步判定samplecode中出现了overfitting\n\n\n这次尝试把epoch增加一倍,测试证明增加一倍epoch","metadata":{}},{"cell_type":"code","source":"# fix random seed for reproducibility\nsame_seeds(0)\n\n# get device \ndevice = get_device()\nprint(f'DEVICE: {device}')\n\n# training parameters\nnum_epoch = 40               # number of training epoch\nlearning_rate = 0.0001       # learning rate\n\n# the path where checkpoint saved\nmodel_path = './model.ckpt'\n\n# create model, define a loss function, and optimizer\nmodel = Classifier().to(device)\ncriterion = nn.CrossEntropyLoss() \noptimizer = torch.optim.NAdam(model.parameters(),lr=learning_rate)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# start training\n\nbest_acc = 0.0\nfor epoch in range(num_epoch):\n    train_acc = 0.0\n    train_loss = 0.0\n    val_acc = 0.0\n    val_loss = 0.0\n\n    # training\n    model.train() # set the model to training mode\n    for i, data in enumerate(train_loader):\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n        optimizer.zero_grad() \n        outputs = model(inputs) \n        batch_loss = criterion(outputs, labels)\n        _, train_pred = torch.max(outputs, 1) # get the index of the class with the highest probability\n        batch_loss.backward() \n        optimizer.step() \n\n        train_acc += (train_pred.cpu() == labels.cpu()).sum().item()\n        train_loss += batch_loss.item()\n\n    # validation\n    if len(val_set) > 0:\n        model.eval() # set the model to evaluation mode\n        with torch.no_grad():\n            for i, data in enumerate(val_loader):\n                inputs, labels = data\n                inputs, labels = inputs.to(device), labels.to(device)\n                outputs = model(inputs)\n                batch_loss = criterion(outputs, labels) \n                _, val_pred = torch.max(outputs, 1) \n            \n                val_acc += (val_pred.cpu() == labels.cpu()).sum().item() # get the index of the class with the highest probability\n                val_loss += batch_loss.item()\n\n            print('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f} | Val Acc: {:3.6f} loss: {:3.6f}'.format(\n                epoch + 1, num_epoch, train_acc/len(train_set), train_loss/len(train_loader), val_acc/len(val_set), val_loss/len(val_loader)\n            ))\n\n            # if the model improves, save a checkpoint at this epoch\n            if val_acc > best_acc:\n                best_acc = val_acc\n                torch.save(model.state_dict(), model_path)\n                print('saving model with acc {:.3f}'.format(best_acc/len(val_set)))\n    else:\n        print('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f}'.format(\n            epoch + 1, num_epoch, train_acc/len(train_set), train_loss/len(train_loader)\n        ))\n\n# if not validating, save the last epoch\nif len(val_set) == 0:\n    torch.save(model.state_dict(), model_path)\n    print('saving model at last epoch')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create testing dataset\ntest_set = TIMITDataset(test, None)\ntest_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=False)\n\n# create model and load weights from checkpoint\nmodel = Classifier().to(device)\nmodel.load_state_dict(torch.load(model_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = []\nmodel.eval() # set the model to evaluation mode\nwith torch.no_grad():\n    for i, data in enumerate(test_loader):\n        inputs = data\n        inputs = inputs.to(device)\n        outputs = model(inputs)\n        _, test_pred = torch.max(outputs, 1) # get the index of the class with the highest probability\n\n        for y in test_pred.cpu().numpy():\n            predict.append(y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('prediction.csv', 'w') as f:\n    f.write('Id,Class\\n')\n    for i, y in enumerate(predict):\n        f.write('{},{}\\n'.format(i, y))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}