{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install kaggle-environments","metadata":{}},{"cell_type":"code","source":"# 1. Enable Internet in the Kernel (Settings side pane)\n\n# 2. Curl cache may need to be purged if v0.1.6 cannot be found (uncomment if needed). \n# !curl -X PURGE https://pypi.org/simple/kaggle-environments\n\n# ConnectX environment was defined in v0.1.6\n!pip install 'kaggle-environments>=0.1.6'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-13T15:05:43.213160Z","iopub.execute_input":"2022-07-13T15:05:43.213798Z","iopub.status.idle":"2022-07-13T15:06:02.567994Z","shell.execute_reply.started":"2022-07-13T15:05:43.213744Z","shell.execute_reply":"2022-07-13T15:06:02.567067Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create ConnectX Environment","metadata":{}},{"cell_type":"code","source":"from kaggle_environments import evaluate, make, utils\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.optim.lr_scheduler as scheduler\nimport numpy as np\nimport random\nimport math\nimport matplotlib.pyplot as plt\nfrom random import choice\nfrom collections import namedtuple, deque","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-07-13T15:06:02.577634Z","iopub.execute_input":"2022-07-13T15:06:02.577934Z","iopub.status.idle":"2022-07-13T15:06:04.050791Z","shell.execute_reply.started":"2022-07-13T15:06:02.577886Z","shell.execute_reply":"2022-07-13T15:06:04.049460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n    lr = 7e-3\n    weight_decay = 1e-5\n    replace_cntr = 500\n    epsilon = 1\n    eps_min = .001\n    eps_decay = 0.9982746 # round(10**(math.log(.01,10)/(config.epochs/2)),7)\n    gamma = .7\n    mem_len = 1000\n    num_actions = 7\n    epochs = 5_000","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:06:29.717762Z","iopub.execute_input":"2022-07-13T15:06:29.718158Z","iopub.status.idle":"2022-07-13T15:06:29.725403Z","shell.execute_reply.started":"2022-07-13T15:06:29.718083Z","shell.execute_reply":"2022-07-13T15:06:29.724221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"round(10**(math.log(config.eps_min,10)/(config.epochs*.8)),7)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:20:36.468018Z","iopub.execute_input":"2022-07-13T16:20:36.468355Z","iopub.status.idle":"2022-07-13T16:20:36.477062Z","shell.execute_reply.started":"2022-07-13T16:20:36.468315Z","shell.execute_reply":"2022-07-13T16:20:36.475762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Callback():\n    def __init__(self):\n        self.reward = []\n        self.total_reward = 0\n        \n    def on_train_begin(self):\n        pass\n    \n    def on_train_end(self):\n        plt.plot(self.reward)\n    \n    def on_epoch_begin(self):\n        pass\n        \n    def on_epoch_end(self):\n        pass\n        \n    def on_game_begin(self):\n        pass\n        \n    def on_game_end(self, reward):\n        agent.scheduler.step()\n        \n        self.reward += [reward]\n        \n        agent.step_counter += 1\n        agent.decrement_epsilon()\n        \n    def on_loss_begin(self):\n        pass\n    \n    def on_loss_end(self):\n        pass\n    \n    def on_step_begin(self):\n        pass\n    \n    def on_step_end(self):\n        pass","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:06:29.751216Z","iopub.execute_input":"2022-07-13T15:06:29.751559Z","iopub.status.idle":"2022-07-13T15:06:29.762524Z","shell.execute_reply.started":"2022-07-13T15:06:29.751501Z","shell.execute_reply":"2022-07-13T15:06:29.761628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RLModel(nn.Module):\n    def __init__(self):\n        super(RLModel,self).__init__()\n        self.fc1 = nn.Linear(3,1)\n        self.fc2 = nn.Linear(42,128)\n        self.fc3 = nn.Linear(128,7)\n        self.type_emb = nn.Embedding(3,3)\n        self.relu = nn.ReLU()\n        \n        self.to(config.device)\n        \n    def forward(self, board):\n        types = self.type_emb(board)\n        types = self.relu(self.fc1(types))\n        types = types.permute([1,0])\n        out = self.relu(self.fc2(types))\n        out = self.fc3(out).squeeze(0)\n        out = out.softmax(dim=0)\n        return out","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-13T15:06:29.764606Z","iopub.execute_input":"2022-07-13T15:06:29.764927Z","iopub.status.idle":"2022-07-13T15:06:29.777641Z","shell.execute_reply.started":"2022-07-13T15:06:29.764870Z","shell.execute_reply":"2022-07-13T15:06:29.776654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RLModel(nn.Module):\n    def __init__(self):\n        super(RLModel,self).__init__()\n        self.conv1 = nn.Conv2d(1,1,kernel_size=3)\n        self.conv2 = nn.Conv2d(1,1,kernel_size=3)\n        self.conv2 = nn.Conv2d(1,1,kernel_size=3)\n        self.conv3 = nn.Conv2d(1,1,kernel_size=3)\n        self.conv2 = nn.Conv2d(7,7,kernel_size=(3,4))\n        self.fc1 = nn.Linear(7,1)\n        self.relu = nn.ReLU()\n        \n        self.to(config.device)\n        \n    def forward(self, board):\n        inp = board.reshape(6,7).unsqueeze(0).unsqueeze(0).float()\n        conv_out = self.relu(self.conv1(inp))\n        conv_out = self.relu(self.conv2(conv_out))\n        conv_out = conv_out.squeeze(-1).squeeze(-1)\n        inp2 = board.reshape(6,7).float()\n        cat_board =  torch.transpose(torch.cat((conv_out,inp2)),0,1)\n        out = self.fc1(cat_board).squeeze(1)\n        out = out.softmax(dim=1)\n        return out","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-13T15:06:29.780077Z","iopub.execute_input":"2022-07-13T15:06:29.780392Z","iopub.status.idle":"2022-07-13T15:06:29.795184Z","shell.execute_reply.started":"2022-07-13T15:06:29.780351Z","shell.execute_reply":"2022-07-13T15:06:29.794043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RLModel(nn.Module):\n    def __init__(self):\n        super(RLModel,self).__init__()\n        self.model = nn.Sequential(\n            nn.Linear(42,128),\n            nn.ReLU(),\n            nn.Linear(128,128),\n            nn.ReLU(),\n            nn.Linear(128,128),\n            nn.ReLU(),\n            nn.Linear(128,128),\n            nn.ReLU(),\n            nn.Linear(128,64),\n            nn.ReLU(),\n            nn.Linear(64,7),\n        )\n        \n        self.to(config.device)\n        \n    def forward(self, board):\n        out = self.model(board)\n        out = out.softmax(dim=0)\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:09:21.459819Z","iopub.execute_input":"2022-07-13T15:09:21.460534Z","iopub.status.idle":"2022-07-13T15:09:21.473050Z","shell.execute_reply.started":"2022-07-13T15:09:21.460180Z","shell.execute_reply":"2022-07-13T15:09:21.471744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Transitions = namedtuple('transitions',\n                        ('states','actions','rewards','states_','dones'))\n\nclass Replay():\n    def __init__(self):\n        self.memory = deque(maxlen=config.mem_len)\n        \n    def store_transition(self, states, actions, rewards, states_, dones):\n        self.memory.append(Transitions(states, actions, rewards, states_, dones))\n        \n    def sample_memory(self):\n        s = random.sample(self.memory, 1)\n        \n        states = s[0].states\n        actions = s[0].actions\n        rewards = s[0].rewards\n        states_ = s[0].states_\n        dones = s[0].dones\n        \n        return states, actions, rewards, states_, dones\n    \n    def __len__(self):\n        return len(self.memory)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:06:29.811326Z","iopub.execute_input":"2022-07-13T15:06:29.811630Z","iopub.status.idle":"2022-07-13T15:06:29.830840Z","shell.execute_reply.started":"2022-07-13T15:06:29.811572Z","shell.execute_reply":"2022-07-13T15:06:29.829680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create an Agent\n\nTo create the submission, an agent function should be fully encapsulated (no external dependencies).  \n\nWhen your agent is being evaluated against others, it will not have access to the Kaggle docker image.  Only the following can be imported: Python Standard Library Modules, gym, numpy, scipy, pytorch (1.3.1, cpu only), and more may be added later.\n\n","metadata":{}},{"cell_type":"code","source":"class RLAgent:\n    def __init__(self):\n        self.qeval = RLModel()\n        self.qtarg = RLModel()\n        self.memory = Replay()\n        self.epsilon = config.epsilon\n        self.gamma = config.gamma\n        self.step_counter = 0\n        self.optimizer = optim.Adam(self.qeval.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n        self.scheduler = scheduler.CosineAnnealingWarmRestarts(self.optimizer, T_0=200)\n    \n    def save_model(self):\n        torch.save({'state_dict': self.qeval.state_dict()}, f'./qeval.pth.tar')\n        torch.save({'state_dict': self.qtarg.state_dict()}, f'./qtarg.pth.tar')\n    \n    def load_model(self):\n        checkpoint_eval = torch.load(f'../input/first-20k-games/qeval.pth.tar')\n        checkpoint_targ = torch.load(f'../input/first-20k-games/qtarg.pth.tar')\n        self.qeval.load_state_dict(checkpoint_eval['state_dict'])\n        self.qtarg.load_state_dict(checkpoint_targ['state_dict'])\n        \n    def store_memory(self, states, actions, rewards, states_, dones):\n        self.memory.store_transition(states, actions, rewards, states_, dones)\n    \n    def get_from_memory(self):\n        return self.memory.sample_memory()\n    \n    def replace_target_network(self):\n        if self.step_counter % config.replace_cntr == 0:\n            self.qtarg.load_state_dict(self.qeval.state_dict())\n\n    def decrement_epsilon(self):\n        self.epsilon = self.epsilon * config.eps_decay if self.epsilon > config.eps_min else config.eps_min\n    \n    def choose_action(self, board):\n        if np.random.random_sample() < self.epsilon:\n            return int(np.random.choice(range(config.num_actions)))\n        return torch.argmax(self.predict(board),dim=0).item()\n        \n    def predict(self, board):\n        return self.qeval.forward(board.to(config.device))\n    \n    def predict_(self, board):\n        return self.qtarg.forward(board.to(config.device))\n        \n    def learn(self):\n        if len(self.memory) < 1:\n            return\n        \n        self.replace_target_network()\n        \n        s, a, r, s_, d = self.get_from_memory()\n        \n        a_ = torch.argmax(self.predict_(s_),dim=0).to(config.device)\n        q_ = self.predict(s_).gather(0,a_.to(config.device))\n        q_star = r + (1-d) * config.gamma * q_\n        \n        q = self.predict(s).gather(0,torch.tensor(a).to(config.device))\n        \n#         callback.on_loss_begin()\n        criterion = nn.MSELoss()\n        loss = criterion(q, q_star)\n        \n#         callback.on_loss_end()\n        self.optimizer.zero_grad()\n        loss.backward()\n        \n#         callback.on_step_begin()\n        \n        self.optimizer.step()\n        \n#         callback.on_step_end()\n        \n#         self.step_counter += 1\n#         self.decrement_epsilon()\n        \ndef model_agent(obs, config):\n    board = torch.tensor(obs['board'], dtype=torch.float)\n    column = agent.choose_action(board)\n    return column","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:06:29.832984Z","iopub.execute_input":"2022-07-13T15:06:29.833612Z","iopub.status.idle":"2022-07-13T15:06:29.861588Z","shell.execute_reply.started":"2022-07-13T15:06:29.833548Z","shell.execute_reply":"2022-07-13T15:06:29.860618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Debug/Train your Agent","metadata":{}},{"cell_type":"code","source":"def change_players(new_players, game_counter):\n    npas = [\"random\", \"negamax\"] # Non-playable Agents\n\n    for idx,player in enumerate(new_players):\n        if player:\n            if game_counter%50 < 5: \n                new_players[idx] = npas[0] # Switches NPA to \"random\" for 10% of games\n            else: \n                new_players[idx] = npas[1] # Switches NPA to \"negamax\" for 90% of games\n    if game_counter%100 == 0 and game_counter != 0:\n        new_players.reverse()\n\n    return new_players\n\ndef reward(observation, won, done):\n    if done:\n        if won == 1: # Won\n            state_reward = 1\n            record['wins'] += 1\n        elif won == -1: # Lost\n            state_reward = -1\n            record['losses'] += 1\n        elif won == 0: # Draw\n            state_reward = .5\n            record['draws'] += 1\n        elif won == None:\n            state_reward = -100\n            record['invalid_actions'] += 1\n    else:\n        state_reward = 1/42\n    return state_reward\n\n\ndef train(games=100):\n    confg = env.configuration\n    players = [None, \"random\"]\n    new_players = [None, \"random\"]\n    \n    trainer = env.train(players)\n\n    for game in range(games):\n        new_players = change_players(players.copy(), game)\n        if new_players != players:\n            players = new_players.copy()\n            trainer = env.train(players)\n\n        obs = trainer.reset()\n\n        if game%500==0:\n            print(f'Game {game}')\n\n        while not env.done:\n            action = model_agent(obs, confg)\n            state = torch.tensor(obs['board'], dtype=torch.float)\n            obs, won, done, _ = trainer.step(action)\n            step_reward = reward(obs, won, done)\n            next_state = torch.tensor(obs['board'], dtype=torch.float)\n            agent.store_memory(state, action, step_reward, next_state, done)\n            agent.store_memory(state.flip(0), config.num_actions - 1 - action, step_reward, next_state.flip(0), done)\n            agent.learn()\n        callback.on_game_end(step_reward)\n    callback.on_train_end()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:06:29.864103Z","iopub.execute_input":"2022-07-13T15:06:29.864720Z","iopub.status.idle":"2022-07-13T15:06:29.887147Z","shell.execute_reply.started":"2022-07-13T15:06:29.864474Z","shell.execute_reply":"2022-07-13T15:06:29.886022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"import warnings\nwarnings.filterwarnings('error')","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"env = make(\"connectx\", debug=False)\nenv.render()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:08:51.237279Z","iopub.execute_input":"2022-07-13T15:08:51.238009Z","iopub.status.idle":"2022-07-13T15:08:51.256272Z","shell.execute_reply.started":"2022-07-13T15:08:51.237953Z","shell.execute_reply":"2022-07-13T15:08:51.254961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# agent.epsilon = 0.\nagent = RLAgent()\ncallback = Callback()\n\n# agent.load_model()\n\nrecord = {'wins':0, 'losses':0,'draws':0,'invalid_actions':0}\nenv.configuration.inarow = 2\ntrain(games=config.epochs)\nprint(record)\n\nrecord = {'wins':0, 'losses':0,'draws':0,'invalid_actions':0}\nenv.configuration.inarow = 3\ntrain(games=config.epochs)\nprint(record)\n\nrecord = {'wins':0, 'losses':0,'draws':0,'invalid_actions':0}\nenv.configuration.inarow = 4\ntrain(games=config.epochs)\nprint(record)\n\nagent.save_model()\n\ndef mean_reward(all_rewards):\n    adjusted_rewards = [[-1,y] if x==None else [x,y] for x,y in all_rewards]\n    return sum(r[0] for r in adjusted_rewards) / float(len(adjusted_rewards))\n\n# Run multiple episodes to estimate its performance.\nagent.epsilon = 0.\nprint(f\"My Agent vs Random Agent: {mean_reward(evaluate('connectx', [model_agent, 'random'], num_episodes=100))}\")\nprint(f\"My Agent vs Negamax Agent: {mean_reward(evaluate('connectx', [model_agent, 'negamax'], num_episodes=100))}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:09:25.567546Z","iopub.execute_input":"2022-07-13T15:09:25.568068Z","iopub.status.idle":"2022-07-13T16:03:02.143380Z","shell.execute_reply.started":"2022-07-13T15:09:25.568023Z","shell.execute_reply":"2022-07-13T16:03:02.142276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run the Agent","metadata":{}},{"cell_type":"code","source":"env.run([model_agent, \"negamax\"])\nenv.render(mode=\"ipython\", width=500, height=450)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:03:02.145495Z","iopub.execute_input":"2022-07-13T16:03:02.145900Z","iopub.status.idle":"2022-07-13T16:03:03.804924Z","shell.execute_reply.started":"2022-07-13T16:03:02.145837Z","shell.execute_reply":"2022-07-13T16:03:03.803732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}