අටාරිබ්රේක්අවුට් සමඟ ඩීQN අත්හදා බැලීම

16import numpy as np
17import torch
18
19from labml import tracker, experiment, logger, monit
20from labml.internal.configs.dynamic_hyperparam import FloatDynamicHyperParam
21from labml_helpers.schedule import Piecewise
22from labml_nn.rl.dqn import QFuncLoss
23from labml_nn.rl.dqn.model import Model
24from labml_nn.rl.dqn.replay_buffer import ReplayBuffer
25from labml_nn.rl.game import Worker

28if torch.cuda.is_available():
29    device = torch.device("cuda:0")
30else:
31    device = torch.device("cpu")

34def obs_to_torch(obs: np.ndarray) -> torch.Tensor:

36    return torch.tensor(obs, dtype=torch.float32, device=device) / 255.

39class Trainer:

44    def __init__(self, *,
45                 updates: int, epochs: int,
46                 n_workers: int, worker_steps: int, mini_batch_size: int,
47                 update_target_model: int,
48                 learning_rate: FloatDynamicHyperParam,
49                 ):

51        self.n_workers = n_workers

53        self.worker_steps = worker_steps

55        self.train_epochs = epochs

58        self.updates = updates

60        self.mini_batch_size = mini_batch_size

63        self.update_target_model = update_target_model

66        self.learning_rate = learning_rate

69        self.exploration_coefficient = Piecewise(
70            [
71                (0, 1.0),
72                (25_000, 0.1),
73                (self.updates / 2, 0.01)
74            ], outside_value=0.01)

77        self.prioritized_replay_beta = Piecewise(
78            [
79                (0, 0.4),
80                (self.updates, 1)
81            ], outside_value=1)

84        self.replay_buffer = ReplayBuffer(2 ** 14, 0.6)

87        self.model = Model().to(device)

89        self.target_model = Model().to(device)

92        self.workers = [Worker(47 + i) for i in range(self.n_workers)]

95        self.obs = np.zeros((self.n_workers, 4, 84, 84), dtype=np.uint8)

98        for worker in self.workers:
99            worker.child.send(("reset", None))

102        for i, worker in enumerate(self.workers):
103            self.obs[i] = worker.child.recv()

106        self.loss_func = QFuncLoss(0.99)

108        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=2.5e-4)

110    def _sample_action(self, q_value: torch.Tensor, exploration_coefficient: float):

120        with torch.no_grad():

122            greedy_action = torch.argmax(q_value, dim=-1)

124            random_action = torch.randint(q_value.shape[-1], greedy_action.shape, device=q_value.device)

126            is_choose_rand = torch.rand(greedy_action.shape, device=q_value.device) < exploration_coefficient

128            return torch.where(is_choose_rand, random_action, greedy_action).cpu().numpy()

130    def sample(self, exploration_coefficient: float):

134        with torch.no_grad():

136            for t in range(self.worker_steps):

138                q_value = self.model(obs_to_torch(self.obs))

140                actions = self._sample_action(q_value, exploration_coefficient)

143                for w, worker in enumerate(self.workers):
144                    worker.child.send(("step", actions[w]))

147                for w, worker in enumerate(self.workers):

149                    next_obs, reward, done, info = worker.child.recv()

152                    self.replay_buffer.add(self.obs[w], actions[w], reward, next_obs, done)

158                    if info:
159                        tracker.add('reward', info['reward'])
160                        tracker.add('length', info['length'])

163                    self.obs[w] = next_obs

165    def train(self, beta: float):

169        for _ in range(self.train_epochs):

171            samples = self.replay_buffer.sample(self.mini_batch_size, beta)

173            q_value = self.model(obs_to_torch(samples['obs']))

177            with torch.no_grad():

179                double_q_value = self.model(obs_to_torch(samples['next_obs']))

181                target_q_value = self.target_model(obs_to_torch(samples['next_obs']))

184            td_errors, loss = self.loss_func(q_value,
185                                             q_value.new_tensor(samples['action']),
186                                             double_q_value, target_q_value,
187                                             q_value.new_tensor(samples['done']),
188                                             q_value.new_tensor(samples['reward']),
189                                             q_value.new_tensor(samples['weights']))

192            new_priorities = np.abs(td_errors.cpu().numpy()) + 1e-6

194            self.replay_buffer.update_priorities(samples['indexes'], new_priorities)

197            for pg in self.optimizer.param_groups:
198                pg['lr'] = self.learning_rate()

200            self.optimizer.zero_grad()

202            loss.backward()

204            torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=0.5)

206            self.optimizer.step()

208    def run_training_loop(self):

214        tracker.set_queue('reward', 100, True)
215        tracker.set_queue('length', 100, True)

218        self.target_model.load_state_dict(self.model.state_dict())
219
220        for update in monit.loop(self.updates):

222            exploration = self.exploration_coefficient(update)
223            tracker.add('exploration', exploration)

225            beta = self.prioritized_replay_beta(update)
226            tracker.add('beta', beta)

229            self.sample(exploration)

232            if self.replay_buffer.is_full():

234                self.train(beta)

237                if update % self.update_target_model == 0:
238                    self.target_model.load_state_dict(self.model.state_dict())

241            tracker.save()

243            if (update + 1) % 1_000 == 0:
244                logger.log()

246    def destroy(self):

251        for worker in self.workers:
252            worker.child.send(("close", None))

255def main():

257    experiment.create(name='dqn')

260    configs = {

262        'updates': 1_000_000,

264        'epochs': 8,

266        'n_workers': 8,

268        'worker_steps': 4,

270        'mini_batch_size': 32,

272        'update_target_model': 250,

274        'learning_rate': FloatDynamicHyperParam(1e-4, (0, 1e-3)),
275    }

278    experiment.configs(configs)

281    m = Trainer(**configs)

283    with experiment.start():
284        m.run_training_loop()

286    m.destroy()

290if __name__ == "__main__":
291    main()