First commit.

Author: tedhuang96

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+results/

README.md

@@ -1,2 +1,50 @@
 # ppo-pytorch
 PPO in pytorch version.
+
+We run multiple episodes with the same policy, and create an experience replay buffer out of trajectories in these episodes to perform on-policy policy gradient updates using PPO. We clear the replay buffer from the last run before we start another run of multiple episodes. 
+
+## Set up Python environment
+Run
+```
+virtualenv -p /usr/bin/python3 ppoenv
+source ppoenv/bin/activate
+pip install -r requirements.txt
+```
+or
+```
+virtualenv -p /usr/bin/python3 ppoenv
+source ppoenv/bin/activate
+pip install gym==0.18.0
+pip install torch
+pip install tqdm
+pip install tensorboard
+```
+
+## Train and evaluate agent in RL (cartpole).
+```
+source ppoenv/bin/activate
+python train.py
+```
+Check training progress by running
+```
+source ppoenv/bin/activate
+tensorboard --logdir results/
+```
+After training is complete, find `[SAVED_LOG]` in `results/` (e.g., `20221023_172239`). To evaluate without visualization, run
+```
+source ppoenv/bin/activate
+python eval.py --log [SAVED_LOG]
+```
+To evaluate with visualization, run
+```
+source ppoenv/bin/activate
+python eval.py --log [SAVED_LOG] --visualize
+```
+If you want to evaluate on a checkpoint at a specific episode (e.g., 1000), run
+```
+source ppoenv/bin/activate
+python eval.py --log [SAVED_LOG] --visualize --training_episodes 1000
+```
+
+## Credits
+Borrowed code from [ikostrikov/pytorch-a2c-ppo-acktr-gail](https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail), [vita-epfl/CrowdNav](https://github.com/vita-epfl/CrowdNav), and [agrimgupta92/sgan](https://github.com/agrimgupta92/sgan).

arg_parse.py

@@ -0,0 +1,33 @@
+import argparse
+
+def arg_parse():
+    parser = argparse.ArgumentParser()
+    # gym environment hyperparameters
+    parser.add_argument('--state_size', type=int, default=4)
+    parser.add_argument('--action_size', type=int, default=2)  
+    # rollout hyperparameters
+    parser.add_argument('--num_episodes_per_run', type=int, default=100)
+    # policy hyperparameters
+    parser.add_argument('--policy_model', type=str, default='actor-critic', help='Policy options: actor-critic.')
+    parser.add_argument('--model_embedding_size', type=int, default=64)
+    parser.add_argument('--model_hidden_size', type=int, default=128)
+    parser.add_argument('--model_dropout', type=float, default=0.1)
+    parser.add_argument('--num_episodes_per_checkpoint', type=int, default=100)
+    # memory hyperparameters
+    parser.add_argument('--experience_replay', action='store_true', default=False)
+    parser.add_argument('--memory_capacity', type=int, default=100000, help='typically equals num_episodes_per_run * step_limit')
+    # training hyperparameters
+    parser.add_argument('--training_episodes', type=int, default=2000, help='equals num_episodes_per_run * num_runs')
+    # ppo parameters
+    parser.add_argument('--lr', type=float, default=1e-4)
+    parser.add_argument('--batch_size', type=int, default=64)
+    parser.add_argument('--clip_param', type=float, default=0.2)
+    parser.add_argument('--ppo_epoch', type=int, default=4)
+    parser.add_argument('--clip_grad', type=float, default=0.5)
+    parser.add_argument('--value_loss_coef', type=float, default=1.)
+    parser.add_argument('--entropy_coef', type=float, default=0.01)
+    parser.add_argument('--use_clipped_value_loss', action='store_true')
+    # evaluation inputs
+    parser.add_argument('--log', default=None)
+    parser.add_argument('--visualize', action='store_true')
+    return parser.parse_args()

eval.py

@@ -0,0 +1,39 @@
+from os.path import join, exists
+
+import gym
+import torch
+
+from arg_parse import arg_parse
+from src.policy import Policy
+from src.rollout import Rollout
+from src.experience_memory import ExperienceMemory
+
+
+def main():
+    ##### Initialization Phase #####
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    print(device)
+    args = arg_parse()
+    experience_memory = ExperienceMemory(args)
+    env = gym.make('CartPole-v0')
+    policy = Policy(args, device=device)
+    rollout = Rollout(env, policy, experience_memory=experience_memory)
+    if args.log is None:
+        raise RuntimeError("The log for evaluation is None.")
+    logdir = join('results', args.log)
+    checkpoint_filepath = join(logdir, 'model_weights', str(args.training_episodes)+'.pt')
+    if not exists(checkpoint_filepath):
+        raise RuntimeError(checkpoint_filepath+" does not exist.")
+    checkpoint = torch.load(checkpoint_filepath, map_location=device)
+    policy.load_model_weights(checkpoint)
+    print("model weights are loaded.")
+    if args.visualize:
+        rollout_mode = 'visualize'
+    else:
+        rollout_mode = 'test'
+    _, _, _, accumulative_reward = rollout.run(args, rollout_mode=rollout_mode)
+    print("accumulative reward: ", accumulative_reward)
+    env.close()
+
+if __name__ == "__main__":
+    main()

log.md

@@ -0,0 +1,8 @@
+# Log
+
+## 221023
+
+- [Todo] Improve coding in `src/experience_memory.py`.
+```
+ppo-pytorch/src/experience_memory.py:82: [TodoserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)
+```

requirements.txt

@@ -0,0 +1,37 @@
+absl-py==1.3.0
+cachetools==4.2.4
+certifi==2022.9.24
+charset-normalizer==2.0.12
+cloudpickle==1.6.0
+dataclasses==0.8
+future==0.18.2
+google-auth==2.13.0
+google-auth-oauthlib==0.4.6
+grpcio==1.48.2
+gym==0.18.0
+idna==3.4
+importlib-metadata==4.8.3
+importlib-resources==5.4.0
+Markdown==3.3.7
+numpy==1.19.5
+oauthlib==3.2.2
+Pillow==7.2.0
+protobuf==3.19.6
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pyglet==1.5.0
+requests==2.27.1
+requests-oauthlib==1.3.1
+rsa==4.9
+scipy==1.5.4
+six==1.16.0
+tdqm==0.0.1
+tensorboard==2.10.1
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+torch==1.10.2
+tqdm==4.64.1
+typing-extensions==4.1.1
+urllib3==1.26.12
+Werkzeug==2.0.3
+zipp==3.6.0

src/experience_memory.py

@@ -0,0 +1,100 @@
+import torch
+from torch.utils.data import Dataset
+
+class ExperienceMemory(Dataset):
+    """
+    reference: https://github.com/vita-epfl/CrowdNav/blob/503173b836d5460e30234df7e14a7c67ee0ebfc7/crowd_nav/utils/memory.py#L4
+    ExperienceMemory can be used to store rollouts for policy update.
+    """
+    def __init__(self, args):
+        self.args = args
+        self.capacity = self.args.memory_capacity
+        self.reset()
+    
+
+    def __len__(self):
+        for k in self.memory.keys():
+            assert self.length == len(self.memory[k])
+        return self.length
+    
+
+    def push(self, experience):
+        """push the newest in and the oldest out. works in experience replay setting."""
+        assert self.memory.keys() == experience.keys()
+        experience_length = len(list(experience.values())[0])
+        for i in range(experience_length):
+            if self.length < self.position + 1: # before capacity is filled.
+                self.length += 1
+                for k in self.memory.keys():
+                    self.memory[k].append(experience[k][i])
+            else: # after capacity is filled, replace old experience with newer ones.
+                for k in self.memory.keys():
+                    self.memory[k][self.position] = experience[k][i]
+            self.position = (self.position + 1) % self.capacity
+    
+
+    def concatenate(self, experience):
+        """concatenate with new memory (list). This means we don't do experience replay.
+        If we do concatenation, we must assure the concatenated won't exceed the capacity."""
+        assert self.memory.keys() == experience.keys()
+        experience_length = len(list(experience.values())[0])
+        if self.length + experience_length > self.capacity:
+            raise RuntimeError("Adding new experience exceeds the memory capacity.")
+        for k in self.memory.keys():
+            self.memory[k] = self.memory[k] + experience[k]
+        self.length += experience_length
+        self.position += experience_length # sync of self.position with self.length when not reach capacity.
+
+    def is_full(self):
+        return self.length == self.capacity
+    
+
+    def reset(self):
+        self.memory = {}
+        self.position = 0
+        self.length = 0
+        if self.args.policy_model == 'actor-critic':
+            self.memory_keys = ['states', 'actions', 'rewards', 'values', 'returns', \
+                'advantages', 'action_log_probs'] # ordered keys
+            for k in self.memory_keys:
+                self.memory[k] = []
+        else:
+            raise RuntimeError("Policy model is not supported.")
+        self.update_tensor()
+    
+    def update_tensor(self):
+        """
+        list2tensor.
+        Called in reset to create empty tensors, or
+        Called after self.memory (which is a dict of list) is updated from rollouts.
+        outputs:
+            - memory_tensor: dict of tensors.
+                - ['states']: (data_len, 5)
+                - ['actions']: (data_len,)
+                - ['rewards']: (data_len,)
+                - ['values']: (data_len,)
+                - ['returns']: (data_len,)
+                - ['advantages']: (data_len,)
+                - ['action_log_probs']: (data_len,)
+        """
+        self.memory_tensor = {}
+        for k in self.memory.keys():
+            self.memory_tensor[k] = torch.Tensor(self.memory[k]) # float32
+    
+    
+    def __getitem__(self, index):
+        """
+        if self.args.policy_model == 'actor-critic':
+            return a list in the following order:
+                - states[index]
+                - actions[index]
+                - rewards[index]
+                - values[index]
+                - returns[index]
+                - advantages[index]
+                - action_log_probs[index] # all are not .to(device) yet
+        """
+        out = []
+        for k in self.memory_keys: # ordered keys
+            out.append(self.memory_tensor[k][index])
+        return out

src/model/actor_critic.py

@@ -0,0 +1,44 @@
+import torch.nn as nn
+import torch.nn.functional as F
+
+from src.model.utils import make_mlp
+
+
+class ActorCritic(nn.Module):
+    """Actor Critic."""
+    def __init__(self, state_size, action_size, \
+        embedding_size=64, hidden_size=128, dropout=0.1):
+        super(ActorCritic, self).__init__()
+        # spatial embeddings
+        self.actor_embedding = make_mlp([state_size, hidden_size, embedding_size], \
+            batchnorm=True, activation='relu', dropout=dropout)
+        self.critic_embedding = make_mlp([state_size, hidden_size, embedding_size], \
+            batchnorm=True, activation='relu', dropout=dropout)
+        # encoder
+        self.actor_encoder = make_mlp([embedding_size, hidden_size], \
+            batchnorm=True, activation='relu', dropout=dropout)
+        self.critic_encoder = make_mlp([embedding_size, hidden_size], \
+            batchnorm=True, activation='relu', dropout=dropout)
+        self.actor_fc = nn.Linear(hidden_size, action_size)        
+        self.critic_fc = nn.Linear(hidden_size, 1)
+
+    
+    def forward(self, x):
+        """
+        inputs:
+            - x 
+                # input data. 
+                # tensor. size: (batch_size, state_size)
+        outputs:
+            - action_prob
+                # probability of actions in the stochastic policy.
+                # tensor. size: (batch_size, action_size)
+            - state_value
+                # values of state from critic.
+                # tensor. size: (batch_size, )
+        """
+        xa = self.actor_embedding(x)
+        action_prob = F.softmax(self.actor_fc(self.actor_encoder(xa)), dim=-1) # (batch_size, action_size)
+        xc = self.critic_embedding(x)
+        state_value = self.critic_fc(self.critic_encoder(xc)).squeeze(-1) # (batch_size, )
+        return action_prob, state_value

src/model/utils.py

@@ -0,0 +1,15 @@
+import torch.nn as nn
+
+def make_mlp(dim_list, batchnorm=False, activation='relu', dropout=0.):
+    layers = []
+    for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]):
+        layers.append(nn.Linear(dim_in, dim_out))
+        if batchnorm:
+            layers.append(nn.BatchNorm1d(dim_out))
+        if activation == 'relu':
+            layers.append(nn.ReLU())
+        elif activation == 'leakyrelu':
+            layers.append(nn.LeakyReLU())
+        if dropout > 0.:
+            layers.append(nn.Dropout(p=dropout))
+    return nn.Sequential(*layers)