Curiosity/icm mario.py

import torch
from torch import nn
from torchvision import transforms as T
from PIL import Image
import numpy as np
from pathlib import Path
from collections import deque
import random, datetime, os, copy
from torch.distributions import Categorical
import collections
import cv2
import torch.nn.functional as f
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter()

# Gym is an OpenAI toolkit for RL
import gym
from gym.spaces import Box
from gym.wrappers import FrameStack

# NES Emulator for OpenAI Gym
from nes_py.wrappers import JoypadSpace

# Super Mario environment for OpenAI Gym
import gym_super_mario_bros
from gym_super_mario_bros.actions import RIGHT_ONLY, SIMPLE_MOVEMENT, COMPLEX_MOVEMENT

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")


class SkipFrame(gym.Wrapper):
    def __init__(self, env, skip):
        """Return only every `skip`-th frame"""
        super().__init__(env)
        self._skip = skip

    def step(self, action):
        """Repeat action, and sum reward"""
        total_reward = 0.0
        for i in range(self._skip):
            # Accumulate reward and repeat the same action
            obs, reward, done, trunk, info = self.env.step(action)
            total_reward += reward
            if done:
                break
        return obs, total_reward, done, trunk, info


class GrayScaleObservation(gym.ObservationWrapper):
    def __init__(self, env):
        super().__init__(env)
        obs_shape = self.observation_space.shape[:2]
        self.observation_space = Box(low=0, high=255, shape=obs_shape, dtype=np.uint8)

    def permute_orientation(self, observation):
        # permute [H, W, C] array to [C, H, W] tensor
        observation = np.transpose(observation, (2, 0, 1))
        observation = torch.tensor(observation.copy(), dtype=torch.float)
        return observation

    def observation(self, observation):
        observation = self.permute_orientation(observation)
        transform = T.Grayscale()
        observation = transform(observation)
        return observation


class ResizeObservation(gym.ObservationWrapper):
    def __init__(self, env, shape):
        super().__init__(env)
        if isinstance(shape, int):
            self.shape = (shape, shape)
        else:
            self.shape = tuple(shape)

        obs_shape = self.shape + self.observation_space.shape[2:]
        self.observation_space = Box(low=0, high=255, shape=obs_shape, dtype=np.uint8)

    def observation(self, observation):
        transforms = T.Compose(
            [T.Resize(self.shape), T.Normalize(0, 255)]
        )
        observation = transforms(observation).squeeze(0)
        return observation
class MaxAndSkipEnv(gym.Wrapper):
    """
        Each action of the agent is repeated over skip frames
        return only every `skip`-th frame
    """
    def __init__(self, env=None, skip=4):
        super(MaxAndSkipEnv, self).__init__(env)
        # most recent raw observations (for max pooling across time steps)
        self._obs_buffer = collections.deque(maxlen=2)
        self._skip = skip

    def step(self, action):
        total_reward = 0.0
        done = None
        for _ in range(self._skip):
            obs, reward, done, info = self.env.step(action)
            self._obs_buffer.append(obs)
            total_reward += reward
            if done:
                break
        max_frame = np.max(np.stack(self._obs_buffer), axis=0)
        return max_frame, total_reward, done, info

    def reset(self):
        """Clear past frame buffer and init to first obs"""
        self._obs_buffer.clear()
        obs = self.env.reset()
        self._obs_buffer.append(obs)
        return obs


class MarioRescale84x84(gym.ObservationWrapper):
    """
    Downsamples/Rescales each frame to size 84x84 with greyscale
    """
    def __init__(self, env=None):
        super(MarioRescale84x84, self).__init__(env)
        self.observation_space = gym.spaces.Box(low=0, high=255, shape=(84, 84, 1), dtype=np.uint8)

    def observation(self, obs):
        return MarioRescale84x84.process(obs)

    @staticmethod
    def process(frame):
        if frame.size == 240 * 256 * 3:
            img = np.reshape(frame, [240, 256, 3]).astype(np.float32)
        else:
            assert False, "Unknown resolution." 
        # image normalization on RBG
        img = img[:, :, 0] * 0.299 + img[:, :, 1] * 0.587 + img[:, :, 2] * 0.114
        resized_screen = cv2.resize(img, (84, 110), interpolation=cv2.INTER_AREA)
        x_t = resized_screen[18:102, :]
        x_t = np.reshape(x_t, [84, 84, 1])
        return x_t.astype(np.uint8)


class ImageToPyTorch(gym.ObservationWrapper):
    """
    Each frame is converted to PyTorch tensors
    """
    def __init__(self, env):
        super(ImageToPyTorch, self).__init__(env)
        old_shape = self.observation_space.shape
        self.observation_space = gym.spaces.Box(low=0.0, high=1.0, shape=(old_shape[-1], old_shape[0], old_shape[1]), dtype=np.float32)

    def observation(self, observation):
        return np.moveaxis(observation, 2, 0)

    
class BufferWrapper(gym.ObservationWrapper):
    """
    Only every k-th frame is collected by the buffer
    """
    def __init__(self, env, n_steps, dtype=np.float32):
        super(BufferWrapper, self).__init__(env)
        self.dtype = dtype
        old_space = env.observation_space
        self.observation_space = gym.spaces.Box(old_space.low.repeat(n_steps, axis=0),
                                                old_space.high.repeat(n_steps, axis=0), dtype=dtype)

    def reset(self):
        self.buffer = np.zeros_like(self.observation_space.low, dtype=self.dtype)
        return self.observation(self.env.reset())

    def observation(self, observation):
        self.buffer[:-1] = self.buffer[1:]
        self.buffer[-1] = observation
        return self.buffer


class PixelNormalization(gym.ObservationWrapper):
    """
    Normalize pixel values in frame --> 0 to 1
    """
    def observation(self, obs):
        return np.array(obs).astype(np.float32) / 255.0


def create_mario_env(env):
    env = MaxAndSkipEnv(env)
    env = MarioRescale84x84(env)
    env = ImageToPyTorch(env)
    env = BufferWrapper(env, 4)
    env = PixelNormalization(env)
    return JoypadSpace(env, COMPLEX_MOVEMENT)

class ActorCritic(nn.Module):
    def __init__(self, input_size, action_size=2):
        super(ActorCritic, self).__init__()
        self.input_size = input_size
        self.action_size = action_size

        self.feature = nn.Sequential(
            nn.Conv2d(in_channels=self.input_size[0], out_channels=32, kernel_size=8, stride=4),
            nn.LeakyReLU(),
            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=4, stride=2),
            nn.LeakyReLU(),
            nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1),
            nn.LeakyReLU(),
            nn.Flatten(),
            nn.Linear(in_features=7*7*64, out_features=512),
            nn.LeakyReLU(),
        )

    def actor(self,state):
        policy = nn.Sequential(
            nn.Linear(in_features=state.shape[1], out_features=state.shape[1]),
            nn.LeakyReLU(),
            nn.Linear(in_features=state.shape[1], out_features=self.action_size),
            nn.Softmax(dim=-1)
        ).to(device)
        return policy(state)

    def critic(self,state):
        value = nn.Sequential(
            nn.Linear(in_features=state.shape[1], out_features=state.shape[1]),
            nn.LeakyReLU(),
            nn.Linear(in_features=state.shape[1], out_features=1)
        ).to(device)
        return value(state)


    def forward(self, state):
        if state.dim() == 3:
            state = state.unsqueeze(0)
        state = self.feature(state)
        value = self.critic(state)
        policy = self.actor(state)
        action_probs = Categorical(policy)
        log_action_probs = torch.log(action_probs.probs)
        return value, action_probs, log_action_probs


class Encoder(nn.Module):
    def __init__(self, input_size, action_size=2):
        super(Encoder, self).__init__()
        self.input_size = input_size[0]
        self.action_size = action_size

        self.feature_encoder = nn.Sequential(
            nn.Conv2d(in_channels=self.input_size, out_channels=32, kernel_size=3, stride=2),
            nn.LeakyReLU(),
            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=2),
            nn.LeakyReLU(),
            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=2),
            nn.LeakyReLU(),
            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=2),
            nn.LeakyReLU(),
            nn.Flatten(),
            nn.Linear(in_features=32*4*4, out_features=256),  
        ).to(device)

    def forward(self, state):
        if state.dim() == 3:
            state = state.unsqueeze(0)
        state = self.feature_encoder(state)
        return state


class InverseModel(nn.Module):
    def __init__(self, input_size, action_size=2):
        super(InverseModel, self).__init__()
        self.input_size = input_size[0]
        self.action_size = action_size
        self.feature_encoder = Encoder(input_size, action_size)

        self.model = nn.Sequential(
            nn.Linear(in_features=256*2, out_features=256),
            nn.LeakyReLU(),
            nn.Linear(in_features=256, out_features=self.action_size),
            nn.Softmax(dim=-1)
        ).to(device)

    def forward(self, state, next_state):

        encoded_state, next_encoded_state = torch.unsqueeze(state, dim=0), torch.unsqueeze(next_state, dim=0)
        encoded_state, next_encoded_state = self.feature_encoder(encoded_state), self.feature_encoder(next_encoded_state)
        encoded_states = torch.cat((encoded_state, next_encoded_state), dim=-1)
        actions = Categorical(self.model(encoded_states))
        a = float(np.array(actions.sample().cpu())[0])
        action = torch.FloatTensor([a])
        one_hot_action = f.one_hot(action.to(torch.int64), self.action_size)
        return one_hot_action, encoded_state, next_encoded_state


class ForwardModel(nn.Module):
    def __init__(self, encoded_state_size, action_size):
        super(ForwardModel, self).__init__()
        self.state_size = encoded_state_size
        self.action_size = action_size
  
        self.model = nn.Sequential(
            nn.Linear(self.state_size + 1, 256),
            nn.LeakyReLU(),
            nn.Linear(256, encoded_state_size)
        ).to(device)

    def forward(self, state, action):
        if state.dim() == 3:
            state = state.unsqueeze(0)
        if action.dim() == 1:
            action = action.unsqueeze(0)
        state = torch.cat((state, action), dim=-1)
        return self.model(state)

class ICM(nn.Module):
    def __init__(self, state_size, action_size, encoded_state_size=256):
        super(ICM, self).__init__()
        self.state_size = state_size
        self.action_size = action_size
        self.inverse_model = InverseModel(state_size, action_size)
        self.forward_model = ForwardModel(encoded_state_size, action_size)
        self.loss = nn.MSELoss().to(device)

        self.feature_encoder = nn.Sequential(
                    nn.Conv2d(in_channels=self.state_size[0], out_channels=32, kernel_size=3, stride=2),
                    nn.LeakyReLU(),
                    nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=2),
                    nn.LeakyReLU(),
                    nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=2),
                    nn.LeakyReLU(),
                    nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, stride=2),
                    nn.LeakyReLU(),
                    nn.Flatten(),
                    nn.Linear(in_features=32*4*4, out_features=256),  
                ).to(device)

        self.model = nn.Sequential(
            nn.Linear(self.state_size[1] + 1, 256),
            nn.LeakyReLU(),
            nn.Linear(256, encoded_state_size)
        ).to(device)

    def forward(self, state, next_state, action):
        if state.dim() == 3:
            state = state.unsqueeze(0)
            next_state = next_state.unsqueeze(0)
        encoded_state, next_encoded_state = self.feature_encoder(state), self.feature_encoder(next_state)
        #_, encoded_state, next_encoded_state = self.inverse_model(state, next_state)
        action = torch.tensor([action]).to(device)
        predicted_next_state = self.forward_model(encoded_state, action)
        
        intrinsic_reward = 0.5 * self.loss(predicted_next_state, next_encoded_state.detach())
        return intrinsic_reward

#env = gym.make('SuperMarioBros-1-1-v0')
#env = GrayScaleObservation(env)
#env = ResizeObservation(env, shape=84)
#env = FrameStack(env, num_stack=4)
env = gym.make('SuperMarioBros-1-1-v0')
env = create_mario_env(env)

ce = nn.CrossEntropyLoss().to(device)
mse = nn.MSELoss().to(device)
icm = ICM(env.observation_space.shape, env.action_space.n).to(device)
ac = ActorCritic(env.observation_space.shape, env.action_space.n).to(device)
optimizer = torch.optim.Adam(list(icm.parameters()) + list(ac.parameters()), lr=0.001)
done = False
t = 0
gamma = 0.99    
for episode in range(1000):
    observation = env.reset()
    total_reward = 0
    t_init = t
    while not done:
        #env.render()
        value, actions, log_action_probs = ac(torch.FloatTensor(np.array(observation)).to(device))    
        action = actions.sample().item()

        next_observation, reward, done, info = env.step(action)  # feedback from environment    
        observation_array, next_observation_array = torch.FloatTensor(np.array(observation)).to(device), torch.FloatTensor(np.array(next_observation)).to(device)

        int_reward = icm(observation_array, next_observation_array, action)

        delta = torch.squeeze(int_reward + gamma * (ac(next_observation_array)[0]*(1-int(done))) - ac(observation_array)[0])
        actor_loss = -log_action_probs[0,action] * int_reward
        critic_loss = delta**2

        reward = torch.FloatTensor([reward]).to(device)
        reward = int_reward
        
        one_hot_action = icm.inverse_model(observation_array, next_observation_array)[0].to(device)
        inverse_loss = ce(one_hot_action.float(), actions.probs)


        loss = actor_loss + critic_loss + inverse_loss
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        writer.add_scalar("loss", loss, t)
        

        observation = next_observation

        total_reward += reward
        t += 1
        #print("timestep: ", t, "reward: ", reward, "loss: ", loss)
        if done:
            done = False
            break
    writer.add_scalar("reward", total_reward/(t-t_init), t)
env.close()