Merge pull request #1 from pockerman/add_actor_critic_algorithm

Add actor critic algorithm

Author: pockerman

.gitignore

@@ -1 +1,2 @@
-preprocessor/__pycache__/
+src/preprocessor/__pycache__/
+src/exceptions/__pycache__/

algorithms/a2c.py

@@ -0,0 +1,99 @@
+import numpy as np
+from typing import TypeVar, Generic
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+Env = TypeVar("Env")
+Optimizer = TypeVar("Optimizer")
+LossFunction = TypeVar("LossFunction")
+State = TypeVar("State")
+Action = TypeVar("Action")
+TimeStep = TypeVar("TimeStep")
+
+
+class A2CNetBase(nn.Module):
+    """
+    Base class for A2C networks
+    """
+
+    def __init__(self, architecture):
+        super(A2CNetBase, self).__init__()
+        self.architecture = architecture
+
+    def forward(self, x):
+        return self.architecture(x)
+
+
+class A2CNet(nn.Module):
+
+    def __init__(self, common_net: A2CNetBase, policy_net: A2CNetBase, value_net: A2CNetBase):
+        super(A2CNet, self).__init__()
+        self.common_net = common_net
+        self.policy_net = policy_net
+        self.value_net = value_net
+
+    def forward(self, x):
+        x = self.common_net(x)
+
+        pol_out = self.policy_net(x)
+        val_out = self.value_net(x)
+        return pol_out, val_out
+
+
+class A2C(Generic[Optimizer]):
+
+    def __init__(self, gamma: float, tau: float, n_workers: int,
+                 n_iterations: int,  optimizer: Optimizer,
+                 a2c_net: A2CNet, loss_function: LossFunction):
+
+        self.gamma = gamma
+        self.tau = tau
+        self.rewards = []
+        self.n_workers = n_workers
+        self.n_iterations = n_iterations
+        self.optimizer = optimizer
+        self.a2c_net = a2c_net
+        self.loss_function = loss_function
+        self.name = "A2C"
+
+    def _optimize_model(self):
+        pass
+
+    def select_action(self, env: Env, observation: State) -> Action:
+        """
+        Select an action
+        :param env: The environment over which the agent is trained
+        :param observation: The current observation of the environment
+        :return: Returns an Action type
+        """
+        return env.sample_action()
+
+    def update(self):
+        pass
+
+    def train(self, env: Env) -> None:
+
+        # reset the environment and obtain the
+        # the time step
+        time_step: TimeStep = env.reset()
+
+        observation = time_step.observation
+
+        for iteration in range(1, self.n_iterations + 1):
+
+            # select an action
+            action = self.select_action(env=env, observation=observation)
+
+            # step in the environment according
+            # to the selected action
+            next_time_step = env.step(action=action)
+
+            # we reached the end of the episode
+            if next_time_step.last():
+                break
+
+            next_state = next_time_step.observation
+            policy_val, v_val = self.a2c_net.forward(x=next_state)
+            self._optimize_model()
+

algorithms/__init__.py src/algorithms/__init__.pyalgorithms/__init__.py renamed to src/algorithms/__init__.py

@@ -6,8 +6,8 @@
 import ray.rllib.agents.a3c as a3c
 from ray.tune.logger import pretty_print
 from ray.rllib.env.env_context import EnvContext
-from spaces.environment import TimeStep, StepType
-from spaces.observation_space import ObsSpace
+from src.spaces.environment import TimeStep, StepType
+from src.spaces.observation_space import ObsSpace
 
 
 class DataSetEnv(gym.Env):

src/algorithms/a2c.py

@@ -2,7 +2,7 @@
 Trainer
 """
 
-from utils import INFO
+from src.utils import INFO
 from typing import TypeVar
 
 Env = TypeVar("Env")
@@ -29,4 +29,9 @@ def train(self):
             # train for a number of iterations
             self.agent.train(self.env)
 
+            # is it time to update the model?
+            if self.configuration["update_frequency"] % episode == 0:
+                self.agent.update()
+
+
         print("{0} Training finished for agent {1}".format(INFO, self.agent.name))

algorithms/anonymity_a2c_ray.py src/algorithms/anonymity_a2c_ray.pyalgorithms/anonymity_a2c_ray.py renamed to src/algorithms/anonymity_a2c_ray.py

@@ -4,10 +4,11 @@
 import pandas as pd
 import numpy as np
 
-from preprocessor.cleanup_utils import read_csv, replace, change_column_types
+from src.preprocessor.cleanup_utils import read_csv, replace, change_column_types
 
 DS = TypeVar("DS")
 HierarchyBase = TypeVar('HierarchyBase')
+Transform = TypeVar("Transform")
 
 
 class DSWrapper(Generic[DS], metaclass=abc.ABCMeta):
@@ -42,21 +43,26 @@ def __init__(self, columns: dir) -> None:
         # on each column in the dataset
         self.column_hierarchy = {}
 
+    @property
     def n_rows(self) -> int:
         """
         Returns the number of rows of the data set
         :return:
         """
-
         return self.ds.shape[0]
 
+    @property
     def n_columns(self) -> int:
         """
         Returns the number of rows of the data set
         :return:
         """
         return self.ds.shape[1]
 
+    @property
+    def schema(self) -> dict:
+        return pd.io.json.build_table_schema(self.ds)
+
     def read(self, filename: Path,  **options) -> None:
         """
         Load a data set from a file
@@ -72,7 +78,7 @@ def read(self, filename: Path,  **options) -> None:
             self.ds = replace(ds=self.ds, options=options["change_col_vals"])
 
         # try to cast to the data types
-        self.ds = change_column_types(ds=self.ds, column_types=self.columns) 
+        self.ds = change_column_types(ds=self.ds, column_types=self.columns)
 
     def set_columns_to_type(self, col_name_types) -> None:
         self.ds.astype(dtype=col_name_types)
@@ -88,7 +94,6 @@ def get_column_unique_values(self, col_name: str):
 
         col = self.get_column(col_name=col_name)
         vals = col.values.ravel()
-
         return pd.unique(vals)
 
     def get_columns_types(self):
@@ -106,3 +111,9 @@ def sample_column(self):
         col_idx = np.random.choice(col_names, 1)
         return self.get_column(col_name=col_names[col_idx])
 
+    def apply_transform(self, transform: Transform) -> None:
+        pass
+
+
+
+

algorithms/trainer.py src/algorithms/trainer.pyalgorithms/trainer.py renamed to src/algorithms/trainer.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+from src.datasets.dataset_wrapper import PandasDSWrapper
+
+
+class MockSubjectsLoader(PandasDSWrapper):
+
+    DEFAULT_COLUMNS = {"gender": str, "ethnicity": str, "education": int,
+                       "salary": int, "diagnosis": int, "preventative_treatment": str,
+                       "mutation_status": int, }
+
+    FILENAME = Path("../../data/mocksubjects.csv")
+
+    FEATURES_DROP_NAMES = ["NHSno", "given_name", "surname", "dob"]
+
+    NAMES = ["NHSno", "given_name", "surname", "gender",
+             "dob", "ethnicity", "education", "salary",
+             "mutation_status", "preventative_treatment", "diagnosis"]
+
+    DROP_NA = True
+
+    CHANGE_COLS_VALS = {"diagnosis": [('N', 0)]}
+
+    def __init__(self):
+        super(MockSubjectsLoader, self).__init__(columns=MockSubjectsLoader.DEFAULT_COLUMNS)
+        self.read(filename=MockSubjectsLoader.FILENAME, **{"features_drop_names": MockSubjectsLoader.FEATURES_DROP_NAMES,
+                                                            "names": MockSubjectsLoader.NAMES,
+                                                            "drop_na": MockSubjectsLoader.DROP_NA,
+                                                            "change_col_vals": MockSubjectsLoader.CHANGE_COLS_VALS})