API updates

Author: pockerman

src/algorithms/sarsa_semi_gradient.py …algorithms/n_step_semi_gradient_sarsa.pysrc/algorithms/sarsa_semi_gradient.py renamed to src/algorithms/n_step_semi_gradient_sarsa.py src/algorithms/sarsa_semi_gradient.py renamed to src/algorithms/n_step_semi_gradient_sarsa.py

@@ -21,7 +21,6 @@
 @dataclass(init=True, repr=True)
 class SARSAnConfig:
     """Configuration class for n-step SARSA algorithm
-
     """
     gamma: float = 1.0
     alpha: float = 0.1

src/algorithms/trainer.py

@@ -47,21 +47,30 @@ def avg_distortion(self) -> np.array:
             avg[i] = self.total_distortions[i] / self.iterations_per_episode[i]
         return avg
 
-    def actions_before_training(self):
-        """
-        Any actions to perform before training begins
-        :return:
+    def actions_before_training(self) -> None:
+        """Any actions to perform before training begins
+
+        Returns
+        -------
+
+        None
         """
+
         self.total_rewards: np.array = np.zeros(self.configuration['n_episodes'])
         self.iterations_per_episode = []
-
         self.agent.actions_before_training(self.env)
 
     def actions_before_episode_begins(self, **options) -> None:
-        """
-        Perform any actions necessary before the training begins
-        :param options:
-        :return:
+        """Perform any actions necessary before the training begins
+
+        Parameters
+        ----------
+        options: Any options passed by the client code
+
+        Returns
+        -------
+
+        None
         """
         self.agent.actions_before_episode_begins(**options)
 

src/datasets/datasets_loaders.py

@@ -5,46 +5,65 @@
 """
 
 from pathlib import Path
+from typing import List
+from dataclasses import dataclass, field
+
 from src.datasets.dataset_wrapper import PandasDSWrapper
 
 
-class MockSubjectsLoader(PandasDSWrapper):
-    """
-    The class MockSubjectsLoader. Loads the  mocksubjects.csv
-    """
+@dataclass(init=True, repr=True)
+class MockSubjectsData(object):
 
     # Path to the dataset file
-    FILENAME = Path("../../data/mocksubjects.csv")
+    FILENAME: Path = Path("../../data/mocksubjects.csv")
 
     # the assumed column types. We use this map to cast
     # the types of the columns
-    COLUMNS_TYPES = {"gender": str, "ethnicity": str, "education": int,
-                       "salary": int, "diagnosis": int, "preventative_treatment": str,
-                       "mutation_status": int, }
+    COLUMNS_TYPES: dict = field(default_factory=lambda: {"gender": str, "ethnicity": str, "education": int,
+                     "salary": int, "diagnosis": int, "preventative_treatment": str,
+                     "mutation_status": int,})
 
     # features to drop
-    FEATURES_DROP_NAMES = ["NHSno", "given_name", "surname", "dob"]
+    FEATURES_DROP_NAMES: List[str] = field(default_factory=lambda: ["NHSno", "given_name", "surname", "dob"])
 
     # Names of the columns in the dataset
-    NAMES = ["NHSno", "given_name", "surname", "gender",
-             "dob", "ethnicity", "education", "salary",
-             "mutation_status", "preventative_treatment", "diagnosis"]
+    NAMES: List[str] = field(default_factory=lambda: ["NHSno", "given_name", "surname", "gender",
+                                              "dob", "ethnicity", "education", "salary",
+                                              "mutation_status", "preventative_treatment", "diagnosis"])
 
     # option to drop NaN
-    DROP_NA = True
+    DROP_NA: bool = True
 
     # Map that holds for each column the transformations
     # we want to apply for each value
-    CHANGE_COLS_VALS = {"diagnosis": [('N', 0)]}
+    CHANGE_COLS_VALS: dict = field(default_factory=lambda: {"diagnosis": [('N', 0)]})
 
     # list of columns to be normalized
-    NORMALIZED_COLUMNS = []
-
-    def __init__(self):
-        super(MockSubjectsLoader, self).__init__(columns=MockSubjectsLoader.COLUMNS_TYPES)
-        self.read(filename=MockSubjectsLoader.FILENAME,
-                  **{"features_drop_names": MockSubjectsLoader.FEATURES_DROP_NAMES,
-                     "names": MockSubjectsLoader.NAMES,
-                     "drop_na": MockSubjectsLoader.DROP_NA,
-                     "change_col_vals": MockSubjectsLoader.CHANGE_COLS_VALS,
-                     "column_normalization": MockSubjectsLoader.NORMALIZED_COLUMNS})
+    NORMALIZED_COLUMNS: List[str] = field(default_factory=list)
+
+
+class MockSubjectsLoader(PandasDSWrapper):
+    """The class MockSubjectsLoader. Loads the  mocksubjects.csv
+    """
+
+    @classmethod
+    def from_options(cls, *, filename: Path,
+                     column_types: dir, features_drop_names: List[str],
+                     names: List[str], drop_na: bool, change_col_vals: dict, column_normalization: List[str]):
+
+        data = MockSubjectsData(FILENAME=filename, COLUMNS_TYPES=column_types,
+                                FEATURES_DROP_NAMES=features_drop_names, NAMES=names,
+                                DROP_NA=drop_na, CHANGE_COLS_VALS=change_col_vals,
+                                NORMALIZED_COLUMNS=column_normalization)
+        return cls(data=data)
+
+    def __init__(self, data: MockSubjectsData, do_read: bool=True):
+        super(MockSubjectsLoader, self).__init__(columns=data.COLUMNS_TYPES)
+
+        if do_read:
+            self.read(filename=data.FILENAME,
+                      **{"features_drop_names": data.FEATURES_DROP_NAMES,
+                         "names": data.NAMES,
+                         "drop_na": data.DROP_NA,
+                         "change_col_vals": data.CHANGE_COLS_VALS,
+                         "column_normalization": data.NORMALIZED_COLUMNS})

src/examples/nstep_semi_grad_sarsa_three_columns.py

@@ -2,7 +2,7 @@
 import numpy as np
 from pathlib import Path
 
-from src.algorithms.sarsa_semi_gradient import SARSAnConfig, SARSAn
+from src.algorithms.n_step_semi_gradient_sarsa import SARSAnConfig, SARSAn
 from src.algorithms.q_estimator import QEstimator
 from src.algorithms.trainer import Trainer
 from src.datasets.datasets_loaders import MockSubjectsLoader

src/spaces/discrete_state_environment.py

@@ -13,7 +13,6 @@
 from src.spaces.actions import ActionBase, ActionType
 from src.spaces.time_step import TimeStep, StepType
 
-
 DataSet = TypeVar("DataSet")
 RewardManager = TypeVar("RewardManager")
 ActionSpace = TypeVar("ActionSpace")
@@ -47,6 +46,24 @@ class DiscreteStateEnvironment(object):
 
     IS_TILED_ENV_CONSTRAINT = False
 
+    @classmethod
+    def from_options(cls, *, data_set: DataSet, action_space: ActionSpace,
+                     reward_manager: RewardManager, distortion_calculator: DistortionCalculator,
+                     average_distortion_constraint: float = 0.0,
+                     gamma: float = 0.99, n_states: int = 10, min_distortion: float = 0.4,
+                     max_distortion: float = 0.7, punish_factor: float = 2.0, reward_factor: float = 0.95,
+                     n_rounds_below_min_distortion: int = 10,
+                     distorted_set_path: Path = None):
+
+        config = DiscreteEnvConfig(data_set=data_set, action_space=action_space, reward_manager=reward_manager,
+                                   distortion_calculator=distortion_calculator, distorted_set_path=distorted_set_path,
+                                   reward_factor=reward_factor,
+                                   n_rounds_below_min_distortion=n_rounds_below_min_distortion,
+                                   punish_factor=punish_factor, max_distortion=max_distortion, gamma=gamma,
+                                   n_states=n_states, min_distortion=min_distortion,
+                                   average_distortion_constraint=average_distortion_constraint)
+        return cls(env_config=config)
+
     def __init__(self, env_config: DiscreteEnvConfig) -> None:
         self.config = env_config
         self.n_rounds_below_min_distortion = 0

src/spaces/tiled_environment.py

@@ -5,6 +5,9 @@
 import copy
 from typing import TypeVar, List
 from dataclasses import dataclass
+
+import numpy as np
+
 from src.extern.tile_coding import IHT, tiles
 from src.spaces.actions import ActionBase, ActionType
 from src.spaces.time_step import TimeStep
@@ -26,6 +29,7 @@ class TiledEnvConfig(object):
     num_tilings: int = 0
     max_size: int = 0
     tiling_dim: int = 0
+    n_bins: int = 1
     column_ranges: dict = None
 
 
@@ -36,12 +40,19 @@ class TiledEnv(object):
 
     IS_TILED_ENV_CONSTRAINT = True
 
+    @classmethod
+    def from_options(cls, *, env: Env, max_size: int, num_tilings: int,
+                     tiling_dim: int, n_bins: int, column_ranges: dict):
+        return cls(TiledEnvConfig(env=env, max_size=max_size, num_tilings=num_tilings,
+                                  tiling_dim=tiling_dim, n_bins=n_bins, column_ranges=column_ranges))
+
     def __init__(self, config: TiledEnvConfig) -> None:
 
         self.env = config.env
         self.max_size = config.max_size
         self.num_tilings = config.num_tilings
         self.tiling_dim = config.tiling_dim
+        self.n_bins = config.n_bins
 
         # set up the columns scaling
         # only the columns that are to be altered participate in the
@@ -55,6 +66,8 @@ def __init__(self, config: TiledEnvConfig) -> None:
         self._create_column_scales()
         self.iht = IHT(self.max_size)
 
+        self.column_bins = {}
+
     @property
     def action_space(self):
         return self.env.action_space
@@ -170,7 +183,20 @@ def create_bins(self) -> None:
         None
 
         """
-        self.env.create_bins()
+
+        # calculate the tile width for each column in the
+        # data set
+
+        tile_widhs = {}
+        for column in self.column_ranges:
+            range_ = self.column_ranges[column]
+            tile_width = (range_[1] + range_[0]) / self.n_bins
+            self.column_bins[column] = np.zeros((self.num_tilings, self.n_bins))
+
+            # for each layer create an offset
+            # bin
+            for i in range(self.num_tilings):
+                self.column_bins[column][i] = np.linspace(range_[0] + i * tile_width, range_[1] + i * tile_width, self.n_bins)
 
     def get_aggregated_state(self, state_val: float) -> int:
         """
@@ -325,10 +351,18 @@ def _validate(self) -> None:
                                     param_value=str(self.max_size) +
                                     " should be >=num_tilings * tiling_dim * tiling_dim")
 
+        if self.column_ranges is None:
+            raise InvalidParamValue(param_name="column_ranges",
+                                    param_value="None")
+
         if len(self.column_ranges) == 0:
             raise InvalidParamValue(param_name="column_scales",
                                     param_value=str(len(self.column_scales)) + " should not be empty")
 
+        if self.env is None:
+            raise InvalidParamValue(param_name="env",
+                                    param_value="None")
+
         if len(self.column_ranges) != len(self.env.column_names):
             raise ValueError("Column ranges is not equal to number of columns")
 

src/tests/test_sarsa_semi_gradient.py …tests/test_n_step_sarsa_semi_gradient.pysrc/tests/test_sarsa_semi_gradient.py renamed to src/tests/test_n_step_sarsa_semi_gradient.py src/tests/test_sarsa_semi_gradient.py renamed to src/tests/test_n_step_sarsa_semi_gradient.py

@@ -1,6 +1,6 @@
 import unittest
 import pytest
-from src.algorithms.sarsa_semi_gradient import SARSAnConfig, SARSAn
+from src.algorithms.n_step_semi_gradient_sarsa import SARSAnConfig, SARSAn
 from src.spaces.tiled_environment import TiledEnv, TiledEnvConfig
 from src.policies.epsilon_greedy_policy import EpsilonGreedyPolicy, EpsilonDecayOption
 from src.exceptions.exceptions import InvalidParamValue

src/tests/test_suite.py

@@ -4,7 +4,8 @@
 from .test_serial_hierarchy import TestSerialHierarchy
 from .test_preprocessor import TestPreprocessor
 from .test_actions import TestActions
-from .test_sarsa_semi_gradient import TestSARSAn
+from .test_n_step_sarsa_semi_gradient import TestSARSAn
+from .test_semi_gradient_sarsa import TestSemiGradSARSA
 from .test_tiled_environment import TestTiledEnv
 
 
@@ -15,6 +16,7 @@ def suite():
     suite.addTest(TestPreprocessor)
     suite.addTest(TestActions)
     suite.addTest(TestSARSAn)
+    suite.addTest(TestSemiGradSARSA)
     suite.addTest(TestTiledEnv)
     return suite
 

src/tests/test_tiled_environment.py

@@ -6,6 +6,10 @@
 from src.spaces.tiled_environment import TiledEnv, TiledEnvConfig
 from src.exceptions.exceptions import InvalidParamValue
 
+class DummyEnv(object):
+
+    def __init__(self):
+        self.column_names = ["col1", "col2"]
 
 class TestTiledEnv(unittest.TestCase):
 
@@ -27,16 +31,47 @@ def test_constructor_raises_invalid_max_size(self):
         with pytest.raises(InvalidParamValue) as e:
             env = TiledEnv(config)
 
-    def test_empty_column_scales(self):
+    def test_none_column_ranges(self):
         config = TiledEnvConfig()
         config.env = None
         config.max_size = 4096
         config.tiling_dim = 2
         config.num_tilings = 5
-        config.columns_scales = {}
+        config.column_ranges = None
         with pytest.raises(InvalidParamValue) as e:
             env = TiledEnv(config)
 
+    def test_empty_column_ranges(self):
+        config = TiledEnvConfig()
+        config.env = None
+        config.max_size = 4096
+        config.tiling_dim = 2
+        config.num_tilings = 5
+        config.column_ranges = {}
+        with pytest.raises(InvalidParamValue) as e:
+            env = TiledEnv(config)
+
+    def test_create_bins(self):
+        config = TiledEnvConfig()
+        config.env = DummyEnv()
+        config.max_size = 4096
+        config.tiling_dim = 2
+        config.num_tilings = 2
+        config.column_ranges = {"col1": [0.0, 1.0], "col2": [0.0, 1.0]}
+        env = TiledEnv(config)
+        env.create_bins()
+
+        tiles = env.column_bins
+        # we must have as many bins as columns
+        self.assertEqual(2, len(tiles))
+
+        for column in tiles:
+            # for each column we must have config.num_tilings
+            self.assertEqual(config.num_tilings, len(tiles[column]))
+
+            # each tiling must have config.n_bins
+            for tile in tiles[column]:
+                self.assertEqual(config.n_bins, len(tile))
 
 
 if __name__ == '__main__':

src/tests/test_trainer.py

@@ -5,7 +5,7 @@
 import pytest
 
 from src.algorithms.trainer import Trainer
-from src.algorithms.sarsa_semi_gradient import SARSAnConfig, SARSAn
+from src.algorithms.n_step_semi_gradient_sarsa import SARSAnConfig, SARSAn
 from src.spaces.tiled_environment import TiledEnv