Skip to content

Commit 3352ff9

Browse files
pockermanpockerman
committed
Update example
1 parent 5f24dda commit 3352ff9

File tree

1 file changed

+167
-0
lines changed

1 file changed

+167
-0
lines changed

‎src/examples/semi_gradient_sarsa.py‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
import random
2+
from pathlib import Path
3+
import numpy as np
4+
5+
from src.algorithms.semi_gradient_sarsa import SemiGradSARSAConfig, SemiGradSARSA
6+
from src.utils.serial_hierarchy import SerialHierarchy
7+
from src.spaces.tiled_environment import TiledEnv, TiledEnvConfig, Layer
8+
from src.spaces.discrete_state_environment import DiscreteStateEnvironment
9+
from src.datasets.datasets_loaders import MockSubjectsLoader, MockSubjectsData
10+
from src.spaces.action_space import ActionSpace
11+
from src.spaces.actions import ActionIdentity, ActionStringGeneralize, ActionNumericBinGeneralize
12+
from src.algorithms.trainer import Trainer
13+
from src.policies.epsilon_greedy_policy import EpsilonDecayOption
14+
from src.algorithms.epsilon_greedy_q_estimator import EpsilonGreedyQEstimatorConfig, EpsilonGreedyQEstimator
15+
from src.utils.distortion_calculator import DistortionCalculationType, DistortionCalculator
16+
from src.utils.numeric_distance_type import NumericDistanceType
17+
from src.utils.string_distance_calculator import StringDistanceType
18+
from src.utils.reward_manager import RewardManager
19+
20+
21+
N_LAYERS = 5
22+
N_BINS = 10
23+
N_EPISODES = 1000
24+
OUTPUT_MSG_FREQUENCY = 100
25+
GAMMA = 0.99
26+
ALPHA = 0.1
27+
N_ITRS_PER_EPISODE = 30
28+
EPS = 1.0
29+
EPSILON_DECAY_OPTION = EpsilonDecayOption.CONSTANT_RATE #.INVERSE_STEP
30+
EPSILON_DECAY_FACTOR = 0.01
31+
MAX_DISTORTION = 0.7
32+
MIN_DISTORTION = 0.3
33+
OUT_OF_MAX_BOUND_REWARD = -1.0
34+
OUT_OF_MIN_BOUND_REWARD = -1.0
35+
IN_BOUNDS_REWARD = 5.0
36+
N_ROUNDS_BELOW_MIN_DISTORTION = 10
37+
SAVE_DISTORTED_SETS_DIR = "/home/alex/qi3/drl_anonymity/src/examples/semi_grad_sarsa/distorted_set"
38+
REWARD_FACTOR = 0.95
39+
PUNISH_FACTOR = 2.0
40+
41+
42+
def get_ethinicity_hierarchy():
43+
ethnicity_hierarchy = SerialHierarchy(values={})
44+
45+
ethnicity_hierarchy["Mixed White/Asian"] = "White/Asian"
46+
ethnicity_hierarchy["White/Asian"] = "Mixed"
47+
48+
ethnicity_hierarchy["Chinese"] = "Asian"
49+
ethnicity_hierarchy["Indian"] = "Asian"
50+
ethnicity_hierarchy["Mixed White/Black African"] = "White/Black"
51+
ethnicity_hierarchy["White/Black"] = "Mixed"
52+
53+
ethnicity_hierarchy["Black African"] = "African"
54+
ethnicity_hierarchy["African"] = "Black"
55+
ethnicity_hierarchy["Asian other"] = "Asian"
56+
ethnicity_hierarchy["Black other"] = "Black"
57+
ethnicity_hierarchy["Mixed White/Black Caribbean"] = "White/Black"
58+
ethnicity_hierarchy["White/Black"] = "Mixed"
59+
60+
ethnicity_hierarchy["Mixed other"] = "Mixed"
61+
ethnicity_hierarchy["Arab"] = "Asian"
62+
ethnicity_hierarchy["White Irish"] = "Irish"
63+
ethnicity_hierarchy["Irish"] = "European"
64+
ethnicity_hierarchy["Not stated"] = "Not stated"
65+
ethnicity_hierarchy["White Gypsy/Traveller"] = "White"
66+
ethnicity_hierarchy["White British"] = "British"
67+
ethnicity_hierarchy["British"] = "European"
68+
ethnicity_hierarchy["Bangladeshi"] = "Asian"
69+
ethnicity_hierarchy["White other"] = "White"
70+
ethnicity_hierarchy["Black Caribbean"] = "Caribbean"
71+
ethnicity_hierarchy["Caribbean"] = "Black"
72+
ethnicity_hierarchy["Pakistani"] = "Asian"
73+
74+
ethnicity_hierarchy["European"] = "European"
75+
ethnicity_hierarchy["Mixed"] = "Mixed"
76+
ethnicity_hierarchy["Asian"] = "Asian"
77+
ethnicity_hierarchy["Black"] = "Black"
78+
ethnicity_hierarchy["White"] = "White"
79+
return ethnicity_hierarchy
80+
81+
82+
def load_mock_subjects() -> MockSubjectsLoader:
83+
84+
mock_data = MockSubjectsData(FILENAME=Path("../../data/mocksubjects.csv"),
85+
COLUMNS_TYPES={"ethnicity": str, "salary": float, "diagnosis": int},
86+
FEATURES_DROP_NAMES=["NHSno", "given_name",
87+
"surname", "dob"] + ["preventative_treatment",
88+
"gender", "education", "mutation_status"],
89+
NORMALIZED_COLUMNS=["salary"])
90+
91+
ds = MockSubjectsLoader(mock_data)
92+
93+
assert ds.n_columns == 3, "Invalid number of columns {0} not equal to 3".format(ds.n_columns)
94+
95+
return ds
96+
97+
98+
def load_discrete_env() -> DiscreteStateEnvironment:
99+
100+
mock_ds = load_mock_subjects()
101+
102+
# create bins for the salary generalization
103+
unique_salary = mock_ds.get_column_unique_values(col_name="salary")
104+
unique_salary.sort()
105+
106+
# modify slightly the max value because
107+
# we get out of bounds for the maximum salary
108+
bins = np.linspace(unique_salary[0], unique_salary[-1] + 1, N_BINS)
109+
110+
action_space = ActionSpace(n=5)
111+
action_space.add_many(ActionIdentity(column_name="ethnicity"),
112+
ActionStringGeneralize(column_name="ethnicity",
113+
generalization_table=get_ethinicity_hierarchy()),
114+
ActionIdentity(column_name="salary"),
115+
ActionNumericBinGeneralize(column_name="salary", generalization_table=bins),
116+
ActionIdentity(column_name="diagnosis"))
117+
118+
action_space.shuffle()
119+
120+
env = DiscreteStateEnvironment.from_options(data_set=mock_ds,
121+
action_space=action_space,
122+
distortion_calculator=DistortionCalculator(
123+
numeric_column_distortion_metric_type=NumericDistanceType.L2_AVG,
124+
string_column_distortion_metric_type=StringDistanceType.COSINE_NORMALIZE,
125+
dataset_distortion_type=DistortionCalculationType.SUM),
126+
reward_manager=RewardManager(bounds=(MIN_DISTORTION, MAX_DISTORTION),
127+
out_of_max_bound_reward=OUT_OF_MAX_BOUND_REWARD,
128+
out_of_min_bound_reward=OUT_OF_MIN_BOUND_REWARD,
129+
in_bounds_reward=IN_BOUNDS_REWARD),
130+
gamma=GAMMA,
131+
reward_factor=REWARD_FACTOR,
132+
punish_factor=PUNISH_FACTOR,
133+
min_distortion=MIN_DISTORTION, max_distortion=MAX_DISTORTION,
134+
n_rounds_below_min_distortion=N_ROUNDS_BELOW_MIN_DISTORTION,
135+
distorted_set_path=Path(SAVE_DISTORTED_SETS_DIR),
136+
n_states=N_LAYERS * Layer.n_tiles_per_action(N_BINS,
137+
mock_ds.n_columns))
138+
139+
return env
140+
141+
142+
if __name__ == '__main__':
143+
144+
# set the seed for random engine
145+
random.seed(42)
146+
147+
discrete_env = load_discrete_env()
148+
tiled_env_config = TiledEnvConfig(n_layers=N_LAYERS, n_bins=N_BINS,
149+
env=discrete_env,
150+
column_ranges={"ethnicity": [0.0, 1.0],
151+
"salary": [0.0, 1.0],
152+
"diagnosis": [0.0, 1.0]})
153+
tiled_env = TiledEnv(tiled_env_config)
154+
tiled_env.create_tiles()
155+
156+
configuration = {"n_episodes": N_EPISODES, "output_msg_frequency": OUTPUT_MSG_FREQUENCY}
157+
158+
agent_config = SemiGradSARSAConfig(gamma=GAMMA, alpha=ALPHA, n_itrs_per_episode=N_ITRS_PER_EPISODE,
159+
policy=EpsilonGreedyQEstimator(EpsilonGreedyQEstimatorConfig(eps=EPS, n_actions=tiled_env.n_actions,
160+
decay_op=EPSILON_DECAY_OPTION,
161+
epsilon_decay_factor=EPSILON_DECAY_FACTOR,
162+
env=tiled_env, gamma=GAMMA, alpha=ALPHA)))
163+
agent = SemiGradSARSA(agent_config)
164+
165+
# create a trainer to train the Qlearning agent
166+
trainer = Trainer(env=tiled_env, agent=agent, configuration=configuration)
167+
trainer.train()

0 commit comments

Comments
 (0)