Skip to content

Commit f254597

Browse files
pockermanpockerman
committed
#27 Add example
1 parent 7dbb2dc commit f254597

File tree

1 file changed

+201
-0
lines changed

1 file changed

+201
-0
lines changed

‎src/examples/qlearning_three_columns.py‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
"""
2+
Simple example that shows how to apply QLearning
3+
on a dataset with three columns
4+
"""
5+
import numpy as np
6+
import random
7+
from pathlib import Path
8+
9+
from src.algorithms.q_learning import QLearning, QLearnConfig
10+
from src.algorithms.trainer import Trainer
11+
from src.datasets.datasets_loaders import MockSubjectsLoader
12+
from src.spaces.action_space import ActionSpace
13+
from src.spaces.actions import ActionIdentity, ActionStringGeneralize, ActionNumericBinGeneralize
14+
from src.utils.reward_manager import RewardManager
15+
from src.utils.serial_hierarchy import SerialHierarchy
16+
from src.policies.epsilon_greedy_policy import EpsilonGreedyPolicy, EpsilonDecreaseOption
17+
from src.policies.softmax_policy import SoftMaxPolicy
18+
from src.utils.numeric_distance_type import NumericDistanceType
19+
from src.utils.string_distance_calculator import StringDistanceType
20+
from src.utils.distortion_calculator import DistortionCalculationType, DistortionCalculator
21+
from src.spaces.discrete_state_environment import DiscreteStateEnvironment, DiscreteEnvConfig
22+
from src.utils.iteration_control import IterationControl
23+
from src.utils.plot_utils import plot_running_avg
24+
from src.utils import INFO
25+
26+
27+
def get_ethinicity_hierarchy():
28+
ethnicity_hierarchy = SerialHierarchy(values={})
29+
ethnicity_hierarchy.add("Mixed White/Asian", "White/Asian")
30+
ethnicity_hierarchy.add("White/Asian", "White")
31+
32+
ethnicity_hierarchy.add("Chinese", "Asian")
33+
ethnicity_hierarchy.add("Indian", "Asian")
34+
35+
ethnicity_hierarchy.add("Mixed White/Black African", "African-Mixed")
36+
ethnicity_hierarchy.add("African-Mixed", "Mixed")
37+
38+
ethnicity_hierarchy.add("Black African", "African")
39+
ethnicity_hierarchy.add("African", "African")
40+
41+
ethnicity_hierarchy.add("Asian other", "Asian")
42+
ethnicity_hierarchy.add("Black other", "Black")
43+
44+
ethnicity_hierarchy.add("Mixed White/Black Caribbean", "Caribbean-Mixed")
45+
ethnicity_hierarchy.add("Caribbean-Mixed", "Mixed")
46+
47+
ethnicity_hierarchy.add("Mixed other", "Mixed")
48+
ethnicity_hierarchy.add("Arab", "Asian")
49+
50+
ethnicity_hierarchy.add("White Irish", "European-White")
51+
ethnicity_hierarchy.add("European-White", "European")
52+
53+
ethnicity_hierarchy.add("Not stated", "Not stated")
54+
ethnicity_hierarchy.add("White Gypsy/Traveller", "White")
55+
56+
ethnicity_hierarchy.add("White British", "British")
57+
ethnicity_hierarchy.add("British", "European")
58+
59+
ethnicity_hierarchy.add("Bangladeshi", "Asian")
60+
ethnicity_hierarchy.add("White other", "White")
61+
ethnicity_hierarchy.add("Black Caribbean", "Black")
62+
ethnicity_hierarchy.add("Pakistani", "Asian")
63+
64+
ethnicity_hierarchy.add("White", "White")
65+
ethnicity_hierarchy.add("Mixed", "Mixed")
66+
ethnicity_hierarchy.add("European", "European")
67+
ethnicity_hierarchy.add("Asian", "Asian")
68+
ethnicity_hierarchy.add("Black", "Black")
69+
ethnicity_hierarchy.add("Not stated", "Not stated")
70+
71+
return ethnicity_hierarchy
72+
73+
74+
if __name__ == '__main__':
75+
random.seed(42)
76+
77+
# configuration params
78+
EPS = 1.0
79+
EPSILON_DECAY_OPTION = EpsilonDecreaseOption.CONSTANT_RATE #.INVERSE_STEP
80+
EPSILON_DECAY_FACTOR = 0.01
81+
GAMMA = 0.99
82+
ALPHA = 0.1
83+
N_EPISODES = 1001
84+
N_ITRS_PER_EPISODE = 30
85+
N_STATES = 10
86+
# fix the rewards. Assume that any average distortion in
87+
# (0.4, 0.7) suits us
88+
MAX_DISTORTION = 0.7
89+
MIN_DISTORTION = 0.4
90+
OUT_OF_MAX_BOUND_REWARD = -1.0
91+
OUT_OF_MIN_BOUND_REWARD = -1.0
92+
IN_BOUNDS_REWARD = 5.0
93+
OUTPUT_MSG_FREQUENCY = 100
94+
N_ROUNDS_BELOW_MIN_DISTORTION = 10
95+
SAVE_DISTORTED_SETS_DIR = "/home/alex/qi3/drl_anonymity/src/examples/q_learn_distorted_sets/distorted_set"
96+
97+
# specify the columns to drop
98+
drop_columns = MockSubjectsLoader.FEATURES_DROP_NAMES + ["preventative_treatment", "gender",
99+
"education", "mutation_status"]
100+
MockSubjectsLoader.FEATURES_DROP_NAMES = drop_columns
101+
102+
# do a salary normalization
103+
MockSubjectsLoader.NORMALIZED_COLUMNS = ["salary"]
104+
105+
# specify the columns to use
106+
MockSubjectsLoader.COLUMNS_TYPES = {"ethnicity": str, "salary": float, "diagnosis": int}
107+
ds = MockSubjectsLoader()
108+
109+
assert ds.n_columns == 3, "Invalid number of columns {0} not equal to 3".format(ds.n_columns)
110+
111+
# create bins for the salary generalization
112+
unique_salary = ds.get_column_unique_values(col_name="salary")
113+
unique_salary.sort()
114+
115+
# modify slightly the max value because
116+
# we get out of bounds
117+
bins = np.linspace(unique_salary[0], unique_salary[-1] + 1, N_STATES)
118+
119+
# establish the action space. For every column
120+
# we assume three actions except for the ```diagnosis```
121+
# which we do not alter
122+
action_space = ActionSpace(n=5)
123+
action_space.add_many(ActionIdentity(column_name="ethnicity"),
124+
ActionStringGeneralize(column_name="ethnicity",
125+
generalization_table=get_ethinicity_hierarchy()),
126+
ActionIdentity(column_name="salary"),
127+
ActionNumericBinGeneralize(column_name="salary", generalization_table=bins),
128+
ActionIdentity(column_name="diagnosis"))
129+
130+
action_space.shuffle()
131+
132+
env_config = DiscreteEnvConfig()
133+
134+
env_config.action_space = action_space
135+
env_config.reward_manager = RewardManager(bounds=(MIN_DISTORTION, MAX_DISTORTION),
136+
out_of_max_bound_reward=OUT_OF_MAX_BOUND_REWARD,
137+
out_of_min_bound_reward=OUT_OF_MIN_BOUND_REWARD,
138+
in_bounds_reward=IN_BOUNDS_REWARD)
139+
env_config.data_set = ds
140+
env_config.gamma = GAMMA
141+
env_config.max_distortion = MAX_DISTORTION
142+
env_config.min_distortion = MIN_DISTORTION
143+
env_config.n_states = N_STATES
144+
env_config.n_rounds_below_min_distortion = N_ROUNDS_BELOW_MIN_DISTORTION
145+
env_config.distorted_set_path = Path(SAVE_DISTORTED_SETS_DIR)
146+
env_config.distortion_calculator = DistortionCalculator(
147+
numeric_column_distortion_metric_type=NumericDistanceType.L2_AVG,
148+
string_column_distortion_metric_type=StringDistanceType.COSINE_NORMALIZE,
149+
dataset_distortion_type=DistortionCalculationType.SUM)
150+
151+
# create the environment
152+
env = DiscreteStateEnvironment(env_config=env_config)
153+
env.reset()
154+
env.save_current_dataset(episode_index=-1)
155+
156+
# save the original dataset for comparison
157+
env.save_current_dataset(episode_index=-1)
158+
env.reset()
159+
160+
# configuration for the Q-learner
161+
algo_config = QLearnConfig()
162+
algo_config.n_itrs_per_episode = N_ITRS_PER_EPISODE
163+
algo_config.gamma = GAMMA
164+
algo_config.alpha = ALPHA
165+
#algo_config.policy = SoftMaxPolicy(n_actions=len(action_space), tau=1.2)
166+
algo_config.policy = EpsilonGreedyPolicy(eps=EPS, env=env,decay_op=EPSILON_DECAY_OPTION,
167+
epsilon_decay_factor=EPSILON_DECAY_FACTOR)
168+
169+
# the learner we want to train
170+
agent = QLearning(algo_config=algo_config)
171+
172+
configuration = {"n_episodes": N_EPISODES, "output_msg_frequency": OUTPUT_MSG_FREQUENCY}
173+
174+
# create a trainer to train the Qlearning agent
175+
trainer = Trainer(env=env, agent=agent, configuration=configuration)
176+
trainer.train()
177+
178+
# avg_rewards = trainer.avg_rewards()
179+
avg_rewards = trainer.total_rewards
180+
plot_running_avg(avg_rewards, steps=100,
181+
xlabel="Episodes", ylabel="Reward",
182+
title="Running reward average over 100 episodes")
183+
184+
avg_episode_dist = np.array(trainer.total_distortions)
185+
print("{0} Max/Min distortion {1}/{2}".format(INFO, np.max(avg_episode_dist), np.min(avg_episode_dist)))
186+
187+
plot_running_avg(avg_episode_dist, steps=100,
188+
xlabel="Episodes", ylabel="Distortion",
189+
title="Running distortion average over 100 episodes")
190+
191+
print("=============================================")
192+
print("{0} Generating distorted dataset".format(INFO))
193+
# Let's play
194+
env.reset()
195+
196+
stop_criterion = IterationControl(n_itrs=10, min_dist=MIN_DISTORTION, max_dist=MAX_DISTORTION)
197+
agent.play(env=env, stop_criterion=stop_criterion)
198+
env.save_current_dataset(episode_index=-2)
199+
200+
201+

0 commit comments

Comments
 (0)