Skip to content

Commit 0e35168

Browse files
dgantdgant
committed
Finished asynchronous unit tests; fixed miscellaneous performance measurements
1 parent af4c986 commit 0e35168

File tree

5 files changed

+102
-52
lines changed

5 files changed

+102
-52
lines changed

‎src/main/java/bwapi/BWClient.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,11 @@ public void startGame(BWClientConfiguration configuration) {
9393
}
9494
botWrapper.onFrame();
9595
performanceMetrics.flushSideEffects.time(() -> getGame().sideEffects.flushTo(liveGameData));
96-
client.update();
96+
performanceMetrics.totalFrameDuration.stopTiming();
97+
performanceMetrics.bwapiResponse.time(client::update);
9798
if (!client.isConnected()) {
9899
client.reconnect();
99100
}
100-
performanceMetrics.totalFrameDuration.stopTiming();
101101
}
102102
botWrapper.endGame();
103103
} while (configuration.autoContinue);

‎src/main/java/bwapi/BotWrapper.java‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,10 @@ void onFrame() {
9090
Add a frame to buffer
9191
If buffer is full, it will wait until it has capacity
9292
Wait for empty buffer OR termination condition
93-
*/
93+
*/
9494
int frame = liveClientData.gameData().getFrameCount();
9595
configuration.log("Main: Enqueuing frame #" + frame);
9696
frameBuffer.enqueueFrame();
97-
performanceMetrics.bwapiResponse.startTiming();
9897
frameBuffer.lockSize.lock();
9998
try {
10099
while (!frameBuffer.empty()) {
@@ -122,7 +121,6 @@ void onFrame() {
122121
} catch(InterruptedException ignored) {
123122
} finally {
124123
frameBuffer.lockSize.unlock();
125-
performanceMetrics.bwapiResponse.stopTiming();
126124
configuration.log("Main: onFrame asynchronous end");
127125
}
128126
} else {
@@ -158,17 +156,17 @@ private Thread createBotThread() {
158156
while (!gameOver) {
159157

160158
configuration.log("Bot: Ready for another frame");
161-
frameBuffer.lockSize.lock();
162-
try {
163-
while (frameBuffer.empty()) {
164-
configuration.log("Bot: Waiting for a frame");
165-
performanceMetrics.botIdle.startTiming();
166-
frameBuffer.conditionSize.awaitUninterruptibly();
159+
performanceMetrics.botIdle.time(() -> {
160+
frameBuffer.lockSize.lock();
161+
try {
162+
while (frameBuffer.empty()) {
163+
configuration.log("Bot: Waiting for a frame");
164+
frameBuffer.conditionSize.awaitUninterruptibly();
165+
}
166+
} finally {
167+
frameBuffer.lockSize.unlock();
167168
}
168-
performanceMetrics.botIdle.stopTiming();
169-
} finally {
170-
frameBuffer.lockSize.unlock();
171-
}
169+
});
172170

173171
configuration.log("Bot: Peeking next frame");
174172
game.clientData().setBuffer(frameBuffer.peek());

‎src/main/java/bwapi/PerformanceMetrics.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,19 @@ public class PerformanceMetrics {
5757
*/
5858
public PerformanceMetric botIdle;
5959

60+
private BWClientConfiguration configuration;
61+
6062
public PerformanceMetrics(BWClientConfiguration configuration) {
63+
this.configuration = configuration;
64+
reset();
65+
}
66+
67+
void reset() {
6168
final int frameDurationBufferMs = 5;
6269
final int sideEffectsBufferMs = 1;
6370
final int realTimeFrameMs = 42;
6471
totalFrameDuration = new PerformanceMetric("Total frame duration", configuration.maxFrameDurationMs + frameDurationBufferMs);
65-
copyingToBuffer = new PerformanceMetric("Time copying to buffer", 5);
72+
copyingToBuffer = new PerformanceMetric("Time copying to buffer", 15);
6673
intentionallyBlocking = new PerformanceMetric("Intentionally blocking", 0);
6774
frameBufferSize = new PerformanceMetric("Frames buffered", 0);
6875
framesBehind = new PerformanceMetric("Frames behind", 0);
@@ -82,6 +89,7 @@ public String toString() {
8289
+ "\n" + framesBehind.toString()
8390
+ "\n" + flushSideEffects.toString()
8491
+ "\n" + botResponse.toString()
85-
+ "\n" + bwapiResponse.toString();
92+
+ "\n" + bwapiResponse.toString()
93+
+ "\n" + botIdle.toString();
8694
}
8795
}

‎src/test/java/bwapi/SynchronizationEnvironment.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,19 @@
44
import java.util.HashMap;
55
import java.util.Map;
66

7-
import static com.sun.javafx.fxml.expression.Expression.greaterThan;
8-
import static org.assertj.core.api.Assertions.assertThat;
97
import static org.junit.Assert.assertTrue;
108
import static org.mockito.ArgumentMatchers.anyBoolean;
119
import static org.mockito.Mockito.doAnswer;
1210
import static org.mockito.Mockito.mock;
1311
import static org.mockito.Mockito.when;
1412

15-
import org.hamcrest.MatcherAssert.*;
16-
1713
public class SynchronizationEnvironment {
1814
BWClientConfiguration configuration;
1915
BWClient bwClient;
2016
private BWEventListener listener;
2117
private Client client;
2218
private int onEndFrame;
19+
private long bwapiDelayMs;
2320
private Map<Integer, Runnable> onFrames;
2421

2522
SynchronizationEnvironment() {
@@ -29,6 +26,7 @@ public class SynchronizationEnvironment {
2926
bwClient = new BWClient(listener);
3027
bwClient.setClient(client);
3128
onEndFrame = -1;
29+
bwapiDelayMs = 0;
3230
onFrames = new HashMap<>();
3331

3432
when(client.mapFile()).thenReturn(GameBuilder.binToBufferUnchecked(GameBuilder.DEFAULT_BUFFER_PATH));
@@ -72,6 +70,10 @@ void onFrame(Integer frame, Runnable runnable) {
7270
onFrames.put(frame, runnable);
7371
}
7472

73+
void setBwapiDelayMs(long milliseconds) {
74+
bwapiDelayMs = milliseconds;
75+
}
76+
7577
void runGame() {
7678
runGame(10);
7779
}
@@ -99,7 +101,8 @@ private int liveFrame() {
99101
return client.clientData().gameData().getFrameCount();
100102
}
101103

102-
private void clientUpdate() {
104+
private void clientUpdate() throws InterruptedException{
105+
Thread.sleep(bwapiDelayMs);
103106
client.clientData().gameData().setFrameCount(liveFrame() + 1);
104107
configuration.log("Test: clientUpdate() to liveFrame #" + liveFrame());
105108
if (liveFrame() == 0) {

‎src/test/java/bwapi/SynchronizationTest.java‎

Lines changed: 71 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
public class SynchronizationTest {
1212

13-
private void sleepUnchecked(int milliseconds) {
13+
private void sleepUnchecked(long milliseconds) {
1414
try {
1515
Thread.sleep(milliseconds);
1616
} catch(InterruptedException exception) {
@@ -26,12 +26,6 @@ private boolean measureApproximateEquality(double expected, double actual, doubl
2626
return expected + margin >= actual && expected - margin <= actual;
2727
}
2828

29-
private void assertWithin(double expected, double actual, double margin) {
30-
assertTrue(
31-
describeApproximateExpectation(expected, actual, margin),
32-
measureApproximateEquality(expected, actual, margin));
33-
}
34-
3529
private void assertWithin(String message, double expected, double actual, double margin) {
3630
assertTrue(
3731
message + ": " + describeApproximateExpectation(expected, actual, margin),
@@ -83,10 +77,10 @@ public void async_IfBotDelay_ThenClientBuffers() {
8377
environment.configuration.asyncFrameBufferSize = 4;
8478

8579
environment.onFrame(1, () -> {
86-
sleepUnchecked(40);
80+
sleepUnchecked(50);
8781
assertEquals("Bot should be observing an old frame", 1, environment.bwClient.getGame().getFrameCount());
88-
assertEquals("Client should be as far ahead as the frame buffer allows", 4, environment.liveGameData().getFrameCount());
89-
assertEquals("Bot should be behind the live game", 3, environment.bwClient.framesBehind());
82+
assertEquals("Client should be as far ahead as the frame buffer allows", 5, environment.liveGameData().getFrameCount());
83+
assertEquals("Bot should be behind the live game", 4, environment.bwClient.framesBehind());
9084
});
9185

9286
environment.onFrame(6, () -> { // Maybe it should be possible to demand that these assertions pass a frame earlier?
@@ -161,17 +155,36 @@ public void async_IfFrameZeroWaitsDisabled_ThenClientBuffers() {
161155

162156
@Test
163157
public void async_MeasurePerformance_TotalFrameDuration() {
164-
158+
final int frames = 10;
159+
final int frameSleep = 20;
160+
SynchronizationEnvironment environment = new SynchronizationEnvironment();
161+
environment.configuration.async = true;
162+
environment.configuration.unlimitedFrameZero = true;
163+
environment.configuration.maxFrameDurationMs = frameSleep + 20;
164+
IntStream.range(0, frames).forEach(i -> environment.onFrame(i, () -> {
165+
sleepUnchecked(frameSleep);
166+
}));
167+
environment.runGame(frames);
168+
169+
// Assume copying accounts for almost all the frame time except what the bot uses
170+
double meanCopy = environment.metrics().copyingToBuffer.avgValue;
171+
assertWithin("Total frame duration: Average", environment.metrics().totalFrameDuration.avgValue, meanCopy + frameSleep, MS_MARGIN);
165172
}
166173

167174
@Test
168175
public void async_MeasurePerformance_CopyingToBuffer() {
169-
170-
}
171-
172-
@Test
173-
public void async_MeasurePerformance_IntentionallyBlocking() {
174-
176+
// Somewhat lazy test; just verify that we're getting sane values
177+
SynchronizationEnvironment environment = new SynchronizationEnvironment();
178+
environment.configuration.async = true;
179+
environment.runGame(20);
180+
final double minObserved = 2;
181+
final double maxObserved = 15;
182+
final double meanObserved = (minObserved + maxObserved) / 2;
183+
final double rangeObserved = (maxObserved - minObserved) / 2;
184+
assertWithin("Copy to buffer: minimum", environment.metrics().copyingToBuffer.minValue, meanObserved, rangeObserved);
185+
assertWithin("Copy to buffer: maximum", environment.metrics().copyingToBuffer.maxValue, meanObserved, rangeObserved);
186+
assertWithin("Copy to buffer: average", environment.metrics().copyingToBuffer.avgValue, meanObserved, rangeObserved);
187+
assertWithin("Copy to buffer: previous", environment.metrics().copyingToBuffer.lastValue, meanObserved, rangeObserved);
175188
}
176189

177190
@Test
@@ -181,8 +194,7 @@ public void async_MeasurePerformance_FrameBufferSizeAndFramesBehind() {
181194
environment.configuration.unlimitedFrameZero = true;
182195
environment.configuration.asyncFrameBufferSize = 3;
183196
environment.configuration.maxFrameDurationMs = 20;
184-
environment.configuration.logVerbosely = true;
185-
197+
186198
environment.onFrame(5, () -> {
187199
assertWithin("5: Frame buffer average", 0, environment.metrics().frameBufferSize.avgValue, 0.1);
188200
assertWithin("5: Frame buffer minimum", 0, environment.metrics().frameBufferSize.minValue, 0.1);
@@ -208,13 +220,8 @@ public void async_MeasurePerformance_FrameBufferSizeAndFramesBehind() {
208220
environment.runGame(8);
209221
}
210222

211-
@Test
212-
public void async_MeasurePerformance_FlushSideEffects() {
213-
214-
}
215-
216223
/**
217-
* Number of milliseconds of leeway to give in performance metrics.
224+
* Number of milliseconds of leeway to give in potentially noisy performance metrics.
218225
* Increase if tests are flaky due to variance in execution speed.
219226
*/
220227
private final static long MS_MARGIN = 10;
@@ -254,14 +261,48 @@ public void MeasurePerformance_BotResponse() {
254261
}
255262

256263
@Test
257-
public void async_MeasurePerformance_BwapiResponse() {
258-
264+
public void MeasurePerformance_BwapiResponse() {
265+
final long bwapiDelayMs = 50;
266+
SynchronizationEnvironment environment = new SynchronizationEnvironment();
267+
environment.setBwapiDelayMs(bwapiDelayMs);
268+
environment.runGame();
269+
System.out.println(environment.metrics());
270+
assertWithin("BWAPI Response: Average", environment.metrics().bwapiResponse.avgValue, bwapiDelayMs, MS_MARGIN);
259271
}
260272

261273
@Test
262-
public void async_MeasurePerformance_BotIdle() {
263-
274+
public void MeasurePerformance_BotIdle() {
275+
final long bwapiDelayMs = 10;
276+
final int frames = 10;
277+
SynchronizationEnvironment environment = new SynchronizationEnvironment();
278+
environment.configuration.async = true;
279+
environment.configuration.asyncFrameBufferSize = 3;
280+
environment.configuration.unlimitedFrameZero = true;
281+
environment.setBwapiDelayMs(bwapiDelayMs);
282+
environment.runGame(frames);
283+
double expected = environment.metrics().copyingToBuffer.avgValue + bwapiDelayMs;
284+
assertWithin("Bot Idle: Average", environment.metrics().botIdle.avgValue, expected, MS_MARGIN);
264285
}
265286

266-
287+
@Test
288+
public void async_MeasurePerformance_IntentionallyBlocking() {
289+
SynchronizationEnvironment environment = new SynchronizationEnvironment();
290+
environment.configuration.async = true;
291+
environment.configuration.unlimitedFrameZero = true;
292+
environment.configuration.asyncFrameBufferSize = 2;
293+
environment.configuration.maxFrameDurationMs = 20;
294+
final int frameDelayMs = 100;
295+
environment.onFrame(1, () -> {
296+
sleepUnchecked(100);
297+
});
298+
environment.onFrame(2, () -> {
299+
assertWithin(
300+
"2: Intentionally blocking previous",
301+
environment.metrics().intentionallyBlocking.lastValue,
302+
frameDelayMs - environment.configuration.asyncFrameBufferSize * environment.configuration.maxFrameDurationMs,
303+
MS_MARGIN);
304+
sleepUnchecked(100);
305+
});
306+
environment.runGame(3);
307+
}
267308
}

0 commit comments

Comments
 (0)