[Tests failing] Finished test environment for sync/async tests. Configured tests to run with sufficient memory to allocate framebuffers.

Author: dgant

pom.xml

@@ -63,6 +63,8 @@
                 <configuration>
                     <parallel>all</parallel> <!-- Run tests in parallel-->
                     <useUnlimitedThreads>true</useUnlimitedThreads>
+                    <argLine>-Xms1g</argLine>
+                    <argLine>-Xmx1g</argLine>
                 </configuration>
             </plugin>
             <plugin>

src/main/java/bwapi/BWClient.java

@@ -25,20 +25,25 @@ public Game getGame() {
     }
 
     /**
-     * Returns JBWAPI performance metrics.
-     * Metrics will be mostly empty if metrics collection isn't timersEnabled in the bot configuration
+     * @return JBWAPI performance metrics.
      */
+    @SuppressWarnings("unused")
     public PerformanceMetrics getPerformanceMetrics() {
         return performanceMetrics;
     }
 
     /**
-     * Number of frames
+     * @return The number of frames between the one exposed to the bot and the most recent received by JBWAPI.
+     * This tracks the size of the frame buffer except when the game is paused (which results in multiple frames arriving with the same count).
      */
+    @SuppressWarnings("unused")
     public int framesBehind() {
-        return botWrapper == null ? 0 : client.clientData().gameData().getFrameCount() - getGame().getFrameCount();
+        return botWrapper == null ? 0 : Math.max(0, client.clientData().gameData().getFrameCount() - getGame().getFrameCount());
     }
 
+    /**
+     * Start the game with default settings.
+     */
     public void startGame() {
         BWClientConfiguration configuration = new BWClientConfiguration();
         startGame(configuration);
@@ -65,7 +70,9 @@ public void startGame(BWClientConfiguration configuration) {
         configuration.validate();
         botWrapper = new BotWrapper(configuration, eventListener);
 
-        client = new Client(configuration);
+        if (client == null) {
+            client = new Client(configuration);
+        }
         client.reconnect();
 
         do {
@@ -95,4 +102,11 @@ public void startGame(BWClientConfiguration configuration) {
             botWrapper.endGame();
         } while (configuration.autoContinue);
     }
+
+    /**
+     * Provides a Client. Intended for test consumers only.
+     */
+    void setClient(Client client) {
+        this.client = client;
+    }
 }

src/main/java/bwapi/BotWrapper.java

@@ -26,6 +26,7 @@ of this software and associated documentation files (the "Software"), to deal
 package bwapi;
 
 import java.nio.ByteBuffer;
+import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * Manages invocation of bot event handlers
@@ -39,6 +40,8 @@ class BotWrapper {
     private Thread botThread;
     private boolean gameOver;
     private PerformanceMetrics performanceMetrics;
+    private Exception lastBotException;
+    private ReentrantLock lastBotExceptionLock = new ReentrantLock();
 
     BotWrapper(BWClientConfiguration configuration, BWEventListener eventListener) {
         this.configuration = configuration;
@@ -50,7 +53,9 @@ class BotWrapper {
      * Resets the BotWrapper for a new game.
      */
     void startNewGame(ByteBuffer dataSource, PerformanceMetrics performanceMetrics) {
-        frameBuffer.initialize(dataSource, performanceMetrics);
+        if (configuration.async) {
+            frameBuffer.initialize(dataSource, performanceMetrics);
+        }
         this.performanceMetrics = performanceMetrics;
         game = new Game(liveClientData);
         liveClientData.setBuffer(dataSource);
@@ -80,7 +85,7 @@ void onFrame() {
             }
             /*
             Add a frame to buffer
-                If buffer is full, it will wait until it has capacity
+            If buffer is full, it will wait until it has capacity
             Wait for empty buffer OR termination condition
              */
             boolean isFrameZero = liveClientData.gameData().getFrameCount() == 0;
@@ -89,6 +94,13 @@ void onFrame() {
             frameBuffer.lockSize.lock();
             try {
                 while (!frameBuffer.empty()) {
+
+                    // Make bot exceptions fall through to the main thread.
+                    Exception lastBotException = getLastBotException();
+                    if (lastBotException != null) {
+                        throw new RuntimeException(lastBotException);
+                    }
+
                     if (configuration.unlimitedFrameZero && isFrameZero) {
                         frameBuffer.conditionSize.await();
                     } else {
@@ -104,6 +116,9 @@ void onFrame() {
             }
         } else {
             handleEvents();
+            if (lastBotException != null) {
+                throw new RuntimeException(lastBotException);
+            }
         }
     }
 
@@ -118,6 +133,13 @@ void endGame() {
         }
     }
 
+    Exception getLastBotException() {
+        lastBotExceptionLock.lock();
+        Exception output = lastBotException;
+        lastBotExceptionLock.unlock();
+        return output;
+    }
+
     private Thread createBotThread() {
         return new Thread(() -> {
             while ( ! gameOver) {
@@ -144,12 +166,18 @@ private void handleEvents() {
         if (gameData.getFrameCount() > 0 || ! configuration.unlimitedFrameZero) {
             performanceMetrics.botResponse.startTiming();
         }
-        for (int i = 0; i < gameData.getEventCount(); i++) {
-            ClientData.Event event = gameData.getEvents(i);
-            EventHandler.operation(eventListener, game, event);
-            if (event.getType() == EventType.MatchEnd) {
-                gameOver = true;
+        try {
+            for (int i = 0; i < gameData.getEventCount(); i++) {
+                ClientData.Event event = gameData.getEvents(i);
+                EventHandler.operation(eventListener, game, event);
+                if (event.getType() == EventType.MatchEnd) {
+                    gameOver = true;
+                }
             }
+        } catch (Exception exception) {
+            lastBotExceptionLock.lock();
+            lastBotException = exception;
+            lastBotExceptionLock.unlock();
         }
         performanceMetrics.botResponse.stopTiming();
     }

src/main/java/bwapi/FrameBuffer.java

@@ -25,6 +25,7 @@ of this software and associated documentation files (the "Software"), to deal
 
 package bwapi;
 
+import com.sun.jna.Platform;
 import sun.nio.ch.DirectBuffer;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -44,6 +45,7 @@ class FrameBuffer {
     private int stepGame = 0;
     private int stepBot = 0;
     private ArrayList<ByteBuffer> dataBuffer = new ArrayList<>();
+    private final String architecture;
 
     // Synchronization locks
     private final Lock lockWrite = new ReentrantLock();
@@ -53,8 +55,10 @@ class FrameBuffer {
     FrameBuffer(int size) {
         this.size = size;
         while(dataBuffer.size() < size) {
+            System.out.println("Allocating " + BUFFER_SIZE / 1024 / 1024 + "mb");
             dataBuffer.add(ByteBuffer.allocateDirect(BUFFER_SIZE));
         }
+        architecture = System.getProperty("sun.arch.data.model");
     }
 
     /**
@@ -124,7 +128,7 @@ void enqueueFrame() {
 
             performanceMetrics.copyingToBuffer.time(() -> {
                 ByteBuffer dataTarget = dataBuffer.get(indexGame());
-                copyBuffer(dataSource, dataTarget, BUFFER_SIZE);
+                copyBuffer(dataSource, dataTarget);
             });
 
             lockSize.lock();
@@ -135,17 +139,6 @@ void enqueueFrame() {
         } finally { lockWrite.unlock(); }
     }
 
-    private void copyBufferOld(ByteBuffer source, ByteBuffer dest, int size) {
-        source.rewind();
-        dest.rewind();
-        dest.put(dataSource);
-    }
-    private void copyBuffer(ByteBuffer source, ByteBuffer dest, int size) {
-        long destAddr = ((DirectBuffer)dest).address();
-        long sourceAddr = ((DirectBuffer)source).address();
-        MSVCRT.INSTANCE.memcpy(destAddr, sourceAddr, size);
-    }
-
     /**
      * Peeks the front-most value in the buffer.
      */
@@ -168,4 +161,38 @@ void dequeue() {
             conditionSize.signalAll();
         } finally { lockSize.unlock(); }
     }
+
+    void copyBuffer(ByteBuffer source, ByteBuffer destination) {
+        /*
+        The speed at which we copy data into the frame buffer is a major cost of JBWAPI's asynchronous operation.
+        Copy times observed in the wild range from 2.6ms - 12ms.
+
+        The normal Java way to execute this copy is via ByteBuffer.put(), which has reasonably good performance characteristics.
+        Some experiments in 64-bit JRE have shown that using a native memcpy achieves a 35% speedup.
+        Some experiments in 32-bit JRE show no difference in performance.
+
+        So, speculatively, we attempt to do a native memcpy.
+         */
+        long addressSource = ((DirectBuffer) source).address();
+        long addressDestination = ((DirectBuffer) destination).address();
+        try {
+            if (Platform.isWindows()) {
+                if (Platform.is64Bit()) {
+                    MSVCRT.INSTANCE.memcpy(addressDestination, addressSource, FrameBuffer.BUFFER_SIZE);
+                    return;
+                } else {
+                    MSVCRT.INSTANCE.memcpy((int) addressDestination, (int) addressSource, FrameBuffer.BUFFER_SIZE);
+                    return;
+                }
+            }
+        }
+        catch(Exception ignored) {}
+
+        // There's no specific case where we expect to fail above,
+        // but this is a safe fallback regardless,
+        // and serves to document the known-good (and cross-platform, for BWAPI 5) way to executing the copy.
+        source.rewind();
+        destination.rewind();
+        destination.put(dataSource);
+    }
 }

src/main/java/bwapi/Game.java

@@ -142,6 +142,7 @@ private static boolean hasPower(final int x, final int y, final UnitType unitTyp
     Call this method in EventHander::OnMatchStart
     */
     void init() {
+        System.out.println("Game.init()"); // TODO: REMOVE!
         visibleUnits.clear();
 
         final int forceCount = gameData().getForceCount();

src/main/java/bwapi/MSVCRT.java

@@ -3,8 +3,25 @@
 import com.sun.jna.Library;
 import com.sun.jna.Native;
 
+/**
+ * JNI interface for access the native MSVC implementation of memcpy.
+ */
 interface MSVCRT extends Library {
     MSVCRT INSTANCE = Native.load("msvcrt.dll", MSVCRT.class);
 
+    /**
+     * 32-bit implementation of memcpy.
+     * @param dest A 32-bit address of a memory block (likely from a ByteBuffer) to copy to
+     * @param src  A 32-bit address of a memory block (likely from a ByteBuffer) to copy from
+     * @param count The number of bytes to copy
+     */
+    long memcpy(int dest, int src, int count);
+
+    /**
+     * 64-bit implementation of memcpy.
+     * @param dest A 64-bit address of a memory block (likely from a ByteBuffer) to copy to
+     * @param src  A 64-bit address of a memory block (likely from a ByteBuffer) to copy from
+     * @param count The number of bytes to copy
+     */
     long memcpy(long dest, long src, int count);
 }

src/test/java/bwapi/GameBuilder.java

@@ -10,15 +10,32 @@
 
 public class GameBuilder {
 
+    public final static String DEFAULT_MAP_FILE = "(2)Benzene.scx";
+    public final static String DEFAULT_BUFFER_PATH = "src/test/resources/" + DEFAULT_MAP_FILE + "_frame0_buffer.bin";
+
     public static Game createGame() throws IOException {
-        return createGame("(2)Benzene.scx");
+        return createGame(DEFAULT_MAP_FILE);
     }
 
     public static Game createGame(String mapName) throws IOException {
-        final ByteBuffer buffer = binToBuffer("src/test/resources/" + mapName + "_frame0_buffer.bin");
+        final ByteBuffer buffer = binToBuffer(DEFAULT_BUFFER_PATH);
         return createGame(new Client(buffer));
     }
 
+    public static Game createGame(Client client) {
+        final Game game = new Game(client.clientData());
+        game.init();
+        return game;
+    }
+
+    public static Game createGameUnchecked() {
+        try {
+            return GameBuilder.createGame();
+        } catch (IOException exception) {
+            throw new RuntimeException(exception);
+        }
+    }
+
     public static ByteBuffer binToBuffer(String binLocation) throws IOException {
         final byte[] compressedBytes = Files.readAllBytes(Paths.get(binLocation));
         final ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -32,9 +49,12 @@ public static ByteBuffer binToBuffer(String binLocation) throws IOException {
         return buffer;
     }
 
-    public static Game createGame(Client client) throws IOException {
-        final Game game = new Game(client.clientData());
-        game.init();
-        return game;
+    public static ByteBuffer binToBufferUnchecked(String binLocation) {
+        try {
+            return binToBuffer(binLocation);
+        } catch(IOException exception) {
+            throw new RuntimeException(exception);
+        }
     }
+
 }