Merge pull request #205 from poissoncorp/RDBC-795

RDBC-795 Make test driver use the embedded server

Author: ml054

.github/workflows/RavenClient.yml

@@ -72,6 +72,9 @@ jobs:
       - name: Install client deps
         run: pip install -e .
 
+      - name: Install embedded RavenDB
+        run: pip install ravendb-embedded
+
       - name: Run certifi script
         run: python ./.github/workflows/add_ca.py
 

ravendb/tests/driver/raven_server_runner.py

@@ -1,23 +1,58 @@
 from __future__ import annotations
-
 import os
 import pathlib
-import subprocess
+import glob
+from typing import TYPE_CHECKING
+
+from ravendb_embedded.embedded_server import EmbeddedServer
+from ravendb_embedded.options import ServerOptions, SecurityOptions
+from ravendb_embedded.provide import ExternalServerProvider
+
 from ravendb.tests.driver.raven_server_locator import RavenServerLocator
 
+if TYPE_CHECKING:
+    from ravendb.tests.test_base import TestBase
+
 
 class RavenServerRunner:
+    @staticmethod  # workaround for places where we cannot import TestBase.TestSecuredServiceLocator
+    def is_locator_secured(locator: RavenServerLocator) -> bool:
+        return hasattr(locator, "client_certificate_path")
+
     @staticmethod
-    def run(locator: RavenServerLocator) -> subprocess.Popen:
+    def get_server_options_for_embedded_server(
+        locator: RavenServerLocator, server_dll_parent_dir: str
+    ) -> ServerOptions:
         process_start_info = RavenServerRunner.get_process_start_info(locator)
+        server_options = ServerOptions()
+        server_options.server_url = "http://127.0.0.1:0"
+        server_options.command_line_args = ["--Features.Availability=Experimental"]
+        server_options.command_line_args.extend(process_start_info.arguments)
+        server_options.provider = ExternalServerProvider(server_dll_parent_dir)
+        server_options.target_server_location = server_dll_parent_dir
+
+        if RavenServerRunner.is_locator_secured(locator):
+            locator: "TestBase.TestSecuredServiceLocator"
+            server_options.security = SecurityOptions()
+            server_options.security.server_pfx_certificate_path = locator.server_certificate_path
+            server_options.security.client_pem_certificate_path = locator.client_certificate_path
+            server_options.security.ca_certificate_path = locator.server_ca_path
+            server_options.server_url = locator.https_server_url
+
+        return server_options
+
+    @staticmethod
+    def get_embedded_server(locator: RavenServerLocator) -> EmbeddedServer:
+        raven_server_dll_pardir = pathlib.Path(locator.get_server_path()).parent.__str__()
+        server_options = RavenServerRunner.get_server_options_for_embedded_server(locator, raven_server_dll_pardir)
 
-        arguments = [process_start_info.command]
-        arguments.extend(process_start_info.arguments)
+        embedded = EmbeddedServer()
+        embedded.start_server(server_options)
 
-        return subprocess.Popen(arguments, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+        return embedded
 
     @staticmethod
-    def get_process_start_info(locator: RavenServerLocator) -> __ProcessStartInfo:
+    def get_process_start_info(locator: RavenServerLocator) -> _ProcessStartInfo:
         server_path = pathlib.Path(locator.get_server_path())
         if not server_path.exists():
             raise FileNotFoundError(f"Server file was not found: {locator.get_server_path()}")
@@ -32,7 +67,7 @@ def get_process_start_info(locator: RavenServerLocator) -> __ProcessStartInfo:
 
         command_arguments.extend(locator.command_arguments)
 
-        process_start_info = RavenServerRunner.__ProcessStartInfo(locator.command, *command_arguments)
+        process_start_info = RavenServerRunner._ProcessStartInfo(locator.command, *command_arguments)
 
         return process_start_info
 
@@ -43,7 +78,7 @@ def get_process_id(fallback: str) -> str:
         except Exception:
             return fallback
 
-    class __ProcessStartInfo:
+    class _ProcessStartInfo:
         def __init__(self, command: str, *arguments: str):
             self.command = command
             self.arguments = arguments

ravendb/tests/driver/raven_test_driver.py

@@ -1,13 +1,13 @@
-import os
 import subprocess
-import time
-from typing import Callable
-
+from typing import TYPE_CHECKING, Tuple
+from ravendb_embedded.embedded_server import EmbeddedServer
 from ravendb.documents.store.definition import DocumentStore
 from ravendb.infrastructure.graph import Genre, Movie, User
-from ravendb.tests.driver.raven_server_locator import RavenServerLocator
 from ravendb.tests.driver.raven_server_runner import RavenServerRunner
 
+if TYPE_CHECKING:
+    from ravendb.tests.driver.raven_server_locator import RavenServerLocator
+
 
 class RavenTestDriver:
     debug = False
@@ -19,46 +19,13 @@ def __init__(self):
     def disposed(self) -> bool:
         return self._disposed
 
-    def _run_server_internal(
-        self, locator: RavenServerLocator, configure_store: Callable[[DocumentStore], None]
-    ) -> (DocumentStore, subprocess.Popen):
-        process = RavenServerRunner.run(locator)
-        self._report_info("Starting global server")
-        url = None
-        stdout = process.stdout
-        startup_duration = time.perf_counter()
-        read_lines = []
-        while True:
-            line = stdout.readline().decode("utf-8")
-            read_lines.append(line)
-
-            if line is None:
-                raise RuntimeError(str.join(os.linesep, read_lines) + process.stdin.read().decode("utf-8"))
-
-            if time.perf_counter() - startup_duration > 60:
-                break
-
-            prefix = "Server available on: "
-            if line.startswith(prefix):
-                url = line[len(prefix) :].rstrip()
-                break
-
-        if url is None:
-            self._report_info("Url is None")
-            try:
-                process.kill()
-            except Exception as e:
-                self._report_error(e)
-
-            raise RuntimeError("Unable to start server")
-        print(url)
-        store = DocumentStore([url], "test.manager")
+    @staticmethod
+    def _run_embedded_server_internal(locator: "RavenServerLocator") -> Tuple[DocumentStore, EmbeddedServer]:
+        embedded_server = RavenServerRunner.get_embedded_server(locator)
+        store = embedded_server.get_document_store("test.manager")
         store.conventions.disable_topology_updates = True
 
-        if configure_store is not None:
-            configure_store(store)
-
-        return store.initialize(), process
+        return store, embedded_server
 
     @staticmethod
     def _kill_process(p: subprocess.Popen) -> None:

ravendb/tests/test_base.py

@@ -6,9 +6,9 @@
 import sys
 import os
 from enum import Enum
-from subprocess import Popen
-from typing import Iterable, List, Union, Optional, Set
+from typing import Iterable, List, Optional, Set
 from datetime import timedelta
+from ravendb_embedded.embedded_server import EmbeddedServer
 
 from ravendb.documents.operations.revisions import (
     ConfigureRevisionsOperationResult,
@@ -147,17 +147,20 @@ def __init__(self, patched):
 
 
 class TestBase(unittest.TestCase, RavenTestDriver):
-    __global_server: Union[None, DocumentStore] = None
-    __global_server_process: Union[None, Popen] = None
+    _global_embedded_server: Optional[EmbeddedServer] = None
+    _global_secured_embedded_server: Optional[EmbeddedServer] = None
 
-    __global_secured_server: Union[None, DocumentStore] = None
-    __global_secured_server_process: Union[None, Popen] = None
+    _global_server_store: Optional[DocumentStore] = None
+    _global_secured_server_store: Optional[DocumentStore] = None
 
     __run_server_lock = threading.Lock()
 
     index = 0
 
-    class __TestServiceLocator(RavenServerLocator):
+    class TestServiceLocator(RavenServerLocator):
+        def get_server_path(self) -> str:
+            return super().get_server_path()
+
         @property
         def server_path(self) -> str:
             return super().get_server_path()
@@ -170,7 +173,7 @@ def command_arguments(self) -> List[str]:
                 "--Features.Availability=Experimental",
             ]
 
-    class __TestSecuredServiceLocator(RavenServerLocator):
+    class TestSecuredServiceLocator(RavenServerLocator):
         ENV_CLIENT_CERTIFICATE_PATH = "RAVENDB_PYTHON_TEST_CLIENT_CERTIFICATE_PATH"
         ENV_SERVER_CERTIFICATE_PATH = "RAVENDB_PYTHON_TEST_SERVER_CERTIFICATE_PATH"
         ENV_TEST_CA_PATH = "RAVENDB_PYTHON_TEST_CA_PATH"
@@ -181,7 +184,7 @@ def get_server_path(self) -> str:
 
         @property
         def command_arguments(self) -> List[str]:
-            https_server_url = self.__https_server_url
+            https_server_url = self.https_server_url
             tcp_server_url = https_server_url.replace("https", "tcp", 1).rsplit(":", 1)[0] + ":38882"
             return [
                 f"--Security.Certificate.Path={self.server_certificate_path}",
@@ -190,7 +193,7 @@ def command_arguments(self) -> List[str]:
             ]
 
         @property
-        def __https_server_url(self) -> str:
+        def https_server_url(self) -> str:
             https_server_url = os.environ[self.ENV_HTTPS_SERVER_URL]
             if https_server_url.isspace():
                 raise ValueError(
@@ -226,27 +229,23 @@ def server_certificate_path(self) -> str:
         def server_ca_path(self) -> str:
             return os.getenv(self.ENV_TEST_CA_PATH)
 
-    def __get_locator(self, secured: bool):
-        return self.__secured_locator if secured else self.__locator
+    def _get_locator(self, secured: bool):
+        return self._secured_locator if secured else self._locator
 
-    def __get_global_server(self, secured: bool):
-        return self.__global_secured_server if secured else self.__global_server
+    def _get_global_server_store(self, secured: bool):
+        return self._global_secured_server_store if secured else self._global_server_store
 
-    def __run_server(self, secured: bool):
-        def __configure_store(s: DocumentStore) -> None:
-            if secured:
-                s.certificate_pem_path = self.test_client_certificate_url
-                s.trust_store_path = self.test_ca_certificate_url
-
-        store, process = self._run_server_internal(self.__get_locator(secured), __configure_store)
-        self.__set_global_server_process(secured, process)
+    def _run_embedded_server(self, secured: bool):
+        store, embedded_server = self._run_embedded_server_internal(self._get_locator(secured))
 
         if secured:
-            TestBase.__global_secured_server = store
+            TestBase._global_secured_server_store = store
+            TestBase._global_secured_embedded_server = embedded_server
         else:
-            TestBase.__global_server = store
+            TestBase._global_server_store = store
+            TestBase._global_embedded_server = embedded_server
 
-        atexit.register(threading.Thread(target=self.__kill_global_server_process, args=[secured]).run)
+        atexit.register(threading.Thread(target=self._discard_embedded_server, args=[secured]).run)
         return store
 
     def _customize_db_record(self, db_record: DatabaseRecord) -> None:
@@ -264,11 +263,11 @@ def secured_document_store(self) -> DocumentStore:
 
     @property
     def test_client_certificate_url(self) -> str:
-        return self.__secured_locator.client_certificate_path
+        return self._secured_locator.client_certificate_path
 
     @property
     def test_ca_certificate_url(self) -> str:
-        return self.__secured_locator.server_ca_path
+        return self._secured_locator.server_ca_path
 
     def get_document_store(
         self,
@@ -280,12 +279,12 @@ def get_document_store(
         name = f"{database}_{TestBase.index}"
         TestBase._report_info(f"get_document_store for db {database}.")
 
-        if self.__get_global_server(secured) is None:
+        if self._get_global_server_store(secured) is None:
             with self.__run_server_lock:
-                if self.__get_global_server(secured) is None:
-                    self.__run_server(secured)
+                if self._get_global_server_store(secured) is None:
+                    self._run_embedded_server(secured)
 
-        document_store = self.__get_global_server(secured)
+        document_store = self._get_global_server_store(secured)
         database_record = DatabaseRecord(name)
 
         self._customize_db_record(database_record)
@@ -303,7 +302,7 @@ def get_document_store(
         store.initialize()
 
         def __after_close():
-            if store not in self.__document_stores:
+            if store not in self._document_stores:
                 return
 
             try:
@@ -317,30 +316,21 @@ def __after_close():
         if wait_for_indexing_timeout is not None:
             self.wait_for_indexing(store, name, wait_for_indexing_timeout)
 
-        self.__document_stores.add(store)
+        self._document_stores.add(store)
         return store
 
-    @staticmethod
-    def __kill_global_server_process(secured: bool) -> None:
-        if secured:
-            p = TestBase.__global_secured_server_process
-            TestBase.__global_secured_server_process = None
-            TestBase.__global_secured_server.close()
-            TestBase.__global_secured_server = None
-        else:
-            p = TestBase.__global_server_process
-            TestBase.__global_server_process = None
-            TestBase.__global_server.close()
-            TestBase.__global_server = None
-
-        RavenTestDriver._kill_process(p)
-
-    @staticmethod
-    def __set_global_server_process(secured: bool, process: Popen) -> None:
+    @classmethod
+    def _discard_embedded_server(cls, secured: bool) -> None:
         if secured:
-            TestBase.__global_secured_server_process = process
+            cls._global_secured_server_store.close()
+            cls._global_secured_server_store = None
+            cls._global_secured_embedded_server.close()
+            cls._global_secured_embedded_server = None
         else:
-            TestBase.__global_server_process = process
+            cls._global_server_store.close()
+            cls._global_server_store = None
+            cls._global_embedded_server.close()
+            cls._global_embedded_server = None
 
     @staticmethod
     def delete_all_topology_files():
@@ -412,11 +402,10 @@ def setConvention(self, conventions):
         self.conventions = conventions
 
     def setUp(self):
-        # todo: investigate if line below is replaceable by more sophisticated code, we don't want to call TestCase init
         RavenTestDriver.__init__(self)
-        self.__locator = TestBase.__TestServiceLocator()
-        self.__secured_locator = TestBase.__TestSecuredServiceLocator()
-        self.__document_stores: Set[DocumentStore] = set()
+        self._locator = TestBase.TestServiceLocator()
+        self._secured_locator = TestBase.TestSecuredServiceLocator()
+        self._document_stores: Set[DocumentStore] = set()
         conventions = getattr(self, "conventions", None)
         self.default_urls = ["http://127.0.0.1:8080"]
         self.default_database = "NorthWindTest"
@@ -435,7 +424,7 @@ def tearDown(self):
 
         exceptions = []
 
-        for document_store in self.__document_stores:
+        for document_store in self._document_stores:
             try:
                 document_store.close()
             except Exception as e: