FEAT: Pipeline & Setup to Build WHL Files and Python Backward Compatibility (#46)

* Added Build steps and Backward compatibility

* checked with main

* restored other files

* restored another file

* changes for ddbc_bindings

* restoring requirements

* tests updated with new build process

* fixed pyd path for artifact

* added auth DLLs and upgraded to latest driver DLLs 18.5

* test pipelines to store python arm64 libs as artifacts

* fixed arch

* debugging arch related things

* win arm whl fix

* setuptools dependency

* import bdist_wheel from wheel

* version change for release

* fix loading path for mssql-auth DLL

* store python_libs for arm64 as artifacts and remove unsupported architectures

* get python arm64 libs from previous successful run

* remove all python arm64 libs

* fix pipeline format to download from latest branch run

* remove artifacts etc. from test pipeline

* clean out pyds directory - minimal now

* updated README

* clean up a bunch of print statements

* cleanup

* cleanup for other archs

Author: bewithgaurav

.gitignore

@@ -3,8 +3,8 @@ mssql_python/pybind/build/
 
 mssql_python/pybind/pymsbuild/build/
 
-# Ignore all pyd files
-**.pyd
+# Ignore pyd file
+mssql_python/ddbc_bindings.pyd
 
 # Ignore pycache files and folders
 __pycache__/
@@ -26,11 +26,25 @@ mssql_python/.vs
 test-*.xml
 **/test-**.xml
 
-# Ignore coverage files
-coverage.xml
-.coverage
-.coverage.*
-
 # Ignore the build & mssql_python.egg-info directories
 build/
-mssql_python.egg-info/
+mssql_python.egg-info/
+
+# Python bytecode
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+
+# C extensions
+*.so
+*.pyd
+
+# IDE files
+.vscode/
+.idea/
+*.swp

eng/pipelines/build-whl-pipeline.yml

@@ -0,0 +1,164 @@
+# Pipeline name shown in ADO UI
+name: build-whl-pipeline
+
+# Trigger the pipeline on pushes to these branches
+trigger:
+  branches:
+    include:
+      - main
+      - dev
+
+# Use Microsoft-hosted Windows VM
+pool:
+  vmImage: 'windows-latest'
+
+jobs:
+- job: BuildPYDs
+  displayName: 'Build -'
+  # Strategy matrix to build all combinations
+  strategy:
+    matrix:
+      # Python 3.10 (only x64)
+      py310_x64:
+        pythonVersion: '3.10'       # Host Python version
+        shortPyVer: '310'           # Used in filenames like cp310
+        architecture: 'x64'         # Host Python architecture
+        targetArch: 'x64'           # Target architecture to pass to build.bat
+
+      # Python 3.11
+      py311_x64:
+        pythonVersion: '3.11'       # Host Python version
+        shortPyVer: '311'           # Used in filenames like cp311
+        architecture: 'x64'         # Host Python architecture
+        targetArch: 'x64'           # Target architecture to pass to build.bat
+      py311_arm64:
+        pythonVersion: '3.11'
+        shortPyVer: '311'
+        architecture: 'x64'         # No arm64 Python, use x64 host
+        targetArch: 'arm64'
+
+      # Python 3.12
+      py312_x64:
+        pythonVersion: '3.12'
+        shortPyVer: '312'
+        architecture: 'x64'
+        targetArch: 'x64'
+      py312_arm64:
+        pythonVersion: '3.12'
+        shortPyVer: '312'
+        architecture: 'x64'
+        targetArch: 'arm64'
+
+      # Python 3.13
+      py313_x64:
+        pythonVersion: '3.13'
+        shortPyVer: '313'
+        architecture: 'x64'
+        targetArch: 'x64'
+      py313_arm64:
+        pythonVersion: '3.13'
+        shortPyVer: '313'
+        architecture: 'x64'
+        targetArch: 'arm64'
+
+  steps:
+    # Use correct Python version and architecture for the current job
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '$(pythonVersion)'
+        architecture: '$(architecture)'
+        addToPath: true
+      displayName: 'Use Python $(pythonVersion) ($(architecture))'
+
+    # Install required packages: pip, CMake, pybind11
+    - script: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install cmake pybind11
+      displayName: 'Install dependencies'
+
+    - task: DownloadPipelineArtifact@2
+      condition: eq(variables['targetArch'], 'arm64')
+      inputs:
+        buildType: 'specific'
+        project: '$(System.TeamProject)'
+        definition: 2134
+        buildVersionToDownload: 'latestFromBranch'
+        branchName: 'refs/heads/bewithgaurav/build_whl_pipeline'                        # Or 'refs/heads/dev'
+        artifactName: 'mssql-python-arm64-libs'
+        targetPath: '$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64'
+      displayName: 'Download ARM64 Python libs from latest successful run on branches'
+
+    # Build the PYD file by calling build.bat
+    - script: |
+        echo "Python Version: $(pythonVersion)"
+        echo "Short Tag: $(shortPyVer)"
+        echo "Architecture: Host=$(architecture), Target=$(targetArch)"
+
+        cd "$(Build.SourcesDirectory)\mssql_python\pybind"
+
+        REM Optional: override lib path if building for ARM64 since we cannot install arm64 python on x64 host
+        if "$(targetArch)"=="arm64" (
+          echo Using arm64-specific Python library...
+          set CUSTOM_PYTHON_LIB_DIR=$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64
+        )
+
+        REM Call build.bat to build the PYD file
+        call build.bat $(targetArch)
+
+        REM Calling keep_single_arch.bat to remove ODBC libs of other architectures
+        call keep_single_arch.bat $(targetArch)
+
+        cd ..\..
+      displayName: 'Build PYD for $(targetArch)'        
+      
+    # Copy the built .pyd file to staging folder for artifacts
+    - task: CopyFiles@2
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)\mssql_python\pybind\build\$(targetArch)\py$(shortPyVer)\Release'
+        Contents: 'ddbc_bindings.cp$(shortPyVer)-*.pyd'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)\all-pyds'
+      displayName: 'Place PYD file into artifacts directory'
+
+    # Build wheel package for the current architecture
+    - script: |
+        python -m pip install --upgrade pip
+        pip install wheel setuptools
+        set ARCHITECTURE=$(targetArch)
+        python setup.py bdist_wheel
+      displayName: 'Build wheel package for Python $(pythonVersion) ($(targetArch))'
+    
+    # Copy the wheel file to the artifacts
+    - task: CopyFiles@2
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)\dist'
+        Contents: '*.whl'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)\dist'
+      displayName: 'Collect wheel package'    
+
+    # Publish the collected .pyd file(s) as build artifacts
+    - task: PublishBuildArtifacts@1
+      condition: succeededOrFailed()
+      inputs:
+        PathtoPublish: '$(Build.ArtifactStagingDirectory)\all-pyds'
+        ArtifactName: 'mssql-python-pyds'
+        publishLocation: 'Container'
+      displayName: 'Publish all PYDs as artifacts'
+
+    # Publish the python arm64 libraries as build artifacts for next builds if ARM64
+    - task: PublishBuildArtifacts@1
+      condition: eq(variables['targetArch'], 'arm64')
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64'
+        ArtifactName: 'mssql-python-arm64-libs'
+        publishLocation: 'Container'
+      displayName: 'Publish arm64 libs as artifacts'
+    
+    # Publish the collected wheel file(s) as build artifacts
+    - task: PublishBuildArtifacts@1
+      condition: succeededOrFailed()
+      inputs:
+        PathtoPublish: '$(Build.ArtifactStagingDirectory)\dist'
+        ArtifactName: 'mssql-python-wheels-dist'
+        publishLocation: 'Container'
+      displayName: 'Publish all wheels as artifacts'

eng/pipelines/test-pipeline.yml

@@ -36,11 +36,7 @@ jobs:
 
   - script: |
       cd mssql_python\pybind
-      mkdir build
-      cd build  
-      cmake -DPython3_EXECUTABLE="python3" -DCMAKE_BUILD_TYPE=Debug ..
-      cmake --build . --config Debug
-      copy Debug\ddbc_bindings.pyd ..\..\ddbc_bindings.pyd
+      build.bat x64
     displayName: 'Build .pyd file'
 
   - script: |
@@ -51,7 +47,7 @@ jobs:
 
   - task: PublishBuildArtifacts@1
     inputs:
-      PathtoPublish: 'mssql_python/ddbc_bindings.pyd'
+      PathtoPublish: 'mssql_python/ddbc_bindings.cp313-amd64.pyd'
       ArtifactName: 'ddbc_bindings'
       publishLocation: 'Container'
     displayName: 'Publish pyd file as artifact'

mssql_python/connection.py

@@ -95,7 +95,6 @@ def _construct_connection_string(self, connection_str: str = "", **kwargs) -> st
             else:
                 continue
             conn_str += f"{key}={value};"
-        print(f"Connection string after adding driver: {conn_str}")
 
         if ENABLE_LOGGING:
             logger.info("Final connection string: %s", conn_str)

mssql_python/ddbc_bindings.py

@@ -0,0 +1,38 @@
+import os
+import importlib.util
+import sys
+import platform
+
+# Get current Python version and architecture
+python_version = f"cp{sys.version_info.major}{sys.version_info.minor}"
+if platform.machine().lower() in ('amd64', 'x86_64', 'x64'):
+    architecture = "amd64"
+elif platform.machine().lower() in ('arm64', 'aarch64'):
+    architecture = "arm64"
+else:
+    architecture = platform.machine().lower()
+
+# Find the specifically matching PYD file
+module_dir = os.path.dirname(__file__)
+expected_pyd = f"ddbc_bindings.{python_version}-{architecture}.pyd"
+pyd_path = os.path.join(module_dir, expected_pyd)
+
+if not os.path.exists(pyd_path):
+    # Fallback to searching for any matching PYD if the specific one isn't found
+    pyd_files = [f for f in os.listdir(module_dir) if f.startswith('ddbc_bindings.') and f.endswith('.pyd')]
+    if not pyd_files:
+        raise ImportError(f"No ddbc_bindings PYD module found for {python_version}-{architecture}")
+    pyd_path = os.path.join(module_dir, pyd_files[0])
+    print(f"Warning: Using fallback PYD file {pyd_files[0]} instead of {expected_pyd}")
+
+# Use the original module name 'ddbc_bindings' that the C extension was compiled with
+name = "ddbc_bindings"
+spec = importlib.util.spec_from_file_location(name, pyd_path)
+module = importlib.util.module_from_spec(spec)
+sys.modules[name] = module
+spec.loader.exec_module(module)
+
+# Copy all attributes from the loaded module to this module
+for attr in dir(module):
+    if not attr.startswith('__'):
+        globals()[attr] = getattr(module, attr)

mssql_python/helpers.py

@@ -46,7 +46,7 @@ def add_driver_to_connection_str(connection_str):
         # Insert the driver attribute at the beginning of the connection string
         final_connection_attributes.insert(0, driver_name)
         connection_str = ";".join(final_connection_attributes)
-        print(f"Connection string after adding driver: {connection_str}")
+
     except Exception as e:
         raise Exception(
             "Invalid connection string, Please follow the format: "