Unify datapath

.azure/OneBranch.Official.yml

@@ -86,22 +86,6 @@ extends:
         parameters:
           config: Debug
 
-    - stage: build_windows_xdp
-      displayName: Build Windows XDP
-      dependsOn: []
-      jobs:
-      - template: .azure/obtemplates/build-winuser-xdp.yml@self
-        parameters:
-          config: Release
-          tls: schannel
-      - template: .azure/obtemplates/build-winuser-xdp.yml@self
-        parameters:
-          config: Release
-          tls: openssl
-      - template: .azure/obtemplates/build-winuser-xdp.yml@self
-        parameters:
-          config: Debug
-
     - stage: build_windows
       displayName: Build Windows
       dependsOn: []
@@ -204,7 +188,6 @@ extends:
       dependsOn:
       - build_windows
       - build_uwp
-      - build_windows_xdp
       jobs:
       - template: .azure/obtemplates/build-nuget.yml@self
         ${{ if startsWith(variables['Build.SourceBranch'], 'refs/tags/') }}:

.azure/OneBranch.PullRequest.yml

@@ -82,22 +82,6 @@ extends:
         parameters:
           config: Debug
 
-    - stage: build_windows_xdp
-      displayName: Build Windows XDP
-      dependsOn: []
-      jobs:
-      - template: .azure/obtemplates/build-winuser-xdp.yml@self
-        parameters:
-          config: Release
-          tls: schannel
-      - template: .azure/obtemplates/build-winuser-xdp.yml@self
-        parameters:
-          config: Release
-          tls: openssl
-      - template: .azure/obtemplates/build-winuser-xdp.yml@self
-        parameters:
-          config: Debug
-
     - stage: build_windows
       displayName: Build Windows
       dependsOn: []
@@ -198,6 +182,5 @@ extends:
       dependsOn:
       - build_windows
       - build_uwp
-      - build_windows_xdp
       jobs:
       - template: .azure/obtemplates/build-nuget.yml@self

.azure/azure-pipelines.perf.yml

@@ -194,9 +194,9 @@ stages:
         extraName: 'xdp'
         extraPrepareArgs: -DisableTest -InstallXdpSdk
         ${{ if eq(parameters.pgo_mode, false) }}:
-          extraBuildArgs: -DisableTest -DisableTools -UseXdp -ExtraArtifactDir Xdp
+          extraBuildArgs: -DisableTest -DisableTools -ExtraArtifactDir Xdp
         ${{ if eq(parameters.pgo_mode, true) }}:
-          extraBuildArgs: -DisableTest -DisableTools -UseXdp -ExtraArtifactDir Xdp -PGO
+          extraBuildArgs: -DisableTest -DisableTools -ExtraArtifactDir Xdp -PGO
 
 - ${{ if eq(parameters.winuser_openssl, true) }}:
   - stage: build_winuser_openssl

.azure/azure-pipelines.periodic.yml

@@ -84,7 +84,7 @@ stages:
       config: Release
       extraName: 'xdp'
       extraPrepareArgs: -DisableTest -InstallXdpSdk
-      extraBuildArgs: -DisableTest -DisableTools -UseXdp -ExtraArtifactDir Xdp -PGO
+      extraBuildArgs: -DisableTest -DisableTools -ExtraArtifactDir Xdp -PGO
 
 #
 # Performance Tests

.azure/obtemplates/build-winuser.yml

@@ -13,6 +13,13 @@ jobs:
     ob_sdl_binskim_break: true # https://aka.ms/obpipelines/sdl
     ob_sdl_codeSignValidation_excludes: -|**\*.exe # Disable signing requirements for test executables
   steps:
+  - task: PowerShell@2
+    displayName: Prepare Build Machine
+    target: windows_build_container2
+    inputs:
+      pwsh: true
+      filePath: scripts/prepare-machine.ps1
+      arguments: -InstallXdpSdk -ForContainerBuild
   - task: PowerShell@2
     displayName: x64
     target: windows_build_container2

.azure/templates/build-config-user.yml

@@ -43,7 +43,7 @@ jobs:
     inputs:
       pwsh: true
       filePath: scripts/prepare-machine.ps1
-      arguments: -Tls ${{ parameters.tls }} -ForBuild ${{ parameters.extraPrepareArgs }}
+      arguments: -Tls ${{ parameters.tls }} -ForBuild ${{ parameters.extraPrepareArgs }} -UseXdp
 
   - task: PowerShell@2
     displayName: Build Source Code (Debug)

.github/workflows/build-reuse-win.yml

@@ -85,15 +85,15 @@ jobs:
       uses: ilammy/setup-nasm@321e6ed62a1fc77024a3bd853deb33645e8b22c4
     - name: Prepare Machine
       shell: pwsh
-      run: scripts/prepare-machine.ps1 -ForBuild -Tls ${{ inputs.tls }}
+      run: scripts/prepare-machine.ps1 -ForBuild -Tls ${{ inputs.tls }} ${{ inputs.xdp }}
     - name: Build For Test
       if: inputs.test == '-Test'
       shell: pwsh
-      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisablePerf -DynamicCRT ${{ inputs.xdp }} ${{ inputs.sanitize }}
+      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} -DisablePerf -DynamicCRT ${{ inputs.sanitize }}
     - name: Build
       if: inputs.test == ''
       shell: pwsh
-      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} ${{ inputs.xdp }} ${{ inputs.sanitize }} ${{ inputs.static }}
+      run: scripts/build.ps1 -Config ${{ inputs.config }} -Platform ${{ inputs.plat }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }} ${{ inputs.sanitize }} ${{ inputs.static }}
     - name: Filter Build Artifacts
       shell: pwsh
       run: |
@@ -103,5 +103,5 @@ jobs:
     - name: Upload build artifacts
       uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
       with:
-        name: ${{ inputs.config }}-${{ inputs.plat }}-${{ inputs.os }}-${{ inputs.arch }}-${{ inputs.tls }}${{ inputs.xdp }}${{ inputs.sanitize }}${{ inputs.static }}${{ inputs.test }}
+        name: ${{ inputs.config }}-${{ inputs.plat }}-${{ inputs.os }}-${{ inputs.arch }}-${{ inputs.tls }}${{ inputs.sanitize }}${{ inputs.static }}${{ inputs.test }}
         path: artifacts

.github/workflows/build.yml

@@ -201,8 +201,6 @@ jobs:
           { plat: "uwp",  tls: "openssl", arg: "-UWP" },
           { plat: "windows",  tls: "openssl" },
           { plat: "windows",  tls: "schannel" },
-          { plat: "windows",  tls: "openssl", xdp: "-UseXdp", arg: "-XDP" },
-          { plat: "windows",  tls: "schannel", xdp: "-UseXdp", arg: "-XDP" },
         ]
     runs-on: windows-2022
     steps:
@@ -211,19 +209,19 @@ jobs:
     - name: Download Build Artifacts
       uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
       with:
-        name: Release-${{ matrix.vec.plat }}-windows-2022-x86-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}
+        name: Release-${{ matrix.vec.plat }}-windows-2022-x86-${{ matrix.vec.tls }}
         path: artifacts
         if_no_artifact_found: ignore
     - name: Download Build Artifacts
       uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
       with:
-        name: Release-${{ matrix.vec.plat }}-windows-2022-x64-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}
+        name: Release-${{ matrix.vec.plat }}-windows-2022-x64-${{ matrix.vec.tls }}
         path: artifacts
         if_no_artifact_found: ignore
     - name: Download Build Artifacts
       uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
       with:
-        name: Release-${{ matrix.vec.plat }}-windows-2022-arm64-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}
+        name: Release-${{ matrix.vec.plat }}-windows-2022-arm64-${{ matrix.vec.tls }}
         path: artifacts
         if_no_artifact_found: ignore
     - name: Build Package
@@ -232,5 +230,5 @@ jobs:
     - name: Upload build artifacts
       uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
       with:
-        name: Nuget-Release-${{ matrix.vec.plat }}-windows-2022-arm64-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}
+        name: Nuget-Release-${{ matrix.vec.plat }}-windows-2022-arm64-${{ matrix.vec.tls }}
         path: artifacts/dist/*.nupkg

.github/workflows/cargo.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
     - name: Prepare Machine
-      run: scripts/prepare-machine.ps1 -Tls openssl -ForBuild
+      run: scripts/prepare-machine.ps1 -Tls openssl -ForBuild -UseXdp
       shell: pwsh
     - name: Install Perl
       if: runner.os == 'Windows'
@@ -45,4 +45,4 @@ jobs:
     - name: Cargo test
       run: cargo test --all
     - name: Cargo Publish (dry run)
-      run: cargo publish --dry-run
+      run: cargo publish --dry-run --allow-dirty

.github/workflows/code-coverage.yml

@@ -59,7 +59,7 @@ jobs:
     - name: Download Build Artifacts
       uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
       with:
-        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
+        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
         path: artifacts
     - name: Prepare Machine
       run: scripts/prepare-machine.ps1 -Tls ${{ matrix.vec.tls }} -ForTest ${{ matrix.vec.xdp }} -InstallCodeCoverage
@@ -79,7 +79,7 @@ jobs:
     - name: Upload Results
       uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
       with:
-        name: BVT-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.qtip }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }}
+        name: BVT-${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.qtip }}${{ matrix.vec.systemcrypto }}${{ matrix.vec.sanitize }}
         path: artifacts/coverage/*.cov
 
   stress-winlatest:
@@ -109,7 +109,7 @@ jobs:
         fetch-depth: 0
     - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
       with:
-        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
+        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
         path: artifacts
     - name: Prepare Machine
       run: scripts/prepare-machine.ps1 -Tls ${{ matrix.vec.tls }} -ForTest ${{ matrix.vec.xdp }} -InstallCodeCoverage
@@ -142,7 +142,7 @@ jobs:
       with:
         fetch-depth: 0
     - name: Prepare Machine
-      run: scripts/prepare-machine.ps1 -ForTest -InstallCodeCoverage
+      run: scripts/prepare-machine.ps1 -ForTest -InstallCodeCoverage -UseXdp
       shell: pwsh
     - name: Create Folder for Artifacts
       shell: pwsh

.github/workflows/stress.yml

@@ -108,7 +108,7 @@ jobs:
     - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
       if: matrix.vec.plat == 'windows'
       with:
-        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os == 'WSPrerelease' && 'windows-2022' || matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
+        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os == 'WSPrerelease' && 'windows-2022' || matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
         path: artifacts
     - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
       if: matrix.vec.plat == 'linux' || matrix.vec.plat == 'macos'

.github/workflows/test.yml

@@ -127,7 +127,7 @@ jobs:
       uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
       if: matrix.vec.plat == 'windows'
       with: # note that BVT for WSPrerelease uses binaries built on windows-2022.
-        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os == 'WSPrerelease' && 'windows-2022' || matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
+        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os == 'WSPrerelease' && 'windows-2022' || matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
         path: artifacts
     - name: Download Build Artifacts
       uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
@@ -234,7 +234,7 @@ jobs:
         fetch-depth: 0
     - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
       with:
-        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.xdp }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
+        name: ${{ matrix.vec.config }}-${{ matrix.vec.plat }}-${{ matrix.vec.os }}-${{ matrix.vec.arch }}-${{ matrix.vec.tls }}${{ matrix.vec.sanitize }}${{ matrix.vec.test }}
         path: artifacts
     - name: Prepare Machine
       run: scripts/prepare-machine.ps1 -ForTest ${{ matrix.vec.xdp }}

CMakeLists.txt

@@ -555,11 +555,6 @@ if(WIN32)
     set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG /IGNORE:4075 /DEBUG /OPT:REF /OPT:ICF")
 
     # Find the right PGO file.
-    if(QUIC_USE_XDP)
-        set(QUIC_PGO_NAME "msquic.${QUIC_TLS}.xdp.pgd")
-        set(QUIC_PGO_FILE "${CMAKE_CURRENT_SOURCE_DIR}/src/bin/winuser/pgo_${SYSTEM_PROCESSOR}/${QUIC_PGO_NAME}")
-    endif()
-
     if(NOT EXISTS "${QUIC_PGO_FILE}")
         set(QUIC_PGO_NAME "msquic.${QUIC_TLS}.pgd")
         set(QUIC_PGO_FILE "${CMAKE_CURRENT_SOURCE_DIR}/src/bin/winuser/pgo_${SYSTEM_PROCESSOR}/${QUIC_PGO_NAME}")

Cargo.toml

@@ -40,6 +40,7 @@ include = [
     "/src/generated",
     "/src/manifest",
     "/src/platform",
+    "/artifacts/xdp/include",
     "/THIRD-PARTY-NOTICES",
 ]
 

scripts/make-pgo-pr.ps1

@@ -43,7 +43,6 @@ git checkout -b $BranchName
 Copy-Item -Path artifacts/PerfDataResults/performance/windows/$($Arch)_$($Config)_schannel/msquic.pgd src/bin/winuser/pgo_$($Arch)/msquic.schannel.pgd -Force
 Copy-Item -Path artifacts/PerfDataResults/performance/windows/$($Arch)_$($Config)_openssl/msquic.pgd src/bin/winuser/pgo_$($Arch)/msquic.openssl.pgd -Force
 Copy-Item -Path artifacts/PerfDataResults/performance/windows/$($Arch)_$($Config)_openssl3/msquic.pgd src/bin/winuser/pgo_$($Arch)/msquic.openssl3.pgd -Force
-Copy-Item -Path artifacts/PerfDataResults/performance/windows/$($Arch)_$($Config)_schannel_Xdp/msquic.pgd src/bin/winuser/pgo_$($Arch)/msquic.schannel.xdp.pgd -Force
 
 # Commit the new PGD files.
 git commit -am "Update PGO data"

src/core/packet_builder.c

@@ -720,7 +720,7 @@ QuicPacketBuilderFinalize(
 
         FinalQuicPacket = TRUE;
 
-        if (!FlushBatchedDatagrams && CxPlatDataPathIsPaddingPreferred(MsQuicLib.Datapath)) {
+        if (!FlushBatchedDatagrams && CxPlatDataPathIsPaddingPreferred(MsQuicLib.Datapath, Builder->SendData)) {
             //
             // When buffering multiple datagrams in a single contiguous buffer
             // (at the datapath layer), all but the last datagram needs to be

src/generated/linux/datapath_raw_socket_win.c.clog.h

@@ -14,6 +14,10 @@
 #include "datapath_raw_socket_win.c.clog.h.lttng.h"
 #endif
 #include <lttng/tracepoint-event.h>
+#ifndef _clog_MACRO_QuicTraceLogVerbose
+#define _clog_MACRO_QuicTraceLogVerbose  1
+#define QuicTraceLogVerbose(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__)))
+#endif
 #ifndef _clog_MACRO_QuicTraceLogConnInfo
 #define _clog_MACRO_QuicTraceLogConnInfo  1
 #define QuicTraceLogConnInfo(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__)))
@@ -25,6 +29,26 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+/*----------------------------------------------------------
+// Decoder Ring for DatapathTcpAuxBinding
+// [data][%p] Binding TCP socket to %s
+// QuicTraceLogVerbose(
+            DatapathTcpAuxBinding,
+            "[data][%p] Binding TCP socket to %s",
+            Socket,
+            LocalAddressString.Address);
+// arg2 = arg2 = Socket = arg2
+// arg3 = arg3 = LocalAddressString.Address = arg3
+----------------------------------------------------------*/
+#ifndef _clog_4_ARGS_TRACE_DatapathTcpAuxBinding
+#define _clog_4_ARGS_TRACE_DatapathTcpAuxBinding(uniqueId, encoded_arg_string, arg2, arg3)\
+tracepoint(CLOG_DATAPATH_RAW_SOCKET_WIN_C, DatapathTcpAuxBinding , arg2, arg3);\
+
+#endif
+
+
+
+
 /*----------------------------------------------------------
 // Decoder Ring for RouteResolutionStart
 // [conn][%p] Starting to look up neighbor on Path[%hhu] with status %u

src/generated/linux/datapath_raw_socket_win.c.clog.h.lttng.h

@@ -1,6 +1,29 @@
 
 
 
+/*----------------------------------------------------------
+// Decoder Ring for DatapathTcpAuxBinding
+// [data][%p] Binding TCP socket to %s
+// QuicTraceLogVerbose(
+            DatapathTcpAuxBinding,
+            "[data][%p] Binding TCP socket to %s",
+            Socket,
+            LocalAddressString.Address);
+// arg2 = arg2 = Socket = arg2
+// arg3 = arg3 = LocalAddressString.Address = arg3
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_SOCKET_WIN_C, DatapathTcpAuxBinding,
+    TP_ARGS(
+        const void *, arg2,
+        const char *, arg3), 
+    TP_FIELDS(
+        ctf_integer_hex(uint64_t, arg2, arg2)
+        ctf_string(arg3, arg3)
+    )
+)
+
+
+
 /*----------------------------------------------------------
 // Decoder Ring for RouteResolutionStart
 // [conn][%p] Starting to look up neighbor on Path[%hhu] with status %u