Skip to content

Commit 31ba163

Browse files
ahojnnesahojnnes
authored
Enable compiler caching in CI (#1972)
* Enable compiler caching in CI * d * d * d * d * d * d * d * d * d * d * d * d * d * d * d * d * d * d * d * d
1 parent 3d64864 commit 31ba163

File tree

6 files changed

+152
-39
lines changed

6 files changed

+152
-39
lines changed

‎.azure-pipelines/build-mac.yaml‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
parameters:
2-
displayName: 'Mac 10.15'
2+
displayName: Mac 10.15
33
macVersion: '10.15'
44

55
jobs:
66
- job: mac_build_${{ replace(parameters.macVersion, '.', '') }}
7-
displayName: '${{ parameters.displayName }}'
7+
displayName: ${{ parameters.displayName }}
88
pool:
99
vmImage: 'macOS-${{ parameters.macVersion }}'
10+
variables:
11+
CCACHE_DIR: $(Pipeline.Workspace)/ccache
12+
CCACHE_BASEDIR: $(Build.SourcesDirectory)
1013
steps:
1114
- script: |
1215
brew install \
@@ -23,8 +26,16 @@ jobs:
2326
qt5 \
2427
glew \
2528
cgal \
26-
sqlite3
27-
displayName: 'Install dependencies'
29+
sqlite3 \
30+
ccache
31+
displayName: Install dependencies
32+
33+
- task: Cache@2
34+
inputs:
35+
key: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
36+
restoreKeys: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
37+
path: $(CCACHE_DIR)
38+
displayName: Restore ccache
2839

2940
- script: |
3041
export PATH="/usr/local/opt/qt@5/bin:$PATH"
@@ -36,7 +47,7 @@ jobs:
3647
-DTESTS_ENABLED=ON \
3748
-DQt5_DIR=/usr/local/opt/qt/lib/cmake/Qt5
3849
ninja
39-
displayName: 'Configure and build'
50+
displayName: Configure and build
4051
4152
- script: |
4253
cd build
@@ -47,4 +58,11 @@ jobs:
4758
ctest --rerun-failed --output-on-failure
4859
fi
4960
exit $tests_pass
50-
displayName: 'Run tests'
61+
displayName: Run tests
62+
63+
- script: |
64+
set -x
65+
ccache --show-stats
66+
ccache --evict-older-than 1d
67+
ccache --show-stats
68+
displayName: Cleanup ccache

‎.azure-pipelines/build-ubuntu.yaml‎

Lines changed: 50 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
parameters:
2-
displayName: 'Ubuntu 20.04'
2+
displayName: Ubuntu 20.04
33
ubuntuVersion: '20.04'
44
cudaEnabled: false
55
asanEnabled: false
66
cmakeBuildType: Release
77

88
jobs:
99
- job: ubuntu_build_${{ replace(parameters.ubuntuVersion, '.', '') }}_cuda_${{ parameters.cudaEnabled }}_asan_${{ parameters.asanEnabled }}_${{ parameters.cmakeBuildType }}
10-
displayName: '${{ parameters.displayName }}'
10+
displayName: ${{ parameters.displayName }}
1111
pool:
1212
vmImage: 'ubuntu-${{ parameters.ubuntuVersion }}'
1313
variables:
14+
CCACHE_DIR: $(Pipeline.Workspace)/ccache
15+
CCACHE_BASEDIR: $(Build.SourcesDirectory)
16+
CCACHE_VERSION: 1
1417
${{ if eq(parameters.asanEnabled, false) }}:
1518
ctestExclusions: "(mvs/gpu_mat_test)"
1619
${{ if eq(parameters.asanEnabled, true) }}:
@@ -41,7 +44,7 @@ jobs:
4144
libgl1-mesa-dri \
4245
libunwind-dev \
4346
xvfb
44-
displayName: 'Install dependencies'
47+
displayName: Install dependencies
4548
4649
- ${{ if eq(parameters.cudaEnabled, true) }}:
4750
- ${{ if eq(parameters.ubuntuVersion, '22.04') }}:
@@ -53,29 +56,47 @@ jobs:
5356
echo '##vso[task.setvariable variable=CC]/usr/bin/gcc-10'
5457
echo '##vso[task.setvariable variable=CXX]/usr/bin/g++-10'
5558
echo '##vso[task.setvariable variable=CUDAHOSTCXX]/usr/bin/g++-10'
56-
displayName: 'Install CUDA'
59+
displayName: Install CUDA
5760
- ${{ if ne(parameters.ubuntuVersion, '22.04') }}:
5861
- script: |
5962
sudo apt-get install -y \
6063
nvidia-cuda-toolkit \
6164
nvidia-cuda-toolkit-gcc
6265
echo '##vso[task.setvariable variable=CC]/usr/bin/cuda-gcc'
6366
echo '##vso[task.setvariable variable=CXX]/usr/bin/cuda-g++'
64-
displayName: 'Install CUDA'
67+
displayName: Install CUDA
6568
6669
- ${{ if eq(parameters.asanEnabled, true) }}:
6770
- script: |
6871
sudo apt-get install -y clang-15
6972
echo '##vso[task.setvariable variable=CC]/usr/bin/clang-15'
7073
echo '##vso[task.setvariable variable=CXX]/usr/bin/clang++-15'
71-
displayName: 'Install Clang'
74+
displayName: Install Clang
7275
7376
- ${{ if eq(parameters.cmakeBuildType, 'ClangTidy') }}:
7477
- script: |
7578
sudo apt-get install -y clang-15 clang-tidy-15
7679
echo '##vso[task.setvariable variable=CC]/usr/bin/clang-15'
7780
echo '##vso[task.setvariable variable=CXX]/usr/bin/clang++-15'
78-
displayName: 'Install Clang'
81+
displayName: Install Clang
82+
83+
- task: Cache@2
84+
inputs:
85+
key: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
86+
restoreKeys: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
87+
path: $(CCACHE_DIR)
88+
displayName: Restore ccache
89+
90+
- script: |
91+
set -x
92+
wget https://github.com/ccache/ccache/releases/download/v4.8.2/ccache-4.8.2-linux-x86_64.tar.xz
93+
echo "0b33f39766fe9db67f40418aed6a5b3d7b2f4f7fab025a8213264b77a2d0e1b1 ccache-4.8.2-linux-x86_64.tar.xz" | sha256sum --check
94+
tar xfv ccache-4.8.2-linux-x86_64.tar.xz
95+
mkdir -p "$(CCACHE_DIR)/bin"
96+
mv ./ccache-4.8.2-linux-x86_64/ccache "$(CCACHE_DIR)/bin"
97+
echo "##vso[task.prependpath]$(CCACHE_DIR)/bin"
98+
displayName: Install CCache
99+
condition: ne(variables.CCACHE_RESTORED, 'true')
79100
80101
- script: |
81102
cmake --version
@@ -88,19 +109,27 @@ jobs:
88109
-DCMAKE_CUDA_ARCHITECTURES=50 \
89110
-DASAN_ENABLED=${{ parameters.asanEnabled }}
90111
ninja
91-
displayName: 'Configure and build'
112+
displayName: Configure and build
113+
114+
- ${{ if ne(parameters.cmakeBuildType, 'ClangTidy') }}:
115+
- script: |
116+
export DISPLAY=":99.0"
117+
export QT_QPA_PLATFORM="offscreen"
118+
Xvfb :99 &
119+
sleep 3
120+
cd build
121+
ctest -E "$(ctestExclusions)"
122+
tests_pass=$?
123+
if [ $tests_pass -ne 0 ]; then
124+
echo "Tests failed, rerunning with verbose output"
125+
ctest --rerun-failed --output-on-failure
126+
fi
127+
exit $tests_pass
128+
displayName: Run tests
92129
93130
- script: |
94-
export DISPLAY=":99.0"
95-
export QT_QPA_PLATFORM="offscreen"
96-
Xvfb :99 &
97-
sleep 3
98-
cd build
99-
ctest -E "$(ctestExclusions)"
100-
tests_pass=$?
101-
if [ $tests_pass -ne 0 ]; then
102-
echo "Tests failed, rerunning with verbose output"
103-
ctest --rerun-failed --output-on-failure
104-
fi
105-
exit $tests_pass
106-
displayName: 'Run tests'
131+
set -x
132+
ccache --show-stats
133+
ccache --evict-older-than 1d
134+
ccache --show-stats
135+
displayName: Cleanup ccache

‎.azure-pipelines/build-windows.yaml‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ jobs:
88
pool:
99
vmImage: 'windows-${{ parameters.visualStudioVersion }}'
1010
variables:
11-
vcpkgGitCommitId: 662dbb50e63af15baa2909b7eac5b1b87e86a0aa
11+
vcpkgGitCommitId: bf64d76062b3004544731fe9c1dc3a13d5b7b59a
1212
VCPKG_BINARY_SOURCES: 'clear;nuget,https://pkgs.dev.azure.com/colmap/colmap/_packaging/vcpkg/nuget/v3/index.json,readwrite'
13+
CCACHE_DIR: $(Pipeline.Workspace)/ccache
14+
CCACHE_BASEDIR: $(Build.SourcesDirectory)
1315
steps:
1416
- task: NuGetAuthenticate@0
1517
displayName: NuGet Authenticate
@@ -24,31 +26,45 @@ jobs:
2426
- pwsh: |
2527
git clone https://github.com/microsoft/vcpkg
2628
cd vcpkg
27-
git checkout --force $(vcpkgGitCommitId)
28-
.\bootstrap-vcpkg.bat
29-
$(Build.SourcesDirectory)\scripts\shell\enter_vs_dev_shell.ps1
30-
.\vcpkg.exe install --recurse @$(Build.SourcesDirectory)/.azure-pipelines/build-windows-vcpkg.txt --clean-after-build
29+
git reset --hard $(vcpkgGitCommitId)
30+
./bootstrap-vcpkg.bat
31+
$(Build.SourcesDirectory)/scripts/shell/enter_vs_dev_shell.ps1
32+
./vcpkg.exe install --recurse @$(Build.SourcesDirectory)/.azure-pipelines/build-windows-vcpkg.txt --clean-after-build
3133
workingDirectory: $(Build.BinariesDirectory)
3234
displayName: Setup vcpkg
3335
36+
- task: Cache@2
37+
inputs:
38+
key: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
39+
restoreKeys: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
40+
path: $(CCACHE_DIR)
41+
cacheHitVar: CCACHE_RESTORED
42+
displayName: Restore ccache
43+
44+
- pwsh: |
45+
$(Build.SourcesDirectory)/.azure-pipelines/install-ccache.ps1 -Destination "$(CCACHE_DIR)/bin"
46+
echo "##vso[task.prependpath]$(CCACHE_DIR)/bin"
47+
displayName: Install CCache
48+
condition: ne(variables.CCACHE_RESTORED, 'true')
49+
3450
- pwsh: |
35-
$(Build.SourcesDirectory)\scripts\shell\enter_vs_dev_shell.ps1
36-
$(Build.BinariesDirectory)\vcpkg\vcpkg.exe integrate install
51+
$(Build.SourcesDirectory)/scripts/shell/enter_vs_dev_shell.ps1
52+
$(Build.BinariesDirectory)/vcpkg/vcpkg.exe integrate install
3753
mkdir build
3854
cd build
3955
cmake .. `
4056
-GNinja `
41-
-DCMAKE_MAKE_PROGRAM=$(Build.BinariesDirectory)\ninja.exe `
57+
-DCMAKE_MAKE_PROGRAM=$(Build.BinariesDirectory)/ninja.exe `
4258
-DCMAKE_BUILD_TYPE=Release `
4359
-DTESTS_ENABLED=ON `
44-
-DCMAKE_TOOLCHAIN_FILE=$(Build.BinariesDirectory)\vcpkg\scripts\buildsystems\vcpkg.cmake `
60+
-DCMAKE_TOOLCHAIN_FILE=$(Build.BinariesDirectory)/vcpkg/scripts/buildsystems/vcpkg.cmake `
4561
-DVCPKG_TARGET_TRIPLET=x64-windows
46-
$(Build.BinariesDirectory)\ninja.exe
62+
$(Build.BinariesDirectory)/ninja.exe
4763
workingDirectory: $(Build.SourcesDirectory)
4864
displayName: Configure and build
4965
5066
- pwsh: |
51-
$(Build.BinariesDirectory)\vcpkg\vcpkg.exe integrate install
67+
$(Build.BinariesDirectory)/vcpkg/vcpkg.exe integrate install
5268
ctest -E "(feature/sift_test)|(util/opengl_utils_test)|(mvs/gpu_mat_test)"
5369
$tests_pass=$LastExitCode
5470
if ($tests_pass -ne 0) {
@@ -58,3 +74,9 @@ jobs:
5874
exit $tests_pass
5975
workingDirectory: $(Build.SourcesDirectory)/build
6076
displayName: Run tests
77+
78+
- pwsh: |
79+
ccache --show-stats
80+
ccache --evict-older-than 1d
81+
ccache --show-stats
82+
displayName: Cleanup ccache

‎.azure-pipelines/install-ccache.ps1‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
[CmdletBinding()]
2+
param (
3+
[Parameter(Mandatory = $true)]
4+
[string] $Destination
5+
)
6+
7+
$version = "4.8"
8+
$folder = "ccache-$version-windows-x86_64"
9+
$url = "https://github.com/ccache/ccache/releases/download/v$version/$folder.zip"
10+
$expectedSha256 = "A2B3BAB4BB8318FFC5B3E4074DC25636258BC7E4B51261F7D9BEF8127FDA8309"
11+
12+
$ErrorActionPreference = "Stop"
13+
14+
try {
15+
New-Item -Path "$Destination" -ItemType Container -ErrorAction SilentlyContinue
16+
17+
Write-Host "Download CCache"
18+
$zipFilePath = Join-Path "$env:TEMP" "$folder.zip"
19+
Invoke-WebRequest -Uri $url -UseBasicParsing -OutFile "$zipFilePath" -MaximumRetryCount 3
20+
21+
$hash = Get-FileHash $zipFilePath -Algorithm "sha256"
22+
if ($hash.Hash -ne $expectedSha256) {
23+
throw "File $Path hash $hash.Hash did not match expected hash $expectedHash"
24+
}
25+
26+
Write-Host "Unzip CCache"
27+
Expand-Archive -Path "$zipFilePath" -DestinationPath "$env:TEMP"
28+
29+
Write-Host "Move CCache"
30+
Move-Item "$env:TEMP/$folder/ccache.exe" "$Destination"
31+
Remove-Item "$zipFilePath"
32+
Remove-Item -Recurse "$env:TEMP/$folder"
33+
}
34+
catch {
35+
Write-Host "Installation failed with an error"
36+
$_.Exception | Format-List
37+
exit -1
38+
}

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ else()
296296
message(STATUS "Disabling OpenGL support")
297297
endif()
298298

299-
if(CCACHE_ENABLED AND NOT "${CMAKE_BUILD_TYPE}" STREQUAL "ClangTidy")
299+
if(CCACHE_ENABLED)
300300
find_program(CCACHE ccache)
301301
if(CCACHE)
302302
message(STATUS "Enabling ccache support")

‎cmake/CMakeHelper.cmake‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@ macro(COLMAP_ADD_CUDA_LIBRARY TARGET_NAME)
129129
target_link_libraries(${TARGET_NAME} CUDA::cudart CUDA::curand)
130130
set_target_properties(${TARGET_NAME} PROPERTIES FOLDER
131131
${COLMAP_TARGETS_ROOT_FOLDER}/${FOLDER_NAME})
132+
if(CLANG_TIDY_EXE)
133+
set_target_properties(${TARGET_NAME} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-header-filter=.*")
134+
endif()
132135
install(TARGETS ${TARGET_NAME} DESTINATION lib/colmap/)
133136
endmacro(COLMAP_ADD_CUDA_LIBRARY)
134137

@@ -177,6 +180,9 @@ macro(COLMAP_ADD_CUDA_TEST TARGET_NAME)
177180
add_executable(${TARGET_NAME} ${ARGN})
178181
set_target_properties(${TARGET_NAME} PROPERTIES FOLDER
179182
${COLMAP_TARGETS_ROOT_FOLDER}/${FOLDER_NAME})
183+
if(CLANG_TIDY_EXE)
184+
set_target_properties(${TARGET_NAME} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-header-filter=.*")
185+
endif()
180186
target_link_libraries(${TARGET_NAME} colmap
181187
${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
182188
add_test("${FOLDER_NAME}/${TARGET_NAME}" ${TARGET_NAME})

0 commit comments

Comments
 (0)
⚡