Skip to content

Commit 44e3ff6

Browse files
DodgySpaniardDodgySpaniard
authored
Fixing CI - Compiling GDAL for non-windows (#1995)
* gdal wip * Patched docker GPU * Fixing CI for v3.6.0 - GDAL is not compiled for non-windows machines (linux/mac). + Added gdal to superbuild. + Updated dependencies in snapcraft. + Set cmake dependencies for Gdal. + Added dummy cmake target for Win setup. - Downgraded CUDA Docker to 12.x (13.0 removed required headers). * Adding review feedback - Fixing format for snapcraft.yaml - Swapping GPU image for runtime - Fixing standard dockerfile runpath
1 parent e2acf27 commit 44e3ff6

14 files changed

+143
-65
lines changed

‎Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ FROM ubuntu:24.04 AS builder
22

33
# Env variables
44
ENV DEBIAN_FRONTEND=noninteractive \
5-
PYTHONPATH="/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
5+
PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/local/lib/python3.12/dist-packages:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
66
LD_LIBRARY_PATH="/code/SuperBuild/install/lib"
77

88
# Prepare directories
@@ -29,7 +29,7 @@ FROM ubuntu:24.04
2929

3030
# Env variables
3131
ENV DEBIAN_FRONTEND=noninteractive \
32-
PYTHONPATH="/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
32+
PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/local/lib/python3.12/dist-packages:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
3333
LD_LIBRARY_PATH="/code/SuperBuild/install/lib" \
3434
PDAL_DRIVER_PATH="/code/SuperBuild/install/bin"
3535

‎SuperBuild/CMakeLists.txt‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,15 @@ if (WIN32)
111111
elseif(APPLE)
112112
set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
113113
set(PYTHON_EXE_PATH "${PYTHON_HOME}/bin/python")
114+
set(GDAL_ROOT "${SB_INSTALL_DIR}")
115+
set(GDAL_LIBRARY "${GDAL_ROOT}/lib/libgdal.dylib")
116+
set(GDAL_INCLUDE_DIR "${GDAL_ROOT}/include")
114117
else()
115-
set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
116-
set(PYTHON_EXE_PATH "${PYTHON_HOME}/bin/python3")
118+
set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
119+
set(PYTHON_EXE_PATH "${PYTHON_HOME}/bin/python3")
120+
set(GDAL_ROOT "${SB_INSTALL_DIR}")
121+
set(GDAL_LIBRARY "${GDAL_ROOT}/lib/libgdal.so")
122+
set(GDAL_INCLUDE_DIR "${GDAL_ROOT}/include")
117123
endif()
118124

119125
# Path to additional CMake modules
@@ -126,6 +132,19 @@ include(ExternalProject-Setup)
126132
# Download and install third party libs #
127133
#########################################
128134

135+
# ---------------------------------------------------------------------------------------------
136+
# Geospatial Data Abstraction Library (GDAL)
137+
#
138+
set(ODM_GDAL_Version 3.11.1)
139+
if(WIN32)
140+
message(STATUS "GDAL: Using pip-installed version from ${GDAL_ROOT}")
141+
# Create a dummy target to satisfy dependencies
142+
add_custom_target(gdal)
143+
else()
144+
option(ODM_BUILD_GDAL "Force to build GDAL library" ON)
145+
SETUP_EXTERNAL_PROJECT(GDAL ${ODM_GDAL_Version} ${ODM_BUILD_GDAL})
146+
endif()
147+
129148
# ---------------------------------------------------------------------------------------------
130149
# Open Source Computer Vision (OpenCV)
131150
#
@@ -218,6 +237,7 @@ externalproject_add(poissonrecon
218237
)
219238

220239
externalproject_add(dem2mesh
240+
DEPENDS gdal
221241
GIT_REPOSITORY https://github.com/OpenDroneMap/dem2mesh.git
222242
GIT_TAG 334
223243
PREFIX ${SB_BINARY_DIR}/dem2mesh
@@ -228,6 +248,7 @@ externalproject_add(dem2mesh
228248
)
229249

230250
externalproject_add(dem2points
251+
DEPENDS gdal
231252
GIT_REPOSITORY https://github.com/OpenDroneMap/dem2points.git
232253
GIT_TAG master
233254
PREFIX ${SB_BINARY_DIR}/dem2points
@@ -238,7 +259,7 @@ externalproject_add(dem2points
238259
)
239260

240261
externalproject_add(odm_orthophoto
241-
DEPENDS opencv
262+
DEPENDS opencv gdal
242263
GIT_REPOSITORY https://github.com/OpenDroneMap/odm_orthophoto.git
243264
GIT_TAG 355
244265
PREFIX ${SB_BINARY_DIR}/odm_orthophoto
@@ -248,6 +269,7 @@ externalproject_add(odm_orthophoto
248269
)
249270

250271
externalproject_add(fastrasterfilter
272+
DEPENDS gdal
251273
GIT_REPOSITORY https://github.com/OpenDroneMap/FastRasterFilter.git
252274
GIT_TAG main
253275
PREFIX ${SB_BINARY_DIR}/fastrasterfilter

‎SuperBuild/cmake/External-FPCFilter.cmake‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ set(_proj_name fpcfilter)
22
set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}")
33

44
ExternalProject_Add(${_proj_name}
5+
DEPENDS gdal
56
PREFIX ${_SB_BINARY_DIR}
67
TMP_DIR ${_SB_BINARY_DIR}/tmp
78
STAMP_DIR ${_SB_BINARY_DIR}/stamp

‎SuperBuild/cmake/External-GDAL.cmake‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
set(_proj_name gdal)
2+
set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}")
3+
4+
ExternalProject_Add(${_proj_name}
5+
PREFIX ${_SB_BINARY_DIR}
6+
TMP_DIR ${_SB_BINARY_DIR}/tmp
7+
STAMP_DIR ${_SB_BINARY_DIR}/stamp
8+
#--Download step--------------
9+
DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}
10+
GIT_REPOSITORY https://github.com/OSGeo/gdal.git
11+
GIT_TAG cf7cef2f1eec2a80c46b0ec0227d8d0cb32e2657
12+
#--Update/Patch step----------
13+
UPDATE_COMMAND ""
14+
#--Configure step-------------
15+
SOURCE_DIR ${SB_SOURCE_DIR}/${_proj_name}
16+
CMAKE_ARGS
17+
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
18+
-DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
19+
-DGDAL_PYTHON_INSTALL_PREFIX=${SB_INSTALL_DIR}
20+
-DBUILD_PYTHON_BINDINGS=ON
21+
${WIN32_CMAKE_ARGS}
22+
#--Build step-----------------
23+
BINARY_DIR ${_SB_BINARY_DIR}
24+
#--Install step---------------
25+
INSTALL_DIR ${SB_INSTALL_DIR}
26+
INSTALL_COMMAND "${CMAKE_COMMAND}" --build . --target install
27+
#--Output logging-------------
28+
LOG_DOWNLOAD OFF
29+
LOG_CONFIGURE OFF
30+
LOG_BUILD OFF
31+
)

‎SuperBuild/cmake/External-Hexer.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ set(_proj_name hexer)
22
set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}")
33

44
ExternalProject_Add(${_proj_name}
5-
DEPENDS
5+
DEPENDS gdal
66
PREFIX ${_SB_BINARY_DIR}
77
TMP_DIR ${_SB_BINARY_DIR}/tmp
88
STAMP_DIR ${_SB_BINARY_DIR}/stamp

‎SuperBuild/cmake/External-PDAL.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ set(LASZIP_LIB "${SB_INSTALL_DIR}/lib/liblaszip.so")
1010
endif()
1111

1212
ExternalProject_Add(${_proj_name}
13-
DEPENDS hexer laszip
13+
DEPENDS gdal hexer laszip
1414
PREFIX ${_SB_BINARY_DIR}
1515
TMP_DIR ${_SB_BINARY_DIR}/tmp
1616
STAMP_DIR ${_SB_BINARY_DIR}/stamp

‎configure.sh‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,18 +54,6 @@ ensure_prereqs() {
5454
echo "Installing tzdata"
5555
sudo $APT_GET install -y -qq tzdata
5656

57-
UBUNTU_VERSION=$(lsb_release -r)
58-
if [[ "$UBUNTU_VERSION" == *"20.04"* ]]; then
59-
echo "Enabling PPA for Ubuntu GIS"
60-
sudo $APT_GET install -y -qq --no-install-recommends software-properties-common
61-
sudo add-apt-repository ppa:ubuntugis/ppa
62-
sudo $APT_GET update
63-
elif [[ "$UBUNTU_VERSION" == *"24.04"* ]]; then
64-
echo "Enabling ubuntugis-unstable PPA for Ubuntu 24.04"
65-
sudo $APT_GET install -y -qq --no-install-recommends software-properties-common
66-
sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
67-
sudo $APT_GET update
68-
fi
6957

7058
echo "Installing Python PIP"
7159
sudo $APT_GET install -y -qq --no-install-recommends \
@@ -113,6 +101,8 @@ installruntimedepsonly() {
113101
installdepsfromsnapcraft runtime opensfm
114102
echo "Installing OpenMVS Dependencies"
115103
installdepsfromsnapcraft runtime openmvs
104+
echo "Installing GDAL Dependencies"
105+
installdepsfromsnapcraft runtime gdal
116106
}
117107

118108
installreqs() {
@@ -134,6 +124,8 @@ installreqs() {
134124
installdepsfromsnapcraft build opensfm
135125
echo "Installing OpenMVS Dependencies"
136126
installdepsfromsnapcraft build openmvs
127+
echo "Installing GDAL Dependencies"
128+
installdepsfromsnapcraft build gdal
137129

138130
set -e
139131

@@ -165,7 +157,11 @@ install() {
165157
echo "Compiling SuperBuild"
166158
cd ${RUNPATH}/SuperBuild
167159
mkdir -p build && cd build
168-
cmake .. && make -j$processes
160+
cmake .. \
161+
-DBUILD_PYTHON_BINDINGS=ON \
162+
-DPython_ROOT=/code/venv \
163+
-DPython_FIND_VIRTUALENV=ONLY \
164+
&& make -j$processes
169165

170166
echo "Configuration Finished"
171167
}

‎gpu.Dockerfile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
FROM nvidia/cuda:13.0.0-devel-ubuntu24.04 AS builder
1+
FROM nvidia/cuda:12.9.1-devel-ubuntu24.04 AS builder
22

33
# Env variables
44
ENV DEBIAN_FRONTEND=noninteractive \
5-
PYTHONPATH="/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
5+
PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/local/lib/python3.12/dist-packages:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
66
LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
77

88
# Prepare directories
@@ -25,11 +25,11 @@ RUN bash configure.sh clean
2525

2626
### Use a second image for the final asset to reduce the number and
2727
# size of the layers.
28-
FROM nvidia/cuda:13.0.0-devel-ubuntu24.04
28+
FROM nvidia/cuda:12.9.1-runtime-ubuntu24.04
2929

3030
# Env variables
3131
ENV DEBIAN_FRONTEND=noninteractive \
32-
PYTHONPATH="/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
32+
PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/local/lib/python3.12/dist-packages:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
3333
LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" \
3434
PDAL_DRIVER_PATH="/code/SuperBuild/install/bin"
3535

@@ -41,7 +41,7 @@ COPY --from=builder /code /code
4141
ENV PATH="/code/venv/bin:$PATH"
4242

4343
RUN apt-get update -y \
44-
&& apt-get install -y ffmpeg libtbb2
44+
&& apt-get install -y ffmpeg libtbbmalloc2
4545
# Install shared libraries that we depend on via APT, but *not*
4646
# the -dev packages to save space!
4747
# Also run a smoke test on ODM and OpenSfM

‎portable.Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ FROM ubuntu:24.04 AS builder
22

33
# Env variables
44
ENV DEBIAN_FRONTEND=noninteractive \
5-
PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
5+
PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/local/lib/python3.12/dist-packages:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
66
LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
77

88
# Prepare directories
@@ -29,7 +29,7 @@ FROM ubuntu:24.04
2929

3030
# Env variables
3131
ENV DEBIAN_FRONTEND=noninteractive \
32-
PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
32+
PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/local/lib/python3.12/dist-packages:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
3333
LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" \
3434
PDAL_DRIVER_PATH="/code/SuperBuild/install/bin"
3535

‎requirements.txt‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@ PyYAML==6.0.2
1515
rasterio==1.4.3 ; sys_platform == 'linux'
1616
rasterio==1.4.3 ; sys_platform == 'darwin'
1717
https://github.com/OpenDroneMap/windows-deps/releases/download/2.6.0/rasterio-1.4.3-cp312-cp312-win_amd64.whl ; sys_platform == 'win32'
18-
gdal[numpy]==3.11.1 ; sys_platform == 'linux'
19-
gdal[numpy]==3.11.1 ; sys_platform == 'darwin'
2018
https://github.com/OpenDroneMap/windows-deps/releases/download/2.6.0/gdal-3.11.1-cp312-cp312-win_amd64.whl ; sys_platform == 'win32'
2119
scikit-learn==1.7.1
2220
scikit-image==0.25.2

0 commit comments

Comments
 (0)
⚡