From 3dde97cba8c7b0e9e06ce0bc4deeac60b8b91508 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Sat, 18 Jan 2020 15:33:24 +0100 Subject: [PATCH 01/15] static build of netcdf --- .gitmodules | 3 +++ Cargo.toml | 1 + netcdf-sys/Cargo.toml | 10 +++++++ netcdf-sys/build.rs | 44 +++++++++++++++++++++++++++++++ netcdf-sys/source | 1 + netcdf-sys/src/lib.rs | 2 ++ netcdf-sys/src/netcdf_bindings.rs | 2 -- 7 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 160000 netcdf-sys/source diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..6fc0a4e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "netcdf-sys/source"] + path = netcdf-sys/source + url = https://github.com/Unidata/netcdf-c.git diff --git a/Cargo.toml b/Cargo.toml index 6300d52..af0c5c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ exclude = ["examples/**", "tests/**"] [features] default = ["ndarray"] memory = ["netcdf-sys/memio"] +static = ["netcdf-sys/static"] [dependencies] lazy_static = "1.4.0" diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index 5f13e59..0fa5419 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -15,8 +15,18 @@ links = "netcdf" build = "build.rs" exclude = ["testdata/**"] +[dependencies] +hdf5-sys = { git = "https://github.com/magnusuMET/hdf5-rust.git", branch = "feature/static_build", features = ["hl", "threadsafe", "deprecated"] } +libz-sys = { version = "1.0.25", optional = true } + [dev-dependencies] lazy_static = "1.4.0" +[build-dependencies] +cmake = "0.1.42" + [features] +default = ["libz"] memio = [] +static = ["libz-sys/static", "hdf5-sys/static"] +libz = ["libz-sys"] diff --git a/netcdf-sys/build.rs b/netcdf-sys/build.rs index 6b5474c..fe0fc0f 100644 --- a/netcdf-sys/build.rs +++ b/netcdf-sys/build.rs @@ -1,4 +1,48 @@ +macro_rules! feature { + ($feature:expr) => { + std::env::var(concat!("CARGO_FEATURE_", $feature)) + }; +} + fn main() { + println!("cargo:rerun-if-changed=build.rs"); + if feature!("STATIC").is_ok() { + build(); + return; + } + // Link to the system netcdf println!("cargo:rustc-link-lib=netcdf"); println!("cargo:rerun-if-changed=build.rs"); } + +fn build() { + let hdf5_incdir = std::env::var("DEP_HDF5_INCLUDE").unwrap(); + let hdf5_lib = std::env::var("DEP_HDF5_LIBRARY").unwrap(); + let hdf5_hl_lib = std::env::var("DEP_HDF5_HL_LIBRARY").unwrap(); + + + let netcdf = cmake::Config::new("source") + .define("BUILD_SHARED_LIBS", "OFF") + .define("NC_FIND_SHARED_LIBS", "OFF") + .define("BUILD_UTILITIES", "OFF") + .define("ENABLE_EXAMPLES", "OFF") + .define("ENABLE_DAP_REMOTE_TESTS", "OFF") + .define("ENABLE_TESTS", "OFF") + .define("ENABLE_EXTREME_NUMBERS", "OFF") + .define("ENABLE_PARALLEL_TESTS", "OFF") + .define("ENABLE_FILTER_TESTING", "OFF") + .define("ENABLE_BASH_SCRIPT_TESTING", "OFF") + // + .define("HDF5_C_LIBRARY", &hdf5_lib) + .define("HDF5_HL_LIBRARY", &hdf5_hl_lib) + .define("HDF5_INCLUDE_DIR", hdf5_incdir) + // + .define("ENABLE_DAP", "OFF") // TODO: feature flag, requires curl + // + .profile("RelWithDebInfo") // TODO: detect opt-level + .very_verbose(true) + .build(); + + println!("cargo:rustc-link-lib=static=netcdf"); + println!("cargo:rustc-link-search=native={}/lib", netcdf.display()); +} diff --git a/netcdf-sys/source b/netcdf-sys/source new file mode 160000 index 0000000..5d34ea5 --- /dev/null +++ b/netcdf-sys/source @@ -0,0 +1 @@ +Subproject commit 5d34ea5b2ce25697826dc599c88afefb94a892a8 diff --git a/netcdf-sys/src/lib.rs b/netcdf-sys/src/lib.rs index 5d4604f..6e214c5 100644 --- a/netcdf-sys/src/lib.rs +++ b/netcdf-sys/src/lib.rs @@ -1,6 +1,8 @@ #![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] +extern crate hdf5_sys; + mod netcdf_bindings; mod netcdf_const; pub use netcdf_bindings::*; diff --git a/netcdf-sys/src/netcdf_bindings.rs b/netcdf-sys/src/netcdf_bindings.rs index aa59f02..c15eb86 100644 --- a/netcdf-sys/src/netcdf_bindings.rs +++ b/netcdf-sys/src/netcdf_bindings.rs @@ -13,12 +13,10 @@ pub struct nc_vlen_t { } pub type nclong = c_int; -#[link(name = "netcdf")] extern "C" { pub static mut ncerr: c_int; pub static mut ncopts: c_int; } -#[link(name = "netcdf")] extern "C" { pub fn nc_inq_libvers() -> *const c_char; pub fn nc_strerror(ncerr: c_int) -> *const c_char; From 123433aeaa4cf92898b3964ebdf642999262c773 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 22 Jan 2020 22:59:16 +0100 Subject: [PATCH 02/15] add DAP support, and use patch section in toml --- netcdf-sys/Cargo.toml | 11 +++++++++-- netcdf-sys/build.rs | 13 +++++++++---- netcdf-sys/source | 2 +- netcdf-sys/src/lib.rs | 3 +++ 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index 0fa5419..94ff5e3 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -8,7 +8,7 @@ authors = [ license = "MIT" description = "FFI bindings to NetCDF" repository = "https://github.com/georust/netcdf" -documentation = "https://github.com/georust/netcdf" +documentation = "https://docs.rs/netcdf" keywords = ["netcdf", "hdf", "hdf5", "cdm", "ffi"] edition = "2018" links = "netcdf" @@ -16,8 +16,9 @@ build = "build.rs" exclude = ["testdata/**"] [dependencies] -hdf5-sys = { git = "https://github.com/magnusuMET/hdf5-rust.git", branch = "feature/static_build", features = ["hl", "threadsafe", "deprecated"] } libz-sys = { version = "1.0.25", optional = true } +curl-sys = { version = "0.4.25", optional = true } +hdf5-sys = { version = "0.5.3", features = ["hl", "threadsafe", "deprecated"] } [dev-dependencies] lazy_static = "1.4.0" @@ -29,4 +30,10 @@ cmake = "0.1.42" default = ["libz"] memio = [] static = ["libz-sys/static", "hdf5-sys/static"] +# TODO: Link static curl (but only when using "dap") "curl-sys/static-curl", "curl-sys/static-ssl" libz = ["libz-sys"] +dap = ["curl-sys"] + +[patch] +[patch.crates-io] +hdf5-sys = { git = "https://github.com/magnusuMET/hdf5-rust.git", branch = "feature/static_build" } diff --git a/netcdf-sys/build.rs b/netcdf-sys/build.rs index fe0fc0f..20fca37 100644 --- a/netcdf-sys/build.rs +++ b/netcdf-sys/build.rs @@ -21,7 +21,8 @@ fn build() { let hdf5_hl_lib = std::env::var("DEP_HDF5_HL_LIBRARY").unwrap(); - let netcdf = cmake::Config::new("source") + let mut netcdf_config = cmake::Config::new("source"); + netcdf_config .define("BUILD_SHARED_LIBS", "OFF") .define("NC_FIND_SHARED_LIBS", "OFF") .define("BUILD_UTILITIES", "OFF") @@ -39,9 +40,13 @@ fn build() { // .define("ENABLE_DAP", "OFF") // TODO: feature flag, requires curl // - .profile("RelWithDebInfo") // TODO: detect opt-level - .very_verbose(true) - .build(); + .profile("RelWithDebInfo"); // TODO: detect opt-level + + if feature!("DAP").is_ok() { + netcdf_config.define("ENABLE_DAP", "ON"); + } + + let netcdf = netcdf_config.build(); println!("cargo:rustc-link-lib=static=netcdf"); println!("cargo:rustc-link-search=native={}/lib", netcdf.display()); diff --git a/netcdf-sys/source b/netcdf-sys/source index 5d34ea5..e1e20ef 160000 --- a/netcdf-sys/source +++ b/netcdf-sys/source @@ -1 +1 @@ -Subproject commit 5d34ea5b2ce25697826dc599c88afefb94a892a8 +Subproject commit e1e20ef6e4c49d2d57d5c7df7118a419ed7e7e41 diff --git a/netcdf-sys/src/lib.rs b/netcdf-sys/src/lib.rs index 6e214c5..3a1c691 100644 --- a/netcdf-sys/src/lib.rs +++ b/netcdf-sys/src/lib.rs @@ -3,6 +3,9 @@ extern crate hdf5_sys; +#[cfg(feature = "dap")] +extern crate curl_sys; + mod netcdf_bindings; mod netcdf_const; pub use netcdf_bindings::*; From fba9151c35551cbadbed4f88ff954056501fe9e8 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 17 Feb 2020 09:34:04 +0100 Subject: [PATCH 03/15] patch section in toplevel --- Cargo.toml | 4 ++++ netcdf-sys/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index af0c5c1..8d5d03d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,3 +32,7 @@ structopt = "0.3.3" members = [ "netcdf-sys", ] + +[patch] +[patch.crates-io] +hdf5-sys = { git = "https://github.com/magnusuMET/hdf5-rust", branch = "feature/static_build" } diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index 94ff5e3..19904bd 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -36,4 +36,4 @@ dap = ["curl-sys"] [patch] [patch.crates-io] -hdf5-sys = { git = "https://github.com/magnusuMET/hdf5-rust.git", branch = "feature/static_build" } +hdf5-sys = { git = "https://github.com/magnusuMET/hdf5-rust", branch = "feature/static_build" } From c92fb7c703831053edbdc49dc18a0315ba7f34cf Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 18 Feb 2020 09:19:04 +0100 Subject: [PATCH 04/15] update hdf5-sys version --- netcdf-sys/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index 19904bd..5ec1e45 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -18,7 +18,7 @@ exclude = ["testdata/**"] [dependencies] libz-sys = { version = "1.0.25", optional = true } curl-sys = { version = "0.4.25", optional = true } -hdf5-sys = { version = "0.5.3", features = ["hl", "threadsafe", "deprecated"] } +hdf5-sys = { version = "0.6.1", features = ["hl", "deprecated"] } [dev-dependencies] lazy_static = "1.4.0" From 384ca015d111b5e2467ff19a6894d891a9c965e5 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 19 Feb 2020 11:19:27 +0100 Subject: [PATCH 05/15] update netcdf-c --- netcdf-sys/source | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netcdf-sys/source b/netcdf-sys/source index e1e20ef..3bcdb5f 160000 --- a/netcdf-sys/source +++ b/netcdf-sys/source @@ -1 +1 @@ -Subproject commit e1e20ef6e4c49d2d57d5c7df7118a419ed7e7e41 +Subproject commit 3bcdb5fbf11af85e63f7916768b13b1f7d02c860 From 9578956878c98c0839b05bbff6a70d43790286d6 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Wed, 19 Feb 2020 11:26:22 +0100 Subject: [PATCH 06/15] require libz --- netcdf-sys/Cargo.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index 5ec1e45..de9d4a2 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -16,9 +16,9 @@ build = "build.rs" exclude = ["testdata/**"] [dependencies] -libz-sys = { version = "1.0.25", optional = true } +libz-sys = { version = "1.0.25" } curl-sys = { version = "0.4.25", optional = true } -hdf5-sys = { version = "0.6.1", features = ["hl", "deprecated"] } +hdf5-sys = { version = "0.6.1", features = ["hl", "deprecated", "zlib"] } [dev-dependencies] lazy_static = "1.4.0" @@ -27,11 +27,10 @@ lazy_static = "1.4.0" cmake = "0.1.42" [features] -default = ["libz"] +default = [] memio = [] static = ["libz-sys/static", "hdf5-sys/static"] # TODO: Link static curl (but only when using "dap") "curl-sys/static-curl", "curl-sys/static-ssl" -libz = ["libz-sys"] dap = ["curl-sys"] [patch] From deb7e855f4a11c90c978638d2b92836ba879f90d Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 21 Feb 2020 13:59:21 +0100 Subject: [PATCH 07/15] limit the files included in archive --- netcdf-sys/Cargo.toml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index de9d4a2..6e557c3 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -13,7 +13,25 @@ keywords = ["netcdf", "hdf", "hdf5", "cdm", "ffi"] edition = "2018" links = "netcdf" build = "build.rs" -exclude = ["testdata/**"] +exclude = [ + "testdata/**", + "source/conda.recipe/**", + "source/ctest_scripts/**", + "source/dap4_test/**", +# "source/docs/**", + "source/debug/**", + "source/examples/**", + "source/h5test/**", + "source/ncdump/**", + "source/ncgen/**", + "source/ncgen3/**", + "source/nc_perf/**", + "source/nctest/**", + "source/nc_test/**", +# "source/nc_test4/**", + "source/NUG/**", + "source/unit_test/**", +] [dependencies] libz-sys = { version = "1.0.25" } From ada7a57d0d3d3267fa21d6584552061d14859c89 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 6 Mar 2020 12:16:43 +0100 Subject: [PATCH 08/15] temporary downgrade of hdf5 version --- netcdf-sys/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index 6e557c3..01cf96e 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -36,7 +36,7 @@ exclude = [ [dependencies] libz-sys = { version = "1.0.25" } curl-sys = { version = "0.4.25", optional = true } -hdf5-sys = { version = "0.6.1", features = ["hl", "deprecated", "zlib"] } +hdf5-sys = { version = "0.6.0", features = ["hl", "deprecated", "zlib"] } [dev-dependencies] lazy_static = "1.4.0" From 032638537e1d1e36cec74589ac62f2578094737a Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 23 Mar 2020 10:43:08 +0100 Subject: [PATCH 09/15] move netcdf-c to netcdf-src --- .gitmodules | 2 +- Cargo.toml | 1 + netcdf-src/Cargo.toml | 18 ++++++++++++++++ netcdf-src/build.rs | 43 +++++++++++++++++++++++++++++++++++++ netcdf-src/source | 1 + netcdf-src/src/lib.rs | 1 + netcdf-sys/Cargo.toml | 10 ++------- netcdf-sys/build.rs | 49 +++++++------------------------------------ netcdf-sys/source | 1 - netcdf-sys/src/lib.rs | 3 +++ 10 files changed, 78 insertions(+), 51 deletions(-) create mode 100644 netcdf-src/Cargo.toml create mode 100644 netcdf-src/build.rs create mode 160000 netcdf-src/source create mode 100644 netcdf-src/src/lib.rs delete mode 160000 netcdf-sys/source diff --git a/.gitmodules b/.gitmodules index 6fc0a4e..53ccd7d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "netcdf-sys/source"] - path = netcdf-sys/source + path = netcdf-src/source url = https://github.com/Unidata/netcdf-c.git diff --git a/Cargo.toml b/Cargo.toml index 8d5d03d..a1aa7b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ structopt = "0.3.3" [workspace] members = [ "netcdf-sys", + "netcdf-src", ] [patch] diff --git a/netcdf-src/Cargo.toml b/netcdf-src/Cargo.toml new file mode 100644 index 0000000..2d6a0fe --- /dev/null +++ b/netcdf-src/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "netcdf-src" +version = "0.1.0" +authors = ["Magnus Ulimoen "] +edition = "2018" +description = "Build scripts for building `netCDF` from source" +build = "build.rs" +license-file = "source/COPYRIGHT" +links = "netcdfsrc" + +[features] +dap = [] + +[dependencies] +hdf5-sys = { version = "*", features=["hl", "deprecated", "zlib"] } + +[build-dependencies] +cmake = "0.1.42" diff --git a/netcdf-src/build.rs b/netcdf-src/build.rs new file mode 100644 index 0000000..dd2ced0 --- /dev/null +++ b/netcdf-src/build.rs @@ -0,0 +1,43 @@ +macro_rules! feature { + ($feature:expr) => { + std::env::var(concat!("CARGO_FEATURE_", $feature)) + }; +} + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + + let hdf5_incdir = std::env::var("DEP_HDF5_INCLUDE").unwrap(); + let hdf5_lib = std::env::var("DEP_HDF5_LIBRARY").unwrap(); + let hdf5_hl_lib = std::env::var("DEP_HDF5_HL_LIBRARY").unwrap(); + + let mut netcdf_config = cmake::Config::new("source"); + netcdf_config + .define("BUILD_SHARED_LIBS", "OFF") + .define("NC_FIND_SHARED_LIBS", "OFF") + .define("BUILD_UTILITIES", "OFF") + .define("ENABLE_EXAMPLES", "OFF") + .define("ENABLE_DAP_REMOTE_TESTS", "OFF") + .define("ENABLE_TESTS", "OFF") + .define("ENABLE_EXTREME_NUMBERS", "OFF") + .define("ENABLE_PARALLEL_TESTS", "OFF") + .define("ENABLE_FILTER_TESTING", "OFF") + .define("ENABLE_BASH_SCRIPT_TESTING", "OFF") + // + .define("HDF5_C_LIBRARY", &hdf5_lib) + .define("HDF5_HL_LIBRARY", &hdf5_hl_lib) + .define("HDF5_INCLUDE_DIR", hdf5_incdir) + // + .define("ENABLE_DAP", "OFF") // TODO: feature flag, requires curl + // + .profile("RelWithDebInfo"); // TODO: detect opt-level + + if feature!("DAP").is_ok() { + netcdf_config.define("ENABLE_DAP", "ON"); + } + + let netcdf = netcdf_config.build(); + + println!("cargo:lib={}", "netcdf"); + println!("cargo:search={}/lib", netcdf.display()); +} diff --git a/netcdf-src/source b/netcdf-src/source new file mode 160000 index 0000000..0014db9 --- /dev/null +++ b/netcdf-src/source @@ -0,0 +1 @@ +Subproject commit 0014db92e93e27b7301163e683f194b8b2fd8d4e diff --git a/netcdf-src/src/lib.rs b/netcdf-src/src/lib.rs new file mode 100644 index 0000000..b61ef8f --- /dev/null +++ b/netcdf-src/src/lib.rs @@ -0,0 +1 @@ +//! Dummy crate for building `netCDF` from source diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index 01cf96e..aa5e7da 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -37,20 +37,14 @@ exclude = [ libz-sys = { version = "1.0.25" } curl-sys = { version = "0.4.25", optional = true } hdf5-sys = { version = "0.6.0", features = ["hl", "deprecated", "zlib"] } +netcdf-src = { path = "../netcdf-src", version = "0.1.0", optional = true } [dev-dependencies] lazy_static = "1.4.0" -[build-dependencies] -cmake = "0.1.42" - [features] default = [] memio = [] -static = ["libz-sys/static", "hdf5-sys/static"] +static = ["libz-sys/static", "hdf5-sys/static", "netcdf-src"] # TODO: Link static curl (but only when using "dap") "curl-sys/static-curl", "curl-sys/static-ssl" dap = ["curl-sys"] - -[patch] -[patch.crates-io] -hdf5-sys = { git = "https://github.com/magnusuMET/hdf5-rust", branch = "feature/static_build" } diff --git a/netcdf-sys/build.rs b/netcdf-sys/build.rs index 20fca37..f1010a6 100644 --- a/netcdf-sys/build.rs +++ b/netcdf-sys/build.rs @@ -6,48 +6,15 @@ macro_rules! feature { fn main() { println!("cargo:rerun-if-changed=build.rs"); - if feature!("STATIC").is_ok() { - build(); - return; - } - // Link to the system netcdf - println!("cargo:rustc-link-lib=netcdf"); - println!("cargo:rerun-if-changed=build.rs"); -} - -fn build() { - let hdf5_incdir = std::env::var("DEP_HDF5_INCLUDE").unwrap(); - let hdf5_lib = std::env::var("DEP_HDF5_LIBRARY").unwrap(); - let hdf5_hl_lib = std::env::var("DEP_HDF5_HL_LIBRARY").unwrap(); + if feature!("STATIC").is_ok() { + let netcdf_lib = std::env::var("DEP_NETCDFSRC_LIB").unwrap(); + let netcdf_path = std::env::var("DEP_NETCDFSRC_SEARCH").unwrap(); - let mut netcdf_config = cmake::Config::new("source"); - netcdf_config - .define("BUILD_SHARED_LIBS", "OFF") - .define("NC_FIND_SHARED_LIBS", "OFF") - .define("BUILD_UTILITIES", "OFF") - .define("ENABLE_EXAMPLES", "OFF") - .define("ENABLE_DAP_REMOTE_TESTS", "OFF") - .define("ENABLE_TESTS", "OFF") - .define("ENABLE_EXTREME_NUMBERS", "OFF") - .define("ENABLE_PARALLEL_TESTS", "OFF") - .define("ENABLE_FILTER_TESTING", "OFF") - .define("ENABLE_BASH_SCRIPT_TESTING", "OFF") - // - .define("HDF5_C_LIBRARY", &hdf5_lib) - .define("HDF5_HL_LIBRARY", &hdf5_hl_lib) - .define("HDF5_INCLUDE_DIR", hdf5_incdir) - // - .define("ENABLE_DAP", "OFF") // TODO: feature flag, requires curl - // - .profile("RelWithDebInfo"); // TODO: detect opt-level - - if feature!("DAP").is_ok() { - netcdf_config.define("ENABLE_DAP", "ON"); + println!("cargo:rustc-link-lib=static={}", netcdf_lib); + println!("cargo:rustc-link-search=native={}", netcdf_path); + } else { + // Link to the system netcdf + println!("cargo:rustc-link-lib=netcdf"); } - - let netcdf = netcdf_config.build(); - - println!("cargo:rustc-link-lib=static=netcdf"); - println!("cargo:rustc-link-search=native={}/lib", netcdf.display()); } diff --git a/netcdf-sys/source b/netcdf-sys/source deleted file mode 160000 index 3bcdb5f..0000000 --- a/netcdf-sys/source +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3bcdb5fbf11af85e63f7916768b13b1f7d02c860 diff --git a/netcdf-sys/src/lib.rs b/netcdf-sys/src/lib.rs index 3a1c691..9681b7b 100644 --- a/netcdf-sys/src/lib.rs +++ b/netcdf-sys/src/lib.rs @@ -6,6 +6,9 @@ extern crate hdf5_sys; #[cfg(feature = "dap")] extern crate curl_sys; +#[cfg(feature = "static")] +extern crate netcdf_src; + mod netcdf_bindings; mod netcdf_const; pub use netcdf_bindings::*; From 2735a60f7efddcc494481b75e0ea234aba6dbbf8 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 30 Mar 2020 17:05:19 +0200 Subject: [PATCH 10/15] update src + can't compress no-dim vars --- netcdf-src/source | 2 +- tests/group.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/netcdf-src/source b/netcdf-src/source index 0014db9..be2f7ca 160000 --- a/netcdf-src/source +++ b/netcdf-src/source @@ -1 +1 @@ -Subproject commit 0014db92e93e27b7301163e683f194b8b2fd8d4e +Subproject commit be2f7ca7b8500486e478d246e3f9b988f2b15804 diff --git a/tests/group.rs b/tests/group.rs index 4d38aba..ceab0d3 100644 --- a/tests/group.rs +++ b/tests/group.rs @@ -63,7 +63,9 @@ fn find_variable() { assert!(group.variable("vvvvv").is_none()); for mut var in group.variables_mut() { - var.compression(3).unwrap(); + if var.dimensions().len() > 0 { + var.compression(3).unwrap(); + } if var.name() == "z" { var.chunking(&[1]).unwrap(); } else { From e96cad5209c2b6f83f66bffcf112230833889408 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 17 Apr 2020 17:16:35 +0200 Subject: [PATCH 11/15] ignore souce files --- netcdf-src/Cargo.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/netcdf-src/Cargo.toml b/netcdf-src/Cargo.toml index 2d6a0fe..c7c1fac 100644 --- a/netcdf-src/Cargo.toml +++ b/netcdf-src/Cargo.toml @@ -5,8 +5,23 @@ authors = ["Magnus Ulimoen "] edition = "2018" description = "Build scripts for building `netCDF` from source" build = "build.rs" +repository = "https://github.com/georust/netcdf" license-file = "source/COPYRIGHT" links = "netcdfsrc" +exclude = [ + "source/unit_test/**", + "source/NUG/**", + "source/dap4_test/**", + "source/examples/**", + "source/nc_test/**", + "source/h5_test/**", + "source/nc_perf/**", + "source/ncdump/**", + "source/hdf4_test/**", + "source/ncgen/**", + "source/ncgen3/**", + "source/nctest/**", +] [features] dap = [] From e2a332ea335f1a52d924d3cd9628a553cd387bff Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 28 May 2020 11:30:22 +0200 Subject: [PATCH 12/15] add fallback to lib64 --- netcdf-src/build.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/netcdf-src/build.rs b/netcdf-src/build.rs index dd2ced0..056f25e 100644 --- a/netcdf-src/build.rs +++ b/netcdf-src/build.rs @@ -39,5 +39,11 @@ fn main() { let netcdf = netcdf_config.build(); println!("cargo:lib={}", "netcdf"); - println!("cargo:search={}/lib", netcdf.display()); + let search_path = format!("{}/lib", netcdf.display()); + if std::path::Path::new(&search_path).exists() { + println!("cargo:search={}", search_path); + } else { + let search_path = format!("{}/lib64", netcdf.display()); + println!("cargo:search={}", search_path); + } } From 68cf93a5014c7321a1ec57b9b95d561dff28ae58 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 10 Aug 2020 16:09:47 +0200 Subject: [PATCH 13/15] Use upstream hdf5-sys --- Cargo.toml | 4 ---- Dockerfile | 18 ------------------ README.md | 9 +++------ netcdf-src/Cargo.toml | 4 ++-- netcdf-src/source | 2 +- netcdf-src/src/lib.rs | 2 ++ netcdf-sys/Cargo.toml | 5 +++-- netcdf-sys/README.md | 4 ++++ 8 files changed, 15 insertions(+), 33 deletions(-) delete mode 100644 Dockerfile create mode 100644 netcdf-sys/README.md diff --git a/Cargo.toml b/Cargo.toml index a1aa7b2..332a4bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,3 @@ members = [ "netcdf-sys", "netcdf-src", ] - -[patch] -[patch.crates-io] -hdf5-sys = { git = "https://github.com/magnusuMET/hdf5-rust", branch = "feature/static_build" } diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 9751a14..0000000 --- a/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# Mirroring travis build (xenial 16.04) for now... -FROM ubuntu:16.04 - -RUN apt-get -y update && apt-get -y install \ - libhdf5-serial-dev \ - netcdf-bin \ - libnetcdf-dev \ - curl - -RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y - -ENV PATH=/root/.cargo/bin:$PATH - -ADD . /code - -WORKDIR /code - -CMD cargo build --verbose && cargo test -j1 --verbose diff --git a/README.md b/README.md index 346fd39..79969ba 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Medium-level [netCDF](http://www.unidata.ucar.edu/software/netcdf/) bindings for Rust, allowing easy reading and writing of array-like structures to a file. +netCDF can read and write `hdf5` files, which is a commonly used file format in scientific computing. ## Status @@ -32,14 +33,10 @@ All variable data is read into a contiguous buffer, or inta an [ndarray](https:/ ## Building -This crate depends on libnetcdf. The Travis build runs on Ubuntu 16.04 Xenial and installs libnetcdf via apt, which results in netcdf v.4.4.0. netcdf is not widely tested on other versions of netcdf. +This crate depends on `libnetcdf`, but a static build from source is also supported, which can be enabled using the `static` feature. -You can build the library and run the tests via Docker like this: +The crate is built on several platforms using github actions. -``` -docker build . -t netcdf -docker run -it --rm netcdf -``` ## Documentation diff --git a/netcdf-src/Cargo.toml b/netcdf-src/Cargo.toml index c7c1fac..5ed73db 100644 --- a/netcdf-src/Cargo.toml +++ b/netcdf-src/Cargo.toml @@ -27,7 +27,7 @@ exclude = [ dap = [] [dependencies] -hdf5-sys = { version = "*", features=["hl", "deprecated", "zlib"] } +hdf5-sys = { version = "0.7.0", features = ["hl", "deprecated", "zlib"] } [build-dependencies] -cmake = "0.1.42" +cmake = "0.1.44" diff --git a/netcdf-src/source b/netcdf-src/source index be2f7ca..26fba54 160000 --- a/netcdf-src/source +++ b/netcdf-src/source @@ -1 +1 @@ -Subproject commit be2f7ca7b8500486e478d246e3f9b988f2b15804 +Subproject commit 26fba54a58fa02af92d84441ed90b417c1d08161 diff --git a/netcdf-src/src/lib.rs b/netcdf-src/src/lib.rs index b61ef8f..96b6cfc 100644 --- a/netcdf-src/src/lib.rs +++ b/netcdf-src/src/lib.rs @@ -1 +1,3 @@ //! Dummy crate for building `netCDF` from source +//! +//! The current pinned version is 4.7.4 diff --git a/netcdf-sys/Cargo.toml b/netcdf-sys/Cargo.toml index aa5e7da..6b7e772 100644 --- a/netcdf-sys/Cargo.toml +++ b/netcdf-sys/Cargo.toml @@ -13,6 +13,7 @@ keywords = ["netcdf", "hdf", "hdf5", "cdm", "ffi"] edition = "2018" links = "netcdf" build = "build.rs" +readme = "README.md" exclude = [ "testdata/**", "source/conda.recipe/**", @@ -36,7 +37,7 @@ exclude = [ [dependencies] libz-sys = { version = "1.0.25" } curl-sys = { version = "0.4.25", optional = true } -hdf5-sys = { version = "0.6.0", features = ["hl", "deprecated", "zlib"] } +hdf5-sys = { version = "0.7.0" } netcdf-src = { path = "../netcdf-src", version = "0.1.0", optional = true } [dev-dependencies] @@ -45,6 +46,6 @@ lazy_static = "1.4.0" [features] default = [] memio = [] -static = ["libz-sys/static", "hdf5-sys/static", "netcdf-src"] +static = ["libz-sys/static", "hdf5-sys/static", "hdf5-sys/hl", "hdf5-sys/deprecated", "hdf5-sys/zlib", "netcdf-src"] # TODO: Link static curl (but only when using "dap") "curl-sys/static-curl", "curl-sys/static-ssl" dap = ["curl-sys"] diff --git a/netcdf-sys/README.md b/netcdf-sys/README.md new file mode 100644 index 0000000..e03a127 --- /dev/null +++ b/netcdf-sys/README.md @@ -0,0 +1,4 @@ +# netcdf-sys + +Rust bindings to `netcdf-c` to locate and link the system libraries neccessary to use `netcdf`. +This library can also build `hdf5` and `netcdf` from source, to allow a fully static linking experience. This is enabled with the `static` feature. From 6e0ffc99a9834a9ab240ce04d22eda42d4bda8d6 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 10 Aug 2020 17:09:48 +0200 Subject: [PATCH 14/15] Clippy lint --- src/error.rs | 6 +----- src/types.rs | 5 +---- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/error.rs b/src/error.rs index eb55d6d..a930963 100644 --- a/src/error.rs +++ b/src/error.rs @@ -52,11 +52,7 @@ impl Error { /// Was the error due to ambiguity of the /// indices or lengths? pub fn is_ambigous(&self) -> bool { - if let Self::Ambiguous = self { - true - } else { - false - } + matches!(self, Self::Ambiguous) } } diff --git a/src/types.rs b/src/types.rs index dac53ea..da0a658 100644 --- a/src/types.rs +++ b/src/types.rs @@ -721,10 +721,7 @@ impl VariableType { #[allow(missing_docs)] impl VariableType { pub fn is_string(&self) -> bool { - match self { - Self::String => true, - _ => false, - } + matches!(self, Self::String) } pub fn is_i8(&self) -> bool { self.as_basic().map_or(false, BasicType::is_i8) From 0c10ba6936ca3b2bd7222ffeea4200155a852ff2 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Mon, 10 Aug 2020 16:41:39 +0200 Subject: [PATCH 15/15] add conda and static to CI --- .github/workflows/ci.yml | 99 ++++++++++++++++++++++------------- .github/workflows/codecov.yml | 34 ++++++++++++ README.md | 2 +- 3 files changed, 98 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/codecov.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4b97d9..9e1b128 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,9 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + with: {submodules: true} + - name: Install netCDF + run: sudo apt-get install libnetcdf-dev - name: Install rust uses: actions-rs/toolchain@v1 with: @@ -28,13 +31,13 @@ jobs: - name: Check formatting run: cargo fmt -- --check - name: Documentation - run: cargo doc --workspace + run: cargo doc --workspace --exclude netcdf-src - name: Clippy - run: cargo clippy --workspace -- -D warnings + run: cargo clippy --workspace --exclude netcdf-src -- -D warnings - test: - name: test - runs-on: ${{ matrix.os }} + test_apt: + name: test apt + runs-on: ubuntu-18.04 strategy: matrix: build: @@ -43,17 +46,15 @@ jobs: - nightly include: - build: stable - os: ubuntu-18.04 rust: stable - build: beta - os: ubuntu-18.04 rust: beta - build: nightly - os: ubuntu-18.04 rust: nightly steps: - name: Checkout repository uses: actions/checkout@v2 + with: {submodules: false} - name: Install netcdf run: sudo apt-get install libnetcdf-dev @@ -66,37 +67,63 @@ jobs: override: true - name: Build - run: cargo build --verbose --workspace + run: cargo build --verbose --workspace --exclude netcdf-src - name: Test - run: cargo test --verbose --workspace + run: cargo test --verbose --workspace --exclude netcdf-src if: matrix.os == 'ubuntu-18.04' - tarpaulin: - name: tarpaulin - runs-on: ubuntu-18.04 + conda: + name: conda + runs-on: ${{matrix.os}}-latest + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu, channel: conda-forge, rust: stable} + - {os: windows, channel: conda-forge, rust: stable} + - {os: macos, channel: conda-forge, rust: stable} + defaults: + run: + shell: bash -l {0} steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Install netcdf - run: sudo apt-get install libnetcdf-dev - - name: Install rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - override: true - - name: Install tarpaulin - uses: actions-rs/install@v0.1 - with: - crate: cargo-tarpaulin - version: latest - use-tool-cache: true - - - name: Tarpaulin - run: cargo tarpaulin --verbose --out Xml --ignore-tests + - name: Checkout repository + uses: actions/checkout@v2 + with: {submodules: true} + - name: Install Rust (${{matrix.rust}}) + uses: actions-rs/toolchain@v1 + with: {toolchain: '${{matrix.rust}}', profile: minimal, override: true} + - name: Install conda + uses: goanpeca/setup-miniconda@v1 + with: {auto-update-conda: false, activate-environment: testenv} + - name: Install netCDF + run: conda install -y -c ${{matrix.channel}} libnetcdf=4.7.4 + - name: Build and test + run: | + export HDF5_DIR="$CONDA_PREFIX" + [ "${{runner.os}}" != "Windows" ] && export RUSTFLAGS="-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib" + cargo test -vv --workspace --exclude netcdf-src - - name: Upload to codecov - uses: codecov/codecov-action@v1.0.2 - with: - token: ${{ secrets.CODECOV_TOKEN }} + static_builds: + name: static builds + runs-on: ${{matrix.os}}-latest + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu, rust: stable} + - {os: windows, rust: stable-msvc} + - {os: windows, rust: stable-gnu} + - {os: macos, rust: stable} + defaults: + run: + shell: bash -l {0} + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: {submodules: true} + - name: Install Rust (${{matrix.rust}}) + uses: actions-rs/toolchain@v1 + with: {toolchain: '${{matrix.rust}}', profile: minimal, override: true} + - name: Build and test + run: cargo test -vv --workspace --features static diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 0000000..86f26f7 --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,34 @@ +name: codecov +on: [push] +env: + CARGO_TERM_COLOR: always + +jobs: + tarpaulin: + name: tarpaulin + runs-on: ubuntu-18.04 + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install netcdf + run: sudo apt-get install libnetcdf-dev + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + - name: Install tarpaulin + uses: actions-rs/install@v0.1 + with: + crate: cargo-tarpaulin + version: latest + use-tool-cache: true + + - name: Tarpaulin + run: cargo tarpaulin --verbose --out Xml --ignore-tests + + - name: Upload to codecov + uses: codecov/codecov-action@v1.0.2 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index 79969ba..adc917a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ All variable data is read into a contiguous buffer, or inta an [ndarray](https:/ This crate depends on `libnetcdf`, but a static build from source is also supported, which can be enabled using the `static` feature. -The crate is built on several platforms using github actions. +The crate is built on several platforms using github actions, and is currently known to build form from source on all major platforms (linux, macos, windows (gnu+msvc)), and through the package installers `conda` and `apt`. ## Documentation