diff options
author | Przemysław Dolata <snowball91b@gmail.com> | 2018-08-21 09:44:25 +0200 |
---|---|---|
committer | Wook Song <wook16.song@samsung.com> | 2020-01-23 22:50:50 +0900 |
commit | d8e9540f7869a03274392247bb82c7b938eb2d9b (patch) | |
tree | ae51c257c86b956260e15c80fc4c747387ddeb45 | |
parent | 985206a68a0a859f98790102e07222cc1ceb116f (diff) | |
parent | 54174529356412dfc2f985775e303421df95b291 (diff) | |
download | caffe-d8e9540f7869a03274392247bb82c7b938eb2d9b.tar.gz caffe-d8e9540f7869a03274392247bb82c7b938eb2d9b.tar.bz2 caffe-d8e9540f7869a03274392247bb82c7b938eb2d9b.zip |
Merge pull request #6346 from jerryz123/HDF5_config
Added USE_HDF5 flag to build
-rw-r--r-- | CMakeLists.txt | 3 | ||||
-rw-r--r-- | Makefile | 12 | ||||
-rw-r--r-- | Makefile.config.example | 2 | ||||
-rw-r--r-- | cmake/ConfigGen.cmake | 12 | ||||
-rw-r--r-- | cmake/Dependencies.cmake | 8 | ||||
-rw-r--r-- | cmake/Summary.cmake | 2 | ||||
-rw-r--r-- | include/caffe/util/hdf5.hpp | 2 | ||||
-rw-r--r-- | src/caffe/layers/hdf5_data_layer.cpp | 2 | ||||
-rw-r--r-- | src/caffe/layers/hdf5_data_layer.cu | 2 | ||||
-rw-r--r-- | src/caffe/layers/hdf5_output_layer.cpp | 2 | ||||
-rw-r--r-- | src/caffe/layers/hdf5_output_layer.cu | 2 | ||||
-rw-r--r-- | src/caffe/net.cpp | 13 | ||||
-rw-r--r-- | src/caffe/solvers/sgd_solver.cpp | 12 | ||||
-rw-r--r-- | src/caffe/test/test_hdf5_output_layer.cpp | 2 | ||||
-rw-r--r-- | src/caffe/test/test_hdf5data_layer.cpp | 2 | ||||
-rw-r--r-- | src/caffe/util/hdf5.cpp | 2 |
16 files changed, 79 insertions, 1 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 08f56a33..27d172f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,9 @@ caffe_option(USE_LMDB "Build with lmdb" ON) caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF) caffe_option(USE_OPENMP "Link with OpenMP (when your BLAS wants OpenMP and you get linker errors)" OFF) +# This code is taken from https://github.com/sh1r0/caffe-android-lib +caffe_option(USE_HDF5 "Build with hdf5" ON) + # ---[ Dependencies include(cmake/Dependencies.cmake) @@ -178,11 +178,13 @@ ifneq ($(CPU_ONLY), 1) LIBRARIES := cudart cublas curand endif -LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_hl hdf5 +LIBRARIES += glog gflags protobuf boost_system boost_filesystem m # handle IO dependencies USE_LEVELDB ?= 1 USE_LMDB ?= 1 +# This code is taken from https://github.com/sh1r0/caffe-android-lib +USE_HDF5 ?= 1 USE_OPENCV ?= 1 ifeq ($(USE_LEVELDB), 1) @@ -191,6 +193,10 @@ endif ifeq ($(USE_LMDB), 1) LIBRARIES += lmdb endif +# This code is taken from https://github.com/sh1r0/caffe-android-lib +ifeq ($(USE_HDF5), 1) + LIBRARIES += hdf5_hl hdf5 +endif ifeq ($(USE_OPENCV), 1) LIBRARIES += opencv_core opencv_highgui opencv_imgproc @@ -347,6 +353,10 @@ ifeq ($(ALLOW_LMDB_NOLOCK), 1) COMMON_FLAGS += -DALLOW_LMDB_NOLOCK endif endif +# This code is taken from https://github.com/sh1r0/caffe-android-lib +ifeq ($(USE_HDF5), 1) + COMMON_FLAGS += -DUSE_HDF5 +endif # CPU-only configuration ifeq ($(CPU_ONLY), 1) diff --git a/Makefile.config.example b/Makefile.config.example index 79905935..24ca6327 100644 --- a/Makefile.config.example +++ b/Makefile.config.example @@ -11,6 +11,8 @@ # USE_OPENCV := 0 # USE_LEVELDB := 0 # USE_LMDB := 0 +# This code is taken from https://github.com/sh1r0/caffe-android-lib +# USE_HDF5 := 0 # uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary) # You should not set this flag if you will be reading LMDBs with any diff --git a/cmake/ConfigGen.cmake b/cmake/ConfigGen.cmake index 09bb09b4..69889c24 100644 --- a/cmake/ConfigGen.cmake +++ b/cmake/ConfigGen.cmake @@ -24,6 +24,18 @@ function(caffe_generate_export_configs) set(HAVE_CUDA FALSE) endif() + set(HDF5_IMPORTED OFF) + foreach(_lib ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) + if(TARGET ${_lib}) + set(HDF5_IMPORTED ON) + endif() + endforeach() + + # This code is taken from https://github.com/sh1r0/caffe-android-lib + if(USE_HDF5) + list(APPEND Caffe_DEFINITIONS -DUSE_HDF5) + endif() + if(NOT HAVE_CUDNN) set(HAVE_CUDNN FALSE) endif() diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index c48255c8..ca2e3ad9 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -47,6 +47,14 @@ find_package(HDF5 COMPONENTS HL REQUIRED) list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${HDF5_INCLUDE_DIRS}) list(APPEND Caffe_LINKER_LIBS PUBLIC ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) +# This code is taken from https://github.com/sh1r0/caffe-android-lib +if(USE_HDF5) + find_package(HDF5 COMPONENTS HL REQUIRED) + include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR}) + list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) + add_definitions(-DUSE_HDF5) +endif() + # ---[ LMDB if(USE_LMDB) find_package(LMDB REQUIRED) diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index ed8c2526..40b8c2f2 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -119,6 +119,8 @@ function(caffe_print_configuration_summary) caffe_status(" USE_LMDB : ${USE_LMDB}") caffe_status(" USE_NCCL : ${USE_NCCL}") caffe_status(" ALLOW_LMDB_NOLOCK : ${ALLOW_LMDB_NOLOCK}") + # This code is taken from https://github.com/sh1r0/caffe-android-lib + caffe_status(" USE_HDF5 : ${USE_HDF5}") caffe_status("") caffe_status("Dependencies:") caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})") diff --git a/include/caffe/util/hdf5.hpp b/include/caffe/util/hdf5.hpp index 71549c1c..dbd8bb6c 100644 --- a/include/caffe/util/hdf5.hpp +++ b/include/caffe/util/hdf5.hpp @@ -1,3 +1,4 @@ +#ifdef USE_HDF5 #ifndef CAFFE_UTIL_HDF5_H_ #define CAFFE_UTIL_HDF5_H_ @@ -37,3 +38,4 @@ string hdf5_get_name_by_idx(hid_t loc_id, int idx); } // namespace caffe #endif // CAFFE_UTIL_HDF5_H_ +#endif // USE_HDF5 diff --git a/src/caffe/layers/hdf5_data_layer.cpp b/src/caffe/layers/hdf5_data_layer.cpp index 00716a92..7668854c 100644 --- a/src/caffe/layers/hdf5_data_layer.cpp +++ b/src/caffe/layers/hdf5_data_layer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_HDF5 /* TODO: - load file in a separate thread ("prefetch") @@ -184,3 +185,4 @@ INSTANTIATE_CLASS(HDF5DataLayer); REGISTER_LAYER_CLASS(HDF5Data); } // namespace caffe +#endif // USE_HDF5 diff --git a/src/caffe/layers/hdf5_data_layer.cu b/src/caffe/layers/hdf5_data_layer.cu index 33eebd41..70cd9f32 100644 --- a/src/caffe/layers/hdf5_data_layer.cu +++ b/src/caffe/layers/hdf5_data_layer.cu @@ -1,3 +1,4 @@ +#ifdef USE_HDF5 /* TODO: - only load parts of the file, in accordance with a prototxt param "max_mem" @@ -34,3 +35,4 @@ void HDF5DataLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom, INSTANTIATE_LAYER_GPU_FUNCS(HDF5DataLayer); } // namespace caffe +#endif // USE_HDF5 diff --git a/src/caffe/layers/hdf5_output_layer.cpp b/src/caffe/layers/hdf5_output_layer.cpp index f8f1edcd..28c453a2 100644 --- a/src/caffe/layers/hdf5_output_layer.cpp +++ b/src/caffe/layers/hdf5_output_layer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_HDF5 #include <vector> #include "hdf5.h" @@ -72,3 +73,4 @@ INSTANTIATE_CLASS(HDF5OutputLayer); REGISTER_LAYER_CLASS(HDF5Output); } // namespace caffe +#endif // USE_HDF5 diff --git a/src/caffe/layers/hdf5_output_layer.cu b/src/caffe/layers/hdf5_output_layer.cu index c1685cd3..891aea03 100644 --- a/src/caffe/layers/hdf5_output_layer.cu +++ b/src/caffe/layers/hdf5_output_layer.cu @@ -1,3 +1,4 @@ +#ifdef USE_HDF5 #include <vector> #include "hdf5.h" @@ -37,3 +38,4 @@ void HDF5OutputLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top, INSTANTIATE_LAYER_GPU_FUNCS(HDF5OutputLayer); } // namespace caffe +#endif // USE_HDF5 diff --git a/src/caffe/net.cpp b/src/caffe/net.cpp index 353c2f95..73adcc6d 100644 --- a/src/caffe/net.cpp +++ b/src/caffe/net.cpp @@ -5,7 +5,9 @@ #include <utility> #include <vector> +#ifdef USE_HDF5 #include "hdf5.h" +#endif // USE_HDF5 #include "caffe/common.hpp" #include "caffe/layer.hpp" @@ -786,6 +788,7 @@ void Net<Dtype>::CopyTrainedLayersFromBinaryProto( template <typename Dtype> void Net<Dtype>::CopyTrainedLayersFromHDF5(const string trained_filename) { +#ifdef USE_HDF5 hid_t file_hid = H5Fopen(trained_filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); CHECK_GE(file_hid, 0) << "Couldn't open " << trained_filename; @@ -832,6 +835,10 @@ void Net<Dtype>::CopyTrainedLayersFromHDF5(const string trained_filename) { } H5Gclose(data_hid); H5Fclose(file_hid); +#else + LOG(FATAL) << "CopyTrainedLayersFromHDF5 requires hdf5;" + << " compile with USE_HDF5."; +#endif // USE_HDF5 } template <typename Dtype> @@ -848,6 +855,8 @@ void Net<Dtype>::ToProto(NetParameter* param, bool write_diff) const { template <typename Dtype> void Net<Dtype>::ToHDF5(const string& filename, bool write_diff) const { +// This code is taken from https://github.com/sh1r0/caffe-android-lib +#ifdef USE_HDF5 hid_t file_hid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK_GE(file_hid, 0) @@ -901,6 +910,10 @@ void Net<Dtype>::ToHDF5(const string& filename, bool write_diff) const { H5Gclose(diff_hid); } H5Fclose(file_hid); +// This code is taken from https://github.com/sh1r0/caffe-android-lib +#else + LOG(FATAL) << "ToHDF5 requires hdf5; compile with USE_HDF5."; +#endif // USE_HDF5 } template <typename Dtype> diff --git a/src/caffe/solvers/sgd_solver.cpp b/src/caffe/solvers/sgd_solver.cpp index a56a2d0b..081c47eb 100644 --- a/src/caffe/solvers/sgd_solver.cpp +++ b/src/caffe/solvers/sgd_solver.cpp @@ -289,6 +289,8 @@ void SGDSolver<Dtype>::SnapshotSolverStateToBinaryProto( template <typename Dtype> void SGDSolver<Dtype>::SnapshotSolverStateToHDF5( const string& model_filename) { +// This code is taken from https://github.com/sh1r0/caffe-android-lib +#ifdef USE_HDF5 string snapshot_filename = Solver<Dtype>::SnapshotFilename(".solverstate.h5"); LOG(INFO) << "Snapshotting solver state to HDF5 file " << snapshot_filename; @@ -310,6 +312,11 @@ void SGDSolver<Dtype>::SnapshotSolverStateToHDF5( } H5Gclose(history_hid); H5Fclose(file_hid); +// This code is taken from https://github.com/sh1r0/caffe-android-lib +#else + LOG(FATAL) << "SnapshotSolverStateToHDF5 requires hdf5;" + << " compile with USE_HDF5."; +#endif // USE_HDF5 } template <typename Dtype> @@ -334,6 +341,7 @@ void SGDSolver<Dtype>::RestoreSolverStateFromBinaryProto( template <typename Dtype> void SGDSolver<Dtype>::RestoreSolverStateFromHDF5(const string& state_file) { +#ifdef USE_HDF5 hid_t file_hid = H5Fopen(state_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); CHECK_GE(file_hid, 0) << "Couldn't open solver state file " << state_file; this->iter_ = hdf5_load_int(file_hid, "iter"); @@ -355,6 +363,10 @@ void SGDSolver<Dtype>::RestoreSolverStateFromHDF5(const string& state_file) { } H5Gclose(history_hid); H5Fclose(file_hid); +#else + LOG(FATAL) << "RestoreSolverStateFromHDF5 requires hdf5;" + << " compile with USE_HDF5."; +#endif // USE_HDF5 } INSTANTIATE_CLASS(SGDSolver); diff --git a/src/caffe/test/test_hdf5_output_layer.cpp b/src/caffe/test/test_hdf5_output_layer.cpp index f94dd57e..11d52310 100644 --- a/src/caffe/test/test_hdf5_output_layer.cpp +++ b/src/caffe/test/test_hdf5_output_layer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_HDF5 #include <string> #include <vector> @@ -120,3 +121,4 @@ TYPED_TEST(HDF5OutputLayerTest, TestForward) { } } // namespace caffe +#endif // USE_HDF5 diff --git a/src/caffe/test/test_hdf5data_layer.cpp b/src/caffe/test/test_hdf5data_layer.cpp index 3977c486..0e5c398f 100644 --- a/src/caffe/test/test_hdf5data_layer.cpp +++ b/src/caffe/test/test_hdf5data_layer.cpp @@ -1,3 +1,4 @@ +#ifdef USE_HDF5 #include <string> #include <vector> @@ -163,3 +164,4 @@ TYPED_TEST(HDF5DataLayerTest, TestSkip) { } } // namespace caffe +#endif // USE_HDF5 diff --git a/src/caffe/util/hdf5.cpp b/src/caffe/util/hdf5.cpp index ed737429..cefd853d 100644 --- a/src/caffe/util/hdf5.cpp +++ b/src/caffe/util/hdf5.cpp @@ -1,3 +1,4 @@ +#ifdef USE_HDF5 #include "caffe/util/hdf5.hpp" #include <string> @@ -207,3 +208,4 @@ string hdf5_get_name_by_idx(hid_t loc_id, int idx) { } } // namespace caffe +#endif // USE_HDF5 |