1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
#include <string>
#include <vector>
#include "leveldb/db.h"
#include "gtest/gtest.h"
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/filler.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/vision_layers.hpp"
#include "caffe/test/test_caffe_main.hpp"
using std::string;
namespace caffe {
template <typename TypeParam>
class HDF5DataLayerTest : public MultiDeviceTest<TypeParam> {
typedef typename TypeParam::Dtype Dtype;
protected:
HDF5DataLayerTest()
: filename(NULL),
blob_top_data_(new Blob<Dtype>()),
blob_top_label_(new Blob<Dtype>()) {}
virtual void SetUp() {
blob_top_vec_.push_back(blob_top_data_);
blob_top_vec_.push_back(blob_top_label_);
// Check out generate_sample_data.py in the same directory.
filename = new string(CMAKE_SOURCE_DIR "caffe/test/test_data/sample_data_list.txt" CMAKE_EXT);
LOG(INFO) << "Using sample HDF5 data file " << filename;
}
virtual ~HDF5DataLayerTest() {
delete blob_top_data_;
delete blob_top_label_;
delete filename;
}
string* filename;
Blob<Dtype>* const blob_top_data_;
Blob<Dtype>* const blob_top_label_;
vector<Blob<Dtype>*> blob_bottom_vec_;
vector<Blob<Dtype>*> blob_top_vec_;
};
TYPED_TEST_CASE(HDF5DataLayerTest, TestDtypesAndDevices);
TYPED_TEST(HDF5DataLayerTest, TestRead) {
typedef typename TypeParam::Dtype Dtype;
// Create LayerParameter with the known parameters.
// The data file we are reading has 10 rows and 8 columns,
// with values from 0 to 10*8 reshaped in row-major order.
LayerParameter param;
HDF5DataParameter* hdf5_data_param = param.mutable_hdf5_data_param();
int batch_size = 5;
hdf5_data_param->set_batch_size(batch_size);
hdf5_data_param->set_source(*(this->filename));
int num_cols = 8;
int height = 5;
int width = 5;
// Test that the layer setup got the correct parameters.
HDF5DataLayer<Dtype> layer(param);
layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_);
EXPECT_EQ(this->blob_top_data_->num(), batch_size);
EXPECT_EQ(this->blob_top_data_->channels(), num_cols);
EXPECT_EQ(this->blob_top_data_->height(), height);
EXPECT_EQ(this->blob_top_data_->width(), width);
EXPECT_EQ(this->blob_top_label_->num(), batch_size);
EXPECT_EQ(this->blob_top_label_->channels(), 1);
EXPECT_EQ(this->blob_top_label_->height(), 1);
EXPECT_EQ(this->blob_top_label_->width(), 1);
layer.SetUp(this->blob_bottom_vec_, &this->blob_top_vec_);
// Go through the data 10 times (5 batches).
const int data_size = num_cols * height * width;
for (int iter = 0; iter < 10; ++iter) {
layer.Forward(this->blob_bottom_vec_, &this->blob_top_vec_);
// On even iterations, we're reading the first half of the data.
// On odd iterations, we're reading the second half of the data.
// NB: label is 1-indexed
int label_offset = 1 + ((iter % 2 == 0) ? 0 : batch_size);
int data_offset = (iter % 2 == 0) ? 0 : batch_size * data_size;
// Every two iterations we are reading the second file,
// which has the same labels, but data is offset by total data size,
// which is 2000 (see generate_sample_data).
int file_offset = (iter % 4 < 2) ? 0 : 2000;
for (int i = 0; i < batch_size; ++i) {
EXPECT_EQ(
label_offset + i,
this->blob_top_label_->cpu_data()[i]);
}
for (int i = 0; i < batch_size; ++i) {
for (int j = 0; j < num_cols; ++j) {
for (int h = 0; h < height; ++h) {
for (int w = 0; w < width; ++w) {
int idx = (
i * num_cols * height * width +
j * height * width +
h * width + w);
EXPECT_EQ(
file_offset + data_offset + idx,
this->blob_top_data_->cpu_data()[idx])
<< "debug: i " << i << " j " << j
<< " iter " << iter;
}
}
}
}
}
}
} // namespace caffe
|
