1/*
2 * Copyright (c) 2016-2019 Arm Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "base/stats/hdf5.hh"
41
42#include "base/logging.hh"
43#include "base/stats/info.hh"
44
45/**
46 * Check if all strings in a container are empty.
47 */
48template<typename T>
49bool emptyStrings(const T &labels)
50{
51 for (const auto &s : labels) {
52 if (!s.empty())
53 return false;
54 }
55 return true;
56}
57
58
59namespace Stats {
60
61Hdf5::Hdf5(const std::string &file, unsigned chunking,
62 bool desc, bool formulas)
63 : fname(file), timeChunk(chunking),
64 enableDescriptions(desc), enableFormula(formulas),
65 dumpCount(0)
66{
67 // Tell the library not to print exceptions by default. There are
68 // cases where we rely on exceptions to determine if we need to
69 // create a node or if we can just open it.
70 H5::Exception::dontPrint();
71}
72
73Hdf5::~Hdf5()
74{
75}
76
77
78void
79Hdf5::begin()
80{
81 h5File = H5::H5File(fname,
82 // Truncate the file if this is the first dump
83 dumpCount > 0 ? H5F_ACC_RDWR : H5F_ACC_TRUNC);
84 path.push(h5File.openGroup("/"));
85}
86
87void
88Hdf5::end()
89{
90 assert(valid());
91
92 dumpCount++;
93}
94
95bool
96Hdf5::valid() const
97{
98 return true;
99}
100
101
102void
103Hdf5::beginGroup(const char *name)
104{
105 auto base = path.top();
106
107 // Try to open an existing stat group corresponding to the
108 // name. Create it if it doesn't exist.
109 H5::Group group;
110 try {
111 group = base.openGroup(name);
| 1/*
2 * Copyright (c) 2016-2019 Arm Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "base/stats/hdf5.hh"
41
42#include "base/logging.hh"
43#include "base/stats/info.hh"
44
45/**
46 * Check if all strings in a container are empty.
47 */
48template<typename T>
49bool emptyStrings(const T &labels)
50{
51 for (const auto &s : labels) {
52 if (!s.empty())
53 return false;
54 }
55 return true;
56}
57
58
59namespace Stats {
60
61Hdf5::Hdf5(const std::string &file, unsigned chunking,
62 bool desc, bool formulas)
63 : fname(file), timeChunk(chunking),
64 enableDescriptions(desc), enableFormula(formulas),
65 dumpCount(0)
66{
67 // Tell the library not to print exceptions by default. There are
68 // cases where we rely on exceptions to determine if we need to
69 // create a node or if we can just open it.
70 H5::Exception::dontPrint();
71}
72
73Hdf5::~Hdf5()
74{
75}
76
77
78void
79Hdf5::begin()
80{
81 h5File = H5::H5File(fname,
82 // Truncate the file if this is the first dump
83 dumpCount > 0 ? H5F_ACC_RDWR : H5F_ACC_TRUNC);
84 path.push(h5File.openGroup("/"));
85}
86
87void
88Hdf5::end()
89{
90 assert(valid());
91
92 dumpCount++;
93}
94
95bool
96Hdf5::valid() const
97{
98 return true;
99}
100
101
102void
103Hdf5::beginGroup(const char *name)
104{
105 auto base = path.top();
106
107 // Try to open an existing stat group corresponding to the
108 // name. Create it if it doesn't exist.
109 H5::Group group;
110 try {
111 group = base.openGroup(name);
|
115 group = base.createGroup(name);
116 }
117
118 path.push(group);
119}
120
121void
122Hdf5::endGroup()
123{
124 assert(!path.empty());
125 path.pop();
126}
127
128void
129Hdf5::visit(const ScalarInfo &info)
130{
131 // Since this stat is a scalar, we need 1-dimensional value in the
132 // stat file. The Hdf5::appendStat helper will populate the size
133 // of the first dimension (time).
134 hsize_t fdims[1] = { 0, };
135 double data[1] = { info.result(), };
136
137 appendStat(info, 1, fdims, data);
138}
139
140void
141Hdf5::visit(const VectorInfo &info)
142{
143 appendVectorInfo(info);
144}
145
146void
147Hdf5::visit(const DistInfo &info)
148{
149 warn_once("HDF5 stat files don't support distributions.\n");
150}
151
152void
153Hdf5::visit(const VectorDistInfo &info)
154{
155 warn_once("HDF5 stat files don't support vector distributions.\n");
156}
157
158void
159Hdf5::visit(const Vector2dInfo &info)
160{
161 // Request a 3-dimensional stat, the first dimension will be
162 // populated by the Hdf5::appendStat() helper. The remaining two
163 // dimensions correspond to the stat instance.
164 hsize_t fdims[3] = { 0, info.x, info.y };
165 H5::DataSet data_set = appendStat(info, 3, fdims, info.cvec.data());
166
167 if (dumpCount == 0) {
168 if (!info.subnames.empty() && !emptyStrings(info.subnames))
169 addMetaData(data_set, "subnames", info.subnames);
170
171 if (!info.y_subnames.empty() && !emptyStrings(info.y_subnames))
172 addMetaData(data_set, "y_subnames", info.y_subnames);
173
174 if (!info.subdescs.empty() && !emptyStrings(info.subdescs))
175 addMetaData(data_set, "subdescs", info.subdescs);
176 }
177}
178
179void
180Hdf5::visit(const FormulaInfo &info)
181{
182 if (!enableFormula)
183 return;
184
185 H5::DataSet data_set = appendVectorInfo(info);
186
187 if (dumpCount == 0)
188 addMetaData(data_set, "equation", info.str());
189}
190
191void
192Hdf5::visit(const SparseHistInfo &info)
193{
194 warn_once("HDF5 stat files don't support sparse histograms.\n");
195}
196
197H5::DataSet
198Hdf5::appendVectorInfo(const VectorInfo &info)
199{
200 const VResult &vr(info.result());
201 // Request a 2-dimensional stat, the first dimension will be
202 // populated by the Hdf5::appendStat() helper. The remaining
203 // dimension correspond to the stat instance.
204 hsize_t fdims[2] = { 0, vr.size() };
205 H5::DataSet data_set = appendStat(info, 2, fdims, vr.data());
206
207 if (dumpCount == 0) {
208 if (!info.subnames.empty() && !emptyStrings(info.subnames))
209 addMetaData(data_set, "subnames", info.subnames);
210
211 if (!info.subdescs.empty() && !emptyStrings(info.subdescs))
212 addMetaData(data_set, "subdescs", info.subdescs);
213 }
214
215 return data_set;
216}
217
218H5::DataSet
219Hdf5::appendStat(const Info &info, int rank, hsize_t *dims, const double *data)
220{
221 H5::Group group = path.top();
222 H5::DataSet data_set;
223 H5::DataSpace fspace;
224
225 dims[0] = dumpCount + 1;
226
227 if (dumpCount > 0) {
228 // Get the existing stat if we have already dumped this stat
229 // before.
230 data_set = group.openDataSet(info.name);
231 data_set.extend(dims);
232 fspace = data_set.getSpace();
233 } else {
234 // We don't have the stat already, create it.
235
236 H5::DSetCreatPropList props;
237
238 // Setup max dimensions based on the requested file dimensions
239 std::vector<hsize_t> max_dims(rank);
240 std::copy(dims, dims + rank, max_dims.begin());
241 max_dims[0] = H5S_UNLIMITED;
242
243 // Setup chunking
244 std::vector<hsize_t> chunk_dims(rank);
245 std::copy(dims, dims + rank, chunk_dims.begin());
246 chunk_dims[0] = timeChunk;
247 props.setChunk(rank, chunk_dims.data());
248
249 // Enable compression
250 props.setDeflate(1);
251
252 fspace = H5::DataSpace(rank, dims, max_dims.data());
253 data_set = group.createDataSet(info.name, H5::PredType::NATIVE_DOUBLE,
254 fspace, props);
255
256 if (enableDescriptions && !info.desc.empty()) {
257 addMetaData(data_set, "description", info.desc);
258 }
259 }
260
261 // The first dimension is time which isn't included in data.
262 dims[0] = 1;
263 H5::DataSpace mspace(rank, dims);
264 std::vector<hsize_t> foffset(rank, 0);
265 foffset[0] = dumpCount;
266
267 fspace.selectHyperslab(H5S_SELECT_SET, dims, foffset.data());
268 data_set.write(data, H5::PredType::NATIVE_DOUBLE, mspace, fspace);
269
270 return data_set;
271}
272
273void
274Hdf5::addMetaData(H5::DataSet &loc, const char *name,
275 const std::vector<const char *> &values)
276{
277 H5::StrType type(H5::PredType::C_S1, H5T_VARIABLE);
278 hsize_t dims[1] = { values.size(), };
279 H5::DataSpace space(1, dims);
280 H5::Attribute attribute = loc.createAttribute(name, type, space);
281 attribute.write(type, values.data());
282}
283
284void
285Hdf5::addMetaData(H5::DataSet &loc, const char *name,
286 const std::vector<std::string> &values)
287{
288 std::vector<const char *> cstrs(values.size());
289 for (int i = 0; i < values.size(); ++i)
290 cstrs[i] = values[i].c_str();
291
292 addMetaData(loc, name, cstrs);
293}
294
295void
296Hdf5::addMetaData(H5::DataSet &loc, const char *name,
297 const std::string &value)
298{
299 H5::StrType type(H5::PredType::C_S1, value.length() + 1);
300 hsize_t dims[1] = { 1, };
301 H5::DataSpace space(1, dims);
302 H5::Attribute attribute = loc.createAttribute(name, type, space);
303 attribute.write(type, value.c_str());
304}
305
306void
307Hdf5::addMetaData(H5::DataSet &loc, const char *name, double value)
308{
309 hsize_t dims[1] = { 1, };
310 H5::DataSpace space(1, dims);
311 H5::Attribute attribute = loc.createAttribute(
312 name, H5::PredType::NATIVE_DOUBLE, space);
313 attribute.write(H5::PredType::NATIVE_DOUBLE, &value);
314}
315
316
317std::unique_ptr<Output>
318initHDF5(const std::string &filename, unsigned chunking,
319 bool desc, bool formulas)
320{
321 return std::unique_ptr<Output>(
322 new Hdf5(filename, chunking, desc, formulas));
323}
324
325}; // namespace Stats
| 115 group = base.createGroup(name);
116 }
117
118 path.push(group);
119}
120
121void
122Hdf5::endGroup()
123{
124 assert(!path.empty());
125 path.pop();
126}
127
128void
129Hdf5::visit(const ScalarInfo &info)
130{
131 // Since this stat is a scalar, we need 1-dimensional value in the
132 // stat file. The Hdf5::appendStat helper will populate the size
133 // of the first dimension (time).
134 hsize_t fdims[1] = { 0, };
135 double data[1] = { info.result(), };
136
137 appendStat(info, 1, fdims, data);
138}
139
140void
141Hdf5::visit(const VectorInfo &info)
142{
143 appendVectorInfo(info);
144}
145
146void
147Hdf5::visit(const DistInfo &info)
148{
149 warn_once("HDF5 stat files don't support distributions.\n");
150}
151
152void
153Hdf5::visit(const VectorDistInfo &info)
154{
155 warn_once("HDF5 stat files don't support vector distributions.\n");
156}
157
158void
159Hdf5::visit(const Vector2dInfo &info)
160{
161 // Request a 3-dimensional stat, the first dimension will be
162 // populated by the Hdf5::appendStat() helper. The remaining two
163 // dimensions correspond to the stat instance.
164 hsize_t fdims[3] = { 0, info.x, info.y };
165 H5::DataSet data_set = appendStat(info, 3, fdims, info.cvec.data());
166
167 if (dumpCount == 0) {
168 if (!info.subnames.empty() && !emptyStrings(info.subnames))
169 addMetaData(data_set, "subnames", info.subnames);
170
171 if (!info.y_subnames.empty() && !emptyStrings(info.y_subnames))
172 addMetaData(data_set, "y_subnames", info.y_subnames);
173
174 if (!info.subdescs.empty() && !emptyStrings(info.subdescs))
175 addMetaData(data_set, "subdescs", info.subdescs);
176 }
177}
178
179void
180Hdf5::visit(const FormulaInfo &info)
181{
182 if (!enableFormula)
183 return;
184
185 H5::DataSet data_set = appendVectorInfo(info);
186
187 if (dumpCount == 0)
188 addMetaData(data_set, "equation", info.str());
189}
190
191void
192Hdf5::visit(const SparseHistInfo &info)
193{
194 warn_once("HDF5 stat files don't support sparse histograms.\n");
195}
196
197H5::DataSet
198Hdf5::appendVectorInfo(const VectorInfo &info)
199{
200 const VResult &vr(info.result());
201 // Request a 2-dimensional stat, the first dimension will be
202 // populated by the Hdf5::appendStat() helper. The remaining
203 // dimension correspond to the stat instance.
204 hsize_t fdims[2] = { 0, vr.size() };
205 H5::DataSet data_set = appendStat(info, 2, fdims, vr.data());
206
207 if (dumpCount == 0) {
208 if (!info.subnames.empty() && !emptyStrings(info.subnames))
209 addMetaData(data_set, "subnames", info.subnames);
210
211 if (!info.subdescs.empty() && !emptyStrings(info.subdescs))
212 addMetaData(data_set, "subdescs", info.subdescs);
213 }
214
215 return data_set;
216}
217
218H5::DataSet
219Hdf5::appendStat(const Info &info, int rank, hsize_t *dims, const double *data)
220{
221 H5::Group group = path.top();
222 H5::DataSet data_set;
223 H5::DataSpace fspace;
224
225 dims[0] = dumpCount + 1;
226
227 if (dumpCount > 0) {
228 // Get the existing stat if we have already dumped this stat
229 // before.
230 data_set = group.openDataSet(info.name);
231 data_set.extend(dims);
232 fspace = data_set.getSpace();
233 } else {
234 // We don't have the stat already, create it.
235
236 H5::DSetCreatPropList props;
237
238 // Setup max dimensions based on the requested file dimensions
239 std::vector<hsize_t> max_dims(rank);
240 std::copy(dims, dims + rank, max_dims.begin());
241 max_dims[0] = H5S_UNLIMITED;
242
243 // Setup chunking
244 std::vector<hsize_t> chunk_dims(rank);
245 std::copy(dims, dims + rank, chunk_dims.begin());
246 chunk_dims[0] = timeChunk;
247 props.setChunk(rank, chunk_dims.data());
248
249 // Enable compression
250 props.setDeflate(1);
251
252 fspace = H5::DataSpace(rank, dims, max_dims.data());
253 data_set = group.createDataSet(info.name, H5::PredType::NATIVE_DOUBLE,
254 fspace, props);
255
256 if (enableDescriptions && !info.desc.empty()) {
257 addMetaData(data_set, "description", info.desc);
258 }
259 }
260
261 // The first dimension is time which isn't included in data.
262 dims[0] = 1;
263 H5::DataSpace mspace(rank, dims);
264 std::vector<hsize_t> foffset(rank, 0);
265 foffset[0] = dumpCount;
266
267 fspace.selectHyperslab(H5S_SELECT_SET, dims, foffset.data());
268 data_set.write(data, H5::PredType::NATIVE_DOUBLE, mspace, fspace);
269
270 return data_set;
271}
272
273void
274Hdf5::addMetaData(H5::DataSet &loc, const char *name,
275 const std::vector<const char *> &values)
276{
277 H5::StrType type(H5::PredType::C_S1, H5T_VARIABLE);
278 hsize_t dims[1] = { values.size(), };
279 H5::DataSpace space(1, dims);
280 H5::Attribute attribute = loc.createAttribute(name, type, space);
281 attribute.write(type, values.data());
282}
283
284void
285Hdf5::addMetaData(H5::DataSet &loc, const char *name,
286 const std::vector<std::string> &values)
287{
288 std::vector<const char *> cstrs(values.size());
289 for (int i = 0; i < values.size(); ++i)
290 cstrs[i] = values[i].c_str();
291
292 addMetaData(loc, name, cstrs);
293}
294
295void
296Hdf5::addMetaData(H5::DataSet &loc, const char *name,
297 const std::string &value)
298{
299 H5::StrType type(H5::PredType::C_S1, value.length() + 1);
300 hsize_t dims[1] = { 1, };
301 H5::DataSpace space(1, dims);
302 H5::Attribute attribute = loc.createAttribute(name, type, space);
303 attribute.write(type, value.c_str());
304}
305
306void
307Hdf5::addMetaData(H5::DataSet &loc, const char *name, double value)
308{
309 hsize_t dims[1] = { 1, };
310 H5::DataSpace space(1, dims);
311 H5::Attribute attribute = loc.createAttribute(
312 name, H5::PredType::NATIVE_DOUBLE, space);
313 attribute.write(H5::PredType::NATIVE_DOUBLE, &value);
314}
315
316
317std::unique_ptr<Output>
318initHDF5(const std::string &filename, unsigned chunking,
319 bool desc, bool formulas)
320{
321 return std::unique_ptr<Output>(
322 new Hdf5(filename, chunking, desc, formulas));
323}
324
325}; // namespace Stats
|