1/*
2 * Copyright (c) 2016-2019 Arm Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andreas Sandberg
38 */
39
40#include "base/stats/hdf5.hh"
41
42#include "base/logging.hh"
43#include "base/stats/info.hh"
44
45/**
46 * Check if all strings in a container are empty.
47 */
48template<typename T>
49bool emptyStrings(const T &labels)
50{
51    for (const auto &s : labels) {
52        if (!s.empty())
53            return false;
54    }
55    return true;
56}
57
58
59namespace Stats {
60
61Hdf5::Hdf5(const std::string &file, unsigned chunking,
62           bool desc, bool formulas)
63    : fname(file), timeChunk(chunking),
64      enableDescriptions(desc), enableFormula(formulas),
65      dumpCount(0)
66{
67    // Tell the library not to print exceptions by default. There are
68    // cases where we rely on exceptions to determine if we need to
69    // create a node or if we can just open it.
70    H5::Exception::dontPrint();
71}
72
73Hdf5::~Hdf5()
74{
75}
76
77
78void
79Hdf5::begin()
80{
81    h5File = H5::H5File(fname,
82                        // Truncate the file if this is the first dump
83                        dumpCount > 0 ? H5F_ACC_RDWR : H5F_ACC_TRUNC);
84    path.push(h5File.openGroup("/"));
85}
86
87void
88Hdf5::end()
89{
90    assert(valid());
91
92    dumpCount++;
93}
94
95bool
96Hdf5::valid() const
97{
98    return true;
99}
100
101
102void
103Hdf5::beginGroup(const char *name)
104{
105    auto base = path.top();
106
107    // Try to open an existing stat group corresponding to the
108    // name. Create it if it doesn't exist.
109    H5::Group group;
110    try {
111        group = base.openGroup(name);
112    } catch (const H5::FileIException& e) {
113        group = base.createGroup(name);
114    } catch (const H5::GroupIException& e) {
115        group = base.createGroup(name);
116    }
117
118    path.push(group);
119}
120
121void
122Hdf5::endGroup()
123{
124    assert(!path.empty());
125    path.pop();
126}
127
128void
129Hdf5::visit(const ScalarInfo &info)
130{
131    // Since this stat is a scalar, we need 1-dimensional value in the
132    // stat file. The Hdf5::appendStat helper will populate the size
133    // of the first dimension (time).
134    hsize_t fdims[1] = { 0, };
135    double data[1] = { info.result(), };
136
137    appendStat(info, 1, fdims, data);
138}
139
140void
141Hdf5::visit(const VectorInfo &info)
142{
143    appendVectorInfo(info);
144}
145
146void
147Hdf5::visit(const DistInfo &info)
148{
149    warn_once("HDF5 stat files don't support distributions.\n");
150}
151
152void
153Hdf5::visit(const VectorDistInfo &info)
154{
155    warn_once("HDF5 stat files don't support vector distributions.\n");
156}
157
158void
159Hdf5::visit(const Vector2dInfo &info)
160{
161    // Request a 3-dimensional stat, the first dimension will be
162    // populated by the Hdf5::appendStat() helper. The remaining two
163    // dimensions correspond to the stat instance.
164    hsize_t fdims[3] = { 0, info.x, info.y };
165    H5::DataSet data_set = appendStat(info, 3, fdims, info.cvec.data());
166
167    if (dumpCount == 0) {
168        if (!info.subnames.empty() && !emptyStrings(info.subnames))
169            addMetaData(data_set, "subnames", info.subnames);
170
171        if (!info.y_subnames.empty() && !emptyStrings(info.y_subnames))
172            addMetaData(data_set, "y_subnames", info.y_subnames);
173
174        if (!info.subdescs.empty() && !emptyStrings(info.subdescs))
175            addMetaData(data_set, "subdescs", info.subdescs);
176    }
177}
178
179void
180Hdf5::visit(const FormulaInfo &info)
181{
182    if (!enableFormula)
183        return;
184
185    H5::DataSet data_set = appendVectorInfo(info);
186
187    if (dumpCount == 0)
188        addMetaData(data_set, "equation", info.str());
189}
190
191void
192Hdf5::visit(const SparseHistInfo &info)
193{
194    warn_once("HDF5 stat files don't support sparse histograms.\n");
195}
196
197H5::DataSet
198Hdf5::appendVectorInfo(const VectorInfo &info)
199{
200    const VResult &vr(info.result());
201    // Request a 2-dimensional stat, the first dimension will be
202    // populated by the Hdf5::appendStat() helper. The remaining
203    // dimension correspond to the stat instance.
204    hsize_t fdims[2] = { 0, vr.size() };
205    H5::DataSet data_set = appendStat(info, 2, fdims, vr.data());
206
207    if (dumpCount == 0) {
208        if (!info.subnames.empty() && !emptyStrings(info.subnames))
209            addMetaData(data_set, "subnames", info.subnames);
210
211        if (!info.subdescs.empty() && !emptyStrings(info.subdescs))
212            addMetaData(data_set, "subdescs", info.subdescs);
213    }
214
215    return data_set;
216}
217
218H5::DataSet
219Hdf5::appendStat(const Info &info, int rank, hsize_t *dims, const double *data)
220{
221    H5::Group group = path.top();
222    H5::DataSet data_set;
223    H5::DataSpace fspace;
224
225    dims[0] = dumpCount + 1;
226
227    if (dumpCount > 0) {
228        // Get the existing stat if we have already dumped this stat
229        // before.
230        data_set = group.openDataSet(info.name);
231        data_set.extend(dims);
232        fspace = data_set.getSpace();
233    } else {
234        // We don't have the stat already, create it.
235
236        H5::DSetCreatPropList props;
237
238        // Setup max dimensions based on the requested file dimensions
239        std::vector<hsize_t> max_dims(rank);
240        std::copy(dims, dims + rank, max_dims.begin());
241        max_dims[0] = H5S_UNLIMITED;
242
243        // Setup chunking
244        std::vector<hsize_t> chunk_dims(rank);
245        std::copy(dims, dims + rank, chunk_dims.begin());
246        chunk_dims[0] = timeChunk;
247        props.setChunk(rank, chunk_dims.data());
248
249        // Enable compression
250        props.setDeflate(1);
251
252        fspace = H5::DataSpace(rank, dims, max_dims.data());
253        data_set = group.createDataSet(info.name, H5::PredType::NATIVE_DOUBLE,
254                                       fspace, props);
255
256        if (enableDescriptions && !info.desc.empty()) {
257            addMetaData(data_set, "description", info.desc);
258        }
259    }
260
261    // The first dimension is time which isn't included in data.
262    dims[0] = 1;
263    H5::DataSpace mspace(rank, dims);
264    std::vector<hsize_t> foffset(rank, 0);
265    foffset[0] = dumpCount;
266
267    fspace.selectHyperslab(H5S_SELECT_SET, dims, foffset.data());
268    data_set.write(data, H5::PredType::NATIVE_DOUBLE, mspace, fspace);
269
270    return data_set;
271}
272
273void
274Hdf5::addMetaData(H5::DataSet &loc, const char *name,
275                  const std::vector<const char *> &values)
276{
277    H5::StrType type(H5::PredType::C_S1, H5T_VARIABLE);
278    hsize_t dims[1] = { values.size(), };
279    H5::DataSpace space(1, dims);
280    H5::Attribute attribute = loc.createAttribute(name, type, space);
281    attribute.write(type, values.data());
282}
283
284void
285Hdf5::addMetaData(H5::DataSet &loc, const char *name,
286                  const std::vector<std::string> &values)
287{
288    std::vector<const char *> cstrs(values.size());
289    for (int i = 0; i < values.size(); ++i)
290        cstrs[i] = values[i].c_str();
291
292    addMetaData(loc, name, cstrs);
293}
294
295void
296Hdf5::addMetaData(H5::DataSet &loc, const char *name,
297                  const std::string &value)
298{
299    H5::StrType type(H5::PredType::C_S1, value.length() + 1);
300    hsize_t dims[1] = { 1, };
301    H5::DataSpace space(1, dims);
302    H5::Attribute attribute = loc.createAttribute(name, type, space);
303    attribute.write(type, value.c_str());
304}
305
306void
307Hdf5::addMetaData(H5::DataSet &loc, const char *name, double value)
308{
309    hsize_t dims[1] = { 1, };
310    H5::DataSpace space(1, dims);
311    H5::Attribute attribute = loc.createAttribute(
312        name, H5::PredType::NATIVE_DOUBLE, space);
313    attribute.write(H5::PredType::NATIVE_DOUBLE, &value);
314}
315
316
317std::unique_ptr<Output>
318initHDF5(const std::string &filename, unsigned chunking,
319         bool desc, bool formulas)
320{
321    return  std::unique_ptr<Output>(
322        new Hdf5(simout.resolve(filename), chunking, desc, formulas));
323}
324
325}; // namespace Stats
326