test_numpy_array.cpp revision 14299:2fbea9df56d2 
1/*
2    tests/test_numpy_array.cpp -- test core array functionality
3
4    Copyright (c) 2016 Ivan Smirnov <i.s.smirnov@gmail.com>
5
6    All rights reserved. Use of this source code is governed by a
7    BSD-style license that can be found in the LICENSE file.
8*/
9
10#include "pybind11_tests.h"
11
12#include <pybind11/numpy.h>
13#include <pybind11/stl.h>
14
15#include <cstdint>
16
17// Size / dtype checks.
18struct DtypeCheck {
19    py::dtype numpy{};
20    py::dtype pybind11{};
21};
22
23template <typename T>
24DtypeCheck get_dtype_check(const char* name) {
25    py::module np = py::module::import("numpy");
26    DtypeCheck check{};
27    check.numpy = np.attr("dtype")(np.attr(name));
28    check.pybind11 = py::dtype::of<T>();
29    return check;
30}
31
32std::vector<DtypeCheck> get_concrete_dtype_checks() {
33    return {
34        // Normalization
35        get_dtype_check<std::int8_t>("int8"),
36        get_dtype_check<std::uint8_t>("uint8"),
37        get_dtype_check<std::int16_t>("int16"),
38        get_dtype_check<std::uint16_t>("uint16"),
39        get_dtype_check<std::int32_t>("int32"),
40        get_dtype_check<std::uint32_t>("uint32"),
41        get_dtype_check<std::int64_t>("int64"),
42        get_dtype_check<std::uint64_t>("uint64")
43    };
44}
45
46struct DtypeSizeCheck {
47    std::string name{};
48    int size_cpp{};
49    int size_numpy{};
50    // For debugging.
51    py::dtype dtype{};
52};
53
54template <typename T>
55DtypeSizeCheck get_dtype_size_check() {
56    DtypeSizeCheck check{};
57    check.name = py::type_id<T>();
58    check.size_cpp = sizeof(T);
59    check.dtype = py::dtype::of<T>();
60    check.size_numpy = check.dtype.attr("itemsize").template cast<int>();
61    return check;
62}
63
64std::vector<DtypeSizeCheck> get_platform_dtype_size_checks() {
65    return {
66        get_dtype_size_check<short>(),
67        get_dtype_size_check<unsigned short>(),
68        get_dtype_size_check<int>(),
69        get_dtype_size_check<unsigned int>(),
70        get_dtype_size_check<long>(),
71        get_dtype_size_check<unsigned long>(),
72        get_dtype_size_check<long long>(),
73        get_dtype_size_check<unsigned long long>(),
74    };
75}
76
77// Arrays.
78using arr = py::array;
79using arr_t = py::array_t<uint16_t, 0>;
80static_assert(std::is_same<arr_t::value_type, uint16_t>::value, "");
81
82template<typename... Ix> arr data(const arr& a, Ix... index) {
83    return arr(a.nbytes() - a.offset_at(index...), (const uint8_t *) a.data(index...));
84}
85
86template<typename... Ix> arr data_t(const arr_t& a, Ix... index) {
87    return arr(a.size() - a.index_at(index...), a.data(index...));
88}
89
90template<typename... Ix> arr& mutate_data(arr& a, Ix... index) {
91    auto ptr = (uint8_t *) a.mutable_data(index...);
92    for (ssize_t i = 0; i < a.nbytes() - a.offset_at(index...); i++)
93        ptr[i] = (uint8_t) (ptr[i] * 2);
94    return a;
95}
96
97template<typename... Ix> arr_t& mutate_data_t(arr_t& a, Ix... index) {
98    auto ptr = a.mutable_data(index...);
99    for (ssize_t i = 0; i < a.size() - a.index_at(index...); i++)
100        ptr[i]++;
101    return a;
102}
103
104template<typename... Ix> ssize_t index_at(const arr& a, Ix... idx) { return a.index_at(idx...); }
105template<typename... Ix> ssize_t index_at_t(const arr_t& a, Ix... idx) { return a.index_at(idx...); }
106template<typename... Ix> ssize_t offset_at(const arr& a, Ix... idx) { return a.offset_at(idx...); }
107template<typename... Ix> ssize_t offset_at_t(const arr_t& a, Ix... idx) { return a.offset_at(idx...); }
108template<typename... Ix> ssize_t at_t(const arr_t& a, Ix... idx) { return a.at(idx...); }
109template<typename... Ix> arr_t& mutate_at_t(arr_t& a, Ix... idx) { a.mutable_at(idx...)++; return a; }
110
111#define def_index_fn(name, type) \
112    sm.def(#name, [](type a) { return name(a); }); \
113    sm.def(#name, [](type a, int i) { return name(a, i); }); \
114    sm.def(#name, [](type a, int i, int j) { return name(a, i, j); }); \
115    sm.def(#name, [](type a, int i, int j, int k) { return name(a, i, j, k); });
116
117template <typename T, typename T2> py::handle auxiliaries(T &&r, T2 &&r2) {
118    if (r.ndim() != 2) throw std::domain_error("error: ndim != 2");
119    py::list l;
120    l.append(*r.data(0, 0));
121    l.append(*r2.mutable_data(0, 0));
122    l.append(r.data(0, 1) == r2.mutable_data(0, 1));
123    l.append(r.ndim());
124    l.append(r.itemsize());
125    l.append(r.shape(0));
126    l.append(r.shape(1));
127    l.append(r.size());
128    l.append(r.nbytes());
129    return l.release();
130}
131
132// note: declaration at local scope would create a dangling reference!
133static int data_i = 42;
134
135TEST_SUBMODULE(numpy_array, sm) {
136    try { py::module::import("numpy"); }
137    catch (...) { return; }
138
139    // test_dtypes
140    py::class_<DtypeCheck>(sm, "DtypeCheck")
141        .def_readonly("numpy", &DtypeCheck::numpy)
142        .def_readonly("pybind11", &DtypeCheck::pybind11)
143        .def("__repr__", [](const DtypeCheck& self) {
144            return py::str("<DtypeCheck numpy={} pybind11={}>").format(
145                self.numpy, self.pybind11);
146        });
147    sm.def("get_concrete_dtype_checks", &get_concrete_dtype_checks);
148
149    py::class_<DtypeSizeCheck>(sm, "DtypeSizeCheck")
150        .def_readonly("name", &DtypeSizeCheck::name)
151        .def_readonly("size_cpp", &DtypeSizeCheck::size_cpp)
152        .def_readonly("size_numpy", &DtypeSizeCheck::size_numpy)
153        .def("__repr__", [](const DtypeSizeCheck& self) {
154            return py::str("<DtypeSizeCheck name='{}' size_cpp={} size_numpy={} dtype={}>").format(
155                self.name, self.size_cpp, self.size_numpy, self.dtype);
156        });
157    sm.def("get_platform_dtype_size_checks", &get_platform_dtype_size_checks);
158
159    // test_array_attributes
160    sm.def("ndim", [](const arr& a) { return a.ndim(); });
161    sm.def("shape", [](const arr& a) { return arr(a.ndim(), a.shape()); });
162    sm.def("shape", [](const arr& a, ssize_t dim) { return a.shape(dim); });
163    sm.def("strides", [](const arr& a) { return arr(a.ndim(), a.strides()); });
164    sm.def("strides", [](const arr& a, ssize_t dim) { return a.strides(dim); });
165    sm.def("writeable", [](const arr& a) { return a.writeable(); });
166    sm.def("size", [](const arr& a) { return a.size(); });
167    sm.def("itemsize", [](const arr& a) { return a.itemsize(); });
168    sm.def("nbytes", [](const arr& a) { return a.nbytes(); });
169    sm.def("owndata", [](const arr& a) { return a.owndata(); });
170
171    // test_index_offset
172    def_index_fn(index_at, const arr&);
173    def_index_fn(index_at_t, const arr_t&);
174    def_index_fn(offset_at, const arr&);
175    def_index_fn(offset_at_t, const arr_t&);
176    // test_data
177    def_index_fn(data, const arr&);
178    def_index_fn(data_t, const arr_t&);
179    // test_mutate_data, test_mutate_readonly
180    def_index_fn(mutate_data, arr&);
181    def_index_fn(mutate_data_t, arr_t&);
182    def_index_fn(at_t, const arr_t&);
183    def_index_fn(mutate_at_t, arr_t&);
184
185    // test_make_c_f_array
186    sm.def("make_f_array", [] { return py::array_t<float>({ 2, 2 }, { 4, 8 }); });
187    sm.def("make_c_array", [] { return py::array_t<float>({ 2, 2 }, { 8, 4 }); });
188
189    // test_empty_shaped_array
190    sm.def("make_empty_shaped_array", [] { return py::array(py::dtype("f"), {}, {}); });
191    // test numpy scalars (empty shape, ndim==0)
192    sm.def("scalar_int", []() { return py::array(py::dtype("i"), {}, {}, &data_i); });
193
194    // test_wrap
195    sm.def("wrap", [](py::array a) {
196        return py::array(
197            a.dtype(),
198            {a.shape(), a.shape() + a.ndim()},
199            {a.strides(), a.strides() + a.ndim()},
200            a.data(),
201            a
202        );
203    });
204
205    // test_numpy_view
206    struct ArrayClass {
207        int data[2] = { 1, 2 };
208        ArrayClass() { py::print("ArrayClass()"); }
209        ~ArrayClass() { py::print("~ArrayClass()"); }
210    };
211    py::class_<ArrayClass>(sm, "ArrayClass")
212        .def(py::init<>())
213        .def("numpy_view", [](py::object &obj) {
214            py::print("ArrayClass::numpy_view()");
215            ArrayClass &a = obj.cast<ArrayClass&>();
216            return py::array_t<int>({2}, {4}, a.data, obj);
217        }
218    );
219
220    // test_cast_numpy_int64_to_uint64
221    sm.def("function_taking_uint64", [](uint64_t) { });
222
223    // test_isinstance
224    sm.def("isinstance_untyped", [](py::object yes, py::object no) {
225        return py::isinstance<py::array>(yes) && !py::isinstance<py::array>(no);
226    });
227    sm.def("isinstance_typed", [](py::object o) {
228        return py::isinstance<py::array_t<double>>(o) && !py::isinstance<py::array_t<int>>(o);
229    });
230
231    // test_constructors
232    sm.def("default_constructors", []() {
233        return py::dict(
234            "array"_a=py::array(),
235            "array_t<int32>"_a=py::array_t<std::int32_t>(),
236            "array_t<double>"_a=py::array_t<double>()
237        );
238    });
239    sm.def("converting_constructors", [](py::object o) {
240        return py::dict(
241            "array"_a=py::array(o),
242            "array_t<int32>"_a=py::array_t<std::int32_t>(o),
243            "array_t<double>"_a=py::array_t<double>(o)
244        );
245    });
246
247    // test_overload_resolution
248    sm.def("overloaded", [](py::array_t<double>) { return "double"; });
249    sm.def("overloaded", [](py::array_t<float>) { return "float"; });
250    sm.def("overloaded", [](py::array_t<int>) { return "int"; });
251    sm.def("overloaded", [](py::array_t<unsigned short>) { return "unsigned short"; });
252    sm.def("overloaded", [](py::array_t<long long>) { return "long long"; });
253    sm.def("overloaded", [](py::array_t<std::complex<double>>) { return "double complex"; });
254    sm.def("overloaded", [](py::array_t<std::complex<float>>) { return "float complex"; });
255
256    sm.def("overloaded2", [](py::array_t<std::complex<double>>) { return "double complex"; });
257    sm.def("overloaded2", [](py::array_t<double>) { return "double"; });
258    sm.def("overloaded2", [](py::array_t<std::complex<float>>) { return "float complex"; });
259    sm.def("overloaded2", [](py::array_t<float>) { return "float"; });
260
261    // Only accept the exact types:
262    sm.def("overloaded3", [](py::array_t<int>) { return "int"; }, py::arg().noconvert());
263    sm.def("overloaded3", [](py::array_t<double>) { return "double"; }, py::arg().noconvert());
264
265    // Make sure we don't do unsafe coercion (e.g. float to int) when not using forcecast, but
266    // rather that float gets converted via the safe (conversion to double) overload:
267    sm.def("overloaded4", [](py::array_t<long long, 0>) { return "long long"; });
268    sm.def("overloaded4", [](py::array_t<double, 0>) { return "double"; });
269
270    // But we do allow conversion to int if forcecast is enabled (but only if no overload matches
271    // without conversion)
272    sm.def("overloaded5", [](py::array_t<unsigned int>) { return "unsigned int"; });
273    sm.def("overloaded5", [](py::array_t<double>) { return "double"; });
274
275    // test_greedy_string_overload
276    // Issue 685: ndarray shouldn't go to std::string overload
277    sm.def("issue685", [](std::string) { return "string"; });
278    sm.def("issue685", [](py::array) { return "array"; });
279    sm.def("issue685", [](py::object) { return "other"; });
280
281    // test_array_unchecked_fixed_dims
282    sm.def("proxy_add2", [](py::array_t<double> a, double v) {
283        auto r = a.mutable_unchecked<2>();
284        for (ssize_t i = 0; i < r.shape(0); i++)
285            for (ssize_t j = 0; j < r.shape(1); j++)
286                r(i, j) += v;
287    }, py::arg().noconvert(), py::arg());
288
289    sm.def("proxy_init3", [](double start) {
290        py::array_t<double, py::array::c_style> a({ 3, 3, 3 });
291        auto r = a.mutable_unchecked<3>();
292        for (ssize_t i = 0; i < r.shape(0); i++)
293        for (ssize_t j = 0; j < r.shape(1); j++)
294        for (ssize_t k = 0; k < r.shape(2); k++)
295            r(i, j, k) = start++;
296        return a;
297    });
298    sm.def("proxy_init3F", [](double start) {
299        py::array_t<double, py::array::f_style> a({ 3, 3, 3 });
300        auto r = a.mutable_unchecked<3>();
301        for (ssize_t k = 0; k < r.shape(2); k++)
302        for (ssize_t j = 0; j < r.shape(1); j++)
303        for (ssize_t i = 0; i < r.shape(0); i++)
304            r(i, j, k) = start++;
305        return a;
306    });
307    sm.def("proxy_squared_L2_norm", [](py::array_t<double> a) {
308        auto r = a.unchecked<1>();
309        double sumsq = 0;
310        for (ssize_t i = 0; i < r.shape(0); i++)
311            sumsq += r[i] * r(i); // Either notation works for a 1D array
312        return sumsq;
313    });
314
315    sm.def("proxy_auxiliaries2", [](py::array_t<double> a) {
316        auto r = a.unchecked<2>();
317        auto r2 = a.mutable_unchecked<2>();
318        return auxiliaries(r, r2);
319    });
320
321    // test_array_unchecked_dyn_dims
322    // Same as the above, but without a compile-time dimensions specification:
323    sm.def("proxy_add2_dyn", [](py::array_t<double> a, double v) {
324        auto r = a.mutable_unchecked();
325        if (r.ndim() != 2) throw std::domain_error("error: ndim != 2");
326        for (ssize_t i = 0; i < r.shape(0); i++)
327            for (ssize_t j = 0; j < r.shape(1); j++)
328                r(i, j) += v;
329    }, py::arg().noconvert(), py::arg());
330    sm.def("proxy_init3_dyn", [](double start) {
331        py::array_t<double, py::array::c_style> a({ 3, 3, 3 });
332        auto r = a.mutable_unchecked();
333        if (r.ndim() != 3) throw std::domain_error("error: ndim != 3");
334        for (ssize_t i = 0; i < r.shape(0); i++)
335        for (ssize_t j = 0; j < r.shape(1); j++)
336        for (ssize_t k = 0; k < r.shape(2); k++)
337            r(i, j, k) = start++;
338        return a;
339    });
340    sm.def("proxy_auxiliaries2_dyn", [](py::array_t<double> a) {
341        return auxiliaries(a.unchecked(), a.mutable_unchecked());
342    });
343
344    sm.def("array_auxiliaries2", [](py::array_t<double> a) {
345        return auxiliaries(a, a);
346    });
347
348    // test_array_failures
349    // Issue #785: Uninformative "Unknown internal error" exception when constructing array from empty object:
350    sm.def("array_fail_test", []() { return py::array(py::object()); });
351    sm.def("array_t_fail_test", []() { return py::array_t<double>(py::object()); });
352    // Make sure the error from numpy is being passed through:
353    sm.def("array_fail_test_negative_size", []() { int c = 0; return py::array(-1, &c); });
354
355    // test_initializer_list
356    // Issue (unnumbered; reported in #788): regression: initializer lists can be ambiguous
357    sm.def("array_initializer_list1", []() { return py::array_t<float>(1); }); // { 1 } also works, but clang warns about it
358    sm.def("array_initializer_list2", []() { return py::array_t<float>({ 1, 2 }); });
359    sm.def("array_initializer_list3", []() { return py::array_t<float>({ 1, 2, 3 }); });
360    sm.def("array_initializer_list4", []() { return py::array_t<float>({ 1, 2, 3, 4 }); });
361
362    // test_array_resize
363    // reshape array to 2D without changing size
364    sm.def("array_reshape2", [](py::array_t<double> a) {
365        const ssize_t dim_sz = (ssize_t)std::sqrt(a.size());
366        if (dim_sz * dim_sz != a.size())
367            throw std::domain_error("array_reshape2: input array total size is not a squared integer");
368        a.resize({dim_sz, dim_sz});
369    });
370
371    // resize to 3D array with each dimension = N
372    sm.def("array_resize3", [](py::array_t<double> a, size_t N, bool refcheck) {
373        a.resize({N, N, N}, refcheck);
374    });
375
376    // test_array_create_and_resize
377    // return 2D array with Nrows = Ncols = N
378    sm.def("create_and_resize", [](size_t N) {
379        py::array_t<double> a;
380        a.resize({N, N});
381        std::fill(a.mutable_data(), a.mutable_data() + a.size(), 42.);
382        return a;
383    });
384
385#if PY_MAJOR_VERSION >= 3
386        sm.def("index_using_ellipsis", [](py::array a) {
387            return a[py::make_tuple(0, py::ellipsis(), 0)];
388        });
389#endif
390}
391