From bf84cb7d1e291c98895283bacce38f181d06b1c4 Mon Sep 17 00:00:00 2001 From: Joel Mason Date: Fri, 7 Dec 2018 03:03:37 +1100 Subject: [PATCH 1/4] Use native sizes by default when parsing buffer format string --- src/pybuffer.jl | 16 ++++++++++------ test/testpybuffer.jl | 8 ++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/pybuffer.jl b/src/pybuffer.jl index 5a25805b..ebda6f64 100644 --- a/src/pybuffer.jl +++ b/src/pybuffer.jl @@ -227,31 +227,35 @@ function array_format(pybuf::PyBuffer) # TODO: handle more cases: https://www.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax # refs: https://github.com/numpy/numpy/blob/v1.14.2/numpy/core/src/multiarray/buffer.c#L966 # https://github.com/numpy/numpy/blob/v1.14.2/numpy/core/_internal.py#L490 - # https://docs.python.org/2/library/struct.html#byte-order-size-and-alignment + # https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment # "NULL implies standard unsigned bytes ("B")" --pep 3118 pybuf.buf.format == C_NULL && return UInt8, true fmt_str = get_format_str(pybuf) native_byteorder = true + use_native_sizes = true type_start_idx = 1 - typestrs = standard_typestrs if length(fmt_str) > 1 type_start_idx = 2 - if fmt_str[1] == '=' + if fmt_str[1] == '@' || fmt_str[1] == '^' + # defaults to native_byteorder: true, is_standard_size: false elseif fmt_str[1] == '<' native_byteorder = ENDIAN_BOM == 0x04030201 + use_native_sizes = false elseif fmt_str[1] == '>' || fmt_str =='!' native_byteorder = ENDIAN_BOM == 0x01020304 - elseif fmt_str[1] == '@' || fmt_str[1] == '^' - typestrs = native_typestrs + use_native_sizes = false + elseif fmt_str[1] == '=' + use_native_sizes = false elseif fmt_str[1] == "Z" type_start_idx = 1 else error("Unsupported format string: \"$fmt_str\"") end end - typestrs[fmt_str[type_start_idx:end]], native_byteorder + strs2types = use_native_sizes ? native_typestrs : standard_typestrs + strs2types[fmt_str[type_start_idx:end]], native_byteorder end ############################################################################# diff --git a/test/testpybuffer.jl b/test/testpybuffer.jl index 8d735a05..233581f4 100644 --- a/test/testpybuffer.jl +++ b/test/testpybuffer.jl @@ -35,6 +35,14 @@ pyutf8(s::String) = pyutf8(PyObject(s)) @test_throws ArgumentError PyArray(wrong_endian_arr) end + @testset "Get the right (native) dtype" begin + nparr = arrpyo(1:10) + npintstr = Int == Int64 ? "int64" : "int32" + + # make sure we're dealing with an Int array + @test pystr(nparr["dtype"]) == npintstr + end + @testset "NoCopyArray 1d" begin ao = arrpyo(1.0:10.0, "d") pybuf = PyBuffer(ao, PyBUF_ND_CONTIGUOUS) From c6b89aea0c8bf1d23b53867540b5a7123ea5facd Mon Sep 17 00:00:00 2001 From: Joel Mason Date: Fri, 7 Dec 2018 04:48:04 +1100 Subject: [PATCH 2/4] Fix a comment --- src/pybuffer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybuffer.jl b/src/pybuffer.jl index ebda6f64..125504b5 100644 --- a/src/pybuffer.jl +++ b/src/pybuffer.jl @@ -239,7 +239,7 @@ function array_format(pybuf::PyBuffer) if length(fmt_str) > 1 type_start_idx = 2 if fmt_str[1] == '@' || fmt_str[1] == '^' - # defaults to native_byteorder: true, is_standard_size: false + # defaults to native_byteorder: true, use_native_sizes: true elseif fmt_str[1] == '<' native_byteorder = ENDIAN_BOM == 0x04030201 use_native_sizes = false From 8176744d8389406c579ebff3617c6210a5aaf4f9 Mon Sep 17 00:00:00 2001 From: Joel Mason Date: Sun, 9 Dec 2018 22:40:52 +1100 Subject: [PATCH 3/4] Slightly better test --- test/testpybuffer.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/testpybuffer.jl b/test/testpybuffer.jl index 233581f4..0162a940 100644 --- a/test/testpybuffer.jl +++ b/test/testpybuffer.jl @@ -35,12 +35,12 @@ pyutf8(s::String) = pyutf8(PyObject(s)) @test_throws ArgumentError PyArray(wrong_endian_arr) end - @testset "Get the right (native) dtype" begin + @testset "dtype should match eltype" begin + npy2jl = Dict(np["int64"][:__name__]=>Int64, + np["int32"][:__name__]=>Int32) nparr = arrpyo(1:10) - npintstr = Int == Int64 ? "int64" : "int32" - - # make sure we're dealing with an Int array - @test pystr(nparr["dtype"]) == npintstr + jltype = npy2jl[pystr(nparr["dtype"])] + @test eltype(convert(PyAny, nparr)) == jltype end @testset "NoCopyArray 1d" begin From 8b1d5ec695fad4cfcf5b803b94a89be392f12a53 Mon Sep 17 00:00:00 2001 From: Joel Mason Date: Mon, 10 Dec 2018 20:41:00 +1100 Subject: [PATCH 4/4] Better tests --- test/testpybuffer.jl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/testpybuffer.jl b/test/testpybuffer.jl index 0162a940..f471c371 100644 --- a/test/testpybuffer.jl +++ b/test/testpybuffer.jl @@ -36,11 +36,18 @@ pyutf8(s::String) = pyutf8(PyObject(s)) end @testset "dtype should match eltype" begin - npy2jl = Dict(np["int64"][:__name__]=>Int64, - np["int32"][:__name__]=>Int32) - nparr = arrpyo(1:10) - jltype = npy2jl[pystr(nparr["dtype"])] - @test eltype(convert(PyAny, nparr)) == jltype + npy2jl = Dict("int32"=>Int32, "float32"=>Float32, + "int64"=>Int64, "float64"=>Float64) + for (pytype, jltype) in npy2jl + @testset "dtype $pytype should match eltype $jltype" begin + jlarr = jltype[1:10;] + nparr = arrpyo(jlarr, dtype=pytype) + @test pystr(nparr["dtype"]) == pytype + jlarr2 = convert(PyAny, nparr) + @test eltype(jlarr2) == jltype + @test jlarr2 == jlarr + end + end end @testset "NoCopyArray 1d" begin