From fba7b54413b414dc09348016ebf1f95ab11e1a11 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Sat, 22 Mar 2014 15:15:49 -0400 Subject: [PATCH 1/5] initial python buffer implementation --- .travis.yml | 3 +- README.md | 2 +- src/PyCall.jl | 2 + src/buffer.jl | 427 ++++++++++++++++++++++++++++++++++++++++++++ src/conversions.jl | 2 +- src/numpy.jl | 3 +- test/test_buffer.jl | 228 +++++++++++++++++++++++ 7 files changed, 663 insertions(+), 4 deletions(-) create mode 100644 src/buffer.jl create mode 100644 test/test_buffer.jl diff --git a/.travis.yml b/.travis.yml index 70e21f42..e08f726e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: python python: - - 2.6 - 2.7 - 3.2 - 3.3 + - 3.4 compiler: - clang notifications: @@ -18,3 +18,4 @@ script: - julia -e 'Pkg.init(); run(`ln -s $(pwd()) $(Pkg.dir())/PyCall`); Pkg.resolve()' - julia -e 'using PyCall; @assert isdefined(:PyCall); @assert typeof(PyCall) === Module' - julia test/test.jl + - julia test/test_buffer.jl diff --git a/README.md b/README.md index 0876cc1b..effc41ef 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Calling Python functions from the Julia language -[![Build Status](https://travis-ci.org/stevengj/PyCall.jl.png)](https://travis-ci.org/stevengj/PyCall.jl) +[![Build Status](https://travis-ci.org/jakebolewski/PyCall.jl.svg)](https://travis-ci.org/jakebolewski/PyCall.jl) This package provides a `@pyimport` macro that mimics a Python `import` statement: it imports a Python module and provides Julia diff --git a/src/PyCall.jl b/src/PyCall.jl index 108d61a0..694e091b 100644 --- a/src/PyCall.jl +++ b/src/PyCall.jl @@ -515,6 +515,8 @@ PyObject(o::PyPtr, keep::Any) = pyembed(PyObject(o), keep) ######################################################################### +include("buffer.jl") + include("conversions.jl") include("pytype.jl") diff --git a/src/buffer.jl b/src/buffer.jl new file mode 100644 index 00000000..518f0512 --- /dev/null +++ b/src/buffer.jl @@ -0,0 +1,427 @@ +# Support for python buffer protocol buffers + +# for versions > 3.2 (TODO: obj field not in struct for python versions <= 3.2) + +type PyBuffer + buf::Ptr{Void} + obj::PyPtr + len::Cssize_t + itemsize::Int + + readonly::Cint + ndim::Cint + format::Ptr{Cchar} + shape::Ptr{Int} + strides::Ptr{Int} + suboffsets::Ptr{Int} + internal::Ptr{Void} + + PyBuffer() = begin + b = new(C_NULL, C_NULL, 0, 0, + 0, 0, C_NULL, C_NULL, + C_NULL, C_NULL, C_NULL) + finalizer(b, release!) + return b + end +end + +release!(b::PyBuffer) = begin + if b.obj != C_NULL + ccall((@pysym :PyBuffer_Release), Void, + (Ptr{PyBuffer},), &b) + end +end + +Base.ndims(b::PyBuffer) = int(b.ndim) +Base.length(b::PyBuffer) = b.ndim >= 1 ? div(b.len, b.itemsize) : 0 +Base.sizeof(b::PyBuffer) = b.len + +Base.size(b::PyBuffer) = begin + if b.ndim == 0 + return (0,) + end + if b.ndim == 1 + return (div(b.len, b.itemsize),) + end + @assert b.shape != C_NULL + return tuple(Int[unsafe_load(b.shape, i) for i=1:b.ndim]...) +end + +Base.strides(b::PyBuffer) = begin + if b.ndim == 0 || b.ndim == 1 + return (1,) + end + @assert b.strides != C_NULL + return tuple(Int[div(unsafe_load(b.strides, i), b.itemsize) for i=1:b.ndim]...) +end + +######################################################################### +# hard coded constant values, copied from + +cint(x) = convert(Cint, x) + +const PyBUF_MAX_NDIM = cint(64) + +const PyBUF_SIMPLE = cint(0) +const PyBUF_WRITABLE = cint(0x0001) +const PyBUF_WRITEABLE = PyBUF_WRITABLE +const PyBUF_FORMAT = cint(0x0004) +const PyBUF_ND = cint(0x0008) + +const PyBUF_STRIDES = cint(0x0010) | PyBUF_ND +const PyBUF_C_CONTIGUOUS = cint(0x0020) | PyBUF_STRIDES +const PyBUF_F_CONTIGUOUS = cint(0x0040) | PyBUF_STRIDES +const PyBUF_ANY_CONTIGUOUS = cint(0x0080) | PyBUF_STRIDES +const PyBUF_INDIRECT = cint(0x0100) | PyBUF_STRIDES + +const PyBUF_CONTIG = cint(PyBUF_ND | PyBUF_WRITABLE) +const PyBUF_CONTIG_RO = cint(PyBUF_ND) + +const PyBUF_STRIDED = cint(PyBUF_STRIDES | PyBUF_WRITABLE) +const PyBUF_STRIDED_RO = cint(PyBUF_STRIDES) + +const PyBUF_RECORDS = cint(PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT) +const PyBUF_RECORDS_RO = cint(PyBUF_STRIDES | PyBUF_FORMAT) + +const PyBUF_FULL = cint(PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT) +const PyBUF_FULL_RO = cint(PyBUF_INDIRECT | PyBUF_FORMAT) + +const PyBUF_READ = cint(0x100) +const PyBUF_WRITE = cint(0x200) + +######################################################## +# parse python's buffer format string +# PEP 3118 format specification +# 't' bit, '4b' specifies # of bits +# '?' platform bool type +# 'g' long double +# 'c' ucs-1 encoding +# 'u' ucs-2 encoding +# 'w' ucs-4 encoding +# 'O' pointer to python object +# 'Z' complex number, Zf (complex float) +# '&' specific pointer, prefix before another character (Ptr{Int8} => '&c') +# 'T{}' structure +# '(k1,k2,...kn) multidim array of whatever follows +# ':name' optional name of preceeding element +# 'X{}' pointer to function (optional function signature placed inside with +# return value preceeded by -> @ end) +# X{b:b}->b + +const pyfmt_byteorder = (Char => Symbol)['@' => :native, + '=' => :native, + '<' => :little, + '>' => :big, + '!' => :big] + +const pyfmt_jltype = (Char => Type)['x' => Uint8, + 'c' => Cchar, + 'b' => Cuchar, + 'B' => Uint8, + '?' => Bool, + 'h' => Cshort, + 'H' => Cushort, + 'i' => Cint, + 'I' => Cuint, + 'l' => Clong, + 'L' => Culong, + 'q' => Clonglong, + 'Q' => Culonglong, + 'n' => Cssize_t, + 'N' => Csize_t, + 'f' => Float32, + 'd' => Float64, + 's' => Ptr{Cchar}, + 'p' => Ptr{Cchar}, + 'P' => Ptr{Void}] + +# for now, heterogenous arrays of a single numeric type +const PyBufType = Union(values(pyfmt_jltype)...) + +const jltype_pyfmt = Dict{Type, Char}(collect(values(pyfmt_jltype)), + collect(keys(pyfmt_jltype))) + +#TODO: this only works for simple struct packing +function parse_pyfmt(fmt::ASCIIString) + types = Type[] + idx = 1 + byteorder = :native + if haskey(pyfmt_byteorder, fmt[idx]) + byteorder = pyfmt_byteorder[fmt[idx]] + idx = 2 + end + len = length(fmt) + while idx <= len + c = fmt[idx] + if isblank(c) + idx += 1 + continue + end + num = 1 + # we punt on overflow checking here for now (num >= C_ssize_t) + if '0' <= c && c <= '9' + num = c - '0' + idx += 1 + if idx > len + return (byteorder, types) + end + c = fmt[idx] + while '0' <= c && c <= '9' + num = num * 10 + (c - '0') + idx += 1 + if idx > len + return (byteorder, types) + end + c = fmt[idx] + end + end + try + ty = pyfmt_jltype[c] + for _ = 1:num + push!(types, ty) + end + catch + throw(ArgumentError("invalid PyBuffer format string $fmt")) + end + idx += 1 + end + return (byteorder, types) +end + +jltype_to_pyfmt{T}(::Type{T}) = jltype_to_pyfmt(IOBuffer(), T) + +function jltype_to_pyfmt{T}(io::IO, ::Type{T}) + length(T.names) == 0 && error("no fields for structure type $T") + write(io, "T{") + idx = 1 + for n in T.names + ty = T.types[idx] + if isbits(ty) + if haskey(jltype_pyfmt, ty) + fty = jltype_pyfmt[ty] + write(io, "$fty:$n:") + elseif Base.isstructtype(T) + jltype_to_pyfmt(io, ty) + else + error("pyfmt unknown conversion for type $T") + end + else + error("pyfmt can only encode bits types") + end + idx += 1 + end + write(io, "}") + return bytestring(io) +end + +pyfmt(b::PyBuffer) = b.format == C_NULL ? bytestring("") : bytestring(b.format) + +sizeof_pyfmt(fmt::ByteString) = ccall((@pysym :PyBuffer_SizeFromFormat), Cint, + (Ptr{Cchar},), &fmt) + +######################################################################### +# TODO: this is the old interface, the new PyObject_CheckBuffer +# is a macro so we cannot link to it +pycheckbuffer(o::PyObject) = ccall((@pysym :PyObject_CheckReadBuffer), Cint, + (PyPtr,), o.o) == cint(1) + +pygetbuffer(o::PyObject, flags::Cint) = begin + view = PyBuffer() + @pycheckzi ccall((@pysym :PyObject_GetBuffer), Cint, + (PyPtr, Ptr{PyBuffer}, Cint), + o.o, &view, flags) + return view +end + +aligned(b::PyBuffer) = begin + if b.strides == C_NULL + throw(ArgumentError("PyBuffer strides field is NULL")) + end + for i=1:b.ndim + if mod(unsafe_load(b.strides, i), b.itemsize) != 0 + return false + end + end + return true +end + +f_contiguous(view::PyBuffer) = + ccall((@pysym :PyBuffer_IsContiguous), Cint, + (Ptr{PyBuffer}, Cchar), &view, 'F') == cint(1) + +c_contiguous(view::PyBuffer) = + ccall((@pysym :PyBuffer_IsContiguous), Cint, + (Ptr{PyBuffer}, Cchar), &view, 'C') == cint(1) + +iscontiguous(view::PyBuffer) = + ccall((@pysym :PyBuffer_IsContiguous), Cint, + (Ptr{PyBuffer}, Cchar), &view, 'A') == cint(1) + +type PyArray{T, N} <: AbstractArray{T, N} + o::PyObject + buf::PyBuffer + native::Bool + readonly::Bool + dims::Dims + strides::Dims + f_contig::Bool + c_contig::Bool + data::Ptr{T} + + function PyArray(o::PyObject, b::PyBuffer) + if !aligned(b) + throw(ArgumentError("only aligned buffer objects are supported")) + throw(ArgumentError("inconsistent type in PyArray constructor")) + elseif ndims(b) != N + throw(ArgumentError("inconsistent ndims in PyArray constructor")) + end + return new(o, b, true, bool(b.readonly), + size(b), strides(b), + f_contiguous(b), + c_contiguous(b), + convert(Ptr{T}, b.buf)) + end +end + +function PyArray(o::PyObject) + view = pygetbuffer(o, PyBUF_FULL) + view.format == C_NULL && error("buffer has no format string") + order, tys = parse_pyfmt(bytestring(view.format)) + length(tys) != 1 && error("PyArray cannot yet handle structure types") + ty = tys[1] + ndim = ndims(view) + return PyArray{ty, ndim}(o, view) +end + +Base.size(a::PyArray) = a.dims +Base.ndims{T,N}(a::PyArray{T,N}) = N +Base.similar(a::PyArray, T, dims::Dims) = Array(T, dims) +Base.stride(a::PyArray, i::Integer) = a.strides[i] +Base.convert{T}(::Type{Ptr{T}}, a::PyArray{T}) = a.data + +Base.summary{T}(a::PyArray{T}) = string(Base.dims2string(size(a)), " ", + string(T), " PyArray") + +#TODO: is this correct for all buffer types other than contig/dense? +Base.copy{T,N}(a::PyArray{T,N}) = begin + if N > 1 && a.c_contig # equivalent to f_contig with reversed dims + B = pointer_to_array(a.data, ntuple(N, n -> a.dims[N - n + 1])) + return N == 2 ? transpose(B) : permutedims(B, (N:-1:1)) + end + A = Array(T, a.dims) + if a.f_contig + ccall(:memcpy, Void, (Ptr{T}, Ptr{T}, Int), A, a, sizeof(T) * length(a)) + return A + else + return copy!(A, a) + end +end + +#TODO: Bounds checking is needed +Base.getindex{T}(a::PyArray{T,0}) = unsafe_load(a.data) + +Base.getindex{T}(a::PyArray{T,1}, i::Integer) = + unsafe_load(a.data, 1 + (i-1) * a.strides[1]) + +Base.getindex{T}(a::PyArray{T,2}, i::Integer, j::Integer) = + unsafe_load(a.data, 1 + (i-1) * a.strides[1] + (j-1) * a.strides[2]) + +Base.getindex(a::PyArray, i::Integer) = begin + if a.f_contig + return unsafe_load(a.data, i) + else + return a[ind2sub(a.dims, i)...] + end +end + +Base.getindex(a::PyArray, is::Integer...) = begin + index = 1 + n = min(length(is), length(a.strides)) + for i = 1:n + index += (is[i] - 1) * a.strides[i] + end + for i = n+1:length(is) + if is[i] != 1 + throw(BoundsError()) + end + end + return unsafe_load(a.data, index) +end + +#TODO: This is only correct for dense, contiguous buffers +Base.pointer{T}(a::PyArray{T}, is::(Int...)) = begin + offset = 0 + for i = 1:length(is) + offset += (is[i] - 1) * a.strides[i] + end + return a.data + (offset * sizeof(T)) +end + +function writeok_assign(a::PyArray, v, i::Integer) + a.readonly && throw(ArgumentError("read-only PyArray")) + unsafe_store!(a.data, v, i) + return a +end + +Base.setindex!{T}(a::PyArray{T,0}, v) = writeok_assign(a, v, 1) + +Base.setindex!{T}(a::PyArray{T,1}, v, i::Integer) = + writeok_assign(a, v, 1 + (i-1) * a.strides[1]) + +Base.setindex!{T}(a::PyArray{T,2}, v, i::Integer, j::Integer) = + writeok_assign(a, v, 1 + (i-1) * a.strides[1] + (j-1) * a.strides[2]) + +Base.setindex!(a::PyArray, v, i::Integer) = begin + if a.f_contig + return writeok_assign(a, v, i) + else + return setindex!(a, v, ind2sub(a.dims, i)...) + end +end + +Base.setindex!{T,N}(a::PyArray{T, N}, v, is::Integer...) = begin + index = 1 + n = min(length(is), N) + for i = 1:n + index += (is[i] - 1) * a.strides[i] + end + for i = n+1:length(is) + if is[i] != 1 + throw(BoundsError()) + end + end + return writeok_assign(a, v, index) +end + +######################################################################### +# PyArray <-> PyObject conversions + +PyObject(a::PyArray) = a.o + +Base.convert(::Type{PyArray}, o::PyObject) = PyArray(o) + +Base.convert{T<:PyBufType}(::Type{Array{T, 1}}, o::PyObject) = begin + try + view = pygetbuffer(o, PyBUF_FULL) + view.format == C_NULL && error("buffer has no format string") + order, tys = parse_pyfmt(bytestring(view.format)) + length(tys) != 1 && error("PyArray cannot yet handle structure types") + tys[1] != T && error("invalid type") + ndim = ndims(view) + ndim != 1 && error("invalid dim") + return copy(PyArray{T, 1}(o, view)) + catch + len = @pycheckzi ccall((@pysym :PySequence_Size), Int, (PyPtr,), o) + A = Array(pyany_toany(T), len) + return py2array(T, A, o, 1, 1) + end +end + +Base.convert(::Type{Array{PyObject}}, o::PyObject) = + map(pyincref, convert(Array{PyPtr}, o)) + +Base.convert(::Type{Array{PyObject,1}}, o::PyObject) = + map(pyincref, convert(Array{PyPtr, 1}, o)) + +Base.convert{N}(::Type{Array{PyObject,N}}, o::PyObject) = + map(pyincref, convert(Array{PyPtr, N}, o)) diff --git a/src/conversions.jl b/src/conversions.jl index 3c989989..9655b591 100644 --- a/src/conversions.jl +++ b/src/conversions.jl @@ -409,7 +409,7 @@ end convert(::Type{Array}, o::PyObject) = py2array(PyAny, o) # NumPy conversions (multidimensional arrays) -include("numpy.jl") +#include("numpy.jl") ######################################################################### # PyDict: no-copy wrapping of a Julia object around a Python dictionary diff --git a/src/numpy.jl b/src/numpy.jl index 54bc5418..253d6c85 100644 --- a/src/numpy.jl +++ b/src/numpy.jl @@ -207,6 +207,7 @@ type PyArray_Info readonly::Bool function PyArray_Info(a::PyObject) + # error("could not convert type $T to fmt string") ai = PyDict{String,PyObject}(a["__array_interface__"]) typestr = convert(String, ai["typestr"]) T = npy_typestrs[typestr[2:end]] @@ -276,7 +277,7 @@ type PyArray{T,N} <: AbstractArray{T,N} if !aligned(info) throw(ArgumentError("only NPY_ARRAY_ALIGNED arrays are supported")) elseif !info.native - throw(ArgumentError("only native byte-order arrays are supported")) + throw(ArgumentError("only native byte-order arrays are supported")) elseif info.T != T throw(ArgumentError("inconsistent type in PyArray constructor")) elseif length(info.sz) != N || length(info.st) != N diff --git a/test/test_buffer.jl b/test/test_buffer.jl new file mode 100644 index 00000000..1e539c92 --- /dev/null +++ b/test/test_buffer.jl @@ -0,0 +1,228 @@ +using Base.Test +using PyCall + +roundtrip(T, x) = convert(T, PyObject(x)) +roundtrip(x) = roundtrip(PyAny, x) +roundtripeq(T, x) = roundtrip(T, x) == x +roundtripeq(x) = roundtrip(x) == x + +@pyimport array +@pyimport numpy as np + +@test roundtripeq({1 2 3; 4 5 6}) +@test roundtripeq([]) +@test convert(Array{PyAny,1}, PyObject({1 2 3; 4 5 6})) == {{1,2,3},{4,5,6}} +@test roundtripeq(begin A = Array(Int, 3); A[1] = 1; A[2] = 2; A[3] = 3; A; end) +@test convert(PyAny, PyObject(begin A = Array(Any); A[1] = 3; A; end)) == 3 + +array2py2arrayeq(x) = PyCall.py2array(Float64, PyCall.array2py(x)) == x +@test array2py2arrayeq(rand(3)) +@test array2py2arrayeq(rand(3,4)) +@test array2py2arrayeq(rand(3,4,5)) + +############################################################# +# PyBuffer Tests + +@test PyCall.pycheckbuffer(array.array("f", [1.0, 2.0, 3.0])) == true +@test PyCall.pycheckbuffer(np.array([1,2,3])) == true + +# Check 1D arrays +a = array.array("f", [1,2,3]) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +@test ndims(view) == 1 +@test length(view) == 3 +@test sizeof(view) == sizeof(Float32) * 3 +@test size(view) == (3,) +@test strides(view) == (1,) +@test PyCall.aligned(view) == true +# a vector is both c/f contiguous +@test PyCall.c_contiguous(view) == true +@test PyCall.f_contiguous(view) == true +@test PyCall.pyfmt(view) == "f" + +# Check 1D numpy arrays +a = np.array([1.0,2.0,3.0]) +@test a[:dtype] == np.dtype("float64") +view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +@test ndims(view) == 1 +@test length(view) == 3 +@test sizeof(view) == sizeof(Float64) * 3 +@test size(view) == (3,) +@test strides(view) == (1,) +@test PyCall.aligned(view) == true +# a vector is both c/f contiguous +@test PyCall.c_contiguous(view) == true +@test PyCall.f_contiguous(view) == true +@test PyCall.pyfmt(view) == "d" + +# Check 2D C ordered arrays +a = np.array([[1 2 3], + [1 2 3]]) +@test a[:dtype] == np.dtype("int64") +view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +@test ndims(view) == 2 +@test length(view) == 6 +@test sizeof(view) == sizeof(Int64) * 6 +@test size(view) == (2,3) +@test strides(view) == (3,1) +@test PyCall.aligned(view) == true +@test PyCall.c_contiguous(view) == true +@test PyCall.f_contiguous(view) == false +@test PyCall.pyfmt(view) == "l" + +# Check Multi-D C ordered arrays +a = np.ones((10,5,3), dtype="float32", order="C") +@test a[:dtype] == np.dtype("float32") +view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +@test ndims(view) == 3 +@test length(view) == (10 * 5 * 3) +@test sizeof(view) == sizeof(Float32) * (10 * 5 * 3) +@test size(view) == (10, 5, 3) +@test strides(view) == (5 * 3, 3, 1) +@test PyCall.aligned(view) == true +@test PyCall.c_contiguous(view) == true +@test PyCall.f_contiguous(view) == false +@test PyCall.pyfmt(view) == "f" + +# Check Multi-D F ordered arrays +a = np.ones((10,5,3), dtype="uint8", order="F") +@test a[:dtype] == np.dtype("uint8") +view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +@test ndims(view) == 3 +@test length(view) == (10 * 5 * 3) +@test sizeof(view) == sizeof(Uint8) * (10 * 5 * 3) +@test size(view) == (10, 5, 3) +@test strides(view) == (1, 10, 50) +@test PyCall.aligned(view) == true +@test PyCall.c_contiguous(view) == false +@test PyCall.f_contiguous(view) == true +@test PyCall.pyfmt(view) == "B" + +################################################### +# PyArray Tests + +npa = np.array([1,2,3]) +a = PyArray(npa) + +@test size(a) == (3,) +@test ndims(a) == 1 +@test strides(a) == (1,) +@test size(similar(a)) == size(a) +@test eltype(similar(a)) == eltype(a) +@test summary(a) == "3-element Int64 PyArray" +@test sprint() do io; show(io, a); end == "[1,2,3]" + +a[1] = 3 +a[2] = 2 +a[3] = 1 +@test sprint() do io; show(io, npa); end == "PyObject array([3, 2, 1])" +aa = np.frombuffer(a) +@test np.shape(aa) == size(a) + +npa = np.ones((10,10), dtype="float32") +a = PyArray(npa) + +@test size(a) == (10,10) +@test ndims(a) == 2 +@test strides(a) == (10, 1) +@test size(similar(a)) == size(a) +@test eltype(similar(a)) == eltype(a) +@test summary(a) == "10x10 Float32 PyArray" +@test sprint() do io; show(io, a); end == sprint() do io + show(io, ones(Float32, (10,10))) + end +aa = np.frombuffer(a) +@test np.shape(aa) == size(a) + +npa = np.ones((10,10,20), dtype="float32") +a = PyArray(npa) +@test size(a) == (10,10,20) +@test strides(a) == (20 * 10, 20, 1) +@test ndims(a) == 3 +@test size(similar(a)) == size(a) +@test eltype(similar(a)) == eltype(a) +@test summary(a) == "10x10x20 Float32 PyArray" + +aa = np.frombuffer(a) +@test np.shape(aa) == size(a) + +# Test fail because of show uses unimplemented subarray methods +#@test sprint() do io; show(io, a); end == sprint() do io +# show(io, ones(Float32, (10,10,20))) +# end + +################################################### +# PyObject Conversion +npa = np.ones(10, dtype="float32") +@test PyObject(a) == a.o + +a = convert(PyArray, npa) +@test typeof(a) === PyArray{Float32,1} + +# PyArray conversion shares the underlying buffer +a[5] = 10 +@test pycall(npa["__getitem__"], Float64, 4) == 10.0 + +npa = np.ones(10, dtype="float32") +a = convert(Array{Float32,1}, npa) +@test typeof(a) === Array{Float32,1} + +# Array conversion should perform a copy +a[5] = 10 +@test pycall(npa["__getitem__"], Float64, 4) == 1.0 + +#TODO: Multi-D conversion +#npa = np.ones((10,10), dtype="float32") +#aa = convert(Array{Float32,2}, npa) +#@test typeof(aa) === Array{Float32,2} +#aa[5,5] = 10 +#@show pycall(npa["__getitem__"], PyCall.PyAny, (4,4)) + +################################################### +#TODO: record array support + +a = np.zeros((2,), dtype=("i4,f4,a10")) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +@test PyCall.pyfmt(view) == "T{=i:f0:f:f1:10s:f2:}" + +a = np.zeros(3, dtype="3int8, float32, (2,3)float64") +view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +@test PyCall.pyfmt(view) == "T{(3)b:f0:=f:f1:(2,3)d:f2:}" + +a = np.zeros(3, dtype={"names" => ["col1", "col2"], + "formats" => ["i4", "f4"]}) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +@test PyCall.pyfmt(view) == "T{i:col1:f:col2:}" + +immutable Test1 + a::Float32 + b::Float64 + c::Bool +end + +@test isbits(Test1) +@test PyCall.jltype_to_pyfmt(Test1) == "T{f:a:d:b:?:c:}" + +immutable Test2 + a::Float32 + b::Test1 +end + +@test isbits(Test2) +@test PyCall.jltype_to_pyfmt(Test2) == "T{f:a:T{f:a:d:b:?:c:}}" + +type Test3{T} + a::T + b::T +end +@test PyCall.jltype_to_pyfmt(Test3{Float32}) == "T{f:a:f:b:}" + +type Dummy end +type Test4 + a::Dummy +end +@test_throws PyCall.jltype_to_pyfmt(Test4) + +immutable Test5 +end +@test_throws PyCall.jltype_to_pyfmt(Test5) From f21dbc6263c14bb55b9d69949ed9e640483a15f5 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Tue, 6 May 2014 00:45:02 -0400 Subject: [PATCH 2/5] cleanup tests , change PyBuffer field types to C typealiases --- src/buffer.jl | 25 ++++++++++--------------- test/test.jl | 6 ------ test/test_buffer.jl | 18 +++++++++--------- 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/src/buffer.jl b/src/buffer.jl index 518f0512..c1497395 100644 --- a/src/buffer.jl +++ b/src/buffer.jl @@ -6,14 +6,14 @@ type PyBuffer buf::Ptr{Void} obj::PyPtr len::Cssize_t - itemsize::Int + itemsize::Cssize_t readonly::Cint ndim::Cint format::Ptr{Cchar} - shape::Ptr{Int} - strides::Ptr{Int} - suboffsets::Ptr{Int} + shape::Ptr{Cssize_t} + strides::Ptr{Cssize_t} + suboffsets::Ptr{Cssize_t} internal::Ptr{Void} PyBuffer() = begin @@ -27,8 +27,7 @@ end release!(b::PyBuffer) = begin if b.obj != C_NULL - ccall((@pysym :PyBuffer_Release), Void, - (Ptr{PyBuffer},), &b) + ccall((@pysym :PyBuffer_Release), Void, (Ptr{PyBuffer},), &b) end end @@ -56,7 +55,7 @@ Base.strides(b::PyBuffer) = begin end ######################################################################### -# hard coded constant values, copied from +# hard coded constant values, copied from Cpython's include/object.h cint(x) = convert(Cint, x) @@ -109,9 +108,9 @@ const PyBUF_WRITE = cint(0x200) # X{b:b}->b const pyfmt_byteorder = (Char => Symbol)['@' => :native, - '=' => :native, - '<' => :little, - '>' => :big, + '=' => :native, + '<' => :little, + '>' => :big, '!' => :big] const pyfmt_jltype = (Char => Type)['x' => Uint8, @@ -220,10 +219,6 @@ sizeof_pyfmt(fmt::ByteString) = ccall((@pysym :PyBuffer_SizeFromFormat), Cint, (Ptr{Cchar},), &fmt) ######################################################################### -# TODO: this is the old interface, the new PyObject_CheckBuffer -# is a macro so we cannot link to it -pycheckbuffer(o::PyObject) = ccall((@pysym :PyObject_CheckReadBuffer), Cint, - (PyPtr,), o.o) == cint(1) pygetbuffer(o::PyObject, flags::Cint) = begin view = PyBuffer() @@ -354,7 +349,7 @@ Base.pointer{T}(a::PyArray{T}, is::(Int...)) = begin for i = 1:length(is) offset += (is[i] - 1) * a.strides[i] end - return a.data + (offset * sizeof(T)) + return a.data + offset * sizeof(T) end function writeok_assign(a::PyArray, v, i::Integer) diff --git a/test/test.jl b/test/test.jl index f082b1c1..aee61a44 100644 --- a/test/test.jl +++ b/test/test.jl @@ -32,9 +32,6 @@ testkw(x; y=0) = x + 2*y @test roundtrip(testkw)(314157) == 314157 @test roundtrip(testkw)(314157, y=1) == 314159 -if PyCall.npy_initialized - @test PyArray(PyObject([1. 2 3;4 5 6])) == [1. 2 3;4 5 6] -end @test PyVector(PyObject([1,3.2,"hello",true])) == [1,3.2,"hello",true] @test PyDict(PyObject([1 => "hello", 2 => "goodbye"])) == [1 => "hello", 2 => "goodbye"] @@ -47,9 +44,6 @@ end @test roundtripeq({1 2 3; 4 5 6}) @test roundtripeq([]) @test convert(Array{PyAny,1}, PyObject({1 2 3; 4 5 6})) == {{1,2,3},{4,5,6}} -if PyCall.npy_initialized - @test roundtripeq(begin A = Array(Int); A[1] = 3; A; end) -end @test convert(PyAny, PyObject(begin A = Array(Any); A[1] = 3; A; end)) == 3 array2py2arrayeq(x) = PyCall.py2array(Float64,PyCall.array2py(x)) == x diff --git a/test/test_buffer.jl b/test/test_buffer.jl index 1e539c92..9b5eec46 100644 --- a/test/test_buffer.jl +++ b/test/test_buffer.jl @@ -9,6 +9,7 @@ roundtripeq(x) = roundtrip(x) == x @pyimport array @pyimport numpy as np +#= @test roundtripeq({1 2 3; 4 5 6}) @test roundtripeq([]) @test convert(Array{PyAny,1}, PyObject({1 2 3; 4 5 6})) == {{1,2,3},{4,5,6}} @@ -19,16 +20,14 @@ array2py2arrayeq(x) = PyCall.py2array(Float64, PyCall.array2py(x)) == x @test array2py2arrayeq(rand(3)) @test array2py2arrayeq(rand(3,4)) @test array2py2arrayeq(rand(3,4,5)) +=# ############################################################# # PyBuffer Tests -@test PyCall.pycheckbuffer(array.array("f", [1.0, 2.0, 3.0])) == true -@test PyCall.pycheckbuffer(np.array([1,2,3])) == true - # Check 1D arrays a = array.array("f", [1,2,3]) -view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) @test ndims(view) == 1 @test length(view) == 3 @test sizeof(view) == sizeof(Float32) * 3 @@ -38,12 +37,13 @@ view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) # a vector is both c/f contiguous @test PyCall.c_contiguous(view) == true @test PyCall.f_contiguous(view) == true +@show view.format, view.ndim, view.readonly @test PyCall.pyfmt(view) == "f" - +error("expected") # Check 1D numpy arrays a = np.array([1.0,2.0,3.0]) @test a[:dtype] == np.dtype("float64") -view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) @test ndims(view) == 1 @test length(view) == 3 @test sizeof(view) == sizeof(Float64) * 3 @@ -59,7 +59,7 @@ view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) a = np.array([[1 2 3], [1 2 3]]) @test a[:dtype] == np.dtype("int64") -view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) @test ndims(view) == 2 @test length(view) == 6 @test sizeof(view) == sizeof(Int64) * 6 @@ -73,7 +73,7 @@ view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) # Check Multi-D C ordered arrays a = np.ones((10,5,3), dtype="float32", order="C") @test a[:dtype] == np.dtype("float32") -view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) @test ndims(view) == 3 @test length(view) == (10 * 5 * 3) @test sizeof(view) == sizeof(Float32) * (10 * 5 * 3) @@ -87,7 +87,7 @@ view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) # Check Multi-D F ordered arrays a = np.ones((10,5,3), dtype="uint8", order="F") @test a[:dtype] == np.dtype("uint8") -view = PyCall.pygetbuffer(a, PyCall.PyBUF_FULL) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) @test ndims(view) == 3 @test length(view) == (10 * 5 * 3) @test sizeof(view) == sizeof(Uint8) * (10 * 5 * 3) From 44023786738b72c40461557491c75947be368363 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Tue, 6 May 2014 02:08:31 -0400 Subject: [PATCH 3/5] make tests pass again turn off np.frombuffer tests and switch to RECORDS buffer protocol support --- src/buffer.jl | 6 +++--- test/test_buffer.jl | 37 ++++++++++++++++++------------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/buffer.jl b/src/buffer.jl index c1497395..59da310a 100644 --- a/src/buffer.jl +++ b/src/buffer.jl @@ -141,7 +141,7 @@ const jltype_pyfmt = Dict{Type, Char}(collect(values(pyfmt_jltype)), collect(keys(pyfmt_jltype))) #TODO: this only works for simple struct packing -function parse_pyfmt(fmt::ASCIIString) +function parse_pyfmt(fmt::ByteString) types = Type[] idx = 1 byteorder = :native @@ -279,7 +279,7 @@ type PyArray{T, N} <: AbstractArray{T, N} end function PyArray(o::PyObject) - view = pygetbuffer(o, PyBUF_FULL) + view = pygetbuffer(o, PyBUF_RECORDS) view.format == C_NULL && error("buffer has no format string") order, tys = parse_pyfmt(bytestring(view.format)) length(tys) != 1 && error("PyArray cannot yet handle structure types") @@ -397,7 +397,7 @@ Base.convert(::Type{PyArray}, o::PyObject) = PyArray(o) Base.convert{T<:PyBufType}(::Type{Array{T, 1}}, o::PyObject) = begin try - view = pygetbuffer(o, PyBUF_FULL) + view = pygetbuffer(o, PyBUF_RECORDS) view.format == C_NULL && error("buffer has no format string") order, tys = parse_pyfmt(bytestring(view.format)) length(tys) != 1 && error("PyArray cannot yet handle structure types") diff --git a/test/test_buffer.jl b/test/test_buffer.jl index 9b5eec46..c281495f 100644 --- a/test/test_buffer.jl +++ b/test/test_buffer.jl @@ -9,7 +9,6 @@ roundtripeq(x) = roundtrip(x) == x @pyimport array @pyimport numpy as np -#= @test roundtripeq({1 2 3; 4 5 6}) @test roundtripeq([]) @test convert(Array{PyAny,1}, PyObject({1 2 3; 4 5 6})) == {{1,2,3},{4,5,6}} @@ -20,33 +19,32 @@ array2py2arrayeq(x) = PyCall.py2array(Float64, PyCall.array2py(x)) == x @test array2py2arrayeq(rand(3)) @test array2py2arrayeq(rand(3,4)) @test array2py2arrayeq(rand(3,4,5)) -=# ############################################################# # PyBuffer Tests # Check 1D arrays a = array.array("f", [1,2,3]) -view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_RECORDS) @test ndims(view) == 1 @test length(view) == 3 @test sizeof(view) == sizeof(Float32) * 3 @test size(view) == (3,) @test strides(view) == (1,) +@test view.suboffsets == C_NULL @test PyCall.aligned(view) == true # a vector is both c/f contiguous @test PyCall.c_contiguous(view) == true @test PyCall.f_contiguous(view) == true -@show view.format, view.ndim, view.readonly @test PyCall.pyfmt(view) == "f" -error("expected") + # Check 1D numpy arrays a = np.array([1.0,2.0,3.0]) @test a[:dtype] == np.dtype("float64") -view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_RECORDS) @test ndims(view) == 1 @test length(view) == 3 -@test sizeof(view) == sizeof(Float64) * 3 +@test sizeof(view) == a[:nbytes] @test size(view) == (3,) @test strides(view) == (1,) @test PyCall.aligned(view) == true @@ -57,12 +55,12 @@ view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) # Check 2D C ordered arrays a = np.array([[1 2 3], - [1 2 3]]) + [1 2 3]]) @test a[:dtype] == np.dtype("int64") -view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_RECORDS) @test ndims(view) == 2 @test length(view) == 6 -@test sizeof(view) == sizeof(Int64) * 6 +@test sizeof(view) == a[:nbytes] @test size(view) == (2,3) @test strides(view) == (3,1) @test PyCall.aligned(view) == true @@ -73,10 +71,10 @@ view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) # Check Multi-D C ordered arrays a = np.ones((10,5,3), dtype="float32", order="C") @test a[:dtype] == np.dtype("float32") -view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_RECORDS) @test ndims(view) == 3 @test length(view) == (10 * 5 * 3) -@test sizeof(view) == sizeof(Float32) * (10 * 5 * 3) +@test sizeof(view) == a[:nbytes] @test size(view) == (10, 5, 3) @test strides(view) == (5 * 3, 3, 1) @test PyCall.aligned(view) == true @@ -87,10 +85,10 @@ view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) # Check Multi-D F ordered arrays a = np.ones((10,5,3), dtype="uint8", order="F") @test a[:dtype] == np.dtype("uint8") -view = PyCall.pygetbuffer(a, PyCall.PyBUF_STRIDED) +view = PyCall.pygetbuffer(a, PyCall.PyBUF_RECORDS) @test ndims(view) == 3 @test length(view) == (10 * 5 * 3) -@test sizeof(view) == sizeof(Uint8) * (10 * 5 * 3) +@test sizeof(view) == a[:nbytes] @test size(view) == (10, 5, 3) @test strides(view) == (1, 10, 50) @test PyCall.aligned(view) == true @@ -131,8 +129,9 @@ a = PyArray(npa) @test sprint() do io; show(io, a); end == sprint() do io show(io, ones(Float32, (10,10))) end -aa = np.frombuffer(a) -@test np.shape(aa) == size(a) +#TODO: +#aa = np.frombuffer(a) +#@test np.shape(aa) == size(a) npa = np.ones((10,10,20), dtype="float32") a = PyArray(npa) @@ -142,9 +141,9 @@ a = PyArray(npa) @test size(similar(a)) == size(a) @test eltype(similar(a)) == eltype(a) @test summary(a) == "10x10x20 Float32 PyArray" - -aa = np.frombuffer(a) -@test np.shape(aa) == size(a) +#TODO: +#aa = np.frombuffer(a) +#@test np.shape(aa) == size(a) # Test fail because of show uses unimplemented subarray methods #@test sprint() do io; show(io, a); end == sprint() do io From 6b70c491b2f81a6240ad7cd68ff7ed53d01fc525 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Tue, 6 May 2014 10:37:32 -0400 Subject: [PATCH 4/5] make the python buffer struct immutable Only immutable types have C-ABI compatability in Julia. Here we make the Py_buffer struct immutable and wrap it with PyBuffer so we can attach a finalizer for auto memory management. This will enable us to reuse the Py_buffer struct for the Julia -> Python buffer implementation. --- src/buffer.jl | 54 ++++++++++++++++++++++++--------------------- test/test_buffer.jl | 1 - 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/buffer.jl b/src/buffer.jl index 59da310a..d32fe12a 100644 --- a/src/buffer.jl +++ b/src/buffer.jl @@ -2,7 +2,7 @@ # for versions > 3.2 (TODO: obj field not in struct for python versions <= 3.2) -type PyBuffer +immutable Py_buffer buf::Ptr{Void} obj::PyPtr len::Cssize_t @@ -15,43 +15,47 @@ type PyBuffer strides::Ptr{Cssize_t} suboffsets::Ptr{Cssize_t} internal::Ptr{Void} +end + +type PyBuffer + buf::Py_buffer PyBuffer() = begin - b = new(C_NULL, C_NULL, 0, 0, - 0, 0, C_NULL, C_NULL, - C_NULL, C_NULL, C_NULL) + b = new(Py_buffer(C_NULL, C_NULL, 0, 0, + 0, 0, C_NULL, C_NULL, + C_NULL, C_NULL, C_NULL)) finalizer(b, release!) return b end end release!(b::PyBuffer) = begin - if b.obj != C_NULL + if b.buf.obj != C_NULL ccall((@pysym :PyBuffer_Release), Void, (Ptr{PyBuffer},), &b) end end -Base.ndims(b::PyBuffer) = int(b.ndim) -Base.length(b::PyBuffer) = b.ndim >= 1 ? div(b.len, b.itemsize) : 0 -Base.sizeof(b::PyBuffer) = b.len +Base.ndims(b::PyBuffer) = int(b.buf.ndim) +Base.length(b::PyBuffer) = b.buf.ndim >= 1 ? div(b.buf.len, b.buf.itemsize) : 0 +Base.sizeof(b::PyBuffer) = b.buf.len Base.size(b::PyBuffer) = begin - if b.ndim == 0 - return (0,) + if b.buf.ndim == 0 + return (0,) end - if b.ndim == 1 - return (div(b.len, b.itemsize),) + if b.buf.ndim == 1 + return (div(b.buf.len, b.buf.itemsize),) end - @assert b.shape != C_NULL - return tuple(Int[unsafe_load(b.shape, i) for i=1:b.ndim]...) + @assert b.buf.shape != C_NULL + return tuple(Int[unsafe_load(b.buf.shape, i) for i=1:b.buf.ndim]...) end Base.strides(b::PyBuffer) = begin - if b.ndim == 0 || b.ndim == 1 + if b.buf.ndim == 0 || b.buf.ndim == 1 return (1,) end - @assert b.strides != C_NULL - return tuple(Int[div(unsafe_load(b.strides, i), b.itemsize) for i=1:b.ndim]...) + @assert b.buf.strides != C_NULL + return tuple(Int[div(unsafe_load(b.buf.strides, i), b.buf.itemsize) for i=1:b.buf.ndim]...) end ######################################################################### @@ -213,7 +217,7 @@ function jltype_to_pyfmt{T}(io::IO, ::Type{T}) return bytestring(io) end -pyfmt(b::PyBuffer) = b.format == C_NULL ? bytestring("") : bytestring(b.format) +pyfmt(b::PyBuffer) = b.buf.format == C_NULL ? bytestring("") : bytestring(b.buf.format) sizeof_pyfmt(fmt::ByteString) = ccall((@pysym :PyBuffer_SizeFromFormat), Cint, (Ptr{Cchar},), &fmt) @@ -229,11 +233,11 @@ pygetbuffer(o::PyObject, flags::Cint) = begin end aligned(b::PyBuffer) = begin - if b.strides == C_NULL + if b.buf.strides == C_NULL throw(ArgumentError("PyBuffer strides field is NULL")) end - for i=1:b.ndim - if mod(unsafe_load(b.strides, i), b.itemsize) != 0 + for i=1:b.buf.ndim + if mod(unsafe_load(b.buf.strides, i), b.buf.itemsize) != 0 return false end end @@ -270,18 +274,18 @@ type PyArray{T, N} <: AbstractArray{T, N} elseif ndims(b) != N throw(ArgumentError("inconsistent ndims in PyArray constructor")) end - return new(o, b, true, bool(b.readonly), + return new(o, b, true, bool(b.buf.readonly), size(b), strides(b), f_contiguous(b), c_contiguous(b), - convert(Ptr{T}, b.buf)) + convert(Ptr{T}, b.buf.buf)) end end function PyArray(o::PyObject) view = pygetbuffer(o, PyBUF_RECORDS) - view.format == C_NULL && error("buffer has no format string") - order, tys = parse_pyfmt(bytestring(view.format)) + view.buf.format == C_NULL && error("buffer has no format string") + order, tys = parse_pyfmt(bytestring(view.buf.format)) length(tys) != 1 && error("PyArray cannot yet handle structure types") ty = tys[1] ndim = ndims(view) diff --git a/test/test_buffer.jl b/test/test_buffer.jl index c281495f..68db33e4 100644 --- a/test/test_buffer.jl +++ b/test/test_buffer.jl @@ -31,7 +31,6 @@ view = PyCall.pygetbuffer(a, PyCall.PyBUF_RECORDS) @test sizeof(view) == sizeof(Float32) * 3 @test size(view) == (3,) @test strides(view) == (1,) -@test view.suboffsets == C_NULL @test PyCall.aligned(view) == true # a vector is both c/f contiguous @test PyCall.c_contiguous(view) == true From 65e65088c4118cdb52324f23936b76296929fb40 Mon Sep 17 00:00:00 2001 From: Jake Bolewski Date: Tue, 6 May 2014 10:44:39 -0400 Subject: [PATCH 5/5] numpy's frombuffer does not seem to preserve shape / stride information of the original buffer. Calling asarray preserves this information. --- test/test_buffer.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/test_buffer.jl b/test/test_buffer.jl index 68db33e4..ef0d11d8 100644 --- a/test/test_buffer.jl +++ b/test/test_buffer.jl @@ -113,7 +113,7 @@ a[1] = 3 a[2] = 2 a[3] = 1 @test sprint() do io; show(io, npa); end == "PyObject array([3, 2, 1])" -aa = np.frombuffer(a) +aa = np.asarray(a) @test np.shape(aa) == size(a) npa = np.ones((10,10), dtype="float32") @@ -128,9 +128,8 @@ a = PyArray(npa) @test sprint() do io; show(io, a); end == sprint() do io show(io, ones(Float32, (10,10))) end -#TODO: -#aa = np.frombuffer(a) -#@test np.shape(aa) == size(a) +aa = np.asarray(a) +@test np.shape(aa) == size(a) npa = np.ones((10,10,20), dtype="float32") a = PyArray(npa) @@ -140,9 +139,9 @@ a = PyArray(npa) @test size(similar(a)) == size(a) @test eltype(similar(a)) == eltype(a) @test summary(a) == "10x10x20 Float32 PyArray" -#TODO: -#aa = np.frombuffer(a) -#@test np.shape(aa) == size(a) + +aa = np.asarray(a) +@test np.shape(aa) == size(a) # Test fail because of show uses unimplemented subarray methods #@test sprint() do io; show(io, a); end == sprint() do io