Merge pull request #75 from static-frame/74/dta-dafault-int

Default type-parsed int is always 64 bit (even on Windows)

Author: flexatone

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
         if: matrix.os == 'macOS' || matrix.os == 'Ubuntu'
       - run: echo '::add-matcher::.github/problem-matchers/msvc.json'
         if: matrix.os == 'Windows'
-      - uses: pypa/cibuildwheel@v2.9.0
+      - uses: pypa/cibuildwheel@v2.11.1
         with:
           output-dir: dist
         env:
@@ -69,7 +69,7 @@ jobs:
         if: matrix.os == 'macOS' || matrix.os == 'Ubuntu'
       - run: echo '::add-matcher::.github/problem-matchers/msvc.json'
         if: matrix.os == 'Windows'
-      - uses: pypa/cibuildwheel@v2.9.0
+      - uses: pypa/cibuildwheel@v2.11.1
         with:
           output-dir: dist
         env:

src/_arraykit.c

@@ -397,36 +397,32 @@ AK_TPS_Resolve(AK_TypeParserState previous, AK_TypeParserState new) {
 PyArray_Descr*
 AK_TPS_ToDtype(AK_TypeParserState state) {
     PyArray_Descr *dtype = NULL;
-    PyArray_Descr *dtype_final;
 
     switch (state) {
         case TPS_UNKNOWN:
-            dtype = PyArray_DescrFromType(NPY_UNICODE);
+            dtype = PyArray_DescrNewFromType(NPY_UNICODE);
             break;
         case TPS_EMPTY: // all empty defaults to string
-            dtype = PyArray_DescrFromType(NPY_UNICODE);
+            dtype = PyArray_DescrNewFromType(NPY_UNICODE);
             break;
         case TPS_STRING:
-            dtype = PyArray_DescrFromType(NPY_UNICODE);
+            dtype = PyArray_DescrNewFromType(NPY_UNICODE);
             break;
         case TPS_BOOL:
-            dtype = PyArray_DescrFromType(NPY_BOOL);
+            dtype = PyArray_DescrNewFromType(NPY_BOOL);
             break;
         case TPS_INT:
-            dtype = PyArray_DescrFromType(NPY_INT64);
+            dtype = PyArray_DescrNewFromType(NPY_INT64);
             break;
         case TPS_FLOAT:
-            dtype = PyArray_DescrFromType(NPY_FLOAT64);
+            dtype = PyArray_DescrNewFromType(NPY_FLOAT64);
             break;
         case TPS_COMPLEX:
-            dtype = PyArray_DescrFromType(NPY_COMPLEX128);
+            dtype = PyArray_DescrNewFromType(NPY_COMPLEX128);
             break;
     }
     if (dtype == NULL) return NULL; // assume error is set by PyArray_DescrFromType
-    // get a fresh instance as we might mutate
-    dtype_final = PyArray_DescrNew(dtype);
-    Py_DECREF(dtype);
-    return dtype_final;
+    return dtype;
 }
 
 //------------------------------------------------------------------------------
@@ -1297,7 +1293,7 @@ AK_CPL_Free(AK_CodePointLine* cpl)
 static inline int
 AK_CPL_resize_buffer(AK_CodePointLine* cpl, Py_ssize_t count)
 {
-    if ((cpl->buffer_count + count) >= cpl->buffer_capacity) {
+    if (AK_UNLIKELY((cpl->buffer_count + count) >= cpl->buffer_capacity)) {
         // realloc
         cpl->buffer_capacity *= 2; // needs to be max of this or element_length
         cpl->buffer = PyMem_Realloc(cpl->buffer,
@@ -1313,7 +1309,7 @@ static inline int
 AK_CPL_resize_offsets(AK_CodePointLine* cpl)
 {
     // increment by at most one, so only need to check if equal
-    if (cpl->offsets_count == cpl->offsets_capacity) {
+    if (AK_UNLIKELY(cpl->offsets_count == cpl->offsets_capacity)) {
         // realloc
         cpl->offsets_capacity *= 2;
         cpl->offsets = PyMem_Realloc(cpl->offsets,
@@ -1923,29 +1919,21 @@ AK_CPL_to_array_bytes(AK_CodePointLine* cpl, PyArray_Descr* dtype)
     return array;
 }
 
-// If we cannot direclty convert bytes to values, create a bytes array and then use PyArray_CastToType to use numpy to interpet it as a new a array. This forces
+// If we cannot direclty convert bytes to values, create a bytes array and then use PyArray_CastToType to use numpy to interpet it as a new a array.
 static inline PyObject*
 AK_CPL_to_array_via_cast(AK_CodePointLine* cpl, PyArray_Descr* dtype)
 {
-    // we cannot use this dtype in array construction as it will mutate a global
-    PyArray_Descr *dtype_bytes_proto = PyArray_DescrFromType(NPY_STRING);
-    if (dtype_bytes_proto == NULL) {
-        Py_DECREF(dtype);
-        return NULL;
-    }
-    PyArray_Descr *dtype_bytes = PyArray_DescrNew(dtype_bytes_proto);
-    Py_DECREF(dtype_bytes_proto);
+    PyArray_Descr *dtype_bytes = PyArray_DescrNewFromType(NPY_STRING);
     if (dtype_bytes == NULL) {
         Py_DECREF(dtype);
         return NULL;
     }
     PyObject* array_bytes = AK_CPL_to_array_bytes(cpl, dtype_bytes);
     if (array_bytes == NULL) {
         Py_DECREF(dtype);
-        Py_DECREF(dtype_bytes); // was not stolen if array creation failed
+        // dtype_bytes stolen even if array creation failed
         return NULL;
     }
-
     PyObject *array = PyArray_CastToType((PyArrayObject*)array_bytes, dtype, 0);
     Py_DECREF(array_bytes);
     if (array == NULL) {
@@ -2096,15 +2084,15 @@ AK_CPG_resize(AK_CodePointGrid* cpg, Py_ssize_t line)
     Py_ssize_t lines_count = cpg->lines_count;
     if (line < lines_count) return 0; // most common scenario
 
-    if (line >= cpg->lines_capacity) {
+    if (AK_UNLIKELY(line >= cpg->lines_capacity)) {
         cpg->lines_capacity *= 2;
         // NOTE: we assume this only copies the pointers, not the data in the CPLs
         cpg->lines = PyMem_Realloc(cpg->lines,
                 sizeof(AK_CodePointLine*) * cpg->lines_capacity);
         if (cpg->lines == NULL) return -1;
     }
     // Create the new CPL; first check if we need to set type_parse by calling into the dtypes function. For now we assume sequential growth, so should only check if equal
-    if (line >= lines_count) {
+    if (AK_UNLIKELY(line >= lines_count)) {
         // determine if we need to parse types
         bool type_parse = false;
         if (cpg->dtypes == NULL) {
@@ -3199,29 +3187,20 @@ dtype_from_element(PyObject *Py_UNUSED(m), PyObject *arg)
 {
     // -------------------------------------------------------------------------
     // 1. Handle fast, exact type checks first.
-
-    // None
     if (arg == Py_None) {
         return (PyObject*)PyArray_DescrFromType(NPY_OBJECT);
     }
-
-    // Float
     if (PyFloat_CheckExact(arg)) {
-        return (PyObject*)PyArray_DescrFromType(NPY_DOUBLE);
+        return (PyObject*)PyArray_DescrFromType(NPY_FLOAT64);
     }
-
-    // Integers
     if (PyLong_CheckExact(arg)) {
-        return (PyObject*)PyArray_DescrFromType(NPY_LONG);
+        return (PyObject*)PyArray_DescrFromType(NPY_INT64);
     }
-
-    // Bool
     if (PyBool_Check(arg)) {
         return (PyObject*)PyArray_DescrFromType(NPY_BOOL);
     }
 
     PyObject* dtype = NULL;
-
     // String
     if (PyUnicode_CheckExact(arg)) {
         PyArray_Descr* descr = PyArray_DescrFromType(NPY_UNICODE);
@@ -3230,27 +3209,23 @@ dtype_from_element(PyObject *Py_UNUSED(m), PyObject *arg)
         Py_DECREF(descr);
         return dtype;
     }
-
     // Bytes
     if (PyBytes_CheckExact(arg)) {
         PyArray_Descr* descr = PyArray_DescrFromType(NPY_STRING);
         if (descr == NULL) return NULL;
-
         dtype = (PyObject*)PyArray_DescrFromObject(arg, descr);
         Py_DECREF(descr);
         return dtype;
     }
 
     // -------------------------------------------------------------------------
     // 2. Construct dtype (slightly more complicated)
-
     // Already known
     dtype = PyObject_GetAttrString(arg, "dtype");
     if (dtype) {
         return dtype;
     }
     PyErr_Clear();
-
     // -------------------------------------------------------------------------
     // 3. Handles everything else.
     return (PyObject*)PyArray_DescrFromType(NPY_OBJECT);
@@ -3956,4 +3931,5 @@ PyInit__arraykit(void)
         return NULL;
     }
     return m;
-}
+}
+

test/test_delimited_to_arrays.py

@@ -738,8 +738,12 @@ def test_delimited_to_arrays_parse_m(self) -> None:
         post1 = delimited_to_arrays(msg, axis=1, skipinitialspace=False)
         self.assertEqual([a.tolist() for a in post1], [['  1', ' 2-'], ['   2', ' 2-0'], [3, -3]])
 
-
-        # import ipdb; ipdb.set_trace()
+    def test_delimited_to_arrays_parse_n(self) -> None:
+        msg = ['1,2,1', '3,4,5']
+        post1 = delimited_to_arrays(msg, axis=1)
+        # NOTE: automatic type parsing should always give an int64
+        self.assertEqual([str(a.dtype) for a in post1],
+                ['int64', 'int64', 'int64'])
 
     #---------------------------------------------------------------------------
     def test_delimited_to_arrays_float_a(self) -> None:

test/test_util.py

@@ -326,6 +326,8 @@ def test_isna_element_false(self) -> None:
         self.assertFalse(isna_element(datetime.date(2020, 12, 31)))
         self.assertFalse(isna_element(False))
 
+    #---------------------------------------------------------------------------
+
     def test_dtype_from_element_core_dtypes(self) -> None:
         dtypes = [
                 np.longlong,
@@ -367,11 +369,11 @@ def test_dtype_from_element_obj_dtypes(self) -> None:
         NT = collections.namedtuple('NT', tuple('abc'))
 
         dtype_obj_pairs = [
-                (np.int_, 12),
-                (np.float_, 12.0),
+                (np.int64, 12),
+                (np.float64, 12.0),
                 (np.bool_, True),
                 (np.dtype('O'), None),
-                (np.float_, float('NaN')),
+                (np.float64, float('NaN')),
                 (np.dtype('O'), object()),
                 (np.dtype('O'), (1, 2, 3)),
                 (np.dtype('O'), NT(1, 2, 3)),
@@ -393,6 +395,12 @@ def test_dtype_from_element_str_and_bytes_dtypes(self) -> None:
             self.assertEqual(np.dtype(f'|S{size}'), dtype_from_element(bytes(size)))
             self.assertEqual(np.dtype(f'<U{size}'), dtype_from_element('x' * size))
 
+    def test_dtype_from_element_int(self) -> None:
+        # make sure all platforms give 64 bit int
+        self.assertEqual(str(dtype_from_element(3)), 'int64')
+
+    #---------------------------------------------------------------------------
+
     def test_get_new_indexers_and_screen_a(self) -> None:
         indexersA = np.array([9, 9, 9, 9, 0, 0, 1, 4, 5, 0, 0, 0, 1], dtype=np.int64)
         postA = get_new_indexers_and_screen_full(indexersA, np.arange(10, dtype=np.int64))