PERF: fastpath for iget from 2dim->1dim

Author: jreback

pandas/core/common.py

@@ -1802,9 +1802,11 @@ def _possibly_infer_to_datetimelike(value):
     v = value
     if not is_list_like(v):
         v = [v]
-    v = np.array(v)
+    if not isinstance(v, np.ndarray):
+        v = np.array(v)
     shape = v.shape
-    v = v.ravel()
+    if not v.ndim == 1:
+        v = v.ravel()
 
     if len(v):
 

pandas/core/internals.py

@@ -2521,7 +2521,7 @@ def _consolidate_inplace(self):
             self._known_consolidated = True
             self._rebuild_blknos_and_blklocs()
 
-    def get(self, item):
+    def get(self, item, fastpath=True):
         """
         Return values for selected item (ndarray or BlockManager).
         """
@@ -2539,7 +2539,7 @@ def get(self, item):
                     else:
                         raise ValueError("cannot label index with a null key")
 
-            return self.iget(loc)
+            return self.iget(loc, fastpath=fastpath)
         else:
 
             if isnull(item):
@@ -2549,8 +2549,25 @@ def get(self, item):
             return self.reindex_indexer(new_axis=self.items[indexer],
                                         indexer=indexer, axis=0, allow_dups=True)
 
-    def iget(self, i):
-        return self.blocks[self._blknos[i]].iget(self._blklocs[i])
+    def iget(self, i, fastpath=True):
+        """
+        Return the data as a SingleBlockManager if fastpath=True and possible
+
+        Otherwise return as a ndarray
+
+        """
+
+        block = self.blocks[self._blknos[i]]
+        values = block.iget(self._blklocs[i])
+        if not fastpath or block.is_sparse or values.ndim != 1:
+            return values
+
+        # fastpath shortcut for select a single-dim from a 2-dim BM
+        return SingleBlockManager([ block.make_block_same_class(values,
+                                                                placement=slice(0, len(values)),
+                                                                fastpath=True) ],
+                                  self.axes[1])
+
 
     def get_scalar(self, tup):
         """

pandas/tests/test_internals.py

@@ -356,7 +356,9 @@ def test_get_scalar(self):
         for item in self.mgr.items:
             for i, index in enumerate(self.mgr.axes[1]):
                 res = self.mgr.get_scalar((item, index))
-                exp = self.mgr.get(item)[i]
+                exp = self.mgr.get(item, fastpath=False)[i]
+                assert_almost_equal(res, exp)
+                exp = self.mgr.get(item).values[i]
                 assert_almost_equal(res, exp)
 
     def test_get(self):
@@ -366,19 +368,22 @@ def test_get(self):
                            placement=np.arange(3))
         mgr = BlockManager(blocks=[block], axes=[cols, np.arange(3)])
 
-        assert_almost_equal(mgr.get('a'), values[0])
-        assert_almost_equal(mgr.get('b'), values[1])
-        assert_almost_equal(mgr.get('c'), values[2])
+        assert_almost_equal(mgr.get('a', fastpath=False), values[0])
+        assert_almost_equal(mgr.get('b', fastpath=False), values[1])
+        assert_almost_equal(mgr.get('c', fastpath=False), values[2])
+        assert_almost_equal(mgr.get('a').values, values[0])
+        assert_almost_equal(mgr.get('b').values, values[1])
+        assert_almost_equal(mgr.get('c').values, values[2])
 
     def test_set(self):
         mgr = create_mgr('a,b,c: int', item_shape=(3,))
 
         mgr.set('d', np.array(['foo'] * 3))
         mgr.set('b', np.array(['bar'] * 3))
-        assert_almost_equal(mgr.get('a'), [0] * 3)
-        assert_almost_equal(mgr.get('b'), ['bar'] * 3)
-        assert_almost_equal(mgr.get('c'), [2] * 3)
-        assert_almost_equal(mgr.get('d'), ['foo'] * 3)
+        assert_almost_equal(mgr.get('a').values, [0] * 3)
+        assert_almost_equal(mgr.get('b').values, ['bar'] * 3)
+        assert_almost_equal(mgr.get('c').values, [2] * 3)
+        assert_almost_equal(mgr.get('d').values, ['foo'] * 3)
 
     def test_insert(self):
         self.mgr.insert(0, 'inserted', np.arange(N))
@@ -580,10 +585,14 @@ def test_reindex_items(self):
         reindexed = mgr.reindex_axis(['g', 'c', 'a', 'd'], axis=0)
         self.assertEqual(reindexed.nblocks, 2)
         assert_almost_equal(reindexed.items, ['g', 'c', 'a', 'd'])
-        assert_almost_equal(mgr.get('g'), reindexed.get('g'))
-        assert_almost_equal(mgr.get('c'), reindexed.get('c'))
-        assert_almost_equal(mgr.get('a'), reindexed.get('a'))
-        assert_almost_equal(mgr.get('d'), reindexed.get('d'))
+        assert_almost_equal(mgr.get('g',fastpath=False), reindexed.get('g',fastpath=False))
+        assert_almost_equal(mgr.get('c',fastpath=False), reindexed.get('c',fastpath=False))
+        assert_almost_equal(mgr.get('a',fastpath=False), reindexed.get('a',fastpath=False))
+        assert_almost_equal(mgr.get('d',fastpath=False), reindexed.get('d',fastpath=False))
+        assert_almost_equal(mgr.get('g').values, reindexed.get('g').values)
+        assert_almost_equal(mgr.get('c').values, reindexed.get('c').values)
+        assert_almost_equal(mgr.get('a').values, reindexed.get('a').values)
+        assert_almost_equal(mgr.get('d').values, reindexed.get('d').values)
 
     def test_multiindex_xs(self):
         mgr = create_mgr('a,b,c: f8; d,e,f: i8')
@@ -608,16 +617,19 @@ def test_get_numeric_data(self):
 
         numeric = mgr.get_numeric_data()
         assert_almost_equal(numeric.items, ['int', 'float', 'complex', 'bool'])
-        assert_almost_equal(mgr.get('float'), numeric.get('float'))
+        assert_almost_equal(mgr.get('float',fastpath=False), numeric.get('float',fastpath=False))
+        assert_almost_equal(mgr.get('float').values, numeric.get('float').values)
 
         # Check sharing
         numeric.set('float', np.array([100., 200., 300.]))
-        assert_almost_equal(mgr.get('float'), np.array([100., 200., 300.]))
+        assert_almost_equal(mgr.get('float',fastpath=False), np.array([100., 200., 300.]))
+        assert_almost_equal(mgr.get('float').values, np.array([100., 200., 300.]))
 
         numeric2 = mgr.get_numeric_data(copy=True)
         assert_almost_equal(numeric.items, ['int', 'float', 'complex', 'bool'])
         numeric2.set('float', np.array([1000., 2000., 3000.]))
-        assert_almost_equal(mgr.get('float'), np.array([100., 200., 300.]))
+        assert_almost_equal(mgr.get('float',fastpath=False), np.array([100., 200., 300.]))
+        assert_almost_equal(mgr.get('float').values, np.array([100., 200., 300.]))
 
     def test_get_bool_data(self):
         mgr = create_mgr('int: int; float: float; complex: complex;'
@@ -627,15 +639,18 @@ def test_get_bool_data(self):
 
         bools = mgr.get_bool_data()
         assert_almost_equal(bools.items, ['bool'])
-        assert_almost_equal(mgr.get('bool'), bools.get('bool'))
+        assert_almost_equal(mgr.get('bool',fastpath=False), bools.get('bool',fastpath=False))
+        assert_almost_equal(mgr.get('bool').values, bools.get('bool').values)
 
         bools.set('bool', np.array([True, False, True]))
-        assert_almost_equal(mgr.get('bool'), [True, False, True])
+        assert_almost_equal(mgr.get('bool',fastpath=False), [True, False, True])
+        assert_almost_equal(mgr.get('bool').values, [True, False, True])
 
         # Check sharing
         bools2 = mgr.get_bool_data(copy=True)
         bools2.set('bool', np.array([False, True, False]))
-        assert_almost_equal(mgr.get('bool'), [True, False, True])
+        assert_almost_equal(mgr.get('bool',fastpath=False), [True, False, True])
+        assert_almost_equal(mgr.get('bool').values, [True, False, True])
 
     def test_unicode_repr_doesnt_raise(self):
         str_repr = repr(create_mgr(u('b,\u05d0: object')))