In [144]: ser = pd.Series(arr[:, 0])
│ │ │ │
│ │ │ │ In [145]: %timeit ser.iloc[indexer]
│ │ │ │ .....: %timeit ser.take(indexer)
│ │ │ │ .....:
│ │ │ │ -356 us +- 34.6 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ -345 us +- 77.4 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ +138 us +- 1.35 us per loop (mean +- std. dev. of 7 runs, 10,000 loops each)
│ │ │ │ +147 us +- 15.1 us per loop (mean +- std. dev. of 7 runs, 10,000 loops each)
│ │ │ │ We have discussed MultiIndex in the previous sections pretty extensively.
│ │ │ │ Documentation about DatetimeIndex and PeriodIndex are shown here,
│ │ │ │ ├── html2text {}
│ │ │ │ │ @@ -1241,23 +1241,23 @@
│ │ │ │ │ In [141]: indexer = np.arange(10000)
│ │ │ │ │
│ │ │ │ │ In [142]: random.shuffle(indexer)
│ │ │ │ │
│ │ │ │ │ In [143]: %timeit arr[indexer]
│ │ │ │ │ .....: %timeit arr.take(indexer, axis=0)
│ │ │ │ │ .....:
│ │ │ │ │ -621 us +- 68.9 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ │ -288 us +- 26.1 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ │ +308 us +- 1.63 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ │ +123 us +- 88.3 ns per loop (mean +- std. dev. of 7 runs, 10,000 loops each)
│ │ │ │ │ In [144]: ser = pd.Series(arr[:, 0])
│ │ │ │ │
│ │ │ │ │ In [145]: %timeit ser.iloc[indexer]
│ │ │ │ │ .....: %timeit ser.take(indexer)
│ │ │ │ │ .....:
│ │ │ │ │ -356 us +- 34.6 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ │ -345 us +- 77.4 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ │ +138 us +- 1.35 us per loop (mean +- std. dev. of 7 runs, 10,000 loops each)
│ │ │ │ │ +147 us +- 15.1 us per loop (mean +- std. dev. of 7 runs, 10,000 loops each)
│ │ │ │ │ ********** IInnddeexx ttyyppeess_## **********
│ │ │ │ │ We have discussed MultiIndex in the previous sections pretty extensively.
│ │ │ │ │ Documentation about DatetimeIndex and PeriodIndex are shown _h_e_r_e, and
│ │ │ │ │ documentation about TimedeltaIndex is found _h_e_r_e.
│ │ │ │ │ In the following sub-sections we will highlight some other index types.
│ │ │ │ │ ******** CCaatteeggoorriiccaallIInnddeexx_## ********
│ │ │ │ │ _C_a_t_e_g_o_r_i_c_a_l_I_n_d_e_x is a type of index that is useful for supporting indexing with
│ │ │ ├── ./usr/share/doc/python-pandas-doc/html/user_guide/enhancingperf.html
│ │ │ │ @@ -601,31 +601,31 @@
│ │ │ │ ...: s += f(a + i * dx)
│ │ │ │ ...: return s * dx
│ │ │ │ ...:
│ │ │ │
│ │ │ │
│ │ │ │
We achieve our result by using DataFrame.apply() (row-wise):
In [5]: %timeit df.apply(lambda x: integrate_f(x["a"], x["b"], x["N"]), axis=1)
│ │ │ │ -210 ms +- 8.6 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ +102 ms +- 196 us per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ Let’s take a look and see where the time is spent during this operation │ │ │ │ using the prun ipython magic function:
│ │ │ │# most time consuming 4 calls
│ │ │ │ In [6]: %prun -l 4 df.apply(lambda x: integrate_f(x["a"], x["b"], x["N"]), axis=1) # noqa E999
│ │ │ │ - 605951 function calls (605933 primitive calls) in 1.613 seconds
│ │ │ │ + 605951 function calls (605933 primitive calls) in 0.283 seconds
│ │ │ │
│ │ │ │ Ordered by: internal time
│ │ │ │ List reduced from 159 to 4 due to restriction <4>
│ │ │ │
│ │ │ │ ncalls tottime percall cumtime percall filename:lineno(function)
│ │ │ │ - 1000 0.834 0.001 1.425 0.001 <ipython-input-4-c2a74e076cf0>:1(integrate_f)
│ │ │ │ - 552423 0.591 0.000 0.591 0.000 <ipython-input-3-c138bdd570e3>:1(f)
│ │ │ │ - 3000 0.029 0.000 0.129 0.000 series.py:1095(__getitem__)
│ │ │ │ - 16098 0.023 0.000 0.030 0.000 {built-in method builtins.isinstance}
│ │ │ │ + 1000 0.165 0.000 0.245 0.000 <ipython-input-4-c2a74e076cf0>:1(integrate_f)
│ │ │ │ + 552423 0.080 0.000 0.080 0.000 <ipython-input-3-c138bdd570e3>:1(f)
│ │ │ │ + 3000 0.006 0.000 0.025 0.000 series.py:1095(__getitem__)
│ │ │ │ + 3000 0.005 0.000 0.005 0.000 base.py:3777(get_loc)
│ │ │ │ By far the majority of time is spend inside either integrate_f or f,
│ │ │ │ hence we’ll concentrate our efforts cythonizing these two functions.
In [9]: %timeit df.apply(lambda x: integrate_f_plain(x["a"], x["b"], x["N"]), axis=1)
│ │ │ │ -175 ms +- 16.2 ms per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ +84.8 ms +- 69.5 us per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ This has improved the performance compared to the pure Python approach by one-third.
│ │ │ │We can annotate the function variables and return types as well as use cdef
│ │ │ │ @@ -667,36 +667,36 @@
│ │ │ │ ....: for i in range(N):
│ │ │ │ ....: s += f_typed(a + i * dx)
│ │ │ │ ....: return s * dx
│ │ │ │ ....:
│ │ │ │
│ │ │ │
│ │ │ │
In [11]: %timeit df.apply(lambda x: integrate_f_typed(x["a"], x["b"], x["N"]), axis=1)
│ │ │ │ -29 ms +- 478 us per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ +14.3 ms +- 17.1 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │ Annotating the functions with C types yields an over ten times performance improvement compared to │ │ │ │ the original Python implementation.
│ │ │ │When re-profiling, time is spent creating a Series from each row, and calling __getitem__ from both
│ │ │ │ the index and the series (three times for each row). These Python function calls are expensive and
│ │ │ │ can be improved by passing an np.ndarray.
In [12]: %prun -l 4 df.apply(lambda x: integrate_f_typed(x["a"], x["b"], x["N"]), axis=1)
│ │ │ │ - 52528 function calls (52510 primitive calls) in 0.158 seconds
│ │ │ │ + 52528 function calls (52510 primitive calls) in 0.037 seconds
│ │ │ │
│ │ │ │ Ordered by: internal time
│ │ │ │ List reduced from 157 to 4 due to restriction <4>
│ │ │ │
│ │ │ │ ncalls tottime percall cumtime percall filename:lineno(function)
│ │ │ │ - 3000 0.024 0.000 0.110 0.000 series.py:1095(__getitem__)
│ │ │ │ - 16098 0.020 0.000 0.025 0.000 {built-in method builtins.isinstance}
│ │ │ │ - 3000 0.017 0.000 0.045 0.000 series.py:1220(_get_value)
│ │ │ │ - 3000 0.015 0.000 0.028 0.000 indexing.py:2765(check_dict_or_set_indexers)
│ │ │ │ + 3000 0.006 0.000 0.024 0.000 series.py:1095(__getitem__)
│ │ │ │ + 3000 0.004 0.000 0.005 0.000 base.py:3777(get_loc)
│ │ │ │ + 3000 0.004 0.000 0.012 0.000 series.py:1220(_get_value)
│ │ │ │ + 16098 0.003 0.000 0.004 0.000 {built-in method builtins.isinstance}
│ │ │ │ In [13]: %%cython
│ │ │ │ ....: cimport numpy as np
│ │ │ │ ....: import numpy as np
│ │ │ │ ....: cdef double f_typed(double x) except? -2:
│ │ │ │ ....: return x * (x - 1)
│ │ │ │ @@ -731,33 +731,33 @@
│ │ │ │ This implementation creates an array of zeros and inserts the result
│ │ │ │ of integrate_f_typed applied over each row. Looping over an ndarray is faster
│ │ │ │ in Cython than looping over a Series object.
Since apply_integrate_f is typed to accept an np.ndarray, Series.to_numpy()
│ │ │ │ calls are needed to utilize this function.
In [14]: %timeit apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df["N"].to_numpy())
│ │ │ │ -2.6 ms +- 44.3 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │ +1.91 ms +- 4.7 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ Performance has improved from the prior implementation by almost ten times.
│ │ │ │ │ │ │ │The majority of the time is now spent in apply_integrate_f. Disabling Cython’s boundscheck
│ │ │ │ and wraparound checks can yield more performance.
In [15]: %prun -l 4 apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df["N"].to_numpy())
│ │ │ │ - 78 function calls in 0.003 seconds
│ │ │ │ + 78 function calls in 0.002 seconds
│ │ │ │
│ │ │ │ Ordered by: internal time
│ │ │ │ List reduced from 21 to 4 due to restriction <4>
│ │ │ │
│ │ │ │ ncalls tottime percall cumtime percall filename:lineno(function)
│ │ │ │ - 1 0.003 0.003 0.003 0.003 <string>:1(<module>)
│ │ │ │ - 1 0.000 0.000 0.003 0.003 {built-in method builtins.exec}
│ │ │ │ + 1 0.002 0.002 0.002 0.002 <string>:1(<module>)
│ │ │ │ 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
│ │ │ │ + 1 0.000 0.000 0.002 0.002 {built-in method builtins.exec}
│ │ │ │ 3 0.000 0.000 0.000 0.000 frame.py:4062(__getitem__)
│ │ │ │ In [16]: %%cython
│ │ │ │ ....: cimport cython
│ │ │ │ ....: cimport numpy as np
│ │ │ │ ....: import numpy as np
│ │ │ │ @@ -1189,19 +1189,19 @@
│ │ │ │ compared to standard Python syntax for large DataFrame. This engine requires the
│ │ │ │ optional dependency numexpr to be installed.
│ │ │ │ The 'python' engine is generally not useful except for testing
│ │ │ │ other evaluation engines against it. You will achieve no performance
│ │ │ │ benefits using eval() with engine='python' and may
│ │ │ │ incur a performance hit.
│ │ │ │ In [40]: %timeit df1 + df2 + df3 + df4
│ │ │ │ -768 ms +- 17.3 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ +458 ms +- 54.2 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │
│ │ │ │
│ │ │ │ In [41]: %timeit pd.eval("df1 + df2 + df3 + df4", engine="python")
│ │ │ │ -753 ms +- 56.8 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ +486 ms +- 82.9 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │
│ │ │ │
│ │ │ │
│ │ │ │
│ │ │ │ The DataFrame.eval() method#
│ │ │ │ In addition to the top level pandas.eval() function you can also
│ │ │ │ evaluate an expression in the “context” of a DataFrame.
│ │ │ │ @@ -1316,39 +1316,39 @@
│ │ │ │ In [58]: nrows, ncols = 20000, 100
│ │ │ │
│ │ │ │ In [59]: df1, df2, df3, df4 = [pd.DataFrame(np.random.randn(nrows, ncols)) for _ in range(4)]
│ │ │ │
│ │ │ │
│ │ │ │ DataFrame arithmetic:
│ │ │ │ In [60]: %timeit df1 + df2 + df3 + df4
│ │ │ │ -808 ms +- 61.1 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ +506 ms +- 135 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │
│ │ │ │
│ │ │ │ In [61]: %timeit pd.eval("df1 + df2 + df3 + df4")
│ │ │ │ -240 ms +- 36 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ +143 ms +- 9.47 ms per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │
│ │ │ │
│ │ │ │ DataFrame comparison:
│ │ │ │ In [62]: %timeit (df1 > 0) & (df2 > 0) & (df3 > 0) & (df4 > 0)
│ │ │ │ -53.6 ms +- 2.66 ms per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ +15.8 ms +- 29.1 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │
│ │ │ │
│ │ │ │ In [63]: %timeit pd.eval("(df1 > 0) & (df2 > 0) & (df3 > 0) & (df4 > 0)")
│ │ │ │ -11.3 ms +- 224 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │ +5.07 ms +- 49.4 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │
│ │ │ │
│ │ │ │ DataFrame arithmetic with unaligned axes.
│ │ │ │ In [64]: s = pd.Series(np.random.randn(50))
│ │ │ │
│ │ │ │ In [65]: %timeit df1 + df2 + df3 + df4 + s
│ │ │ │ -1.23 s +- 63.9 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ +762 ms +- 43.3 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │
│ │ │ │
│ │ │ │ In [66]: %timeit pd.eval("df1 + df2 + df3 + df4 + s")
│ │ │ │ -252 ms +- 13.4 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ +161 ms +- 4.23 ms per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │
│ │ │ │
│ │ │ │
│ │ │ │ Note
│ │ │ │ Operations such as
│ │ │ │ 1 and 2 # would parse to 1 & 2, but should evaluate to 2
│ │ │ │ 3 or 4 # would parse to 3 | 4, but should evaluate to 3
│ │ │ │ ├── html2text {}
│ │ │ │ │ @@ -106,33 +106,32 @@
│ │ │ │ │ ...: dx = (b - a) / N
│ │ │ │ │ ...: for i in range(N):
│ │ │ │ │ ...: s += f(a + i * dx)
│ │ │ │ │ ...: return s * dx
│ │ │ │ │ ...:
│ │ │ │ │ We achieve our result by using _D_a_t_a_F_r_a_m_e_._a_p_p_l_y_(_) (row-wise):
│ │ │ │ │ In [5]: %timeit df.apply(lambda x: integrate_f(x["a"], x["b"], x["N"]), axis=1)
│ │ │ │ │ -210 ms +- 8.6 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ +102 ms +- 196 us per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ │ Let’s take a look and see where the time is spent during this operation using
│ │ │ │ │ the _p_r_u_n_ _i_p_y_t_h_o_n_ _m_a_g_i_c_ _f_u_n_c_t_i_o_n:
│ │ │ │ │ # most time consuming 4 calls
│ │ │ │ │ In [6]: %prun -l 4 df.apply(lambda x: integrate_f(x["a"], x["b"], x["N"]),
│ │ │ │ │ axis=1) # noqa E999
│ │ │ │ │ - 605951 function calls (605933 primitive calls) in 1.613 seconds
│ │ │ │ │ + 605951 function calls (605933 primitive calls) in 0.283 seconds
│ │ │ │ │
│ │ │ │ │ Ordered by: internal time
│ │ │ │ │ List reduced from 159 to 4 due to restriction <4>
│ │ │ │ │
│ │ │ │ │ ncalls tottime percall cumtime percall filename:lineno(function)
│ │ │ │ │ - 1000 0.834 0.001 1.425 0.001 :1
│ │ │ │ │ + 1000 0.165 0.000 0.245 0.000 :1
│ │ │ │ │ (integrate_f)
│ │ │ │ │ - 552423 0.591 0.000 0.591 0.000 :1
│ │ │ │ │ + 552423 0.080 0.000 0.080 0.000 :1
│ │ │ │ │ (f)
│ │ │ │ │ - 3000 0.029 0.000 0.129 0.000 series.py:1095(__getitem__)
│ │ │ │ │ - 16098 0.023 0.000 0.030 0.000 {built-in method
│ │ │ │ │ -builtins.isinstance}
│ │ │ │ │ + 3000 0.006 0.000 0.025 0.000 series.py:1095(__getitem__)
│ │ │ │ │ + 3000 0.005 0.000 0.005 0.000 base.py:3777(get_loc)
│ │ │ │ │ By far the majority of time is spend inside either integrate_f or f, hence
│ │ │ │ │ we’ll concentrate our efforts cythonizing these two functions.
│ │ │ │ │ ******** PPllaaiinn CCyytthhoonn_## ********
│ │ │ │ │ First we’re going to need to import the Cython magic function to IPython:
│ │ │ │ │ In [7]: %load_ext Cython
│ │ │ │ │ Now, let’s simply copy our functions over to Cython:
│ │ │ │ │ In [8]: %%cython
│ │ │ │ │ @@ -143,15 +142,15 @@
│ │ │ │ │ ...: dx = (b - a) / N
│ │ │ │ │ ...: for i in range(N):
│ │ │ │ │ ...: s += f_plain(a + i * dx)
│ │ │ │ │ ...: return s * dx
│ │ │ │ │ ...:
│ │ │ │ │ In [9]: %timeit df.apply(lambda x: integrate_f_plain(x["a"], x["b"], x["N"]),
│ │ │ │ │ axis=1)
│ │ │ │ │ -175 ms +- 16.2 ms per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ │ +84.8 ms +- 69.5 us per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ │ This has improved the performance compared to the pure Python approach by one-
│ │ │ │ │ third.
│ │ │ │ │ ******** DDeeccllaarriinngg CC ttyyppeess_## ********
│ │ │ │ │ We can annotate the function variables and return types as well as use cdef and
│ │ │ │ │ cpdef to improve performance:
│ │ │ │ │ In [10]: %%cython
│ │ │ │ │ ....: cdef double f_typed(double x) except? -2:
│ │ │ │ │ @@ -163,36 +162,35 @@
│ │ │ │ │ ....: dx = (b - a) / N
│ │ │ │ │ ....: for i in range(N):
│ │ │ │ │ ....: s += f_typed(a + i * dx)
│ │ │ │ │ ....: return s * dx
│ │ │ │ │ ....:
│ │ │ │ │ In [11]: %timeit df.apply(lambda x: integrate_f_typed(x["a"], x["b"], x["N"]),
│ │ │ │ │ axis=1)
│ │ │ │ │ -29 ms +- 478 us per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ │ +14.3 ms +- 17.1 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │ │ Annotating the functions with C types yields an over ten times performance
│ │ │ │ │ improvement compared to the original Python implementation.
│ │ │ │ │ ******** UUssiinngg nnddaarrrraayy_## ********
│ │ │ │ │ When re-profiling, time is spent creating a _S_e_r_i_e_s from each row, and calling
│ │ │ │ │ __getitem__ from both the index and the series (three times for each row).
│ │ │ │ │ These Python function calls are expensive and can be improved by passing an
│ │ │ │ │ np.ndarray.
│ │ │ │ │ In [12]: %prun -l 4 df.apply(lambda x: integrate_f_typed(x["a"], x["b"], x
│ │ │ │ │ ["N"]), axis=1)
│ │ │ │ │ - 52528 function calls (52510 primitive calls) in 0.158 seconds
│ │ │ │ │ + 52528 function calls (52510 primitive calls) in 0.037 seconds
│ │ │ │ │
│ │ │ │ │ Ordered by: internal time
│ │ │ │ │ List reduced from 157 to 4 due to restriction <4>
│ │ │ │ │
│ │ │ │ │ ncalls tottime percall cumtime percall filename:lineno(function)
│ │ │ │ │ - 3000 0.024 0.000 0.110 0.000 series.py:1095(__getitem__)
│ │ │ │ │ - 16098 0.020 0.000 0.025 0.000 {built-in method
│ │ │ │ │ + 3000 0.006 0.000 0.024 0.000 series.py:1095(__getitem__)
│ │ │ │ │ + 3000 0.004 0.000 0.005 0.000 base.py:3777(get_loc)
│ │ │ │ │ + 3000 0.004 0.000 0.012 0.000 series.py:1220(_get_value)
│ │ │ │ │ + 16098 0.003 0.000 0.004 0.000 {built-in method
│ │ │ │ │ builtins.isinstance}
│ │ │ │ │ - 3000 0.017 0.000 0.045 0.000 series.py:1220(_get_value)
│ │ │ │ │ - 3000 0.015 0.000 0.028 0.000 indexing.py:2765
│ │ │ │ │ -(check_dict_or_set_indexers)
│ │ │ │ │ In [13]: %%cython
│ │ │ │ │ ....: cimport numpy as np
│ │ │ │ │ ....: import numpy as np
│ │ │ │ │ ....: cdef double f_typed(double x) except? -2:
│ │ │ │ │ ....: return x * (x - 1)
│ │ │ │ │ ....: cpdef double integrate_f_typed(double a, double b, int N):
│ │ │ │ │ ....: cdef int i
│ │ │ │ │ @@ -233,31 +231,31 @@
│ │ │ │ │ This implementation creates an array of zeros and inserts the result of
│ │ │ │ │ integrate_f_typed applied over each row. Looping over an ndarray is faster in
│ │ │ │ │ Cython than looping over a _S_e_r_i_e_s object.
│ │ │ │ │ Since apply_integrate_f is typed to accept an np.ndarray, _S_e_r_i_e_s_._t_o___n_u_m_p_y_(_)
│ │ │ │ │ calls are needed to utilize this function.
│ │ │ │ │ In [14]: %timeit apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df
│ │ │ │ │ ["N"].to_numpy())
│ │ │ │ │ -2.6 ms +- 44.3 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │ │ +1.91 ms +- 4.7 us per loop (mean +- std. dev. of 7 runs, 1,000 loops each)
│ │ │ │ │ Performance has improved from the prior implementation by almost ten times.
│ │ │ │ │ ******** DDiissaabblliinngg ccoommppiilleerr ddiirreeccttiivveess_## ********
│ │ │ │ │ The majority of the time is now spent in apply_integrate_f. Disabling Cython’s
│ │ │ │ │ boundscheck and wraparound checks can yield more performance.
│ │ │ │ │ In [15]: %prun -l 4 apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(),
│ │ │ │ │ df["N"].to_numpy())
│ │ │ │ │ - 78 function calls in 0.003 seconds
│ │ │ │ │ + 78 function calls in 0.002 seconds
│ │ │ │ │
│ │ │ │ │ Ordered by: internal time
│ │ │ │ │ List reduced from 21 to 4 due to restriction <4>
│ │ │ │ │
│ │ │ │ │ ncalls tottime percall cumtime percall filename:lineno(function)
│ │ │ │ │ - 1 0.003 0.003 0.003 0.003 :1()
│ │ │ │ │ - 1 0.000 0.000 0.003 0.003 {built-in method builtins.exec}
│ │ │ │ │ + 1 0.002 0.002 0.002 0.002 :1()
│ │ │ │ │ 1 0.000 0.000 0.000 0.000 {method 'disable' of
│ │ │ │ │ '_lsprof.Profiler' objects}
│ │ │ │ │ + 1 0.000 0.000 0.002 0.002 {built-in method builtins.exec}
│ │ │ │ │ 3 0.000 0.000 0.000 0.000 frame.py:4062(__getitem__)
│ │ │ │ │ In [16]: %%cython
│ │ │ │ │ ....: cimport cython
│ │ │ │ │ ....: cimport numpy as np
│ │ │ │ │ ....: import numpy as np
│ │ │ │ │ ....: cdef np.float64_t f_typed(np.float64_t x) except? -2:
│ │ │ │ │ ....: return x * (x - 1)
│ │ │ │ │ @@ -644,17 +642,17 @@
│ │ │ │ │ The 'numexpr' engine is the more performant engine that can yield performance
│ │ │ │ │ improvements compared to standard Python syntax for large _D_a_t_a_F_r_a_m_e. This
│ │ │ │ │ engine requires the optional dependency numexpr to be installed.
│ │ │ │ │ The 'python' engine is generally nnoott useful except for testing other evaluation
│ │ │ │ │ engines against it. You will achieve nnoo performance benefits using _e_v_a_l_(_) with
│ │ │ │ │ engine='python' and may incur a performance hit.
│ │ │ │ │ In [40]: %timeit df1 + df2 + df3 + df4
│ │ │ │ │ -768 ms +- 17.3 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ +458 ms +- 54.2 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ In [41]: %timeit pd.eval("df1 + df2 + df3 + df4", engine="python")
│ │ │ │ │ -753 ms +- 56.8 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ +486 ms +- 82.9 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ ******** TThhee _DD_aa_tt_aa_FF_rr_aa_mm_ee_.._ee_vv_aa_ll_((_)) mmeetthhoodd_## ********
│ │ │ │ │ In addition to the top level _p_a_n_d_a_s_._e_v_a_l_(_) function you can also evaluate an
│ │ │ │ │ expression in the “context” of a _D_a_t_a_F_r_a_m_e.
│ │ │ │ │ In [42]: df = pd.DataFrame(np.random.randn(5, 2), columns=["a", "b"])
│ │ │ │ │
│ │ │ │ │ In [43]: df.eval("a + b")
│ │ │ │ │ Out[43]:
│ │ │ │ │ @@ -751,29 +749,29 @@
│ │ │ │ │ _p_a_n_d_a_s_._e_v_a_l_(_) works well with expressions containing large arrays.
│ │ │ │ │ In [58]: nrows, ncols = 20000, 100
│ │ │ │ │
│ │ │ │ │ In [59]: df1, df2, df3, df4 = [pd.DataFrame(np.random.randn(nrows, ncols)) for
│ │ │ │ │ _ in range(4)]
│ │ │ │ │ _D_a_t_a_F_r_a_m_e arithmetic:
│ │ │ │ │ In [60]: %timeit df1 + df2 + df3 + df4
│ │ │ │ │ -808 ms +- 61.1 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ +506 ms +- 135 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ In [61]: %timeit pd.eval("df1 + df2 + df3 + df4")
│ │ │ │ │ -240 ms +- 36 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ +143 ms +- 9.47 ms per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ │ _D_a_t_a_F_r_a_m_e comparison:
│ │ │ │ │ In [62]: %timeit (df1 > 0) & (df2 > 0) & (df3 > 0) & (df4 > 0)
│ │ │ │ │ -53.6 ms +- 2.66 ms per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ │ +15.8 ms +- 29.1 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │ │ In [63]: %timeit pd.eval("(df1 > 0) & (df2 > 0) & (df3 > 0) & (df4 > 0)")
│ │ │ │ │ -11.3 ms +- 224 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │ │ +5.07 ms +- 49.4 us per loop (mean +- std. dev. of 7 runs, 100 loops each)
│ │ │ │ │ _D_a_t_a_F_r_a_m_e arithmetic with unaligned axes.
│ │ │ │ │ In [64]: s = pd.Series(np.random.randn(50))
│ │ │ │ │
│ │ │ │ │ In [65]: %timeit df1 + df2 + df3 + df4 + s
│ │ │ │ │ -1.23 s +- 63.9 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ +762 ms +- 43.3 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ In [66]: %timeit pd.eval("df1 + df2 + df3 + df4 + s")
│ │ │ │ │ -252 ms +- 13.4 ms per loop (mean +- std. dev. of 7 runs, 1 loop each)
│ │ │ │ │ +161 ms +- 4.23 ms per loop (mean +- std. dev. of 7 runs, 10 loops each)
│ │ │ │ │ Note
│ │ │ │ │ Operations such as
│ │ │ │ │ 1 and 2 # would parse to 1 & 2, but should evaluate to 2
│ │ │ │ │ 3 or 4 # would parse to 3 | 4, but should evaluate to 3
│ │ │ │ │ ~1 # this is okay, but slower when using eval
│ │ │ │ │ should be performed in Python. An exception will be raised if you try to
│ │ │ │ │ perform any boolean/bitwise operations with scalar operands that are not of
│ │ │ ├── ./usr/share/doc/python-pandas-doc/html/user_guide/pyarrow.html
│ │ │ │ @@ -995,26 +995,19 @@
│ │ │ │ Cell In[33], line 1
│ │ │ │ ----> 1 table = pa.table([pa.array([1, 2, 3], type=pa.int64())], names=["a"])
│ │ │ │
│ │ │ │ NameError: name 'pa' is not defined
│ │ │ │
│ │ │ │ In [34]: df = table.to_pandas(types_mapper=pd.ArrowDtype)
│ │ │ │ ---------------------------------------------------------------------------
│ │ │ │ -AttributeError Traceback (most recent call last)
│ │ │ │ -<ipython-input-34-64ec62289cb4> in ?()
│ │ │ │ +NameError Traceback (most recent call last)
│ │ │ │ +Cell In[34], line 1
│ │ │ │ ----> 1 df = table.to_pandas(types_mapper=pd.ArrowDtype)
│ │ │ │
│ │ │ │ -/usr/lib/python3/dist-packages/pandas/core/generic.py in ?(self, name)
│ │ │ │ - 6295 and name not in self._accessors
│ │ │ │ - 6296 and self._info_axis._can_hold_identifiers_and_holds_name(name)
│ │ │ │ - 6297 ):
│ │ │ │ - 6298 return self[name]
│ │ │ │ --> 6299 return object.__getattribute__(self, name)
│ │ │ │ -
│ │ │ │ -AttributeError: 'DataFrame' object has no attribute 'to_pandas'
│ │ │ │ +NameError: name 'table' is not defined
│ │ │ │
│ │ │ │ In [35]: df
│ │ │ │ Out[35]:
│ │ │ │ a b
│ │ │ │ 0 xxx yyy
│ │ │ │ 1 ¡¡ ¡¡
│ │ │ │ ├── html2text {}
│ │ │ │ │ @@ -522,27 +522,19 @@
│ │ │ │ │ Cell In[33], line 1
│ │ │ │ │ ----> 1 table = pa.table([pa.array([1, 2, 3], type=pa.int64())], names=["a"])
│ │ │ │ │
│ │ │ │ │ NameError: name 'pa' is not defined
│ │ │ │ │
│ │ │ │ │ In [34]: df = table.to_pandas(types_mapper=pd.ArrowDtype)
│ │ │ │ │ ---------------------------------------------------------------------------
│ │ │ │ │ -AttributeError Traceback (most recent call last)
│ │ │ │ │ - in ?()
│ │ │ │ │ +NameError Traceback (most recent call last)
│ │ │ │ │ +Cell In[34], line 1
│ │ │ │ │ ----> 1 df = table.to_pandas(types_mapper=pd.ArrowDtype)
│ │ │ │ │
│ │ │ │ │ -/usr/lib/python3/dist-packages/pandas/core/generic.py in ?(self, name)
│ │ │ │ │ - 6295 and name not in self._accessors
│ │ │ │ │ - 6296 and self._info_axis._can_hold_identifiers_and_holds_name
│ │ │ │ │ -(name)
│ │ │ │ │ - 6297 ):
│ │ │ │ │ - 6298 return self[name]
│ │ │ │ │ --> 6299 return object.__getattribute__(self, name)
│ │ │ │ │ -
│ │ │ │ │ -AttributeError: 'DataFrame' object has no attribute 'to_pandas'
│ │ │ │ │ +NameError: name 'table' is not defined
│ │ │ │ │
│ │ │ │ │ In [35]: df
│ │ │ │ │ Out[35]:
│ │ │ │ │ a b
│ │ │ │ │ 0 xxx yyy
│ │ │ │ │ 1 ¡¡ ¡¡
│ │ │ ├── ./usr/share/doc/python-pandas-doc/html/user_guide/scale.html
│ │ │ │ @@ -1095,16 +1095,16 @@
│ │ │ │ ....: files = pathlib.Path("data/timeseries/").glob("ts*.parquet")
│ │ │ │ ....: counts = pd.Series(dtype=int)
│ │ │ │ ....: for path in files:
│ │ │ │ ....: df = pd.read_parquet(path)
│ │ │ │ ....: counts = counts.add(df["name"].value_counts(), fill_value=0)
│ │ │ │ ....: counts.astype(int)
│ │ │ │ ....:
│ │ │ │ -CPU times: user 680 us, sys: 0 ns, total: 680 us
│ │ │ │ -Wall time: 684 us
│ │ │ │ +CPU times: user 408 us, sys: 362 us, total: 770 us
│ │ │ │ +Wall time: 2.18 ms
│ │ │ │ Out[32]: Series([], dtype: int32)
│ │ │ │
│ │ │ │
│ │ │ │ Some readers, like pandas.read_csv(), offer parameters to control the
│ │ │ │ chunksize when reading a single file.
│ │ │ │ Manually chunking is an OK option for workflows that don’t
│ │ │ │ require too sophisticated of operations. Some operations, like pandas.DataFrame.groupby(), are
│ │ │ │ ├── html2text {}
│ │ │ │ │ @@ -640,16 +640,16 @@
│ │ │ │ │ ....: files = pathlib.Path("data/timeseries/").glob("ts*.parquet")
│ │ │ │ │ ....: counts = pd.Series(dtype=int)
│ │ │ │ │ ....: for path in files:
│ │ │ │ │ ....: df = pd.read_parquet(path)
│ │ │ │ │ ....: counts = counts.add(df["name"].value_counts(), fill_value=0)
│ │ │ │ │ ....: counts.astype(int)
│ │ │ │ │ ....:
│ │ │ │ │ -CPU times: user 680 us, sys: 0 ns, total: 680 us
│ │ │ │ │ -Wall time: 684 us
│ │ │ │ │ +CPU times: user 408 us, sys: 362 us, total: 770 us
│ │ │ │ │ +Wall time: 2.18 ms
│ │ │ │ │ Out[32]: Series([], dtype: int32)
│ │ │ │ │ Some readers, like _p_a_n_d_a_s_._r_e_a_d___c_s_v_(_), offer parameters to control the chunksize
│ │ │ │ │ when reading a single file.
│ │ │ │ │ Manually chunking is an OK option for workflows that don’t require too
│ │ │ │ │ sophisticated of operations. Some operations, like _p_a_n_d_a_s_._D_a_t_a_F_r_a_m_e_._g_r_o_u_p_b_y_(_),
│ │ │ │ │ are much harder to do chunkwise. In these cases, you may be better switching to
│ │ │ │ │ a different library that implements these out-of-core algorithms for you.
│ │ │ ├── ./usr/share/doc/python-pandas-doc/html/user_guide/style.ipynb.gz
│ │ │ │ ├── style.ipynb
│ │ │ │ │ ├── Pretty-printed
│ │ │ │ │ │┄ Similarity: 0.9985610875706213%
│ │ │ │ │ │┄ Differences: {"'cells'": "{1: {'metadata': {'execution': {'iopub.execute_input': '2025-12-11T23:36:27.300629Z', "
│ │ │ │ │ │┄ "'iopub.status.busy': '2025-12-11T23:36:27.300243Z', 'iopub.status.idle': "
│ │ │ │ │ │┄ "'2025-12-11T23:36:27.889580Z', 'shell.execute_reply': "
│ │ │ │ │ │┄ "'2025-12-11T23:36:27.888991Z'}}}, 3: {'metadata': {'execution': "
│ │ │ │ │ │┄ "{'iopub.execute_input': '2025-12-11T23:36:27.893615Z', 'iopub.status.busy': "
│ │ │ │ │ │┄ "'2025-12-11T23:36:27.893061Z', 'iopub.status.idle': '2025-12-11T23:36:2 […]
│ │ │ │ │ │ @@ -39,18 +39,18 @@
│ │ │ │ │ │ ]
│ │ │ │ │ │ },
│ │ │ │ │ │ {
│ │ │ │ │ │ "cell_type": "code",
│ │ │ │ │ │ "execution_count": 1,
│ │ │ │ │ │ "metadata": {
│ │ │ │ │ │ "execution": {
│ │ │ │ │ │ - "iopub.execute_input": "2024-11-08T14:51:53.436265Z",
│ │ │ │ │ │ - "iopub.status.busy": "2024-11-08T14:51:53.435850Z",
│ │ │ │ │ │ - "iopub.status.idle": "2024-11-08T14:51:54.474028Z",
│ │ │ │ │ │ - "shell.execute_reply": "2024-11-08T14:51:54.472870Z"
│ │ │ │ │ │ + "iopub.execute_input": "2025-12-11T23:36:27.300629Z",
│ │ │ │ │ │ + "iopub.status.busy": "2025-12-11T23:36:27.300243Z",
│ │ │ │ │ │ + "iopub.status.idle": "2025-12-11T23:36:27.889580Z",
│ │ │ │ │ │ + "shell.execute_reply": "2025-12-11T23:36:27.888991Z"
│ │ │ │ │ │ },
│ │ │ │ │ │ "nbsphinx": "hidden"
│ │ │ │ │ │ },
│ │ │ │ │ │ "outputs": [],
│ │ │ │ │ │ "source": [
│ │ │ │ │ │ "import matplotlib.pyplot\n",
│ │ │ │ │ │ "# We have this here to trigger matplotlib's font cache stuff.\n",
│ │ │ │ │ │ @@ -77,36 +77,36 @@
│ │ │ │ │ │ ]
│ │ │ │ │ │ },
│ │ │ │ │ │ {
│ │ │ │ │ │ "cell_type": "code",
│ │ │ │ │ │ "execution_count": 2,
│ │ │ │ │ │ "metadata": {
│ │ │ │ │ │ "execution": {
│ │ │ │ │ │ - "iopub.execute_input": "2024-11-08T14:51:54.479097Z",
│ │ │ │ │ │ - "iopub.status.busy": "2024-11-08T14:51:54.478394Z",
│ │ │ │ │ │ - "iopub.status.idle": "2024-11-08T14:51:55.099332Z",
│ │ │ │ │ │ - "shell.execute_reply": "2024-11-08T14:51:55.098252Z"
│ │ │ │ │ │ + "iopub.execute_input": "2025-12-11T23:36:27.893615Z",
│ │ │ │ │ │ + "iopub.status.busy": "2025-12-11T23:36:27.893061Z",
│ │ │ │ │ │ + "iopub.status.idle": "2025-12-11T23:36:28.358030Z",
│ │ │ │ │ │ + "shell.execute_reply": "2025-12-11T23:36:28.357414Z"
│ │ │ │ │ │ }
│ │ │ │ │ │ },
│ │ │ │ │ │ "outputs": [],
│ │ │ │ │ │ "source": [
│ │ │ │ │ │ "import pandas as pd\n",
│ │ │ │ │ │ "import numpy as np\n",
│ │ │ │ │ │ "import matplotlib as mpl\n"
│ │ │ │ │ │ ]
│ │ │ │ │ │ },
│ │ │ │ │ │ {
│ │ │ │ │ │ "cell_type": "code",
│ │ │ │ │ │ "execution_count": 3,
│ │ │ │ │ │ "metadata": {
│ │ │ │ │ │ "execution": {
│ │ │ │ │ │ - "iopub.execute_input": "2024-11-08T14:51:55.104847Z",
│ │ │ │ │ │ - "iopub.status.busy": "2024-11-08T14:51:55.104268Z",
│ │ │ │ │ │ - "iopub.status.idle": "2024-11-08T14:51:55.230295Z",
│ │ │ │ │ │ - "shell.execute_reply": "2024-11-08T14:51:55.229357Z"
│ │ │ │ │ │ + "iopub.execute_input": "2025-12-11T23:36:28.360993Z",
│ │ │ │ │ │ + "iopub.status.busy": "2025-12-11T23:36:28.360514Z",
│ │ │ │ │ │ + "iopub.status.idle": "2025-12-11T23:36:28.431529Z",
│ │ │ │ │ │ + "shell.execute_reply": "2025-12-11T23:36:28.430970Z"
│ │ │ │ │ │ },
│ │ │ │ │ │ "nbsphinx": "hidden"
│ │ │ │ │ │ },
│ │ │ │ │ │ "outputs": [],
│ │ │ │ │ │ "source": [
│ │ │ │ │ │ "# For reproducibility - this doesn't respect uuid_len or positionally-passed uuid but the places here that use that coincidentally bypass this anyway\n",
│ │ │ │ │ │ "from pandas.io.formats.style import Styler\n",
│ │ │ │ │ │ @@ -123,18 +123,18 @@
│ │ │ │ │ │ ]
│ │ │ │ │ │ },
│ │ │ │ │ │ {
│ │ │ │ │ │ "cell_type": "code",
│ │ │ │ │ │ "execution_count": 4,
│ │ │ │ │ │ "metadata": {
│ │ │ │ │ │ "execution": {
│ │ │ │ │ │ - "iopub.execute_input": "2024-11-08T14:51:55.236100Z",
│ │ │ │ │ │ - "iopub.status.busy": "2024-11-08T14:51:55.234984Z",
│ │ │ │ │ │ - "iopub.status.idle": "2024-11-08T14:51:55.251807Z",
│ │ │ │ │ │ - "shell.execute_reply": "2024-11-08T14:51:55.250847Z"
│ │ │ │ │ │ + "iopub.execute_input": "2025-12-11T23:36:28.434079Z",
│ │ │ │ │ │ + "iopub.status.busy": "2025-12-11T23:36:28.433553Z",
│ │ │ │ │ │ + "iopub.status.idle": "2025-12-11T23:36:28.442917Z",
│ │ │ │ │ │ + "shell.execute_reply": "2025-12-11T23:36:28.442347Z"
│ │ │ │ │ │ }
│ │ │ │ │ │ },
│ │ │ │ │ │ "outputs": [
│ │ │ │ │ │ {
│ │ │ │ │ │ "data": {
│ │ │ │ │ │ "text/html": [
│ │ │ │ │ │ "