done

lib/python3.11/site-packages/numpy/random/_examples/cffi/extending.py

@@ -0,0 +1,44 @@
+"""
+Use cffi to access any of the underlying C functions from distributions.h
+"""
+import os
+
+import cffi
+
+import numpy as np
+
+from .parse import parse_distributions_h
+
+ffi = cffi.FFI()
+
+inc_dir = os.path.join(np.get_include(), 'numpy')
+
+# Basic numpy types
+ffi.cdef('''
+    typedef intptr_t npy_intp;
+    typedef unsigned char npy_bool;
+
+''')
+
+parse_distributions_h(ffi, inc_dir)
+
+lib = ffi.dlopen(np.random._generator.__file__)
+
+# Compare the distributions.h random_standard_normal_fill to
+# Generator.standard_random
+bit_gen = np.random.PCG64()
+rng = np.random.Generator(bit_gen)
+state = bit_gen.state
+
+interface = rng.bit_generator.cffi
+n = 100
+vals_cffi = ffi.new('double[%d]' % n)
+lib.random_standard_normal_fill(interface.bit_generator, n, vals_cffi)
+
+# reset the state
+bit_gen.state = state
+
+vals = rng.standard_normal(n)
+
+for i in range(n):
+    assert vals[i] == vals_cffi[i]

lib/python3.11/site-packages/numpy/random/_examples/cffi/parse.py

@@ -0,0 +1,53 @@
+import os
+
+
+def parse_distributions_h(ffi, inc_dir):
+    """
+    Parse distributions.h located in inc_dir for CFFI, filling in the ffi.cdef
+
+    Read the function declarations without the "#define ..." macros that will
+    be filled in when loading the library.
+    """
+
+    with open(os.path.join(inc_dir, 'random', 'bitgen.h')) as fid:
+        s = []
+        for line in fid:
+            # massage the include file
+            if line.strip().startswith('#'):
+                continue
+            s.append(line)
+        ffi.cdef('\n'.join(s))
+
+    with open(os.path.join(inc_dir, 'random', 'distributions.h')) as fid:
+        s = []
+        in_skip = 0
+        ignoring = False
+        for line in fid:
+            # check for and remove extern "C" guards
+            if ignoring:
+                if line.strip().startswith('#endif'):
+                    ignoring = False
+                continue
+            if line.strip().startswith('#ifdef __cplusplus'):
+                ignoring = True
+
+            # massage the include file
+            if line.strip().startswith('#'):
+                continue
+
+            # skip any inlined function definition
+            # which starts with 'static inline xxx(...) {'
+            # and ends with a closing '}'
+            if line.strip().startswith('static inline'):
+                in_skip += line.count('{')
+                continue
+            elif in_skip > 0:
+                in_skip += line.count('{')
+                in_skip -= line.count('}')
+                continue
+
+            # replace defines with their value or remove them
+            line = line.replace('DECLDIR', '')
+            line = line.replace('RAND_INT_TYPE', 'int64_t')
+            s.append(line)
+        ffi.cdef('\n'.join(s))

lib/python3.11/site-packages/numpy/random/_examples/cython/extending.pyx

@@ -0,0 +1,77 @@
+#cython: language_level=3
+
+from libc.stdint cimport uint32_t
+from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer
+
+import numpy as np
+cimport numpy as np
+cimport cython
+
+from numpy.random cimport bitgen_t
+from numpy.random import PCG64
+
+np.import_array()
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+def uniform_mean(Py_ssize_t n):
+    cdef Py_ssize_t i
+    cdef bitgen_t *rng
+    cdef const char *capsule_name = "BitGenerator"
+    cdef double[::1] random_values
+    cdef np.ndarray randoms
+
+    x = PCG64()
+    capsule = x.capsule
+    if not PyCapsule_IsValid(capsule, capsule_name):
+        raise ValueError("Invalid pointer to anon_func_state")
+    rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name)
+    random_values = np.empty(n)
+    # Best practice is to acquire the lock whenever generating random values.
+    # This prevents other threads from modifying the state. Acquiring the lock
+    # is only necessary if the GIL is also released, as in this example.
+    with x.lock, nogil:
+        for i in range(n):
+            random_values[i] = rng.next_double(rng.state)
+    randoms = np.asarray(random_values)
+    return randoms.mean()
+
+
+# This function is declared nogil so it can be used without the GIL below
+cdef uint32_t bounded_uint(uint32_t lb, uint32_t ub, bitgen_t *rng) nogil:
+    cdef uint32_t mask, delta, val
+    mask = delta = ub - lb
+    mask |= mask >> 1
+    mask |= mask >> 2
+    mask |= mask >> 4
+    mask |= mask >> 8
+    mask |= mask >> 16
+
+    val = rng.next_uint32(rng.state) & mask
+    while val > delta:
+        val = rng.next_uint32(rng.state) & mask
+
+    return lb + val
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+def bounded_uints(uint32_t lb, uint32_t ub, Py_ssize_t n):
+    cdef Py_ssize_t i
+    cdef bitgen_t *rng
+    cdef uint32_t[::1] out
+    cdef const char *capsule_name = "BitGenerator"
+
+    x = PCG64()
+    out = np.empty(n, dtype=np.uint32)
+    capsule = x.capsule
+
+    if not PyCapsule_IsValid(capsule, capsule_name):
+        raise ValueError("Invalid pointer to anon_func_state")
+    rng = <bitgen_t *>PyCapsule_GetPointer(capsule, capsule_name)
+
+    with x.lock, nogil:
+        for i in range(n):
+            out[i] = bounded_uint(lb, ub, rng)
+    return np.asarray(out)

lib/python3.11/site-packages/numpy/random/_examples/cython/extending_distributions.pyx

@@ -0,0 +1,118 @@
+#cython: language_level=3
+"""
+This file shows how the to use a BitGenerator to create a distribution.
+"""
+import numpy as np
+cimport numpy as np
+cimport cython
+from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer
+from libc.stdint cimport uint16_t, uint64_t
+from numpy.random cimport bitgen_t
+from numpy.random import PCG64
+from numpy.random.c_distributions cimport (
+      random_standard_uniform_fill, random_standard_uniform_fill_f)
+
+np.import_array()
+
+
+@cython.boundscheck(False)
+@cython.wraparound(False)
+def uniforms(Py_ssize_t n):
+    """
+    Create an array of `n` uniformly distributed doubles.
+    A 'real' distribution would want to process the values into
+    some non-uniform distribution
+    """
+    cdef Py_ssize_t i
+    cdef bitgen_t *rng
+    cdef const char *capsule_name = "BitGenerator"
+    cdef double[::1] random_values
+
+    x = PCG64()
+    capsule = x.capsule
+    # Optional check that the capsule if from a BitGenerator
+    if not PyCapsule_IsValid(capsule, capsule_name):
+        raise ValueError("Invalid pointer to anon_func_state")
+    # Cast the pointer
+    rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name)
+    random_values = np.empty(n, dtype='float64')
+    with x.lock, nogil:
+        for i in range(n):
+            # Call the function
+            random_values[i] = rng.next_double(rng.state)
+    randoms = np.asarray(random_values)
+
+    return randoms
+
+# cython example 2
+@cython.boundscheck(False)
+@cython.wraparound(False)
+def uint10_uniforms(Py_ssize_t n):
+    """Uniform 10 bit integers stored as 16-bit unsigned integers"""
+    cdef Py_ssize_t i
+    cdef bitgen_t *rng
+    cdef const char *capsule_name = "BitGenerator"
+    cdef uint16_t[::1] random_values
+    cdef int bits_remaining
+    cdef int width = 10
+    cdef uint64_t buff, mask = 0x3FF
+
+    x = PCG64()
+    capsule = x.capsule
+    if not PyCapsule_IsValid(capsule, capsule_name):
+        raise ValueError("Invalid pointer to anon_func_state")
+    rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name)
+    random_values = np.empty(n, dtype='uint16')
+    # Best practice is to release GIL and acquire the lock
+    bits_remaining = 0
+    with x.lock, nogil:
+        for i in range(n):
+            if bits_remaining < width:
+                buff = rng.next_uint64(rng.state)
+            random_values[i] = buff & mask
+            buff >>= width
+
+    randoms = np.asarray(random_values)
+    return randoms
+
+# cython example 3
+def uniforms_ex(bit_generator, Py_ssize_t n, dtype=np.float64):
+    """
+    Create an array of `n` uniformly distributed doubles via a "fill" function.
+
+    A 'real' distribution would want to process the values into
+    some non-uniform distribution
+
+    Parameters
+    ----------
+    bit_generator: BitGenerator instance
+    n: int
+        Output vector length
+    dtype: {str, dtype}, optional
+        Desired dtype, either 'd' (or 'float64') or 'f' (or 'float32'). The
+        default dtype value is 'd'
+    """
+    cdef Py_ssize_t i
+    cdef bitgen_t *rng
+    cdef const char *capsule_name = "BitGenerator"
+    cdef np.ndarray randoms
+
+    capsule = bit_generator.capsule
+    # Optional check that the capsule if from a BitGenerator
+    if not PyCapsule_IsValid(capsule, capsule_name):
+        raise ValueError("Invalid pointer to anon_func_state")
+    # Cast the pointer
+    rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name)
+
+    _dtype = np.dtype(dtype)
+    randoms = np.empty(n, dtype=_dtype)
+    if _dtype == np.float32:
+        with bit_generator.lock:
+            random_standard_uniform_fill_f(rng, n, <float*>np.PyArray_DATA(randoms))
+    elif _dtype == np.float64:
+        with bit_generator.lock:
+            random_standard_uniform_fill(rng, n, <double*>np.PyArray_DATA(randoms))
+    else:
+        raise TypeError('Unsupported dtype %r for random' % _dtype)
+    return randoms
+

lib/python3.11/site-packages/numpy/random/_examples/cython/meson.build

@@ -0,0 +1,53 @@
+project('random-build-examples', 'c', 'cpp', 'cython')
+
+py_mod = import('python')
+py3 = py_mod.find_installation(pure: false)
+
+cc = meson.get_compiler('c')
+cy = meson.get_compiler('cython')
+
+# Keep synced with pyproject.toml
+if not cy.version().version_compare('>=3.0.6')
+  error('tests requires Cython >= 3.0.6')
+endif
+
+base_cython_args = []
+if cy.version().version_compare('>=3.1.0')
+  base_cython_args += ['-Xfreethreading_compatible=True']
+endif
+
+_numpy_abs = run_command(py3, ['-c',
+               'import os; os.chdir(".."); import numpy; print(os.path.abspath(numpy.get_include() + "../../.."))'],
+                         check: true).stdout().strip()
+
+npymath_path = _numpy_abs / '_core' / 'lib'
+npy_include_path = _numpy_abs / '_core' / 'include'
+npyrandom_path = _numpy_abs / 'random' / 'lib'
+npymath_lib = cc.find_library('npymath', dirs: npymath_path)
+npyrandom_lib = cc.find_library('npyrandom', dirs: npyrandom_path)
+
+py3.extension_module(
+    'extending_distributions',
+    'extending_distributions.pyx',
+    install: false,
+    include_directories: [npy_include_path],
+    dependencies: [npyrandom_lib, npymath_lib],
+    cython_args: base_cython_args,
+)
+py3.extension_module(
+    'extending',
+    'extending.pyx',
+    install: false,
+    include_directories: [npy_include_path],
+    dependencies: [npyrandom_lib, npymath_lib],
+    cython_args: base_cython_args,
+)
+py3.extension_module(
+    'extending_cpp',
+    'extending_distributions.pyx',
+    install: false,
+    override_options : ['cython_language=cpp'],
+    cython_args: base_cython_args + ['--module-name', 'extending_cpp'],
+    include_directories: [npy_include_path],
+    dependencies: [npyrandom_lib, npymath_lib],
+)

lib/python3.11/site-packages/numpy/random/_examples/numba/extending.py

@@ -0,0 +1,86 @@
+from timeit import timeit
+
+import numba as nb
+
+import numpy as np
+from numpy.random import PCG64
+
+bit_gen = PCG64()
+next_d = bit_gen.cffi.next_double
+state_addr = bit_gen.cffi.state_address
+
+def normals(n, state):
+    out = np.empty(n)
+    for i in range((n + 1) // 2):
+        x1 = 2.0 * next_d(state) - 1.0
+        x2 = 2.0 * next_d(state) - 1.0
+        r2 = x1 * x1 + x2 * x2
+        while r2 >= 1.0 or r2 == 0.0:
+            x1 = 2.0 * next_d(state) - 1.0
+            x2 = 2.0 * next_d(state) - 1.0
+            r2 = x1 * x1 + x2 * x2
+        f = np.sqrt(-2.0 * np.log(r2) / r2)
+        out[2 * i] = f * x1
+        if 2 * i + 1 < n:
+            out[2 * i + 1] = f * x2
+    return out
+
+
+# Compile using Numba
+normalsj = nb.jit(normals, nopython=True)
+# Must use state address not state with numba
+n = 10000
+
+def numbacall():
+    return normalsj(n, state_addr)
+
+
+rg = np.random.Generator(PCG64())
+
+def numpycall():
+    return rg.normal(size=n)
+
+
+# Check that the functions work
+r1 = numbacall()
+r2 = numpycall()
+assert r1.shape == (n,)
+assert r1.shape == r2.shape
+
+t1 = timeit(numbacall, number=1000)
+print(f'{t1:.2f} secs for {n} PCG64 (Numba/PCG64) gaussian randoms')
+t2 = timeit(numpycall, number=1000)
+print(f'{t2:.2f} secs for {n} PCG64 (NumPy/PCG64) gaussian randoms')
+
+# example 2
+
+next_u32 = bit_gen.ctypes.next_uint32
+ctypes_state = bit_gen.ctypes.state
+
+@nb.jit(nopython=True)
+def bounded_uint(lb, ub, state):
+    mask = delta = ub - lb
+    mask |= mask >> 1
+    mask |= mask >> 2
+    mask |= mask >> 4
+    mask |= mask >> 8
+    mask |= mask >> 16
+
+    val = next_u32(state) & mask
+    while val > delta:
+        val = next_u32(state) & mask
+
+    return lb + val
+
+
+print(bounded_uint(323, 2394691, ctypes_state.value))
+
+
+@nb.jit(nopython=True)
+def bounded_uints(lb, ub, n, state):
+    out = np.empty(n, dtype=np.uint32)
+    for i in range(n):
+        out[i] = bounded_uint(lb, ub, state)
+
+
+bounded_uints(323, 2394691, 10000000, ctypes_state.value)

lib/python3.11/site-packages/numpy/random/_examples/numba/extending_distributions.py

@@ -0,0 +1,67 @@
+r"""
+Building the required library in this example requires a source distribution
+of NumPy or clone of the NumPy git repository since distributions.c is not
+included in binary distributions.
+
+On *nix, execute in numpy/random/src/distributions
+
+export ${PYTHON_VERSION}=3.8 # Python version
+export PYTHON_INCLUDE=#path to Python's include folder, usually \
+    ${PYTHON_HOME}/include/python${PYTHON_VERSION}m
+export NUMPY_INCLUDE=#path to numpy's include folder, usually \
+    ${PYTHON_HOME}/lib/python${PYTHON_VERSION}/site-packages/numpy/_core/include
+gcc -shared -o libdistributions.so -fPIC distributions.c \
+    -I${NUMPY_INCLUDE} -I${PYTHON_INCLUDE}
+mv libdistributions.so ../../_examples/numba/
+
+On Windows
+
+rem PYTHON_HOME and PYTHON_VERSION are setup dependent, this is an example
+set PYTHON_HOME=c:\Anaconda
+set PYTHON_VERSION=38
+cl.exe /LD .\distributions.c -DDLL_EXPORT \
+    -I%PYTHON_HOME%\lib\site-packages\numpy\_core\include \
+    -I%PYTHON_HOME%\include %PYTHON_HOME%\libs\python%PYTHON_VERSION%.lib
+move distributions.dll ../../_examples/numba/
+"""
+import os
+
+import numba as nb
+from cffi import FFI
+
+import numpy as np
+from numpy.random import PCG64
+
+ffi = FFI()
+if os.path.exists('./distributions.dll'):
+    lib = ffi.dlopen('./distributions.dll')
+elif os.path.exists('./libdistributions.so'):
+    lib = ffi.dlopen('./libdistributions.so')
+else:
+    raise RuntimeError('Required DLL/so file was not found.')
+
+ffi.cdef("""
+double random_standard_normal(void *bitgen_state);
+""")
+x = PCG64()
+xffi = x.cffi
+bit_generator = xffi.bit_generator
+
+random_standard_normal = lib.random_standard_normal
+
+
+def normals(n, bit_generator):
+    out = np.empty(n)
+    for i in range(n):
+        out[i] = random_standard_normal(bit_generator)
+    return out
+
+
+normalsj = nb.jit(normals, nopython=True)
+
+# Numba requires a memory address for void *
+# Can also get address from x.ctypes.bit_generator.value
+bit_generator_address = int(ffi.cast('uintptr_t', bit_generator))
+
+norm = normalsj(1000, bit_generator_address)
+print(norm[:12])