blender
/
blender
129
398
Fork 572
Code Issues 6.2k Pull Requests 748 Projects 19 Wiki Activity

Python: add foreach_get and foreach_set methods to pyrna_prop_array 

This allows fast access to various arrays in the Python API.
Most notably, `image.pixels` can be accessed much more efficiently now.

**Benchmark**

Below are the results of a benchmark that compares different ways to
set/get all pixel values. I do the tests on 2048x2048 rgba images.
The benchmark tests the following dimensions:
- Byte vs. float per color channel
- Python list vs. numpy array containing floats
- `foreach_set` (new) vs. `image.pixels = ...` (old)

```
Pixel amount: 2048 * 2048 = 4.194.304
Byte buffer size:  16.8 mb
Float buffer size: 67.1 mb

Set pixel colors:
    byte  - new - list:    271 ms
    byte  - new - buffer:   29 ms
    byte  - old - list:    350 ms
    byte  - old - buffer: 2900 ms

    float - new - list:    249 ms
    float - new - buffer:    8 ms
    float - old - list:    330 ms
    float - old - buffer: 2880 ms

Get pixel colors:
    byte - list:   128 ms
    byte - buffer:   9 ms
    float - list:  125 ms
    float - buffer:  8 ms
```

**Observations**

The best set and get speed can be achieved with buffers and a float image,
at the cost of higher memory consumption. Furthermore, using buffers when
using `pixels = ...` is incredibly slow, because it is not optimized.
Optimizing this is possible, but might not be trivial (there were multiple
attempts afaik).

Float images are faster due to overhead introduced by the api for byte images.
If I profiled it correctly, a lot of time is spend in the `[0, 1] -> {0, ..., 255}`
conversion. The functions doing that conversion is `unit_float_to_uchar_clamp`.
While I have an idea on how it can be optimized, I do not know if it can be done
without changing its functionality slightly. Performance wise the best solution
would be to not do this conversion at all and accept byte input from the api
user directly, but that seems to be a more involved task as well.

Differential Revision: https://developer.blender.org/D7053

Reviewers: JacquesLucke, mont29
This commit is contained in:
Bogdan Nagirniak 2020-03-13 12:57:12 +01:00 committed by Jacques Lucke
parent db4298c6be
commit 9075ec8269
3 changed files with 267 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

177
source/blender/python/intern/bpy_rna.c
View File

@ -5424,6 +5424,174 @@ static PyObject *pyrna_prop_collection_foreach_set(BPy_PropertyRNA *self, PyObje
return foreach_getset(self, args, 1);
}
static PyObject *pyprop_array_foreach_getset(BPy_PropertyArrayRNA *self,
PyObject *args,
const bool do_set)
{
PyObject *item = NULL;
Py_ssize_t i, seq_size, size;
void *array = NULL;
PropertyType prop_type = RNA_property_type(self->prop);
/* Get/set both take the same args currently. */
PyObject *seq;
if (prop_type != PROP_INT && prop_type != PROP_FLOAT) {
PyErr_Format(PyExc_TypeError, "foreach_get/set available only for int and float");
return NULL;
}
if (!PyArg_ParseTuple(args, "O:foreach_get/set", &seq)) {
return NULL;
}
if (!PySequence_Check(seq) && PyObject_CheckBuffer(seq)) {
PyErr_Format(
PyExc_TypeError,
"foreach_get/set expected second argument to be a sequence or buffer, not a %.200s",
Py_TYPE(seq)->tp_name);
return NULL;
}
size = pyrna_prop_array_length(self);
seq_size = PySequence_Size(seq);
if (size != seq_size) {
PyErr_Format(PyExc_TypeError, "expected sequence size %d, got %d", size, seq_size);
return NULL;
}
Py_buffer buf;
if (PyObject_GetBuffer(seq, &buf, PyBUF_SIMPLE | PyBUF_FORMAT) == -1) {
PyErr_Clear();
switch (prop_type) {
case PROP_INT:
array = PyMem_Malloc(sizeof(int) * size);
if (do_set) {
for (i = 0; i < size; i++) {
item = PySequence_GetItem(seq, i);
((int *)array)[i] = (int)PyLong_AsLong(item);
Py_DECREF(item);
}
RNA_property_int_set_array(&self->ptr, self->prop, array);
}
else {
RNA_property_int_get_array(&self->ptr, self->prop, array);
for (i = 0; i < size; i++) {
item = PyLong_FromLong((long)((int *)array)[i]);
PySequence_SetItem(seq, i, item);
Py_DECREF(item);
}
}
break;
case PROP_FLOAT:
array = PyMem_Malloc(sizeof(float) * size);
if (do_set) {
for (i = 0; i < size; i++) {
item = PySequence_GetItem(seq, i);
((float *)array)[i] = (float)PyFloat_AsDouble(item);
Py_DECREF(item);
}
RNA_property_float_set_array(&self->ptr, self->prop, array);
}
else {
RNA_property_float_get_array(&self->ptr, self->prop, array);
for (i = 0; i < size; i++) {
item = PyFloat_FromDouble((double)((float *)array)[i]);
PySequence_SetItem(seq, i, item);
Py_DECREF(item);
}
}
break;
case PROP_BOOLEAN:
case PROP_STRING:
case PROP_ENUM:
case PROP_POINTER:
case PROP_COLLECTION:
/* Should never happen. */
BLI_assert(false);
break;
}
PyMem_Free(array);
if (PyErr_Occurred()) {
/* Maybe we could make our own error. */
PyErr_Print();
PyErr_SetString(PyExc_TypeError, "couldn't access the py sequence");
return NULL;
}
}
else {
char f = buf.format ? buf.format[0] : 0;
if ((prop_type == PROP_INT && (buf.itemsize != sizeof(int) || (f != 'l' && f != 'i'))) ||
(prop_type == PROP_FLOAT && (buf.itemsize != sizeof(float) || f != 'f'))) {
PyBuffer_Release(&buf);
PyErr_Format(PyExc_TypeError, "incorrect sequence item type: %s", buf.format);
return NULL;
}
switch (prop_type) {
case PROP_INT:
if (do_set) {
RNA_property_int_set_array(&self->ptr, self->prop, buf.buf);
}
else {
RNA_property_int_get_array(&self->ptr, self->prop, buf.buf);
}
break;
case PROP_FLOAT:
if (do_set) {
RNA_property_float_set_array(&self->ptr, self->prop, buf.buf);
}
else {
RNA_property_float_get_array(&self->ptr, self->prop, buf.buf);
}
break;
case PROP_BOOLEAN:
case PROP_STRING:
case PROP_ENUM:
case PROP_POINTER:
case PROP_COLLECTION:
/* Should never happen. */
BLI_assert(false);
break;
}
PyBuffer_Release(&buf);
}
Py_RETURN_NONE;
}
PyDoc_STRVAR(pyrna_prop_array_foreach_get_doc,
".. method:: foreach_get(seq)\n"
"\n"
" This is a function to give fast access to array data.\n");
static PyObject *pyrna_prop_array_foreach_get(BPy_PropertyArrayRNA *self, PyObject *args)
{
PYRNA_PROP_CHECK_OBJ((BPy_PropertyRNA *)self);
return pyprop_array_foreach_getset(self, args, false);
}
PyDoc_STRVAR(pyrna_prop_array_foreach_set_doc,
".. method:: foreach_set(seq)\n"
"\n"
" This is a function to give fast access to array data.\n");
static PyObject *pyrna_prop_array_foreach_set(BPy_PropertyArrayRNA *self, PyObject *args)
{
PYRNA_PROP_CHECK_OBJ((BPy_PropertyRNA *)self);
return pyprop_array_foreach_getset(self, args, true);
}
/* A bit of a kludge, make a list out of a collection or array,
* then return the list's iter function, not especially fast, but convenient for now. */
static PyObject *pyrna_prop_array_iter(BPy_PropertyArrayRNA *self)
@ -5572,6 +5740,15 @@ static struct PyMethodDef pyrna_prop_methods[] = {
};
static struct PyMethodDef pyrna_prop_array_methods[] = {
{"foreach_get",
(PyCFunction)pyrna_prop_array_foreach_get,
METH_VARARGS,
pyrna_prop_array_foreach_get_doc},
{"foreach_set",
(PyCFunction)pyrna_prop_array_foreach_set,
METH_VARARGS,
pyrna_prop_array_foreach_set_doc},
{NULL, NULL, 0, NULL},
};

5
tests/python/CMakeLists.txt
View File

@ -120,6 +120,11 @@ add_blender_test(
--python ${CMAKE_CURRENT_LIST_DIR}/bl_pyapi_idprop_datablock.py
)
add_blender_test(
script_pyapi_prop_array
--python ${CMAKE_CURRENT_LIST_DIR}/bl_pyapi_prop_array.py
)
# ------------------------------------------------------------------------------
# DATA MANAGEMENT TESTS

85
tests/python/bl_pyapi_prop_array.py Normal file
View File

@ -0,0 +1,85 @@
# Apache License, Version 2.0
# ./blender.bin --background -noaudio --python tests/python/bl_pyapi_prop_array.py -- --verbose
import bpy
import unittest
import numpy as np
class TestPropArray(unittest.TestCase):
def setUp(self):
bpy.types.Scene.test_array_f = bpy.props.FloatVectorProperty(size=10)
bpy.types.Scene.test_array_i = bpy.props.IntVectorProperty(size=10)
scene = bpy.context.scene
self.array_f = scene.test_array_f
self.array_i = scene.test_array_i
def test_foreach_getset_i(self):
with self.assertRaises(TypeError):
self.array_i.foreach_set(range(5))
self.array_i.foreach_set(range(5, 15))
with self.assertRaises(TypeError):
self.array_i.foreach_set(np.arange(5, dtype=np.int32))
with self.assertRaises(TypeError):
self.array_i.foreach_set(np.arange(10, dtype=np.int64))
with self.assertRaises(TypeError):
self.array_i.foreach_get(np.arange(10, dtype=np.float32))
a = np.arange(10, dtype=np.int32)
self.array_i.foreach_set(a)
with self.assertRaises(TypeError):
self.array_i.foreach_set(a[:5])
for v1, v2 in zip(a, self.array_i[:]):
self.assertEqual(v1, v2)
b = np.empty(10, dtype=np.int32)
self.array_i.foreach_get(b)
for v1, v2 in zip(a, b):
self.assertEqual(v1, v2)
b = [None] * 10
self.array_f.foreach_get(b)
for v1, v2 in zip(a, b):
self.assertEqual(v1, v2)
def test_foreach_getset_f(self):
with self.assertRaises(TypeError):
self.array_i.foreach_set(range(5))
self.array_f.foreach_set(range(5, 15))
with self.assertRaises(TypeError):
self.array_f.foreach_set(np.arange(5, dtype=np.float32))
with self.assertRaises(TypeError):
self.array_f.foreach_set(np.arange(10, dtype=np.int32))
with self.assertRaises(TypeError):
self.array_f.foreach_get(np.arange(10, dtype=np.float64))
a = np.arange(10, dtype=np.float32)
self.array_f.foreach_set(a)
for v1, v2 in zip(a, self.array_f[:]):
self.assertEqual(v1, v2)
b = np.empty(10, dtype=np.float32)
self.array_f.foreach_get(b)
for v1, v2 in zip(a, b):
self.assertEqual(v1, v2)
b = [None] * 10
self.array_f.foreach_get(b)
for v1, v2 in zip(a, b):
self.assertEqual(v1, v2)
if __name__ == '__main__':
import sys
sys.argv = [__file__] + (sys.argv[sys.argv.index("--") + 1:] if "--" in sys.argv else [])
unittest.main()