Fix T55964: Direction not normalized in isect_ray_aabb_v3_simple()?

RNA API Object.ray_cast would not normalize direction vector before
doing first quick bbox intersection test, while using its returned
distance value. This could lead to wrong exclusion of object.
Thanks to @codemanx for finding that issue.
This commit is contained in:
Bastien Montagne 2018-07-25 21:05:44 +02:00
parent 5ba4d059c0
commit 29d1db9ed6
Notes: blender-bot 2023-02-14 19:27:44 +01:00
Referenced by issue blender/blender-addons#55964, Direction not normalized in isect_ray_aabb_v3_simple()?
1 changed files with 4 additions and 5 deletions

View File

@ -332,8 +332,10 @@ static void rna_Object_ray_cast(
/* Test BoundBox first (efficiency) */
BoundBox *bb = BKE_object_boundbox_get(ob);
float distmin;
if (!bb || (isect_ray_aabb_v3_simple(origin, direction, bb->vec[0], bb->vec[6], &distmin, NULL) && distmin <= distance)) {
normalize_v3(direction); /* Needed for valid distance check from isect_ray_aabb_v3_simple() call. */
if (!bb ||
(isect_ray_aabb_v3_simple(origin, direction, bb->vec[0], bb->vec[6], &distmin, NULL) && distmin <= distance))
{
BVHTreeFromMesh treeData = {NULL};
/* no need to managing allocation or freeing of the BVH data. this is generated and freed as needed */
@ -346,9 +348,6 @@ static void rna_Object_ray_cast(
hit.index = -1;
hit.dist = distance;
normalize_v3(direction);
if (BLI_bvhtree_ray_cast(treeData.tree, origin, direction, 0.0f, &hit,
treeData.raycast_callback, &treeData) != -1)
{