"""Useful functions ot evaluate the objects in the camera's viewport."""
import numbers
import sys
from typing import Union, List, Set
from collections import defaultdict
import bpy
import numpy as np
from mathutils import Matrix
from mathutils.bvhtree import BVHTree
from blenderproc.python.types.MeshObjectUtility import MeshObject
[docs]
def visible_objects(cam2world_matrix: Union[Matrix, np.ndarray], sqrt_number_of_rays: int = 10) -> Set[MeshObject]:
""" Returns a set of objects visible from the given camera pose.
Sends a grid of rays through the camera frame and returns all objects hit by at least one ray.
:param cam2world_matrix: The world matrix which describes the camera orientation to check.
:param sqrt_number_of_rays: The square root of the number of rays which will be used to determine the
visible objects.
:return: A set of objects visible hit by the sent rays.
"""
cam2world_matrix = Matrix(cam2world_matrix)
visible_objects_set = set()
cam_ob = bpy.context.scene.camera
cam = cam_ob.data
# Get position of the corners of the near plane
frame = cam.view_frame(scene=bpy.context.scene)
# Bring to world space
frame = [cam2world_matrix @ v for v in frame]
# Compute vectors along both sides of the plane
vec_x = frame[1] - frame[0]
vec_y = frame[3] - frame[0]
# Go in discrete grid-like steps over plane
position = cam2world_matrix.to_translation()
for x in range(0, sqrt_number_of_rays):
for y in range(0, sqrt_number_of_rays):
# Compute current point on plane
end = frame[0] + vec_x * x / float(sqrt_number_of_rays - 1) + vec_y * y / float(sqrt_number_of_rays - 1)
# Send ray from the camera position through the current point on the plane
_, _, _, _, hit_object, _ = bpy.context.scene.ray_cast(bpy.context.evaluated_depsgraph_get(),
position, end - position)
# Add hit object to set
if hit_object:
visible_objects_set.add(MeshObject(hit_object))
return visible_objects_set
[docs]
def scene_coverage_score(cam2world_matrix: Union[Matrix, np.ndarray], special_objects: list = None,
special_objects_weight: float = 2, sqrt_number_of_rays: int = 10) -> float:
""" Evaluate the interestingness/coverage of the scene.
This module tries to look at as many objects at possible, this might lead to
a focus on the same objects from similar angles.
Only for SUNCG and 3D Front:
The least interesting objects: walls, ceilings, floors.
:param cam2world_matrix: The world matrix which describes the camera pose to check.
:param special_objects: Objects that weights differently in calculating whether the scene is interesting or not,
uses the coarse_grained_class or if not SUNCG, 3D Front, the category_id.
:param special_objects_weight: Weighting factor for more special objects, used to estimate how interesting the
scene is. Default: 2.0.
:param sqrt_number_of_rays: The square root of the number of rays which will be used to determine the
visible objects.
:return: the scoring of the scene.
"""
cam2world_matrix = Matrix(cam2world_matrix)
if special_objects is None:
special_objects = []
cam_ob = bpy.context.scene.camera
cam = cam_ob.data
num_of_rays = sqrt_number_of_rays * sqrt_number_of_rays
score = 0.0
objects_hit: defaultdict = defaultdict(int)
# Get position of the corners of the near plane
frame = cam.view_frame(scene=bpy.context.scene)
# Bring to world space
frame = [cam2world_matrix @ v for v in frame]
# Compute vectors along both sides of the plane
vec_x = frame[1] - frame[0]
vec_y = frame[3] - frame[0]
# Go in discrete grid-like steps over plane
position = cam2world_matrix.to_translation()
for x in range(0, sqrt_number_of_rays):
for y in range(0, sqrt_number_of_rays):
# Compute current point on plane
end = frame[0] + vec_x * x / float(sqrt_number_of_rays - 1) + vec_y * y / float(sqrt_number_of_rays - 1)
# Send ray from the camera position through the current point on the plane
hit, _, _, _, hit_object, _ = bpy.context.scene.ray_cast(bpy.context.evaluated_depsgraph_get(),
position, end - position)
if hit:
is_of_special_dataset = "is_suncg" in hit_object or "is_3d_front" in hit_object
is_suncg_object = "suncg_type" in hit_object and hit_object["suncg_type"] == "Object"
is_front_3d_object = "3D_future_type" in hit_object and hit_object["3D_future_type"] == "Object"
if is_of_special_dataset and is_suncg_object or is_of_special_dataset and is_front_3d_object:
# calculate the score based on the type of the object,
# wall, floor and ceiling objects have 0 score
if "coarse_grained_class" in hit_object:
object_class = hit_object["coarse_grained_class"]
objects_hit[object_class] += 1
if object_class in special_objects:
score += special_objects_weight
else:
score += 1
else:
score += 1
elif "category_id" in hit_object:
object_class = hit_object["category_id"]
if object_class in special_objects:
score += special_objects_weight
else:
score += 1
objects_hit[object_class] += 1
else:
objects_hit[hit_object] += 1
score += 1
# For a scene with three different objects, the starting variance is 1.0, increases/decreases by '1/3' for
# each object more/less, excluding floor, ceiling and walls
scene_variance = len(objects_hit) / 3.0
for object_hit_value in objects_hit.values():
# For an object taking half of the scene, the scene_variance is halved, this penalizes non-even
# distribution of the objects in the scene
scene_variance *= 1.0 - object_hit_value / float(num_of_rays)
score = scene_variance * (score / float(num_of_rays))
return score
[docs]
def decrease_interest_score(interest_score: float, min_interest_score: float, interest_score_step: float):
""" Decreases the interest scores in the given interval
:param interest_score: The current interest score.
:param min_interest_score: The minimum desired interest scores.
:param interest_score_step: The step size in which the interest score should be reduced.
:return: Returns the new interest score, and True/False if minimum has not been reached.
"""
if interest_score <= min_interest_score:
return False, interest_score
return True, interest_score - interest_score_step
[docs]
def check_novel_pose(cam2world_matrix: Union[Matrix, np.ndarray], existing_poses: List[Union[Matrix, np.ndarray]],
check_pose_novelty_rot: bool, check_pose_novelty_translation: bool,
min_var_diff_rot: float = -1, min_var_diff_translation: float = -1):
""" Checks if a newly sampled pose is novel based on variance checks.
:param cam2world_matrix: The world matrix which describes the camera pose to check.
:param existing_poses: The list of already sampled valid poses.
:param check_pose_novelty_rot: Checks that a sampled new pose is novel with respect to the rotation component.
:param check_pose_novelty_translation: Checks that a sampled new pose is novel with respect to the
translation component.
:param min_var_diff_rot: Considers a pose novel if it increases the variance of the rotation component of all
poses sampled by this parameter's value in percentage. If set to -1, then it would only
check that the variance is increased. Default: sys.float_info.min.
:param min_var_diff_translation: Same as min_var_diff_rot but for translation. If set to -1, then it would only
check that the variance is increased. Default: sys.float_info.min.
:return: True, if the given pose is novel.
"""
def _variance_constraint(array, new_val, old_var, diff_threshold, mode):
array.append(new_val)
var = np.var(array)
if var < old_var:
array.pop()
return False
diff = ((var - old_var) / old_var) * 100.0
print(f"Variance difference {mode}: {diff}")
if diff < diff_threshold: # Check if the variance increased sufficiently
array.pop()
return False
return True
if len(existing_poses) > 0: # First pose is always novel
cam2world_matrix = Matrix(cam2world_matrix)
if check_pose_novelty_rot:
rotations = [Matrix(pose).to_euler() for pose in existing_poses]
var_rot = np.var(rotations)
if not _variance_constraint(rotations, cam2world_matrix.to_euler(), var_rot, min_var_diff_rot, "rotation"):
return False
if check_pose_novelty_translation:
translations = [Matrix(pose).to_translation() for pose in existing_poses]
var_translation = np.var(translations)
if not _variance_constraint(translations, cam2world_matrix.to_translation(), var_translation,
min_var_diff_translation, "translation"):
return False
return True