mitsuba-visualize/cameras.py

import sys
import argparse
from pathlib import Path
from typing import List
import math

import numpy as np
import xml.etree.ElementTree as etree

sys.path.append('')
import set_python_path
from mitsuba.core import Transform, Matrix4x4, Vector3, PluginManager, normalize, AnimatedTransform, Point3
from mitsuba.render import Sensor

import util
from math_util.catmull_rom_spline import catmull_rom_spline
from math_util.bezier_curve import bezier_curve


def convert_meshlab2mitsuba_camera(camera_def: dict) -> Transform:
    """
    Takes a meshlab camera dict (usually loaded from a meshlab xml file) and turns it into a valid mitsuba Transform
    which can then be applied to a sensor
    :param camera_def: The camera def dict, containing at least keys RotationMatrix and TranslationVector
    :return: A mitsuba transform constructed from the given values
    """
    # Meshlab camera matrix is transposed
    matrix = Matrix4x4([
        float(elem) for elem in camera_def['RotationMatrix'].split(' ')[:16]
    ]).transpose()

    # Add translation vector
    translation = [-float(elem) for elem in camera_def['TranslationVector'].split(' ')]
    for i in range(3):
        matrix[i, 3] = translation[i]

    # Make Mitsuba transform and flip rotation signs (y is not flipped, otherwise resulting image will be flipped vertically)
    transform = Transform(matrix)
    transform *= transform.scale(Vector3(-1, 1, -1))

    return transform


def read_meshlab_sensor_transforms(file_path: Path) -> List[Transform]:
    """
    Reads meshlab camera properties from an xml file. To obtain those, simply press ctrl/cmd+c in meshlab, and paste into a text editor.
    If you paste multiple cameras into one file, make sure to have a valid document structure with one root node, e.g.
    <!DOCTYPE ViewState>
    <project>
     <VCGCamera ... />
     <ViewSettings ... />
    </project>

    :param file_path: Path to the xml file to read
    :return: A list of Mitsuba transforms
    """
    assert file_path.suffix == '.xml'
    root_node = etree.parse(file_path).getroot()

    transforms = [convert_meshlab2mitsuba_camera(elem.attrib) for elem in root_node if elem.tag == 'VCGCamera']

    return transforms


def create_transform_on_bbsphere(bbox, radius_multiplier: float, positioning_vector: Vector3, tilt=None) -> Transform:
    """
    Create a camera with origin on the bounding sphere around an object and looking towards the center of that sphere
    Camera center is calculated as: bbsphere_center + radius_multiplier * bbsphere_radius * normalized_positioning_vector
    :param bbox: The bounding box of the object from which to calculate the bounding sphere
    :param radius_multiplier: The value to multiply the bounding sphere's radius with
    (= distance of the camera origin to the center of the bounding sphere)
    :param positioning_vector: The vector pointing towards the camera center
    :param tilt: Transform to apply to camera origin. Usually a small rotation to tilt the camera perspective a little
    :return: A transform on the bbsphere
    """
    bsphere = bbox.getBSphere()
    camera_target = bsphere.center

    camera_offset = radius_multiplier * bsphere.radius * normalize(positioning_vector)
    camera_origin = camera_target + camera_offset

    camera_transform = Transform.lookAt(tilt * camera_origin if tilt is not None else camera_origin, camera_target, Vector3(0., 0., 1.))

    return camera_transform


def create_trajectory_on_bbsphere(bbox, initial_positioning_vector: Vector3, rotation_around: Vector3, num_cameras: int,
                                  radius_multiplier: float, tilt: Transform = None) -> List[Transform]:
    """
    Creates an interpolated trajectory on the bounding sphere of a scene/object
    :param bbox: The bounding box of the object/scene to render
    :param initial_positioning_vector: Controls where to position the camera initially
    :param rotation_around: Specify axis to rotate camera around
    :param num_cameras: Number of cameras to generate
    :param radius_multiplier: Cameras will be placed at distance multiplier * bbsphere radius
    :param tilt: Additional transformation meant to tilt the camera a bit
    :return: List with generated transforms
    """
    transforms = []

    step_angle = 360. / float(num_cameras)
    for camera_idx in range(num_cameras):
        transform = create_transform_on_bbsphere(bbox, radius_multiplier, Transform.rotate(rotation_around, step_angle * camera_idx) * tilt * initial_positioning_vector)
        transforms.append(transform)

    return transforms


def create_interpolated_trajectory(cameras: List[Transform], method: str,
                                   num_cameras_per_m: int = 75, num_cameras_per_rad: int = 225,
                                   num_total_cameras: int = 1000) -> List[Transform]:
    """
    Creates an interpolated camera trajectory using supplied key camera transforms
    :param cameras: Camera transformations to use as control points for interpolation
    :param method: How to interpolate: *bezier* (used for both camera centers and rotations):
    Smooth trajectory, but does not necessarily pass through the control points (except the first and last) or
    *catmullrom* (used for camera centers, using quaternion slerp for rotations):
    Passes through all control points, but needs more tuning to prevent weird behaviour
    :param num_cameras_per_m: catmullrom parameter: Camera centers per meter
    :param num_cameras_per_rad: catmullrom parameter: Camera rotations per radiant
    :param num_total_cameras: bezier parameter: Number of interpolated cameras between first and last key camera pose
    :return: A list of interpolated transforms
    """
    all_interpolated = []

    camera_pose_matrices = [util.convert_transform2numpy(camera) for camera in cameras]
    if method == 'bezier':
        interpolated_cameras = bezier_curve(camera_pose_matrices, num_steps=num_total_cameras)
        for elem in interpolated_cameras:
            position = Point3(*elem[:3, 3].tolist())
            look = Vector3(*elem[:3, :3].dot(np.array([0, 0, 1])).tolist())
            up = Vector3(*(elem[:3, :3].dot(np.array([0, 1, 0]))).tolist())
            all_interpolated.append(Transform.lookAt(position, position + look, up))

    elif method == 'catmullrom':
        assert len(cameras) >= 4
        camera_groups = [camera_pose_matrices[idx: idx + 4] for idx in range(len(cameras) - 3)]
        for camera_group in camera_groups:
            key_positions = (camera_group[1][:3, 3], camera_group[2][:3, 3])
            key_looks = (camera_group[1][:3, :3] * np.array([0, 0, 1]), camera_group[2][:3, :3] * np.array([0, 0, 1]))
            dist = np.linalg.norm(key_positions[1] - key_positions[0])
            angle = math.acos(np.clip(np.sum(key_looks[1] @ key_looks[0]), -1., 1.))

            num_t = int(np.round(dist / 100. * num_cameras_per_m))
            num_r = int(np.round(angle * num_cameras_per_rad))
            num = max(40, max(num_t, num_r))

            interpolated_cameras = catmull_rom_spline(camera_group, num)

            for elem in interpolated_cameras:
                position = Point3(*elem[:3, 3].tolist())
                look = Vector3(*elem[:3, :3].dot(np.array([0, 0, 1])).tolist())
                up = Vector3(*(elem[:3, :3].dot(np.array([0, 1, 0]))).tolist())
                all_interpolated.append(Transform.lookAt(position, position + look, up))

    else:
        raise NotImplementedError(f'The method you chose ({method}) is not implemented')

    return all_interpolated


def create_animated_sensors(trajectory: List[Transform], shutter_time: float = 1., width: int = 1920, height: int = 1440,
                            fov: float = 45., num_samples: int = 256) -> List[Sensor]:
    """
    Create an animated sensor (Applies motion blur to the rendered image)
    :param trajectory: The trajectory containing all Transforms
    :param shutter_time: The shutter time to be used to set the Mitsuba setShutterOpenTime
    :param width: Width of the generated image
    :param height: Height of the generated image
    :param fov: The sensor field of view
    :param num_samples: Number of samples per pixel (controls the noise in the resulting image)
    :return: A list of Mitsuba animated sensors
    """
    animated_sensors = []
    for transform_idx in range(len(trajectory)):
        atrafo = AnimatedTransform()
        atrafo.appendTransform(0, trajectory[transform_idx])
        atrafo.appendTransform(1, trajectory[min(transform_idx + 1, len(trajectory) - 1)])
        atrafo.sortAndSimplify()
        sensor = create_sensor_from_transform(atrafo, width, height, fov, num_samples)
        sensor.setShutterOpen(0)
        sensor.setShutterOpenTime(shutter_time)
        animated_sensors.append(sensor)

    return animated_sensors


def create_sensor_from_transform(transform, width=1920, height=1440, fov=45., num_samples=256) -> Sensor:
    """
    Create a Mitsuba sensor from a camera transform
    :param transform: The transform (camera to world) that this sensor uses
    :param width: The width of the resulting image
    :param height: The height of the resulting image
    :param fov: The field of view in degrees (default 45, meshlab uses 60)
    :param num_samples: Number of samples per pixel
    :return: A Mitsuba sensor
    """
    sensor = PluginManager.getInstance().create({
        'type': 'perspective',
        'film': {
            'type': 'ldrfilm',
            'width': width,
            'height': height,
            'pixelFormat': 'rgba',
            'exposure': 1.0,
            'banner': False
        },
        'sampler': {
            'type': 'ldsampler',
            'sampleCount': num_samples
        },
        'toWorld': transform,
        'fov': fov,
    })

    return sensor


if __name__ == '__main__':
    parser = argparse.ArgumentParser("Load and parse cameras from different formats for mitsuba. Called directly, it will print the parsed mitsuba camera")
    parser.add_argument('camera_filename', type=str, help="The file containing camera definitions")
    parser.add_argument('--type', default='meshlab', choices=['meshlab'], help="Which type the camera definitions are")
    parser.add_argument('--interpolate', action='store_true', help="Whether to interpolate cameras between the given one (need at least 4 in this case)")

    args = parser.parse_args()

    camera_filepath = Path(args.camera_filename)
    assert camera_filepath.is_file(), "Camera file has to exist"

    if args.type == 'meshlab':
        transforms = read_meshlab_sensor_transforms(camera_filepath)
    else:
        raise NotImplementedError

    print(f"Read {len(transforms)} camera transformations from type {args.type}:")
    print(transforms)

    if args.interpolate:
        interpolated_cameras = create_interpolated_trajectory(transforms, method='bezier')

        points = np.array([elem[3, :3] for elem in interpolated_cameras])
        from io_util import ply

        ply.write_ply('/home/christian/tmp/cameras.ply', points=points, as_text=True)

        print(f"Interpolated {len(interpolated_cameras)} camera transformations:")
        print(interpolated_cameras)