Initial commit

CMakeLists.txt

+project(sibr_gk_all)
+
+add_subdirectory(apps)
+add_subdirectory(renderer)
+
+include(install_runtime)
+subdirectory_target(${PROJECT_NAME} ${CMAKE_CURRENT_LIST_DIR} "projects/gk")

README.txt

+* git clone https://gitlab.inria.fr/sibr/sibr_core.git
+* cd sibr_core\src\projects
+* git clone [TORCHGL REPO]
+* git clone [GK PROJECT]
+* CMAKE
+* CMAKE CLICK ALL PROJECTS
+* set gk_sibr project as start-up
+* buid in RelWithDebugInfo
+* build INSTALL target
+* [maybe optional] copy caffe2_nvrtc.dll nvrtc464_101_0.dll nvrtc-builtins64_101.dll from extlibs/libtorch/lib to install/bin
+
+\\FOLEGANDROS\scenes <- for scenes
+\\FOLEGANDROS\tensorboard_3d_cluster <- for the trainings
+
+arguments:
+--colmap_fovXfovY_flag
+--splat_layers 10
+--ogl_data_path F:\tensorboard_3d_cluster\deep_blending\ponche\ogl_data.dat
+--iteration 100000
+--scene_name ponche
+--tensorboard_path F:\tensorboard_3d_cluster\deep_blending\ponche\ponche_eigen_8resblocks_16width
+--path F:\gkopanas\scenes\deep_blending\ponche\Ponche_perview
+--tensorboard_path F:\tensorboard_3d_cluster\deep_blending\tree\tree_eigen_8resblocks_16width

apps/CMakeLists.txt

+# Copyright (C) 2020, Inria
+# GRAPHDECO research group, https://team.inria.fr/graphdeco
+# All rights reserved.
+# 
+# This software is free for non-commercial, research and evaluation use 
+# under the terms of the LICENSE.md file.
+# 
+# For inquiries contact sibr@inria.fr and/or George.Drettakis@inria.fr
+
+
+
+project(SIBR_gk_sibr_apps)
+
+add_subdirectory(pytorch_impl/)
+add_subdirectory(opengl_impl/)
\ No newline at end of file

apps/opengl_impl/CMakeLists.txt

+project(SIBR_gk_sibr_opengl_impl)
+
+file(GLOB SOURCES "*.cpp" "*.h" "*.hpp")
+source_group("Source Files" FILES ${SOURCES})
+
+# Define build output for project
+add_executable(${PROJECT_NAME} ${SOURCES})
+
+# Define dependencies
+target_link_libraries(${PROJECT_NAME}
+	${Boost_LIBRARIES}
+	${ASSIMP_LIBRARIES}
+	${GLEW_LIBRARIES}
+	${OPENGL_LIBRARIES}
+    ${OpenCV_LIBRARIES}
+	sibr_view
+	sibr_assets
+	sibr_ulr
+	sibr_renderer
+	sibr_gk
+)
+
+include_directories(
+  ${TORCH_INCLUDE_DIRS}
+  ${TORCH_INCLUDE_BASE_DIRS}
+  ${CUDA_INCLUDE_DIRS}
+)
+
+# Define location in solution.
+set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER "projects/gk/apps")
+
+
+
+## High level macro to install in an homogen way all our ibr targets
+include(install_runtime)
+ibr_install_target(${PROJECT_NAME}
+    INSTALL_PDB                         ## mean install also MSVC IDE *.pdb file (DEST according to target type)
+	RESOURCES  	${RESOURCES}
+	RSC_FOLDER 	"gk"
+    STANDALONE  ${INSTALL_STANDALONE}   ## mean call install_runtime with bundle dependencies resolution
+    COMPONENT   ${PROJECT_NAME}_install ## will create custom target to install only this project
+)
+
+

apps/opengl_impl/main.cpp

+#include <fstream>
+
+#include <core/graphics/Window.hpp>
+#include <core/view/SceneDebugView.hpp>
+#include <core/view/MultiViewManager.hpp>
+#include <core/system/String.hpp>
+#include <core/renderer/DepthRenderer.hpp>
+#include <core/raycaster/Raycaster.hpp>
+#include <core/raycaster/CameraRaycaster.hpp>
+#include <core/system/Utils.hpp>
+
+#include <projects/gk_sibr_project/renderer/DeepLearningPointViewOGL.hpp>
+#include <projects/gk_sibr_project/renderer/GkIBRScene.hpp>
+#include "projects/torchgl_interop/renderer/torchgl_interop.h"
+
+#include <Windows.h>
+#define PROGRAM_NAME "gk_sibr_app"
+using namespace sibr;
+
+const char* usage = ""
+"Usage: " PROGRAM_NAME " -path <dataset-path>"    	                                "\n"
+;
+
+bool sortByDistance(sibr::InputCamera lhs, sibr::InputCamera rhs, Vector3f ref_point) {
+	float lhs_dist = (lhs.position() - ref_point).squaredNorm();
+	float rhs_dist = (rhs.position() - ref_point).squaredNorm();
+	return lhs_dist < rhs_dist;
+};
+
+int main(int ac, char** av) {
+	torch::NoGradGuard no_grad_guard;
+	std::cout << std::setprecision(6);
+	// Parse Commad-line Args
+	CommandLineArgs::parseMainArgs(ac, av);
+	GkIBRAppArgs myArgs;
+	myArgs.displayHelpIfRequired();
+	
+	// Window setup
+	sibr::Window window(PROGRAM_NAME, sibr::Vector2i(50, 50), myArgs);
+	sibr::Window::Ptr winptr;
+	winptr.reset(&window);
+	window.makeContextCurrent();
+		
+	Vector2u neuralRenderResolutionT(200, 200);
+	auto _outTex = std::make_shared<sibr::Texture2DRGB32F>(sibr::ImageRGB32F(neuralRenderResolutionT[0], neuralRenderResolutionT[1], sibr::Vector3f(1.0f, 0.7f, 0.7f)),  SIBR_GPU_LINEAR_SAMPLING);
+	auto _copyToOutTex = std::make_shared<CopyToTextureOp>(_outTex->handle());
+
+	GkIBRScene::Ptr scene(new GkIBRScene(myArgs, myArgs.preprocess_mode));
+	if (myArgs.preprocess_mode) {
+		return 0;
+	}
+
+	// Raycaster.
+	std::shared_ptr<sibr::Raycaster> raycaster = std::make_shared<sibr::Raycaster>();
+	raycaster->init();
+	raycaster->addMesh(scene->proxies()->proxy());
+
+	Vector2u neuralRenderResolution(900,
+	                                600);
+	Vector2u totalResolution;
+	if (myArgs.debug_mode) {
+		totalResolution[0] = 3 * neuralRenderResolution[0];
+		totalResolution[1] = 4 * neuralRenderResolution[1];
+	}
+	else {
+		totalResolution = neuralRenderResolution;
+	}
+
+	// Camera handler for main view.
+	sibr::InteractiveCameraHandler::Ptr generalCamera(new InteractiveCameraHandler());
+	generalCamera->setup(scene->cameras()->inputCameras(),
+		                 Viewport(0, 0, neuralRenderResolution[0], neuralRenderResolution[1]),
+		                 raycaster, {0.1, 100.0});
+
+	const std::shared_ptr<sibr::DeepLearningPointViewOGL>
+	    deepLearningPointView(new DeepLearningPointViewOGL(winptr,
+														   scene,
+	                                                       neuralRenderResolution,
+		                                                   totalResolution,
+		                                                   generalCamera,
+														   myArgs.ogl_data_path,
+														   myArgs.splatLayers,
+	                                                       myArgs.tensorboard_path.get() + "/neural_renderer/model_" + myArgs.iteration.get(),
+			                                               myArgs.debug_mode));
+	// Add views to mvm.
+	MultiViewManager        multiViewManager(window, false);
+
+	multiViewManager.addIBRSubView("DL view", deepLearningPointView, totalResolution, ImGuiWindowFlags_ResizeFromAnySide);
+	multiViewManager.addCameraForView("DL view", generalCamera);
+
+
+	if (myArgs.leave_one_out) {
+		std::string outpathd = myArgs.outPath;
+
+		sibr::RenderTargetRGBA32F::Ptr outFrame;
+		outFrame.reset(new RenderTargetRGBA32F(neuralRenderResolution[0], neuralRenderResolution[1]));
+		sibr::ImageRGBA32F::Ptr outImage;
+		outImage.reset(new ImageRGBA32F(neuralRenderResolution[0], neuralRenderResolution[1]));
+
+		for (int idx = 0; idx < scene->cameras()->inputCameras().size(); idx++) {
+			deepLearningPointView->exclude_view = idx;
+			outFrame->clear();
+			sibr::Camera::Ptr view_cam = scene->cameras()->inputCameras()[idx];
+			deepLearningPointView->onRenderIBR(*outFrame, *view_cam);
+			std::ostringstream ssZeroPad;
+			ssZeroPad << std::setw(8) << std::setfill('0') << idx;
+			std::string outFileName = outpathd + "/ours_interactive/renders/" + scene->cameras()->inputCameras()[idx]->name();
+			outFrame->readBack(*outImage);
+			outImage->save(outFileName, false);
+
+			ImageRGB32F image1;
+			image1 = tensorToIm(scene->_images[idx] * scene->_exp_coefs[idx]);
+			outFileName = outpathd + "/ours_interactive/gt/" + scene->cameras()->inputCameras()[idx]->name();
+			image1.save(outFileName);
+
+		}
+		exit(0);
+	}
+
+
+	if (myArgs.pathFile.get() != "") {
+		generalCamera->getCameraRecorder().loadPath(myArgs.pathFile.get(), neuralRenderResolution[0], neuralRenderResolution[1]);
+		generalCamera->getCameraRecorder().recordOfflinePath(myArgs.outPath, multiViewManager.getIBRSubView("DL view"), "dl");
+		if (!myArgs.noExit)
+			exit(0);
+	}
+
+	// Main looooooop.
+	while (window.isOpened()) {
+
+		sibr::Input::poll();
+		window.makeContextCurrent();
+		if (sibr::Input::global().key().isPressed(sibr::Key::Escape)) {
+			window.close();
+		}
+
+		multiViewManager.onUpdate(sibr::Input::global());
+		multiViewManager.onRender(window);
+
+		window.swapBuffer();
+		CHECK_GL_ERROR;
+	}
+
+	return EXIT_SUCCESS;
+}

apps/pytorch_impl/CMakeLists.txt

+project(SIBR_gk_sibr_pytorch_impl)
+
+file(GLOB SOURCES "*.cpp" "*.h" "*.hpp")
+source_group("Source Files" FILES ${SOURCES})
+
+# Define build output for project
+add_executable(${PROJECT_NAME} ${SOURCES})
+
+# Define dependencies
+target_link_libraries(${PROJECT_NAME}
+	${Boost_LIBRARIES}
+	${ASSIMP_LIBRARIES}
+	${GLEW_LIBRARIES}
+	${OPENGL_LIBRARIES}
+    ${OpenCV_LIBRARIES}
+	sibr_view
+	sibr_assets
+	sibr_ulr
+	sibr_renderer
+	sibr_gk
+)
+
+include_directories(
+  ${TORCH_INCLUDE_DIRS}
+  ${TORCH_INCLUDE_BASE_DIRS}
+  ${CUDA_INCLUDE_DIRS}
+)
+
+# Define location in solution.
+set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER "projects/gk/apps")
+
+
+
+## High level macro to install in an homogen way all our ibr targets
+include(install_runtime)
+ibr_install_target(${PROJECT_NAME}
+    INSTALL_PDB                         ## mean install also MSVC IDE *.pdb file (DEST according to target type)
+	RESOURCES  	${RESOURCES}
+	RSC_FOLDER 	"gk"
+    STANDALONE  ${INSTALL_STANDALONE}   ## mean call install_runtime with bundle dependencies resolution
+    COMPONENT   ${PROJECT_NAME}_install ## will create custom target to install only this project
+)
+
+

apps/pytorch_impl/main.cpp

+#include <fstream>
+
+#include <core/graphics/Window.hpp>
+#include <core/view/SceneDebugView.hpp>
+#include <core/view/MultiViewManager.hpp>
+#include <core/system/String.hpp>
+#include <core/renderer/DepthRenderer.hpp>
+#include <core/raycaster/Raycaster.hpp>
+#include <core/raycaster/CameraRaycaster.hpp>
+#include <core/system/Utils.hpp>
+
+#include <projects/gk_sibr_project/renderer/DeepLearningPointView.hpp>
+#include <projects/gk_sibr_project/renderer/GkIBRScene.hpp>
+#include "projects/torchgl_interop/renderer/torchgl_interop.h"
+
+#include <Windows.h>
+#define PROGRAM_NAME "gk_sibr_app"
+using namespace sibr;
+
+const char* usage = ""
+"Usage: " PROGRAM_NAME " -path <dataset-path>"    	                                "\n"
+;
+
+bool sortByDistance(sibr::InputCamera lhs, sibr::InputCamera rhs, Vector3f ref_point) {
+	float lhs_dist = (lhs.position() - ref_point).squaredNorm();
+	float rhs_dist = (rhs.position() - ref_point).squaredNorm();
+	return lhs_dist < rhs_dist;
+};
+
+int main(int ac, char** av) {
+	torch::NoGradGuard no_grad_guard;
+	std::cout << std::setprecision(6);
+	// Parse Commad-line Args
+	CommandLineArgs::parseMainArgs(ac, av);
+	GkIBRAppArgs myArgs;
+	myArgs.displayHelpIfRequired();
+	
+	// Window setup
+	sibr::Window window(PROGRAM_NAME, sibr::Vector2i(50, 50), myArgs);
+	sibr::Window::Ptr winptr;
+	winptr.reset(&window);
+	window.makeContextCurrent();
+		
+	Vector2u neuralRenderResolutionT(200, 200);
+	auto _outTex = std::make_shared<sibr::Texture2DRGB32F>(sibr::ImageRGB32F(neuralRenderResolutionT[0], neuralRenderResolutionT[1], sibr::Vector3f(1.0f, 0.7f, 0.7f)),  SIBR_GPU_LINEAR_SAMPLING);
+	auto _copyToOutTex = std::make_shared<CopyToTextureOp>(_outTex->handle());
+
+	GkIBRScene::Ptr scene(new GkIBRScene(myArgs, myArgs.preprocess_mode));
+	if (myArgs.preprocess_mode) {
+		return 0;
+	}
+
+	// Raycaster.
+	std::shared_ptr<sibr::Raycaster> raycaster = std::make_shared<sibr::Raycaster>();
+	raycaster->init();
+	raycaster->addMesh(scene->proxies()->proxy());
+
+	Vector2u neuralRenderResolution(900,
+	                                600);
+	Vector2u totalResolution;
+	if (myArgs.debug_mode) {
+		totalResolution[0] = 3 * neuralRenderResolution[0];
+		totalResolution[1] = 4 * neuralRenderResolution[1];
+	}
+	else {
+		totalResolution = neuralRenderResolution;
+	}
+
+	// Camera handler for main view.
+	sibr::InteractiveCameraHandler::Ptr generalCamera(new InteractiveCameraHandler());
+	generalCamera->setup(scene->cameras()->inputCameras(),
+		                 Viewport(0, 0, neuralRenderResolution[0], neuralRenderResolution[1]),
+		                 raycaster, {0.1, 100.0});
+
+	const std::shared_ptr<sibr::DeepLearningPointView>
+	    deepLearningPointView(new DeepLearningPointView(scene,
+	                                                    neuralRenderResolution,
+		                                                totalResolution,
+		                                                generalCamera,
+	                                                    myArgs.tensorboard_path.get() + "/neural_renderer/model_" + myArgs.iteration.get(),
+	                                                    myArgs.debug_mode));
+	// Add views to mvm.
+	MultiViewManager        multiViewManager(window, false);
+
+	multiViewManager.addIBRSubView("DL view", deepLearningPointView, totalResolution, ImGuiWindowFlags_ResizeFromAnySide);
+	multiViewManager.addCameraForView("DL view", generalCamera);
+
+
+	/*
+	std::cout << scene->exp_coef_mean << std::endl;
+	std::vector<torch::Tensor> harm_images_stack;
+	for (int idx = 0; idx < scene->cameras()->inputCameras().size(); idx++) {
+		std::tuple<torch::Tensor, torch::Tensor> packed_tensors = scene->_points3d_color_tuple[idx];
+		torch::Tensor image = std::get<1>(packed_tensors).view({ 1, 600, 900, 3 }).permute({ 0, 3, 1, 2 });
+		
+		std::cout << scene->_exp_coefs[idx] << std::endl;
+		torch::Tensor image_harmonized = image * scene->_exp_coefs[idx];
+		image_harmonized += image_harmonized * (1.0 - scene->exp_coef_mean);
+
+		ImageRGB32F image1;
+		image1 = tensorToIm(image_harmonized);
+		std::stringstream stream1;
+		stream1 << myArgs.dataset_path.get() + "/colmap/stereo/images_harmonized/" << scene->cameras()->inputCameras()[idx]->name();
+		image1.save(stream1.str());
+
+		harm_images_stack.push_back(image_harmonized);
+	}
+	torch::Tensor harmonized_grid = makeGrid(torch::TensorList(harm_images_stack), 5);
+
+	ImageRGB32F image1;
+	image1 = tensorToIm(harmonized_grid);
+	std::stringstream stream1;
+	stream1 << "F:/harmonized_grid.png";
+	image1.save(stream1.str());
+	*/
+	/*
+	for (int idx = 0; idx < scene->cameras()->inputCameras().size(); idx++) {
+		const sibr::InputCamera::Ptr cam = scene->cameras()->inputCameras()[idx];
+		std::cout << cam->name() << std::endl;
+		generalCamera->fromCamera(*cam, false, false);
+		window.makeContextCurrent();
+		multiViewManager.onUpdate(sibr::Input::global());
+		multiViewManager.onRender(window);
+		char title[80];
+		sprintf(title, 80, "title_cam_%s.png", cam->name().substr(0, cam->name().find_last_of(".")));
+		multiViewManager.captureView("DL view", "F:/", title);
+		window.swapBuffer();
+		CHECK_GL_ERROR;
+		std::cout << "--------------------" << std::endl;
+
+	}*/
+	if (myArgs.pathFile.get() != "") {
+		generalCamera->getCameraRecorder().loadPath(myArgs.pathFile.get(), neuralRenderResolution[0], neuralRenderResolution[1]);
+		generalCamera->getCameraRecorder().recordOfflinePath(myArgs.outPath, multiViewManager.getIBRSubView("DL view"), "dl");
+		if (!myArgs.noExit)
+			exit(0);
+	}
+
+	// Main looooooop.
+	while (window.isOpened()) {
+
+		sibr::Input::poll();
+		window.makeContextCurrent();
+		if (sibr::Input::global().key().isPressed(sibr::Key::Escape)) {
+			window.close();
+		}
+
+		multiViewManager.onUpdate(sibr::Input::global());
+		multiViewManager.onRender(window);
+
+		window.swapBuffer();
+		CHECK_GL_ERROR;
+	}
+
+	return EXIT_SUCCESS;
+}

renderer/CMakeLists.txt

+set(SIBR_PROJECT "gk")
+project(sibr_${SIBR_PROJECT})
+
+file(GLOB SOURCES "*.cpp" "*.h" "*.hpp")
+source_group("Source Files" FILES ${SOURCES})
+
+file (GLOB RASTER "p3d_rasterizer/*.cu" "p3d_rasterizer/*.cuh" "p3d_rasterizer/*.h", "p3d_rasterizer/*.cpp")
+source_group("Source Files\\p3d_rasterizer" FILES ${RASTER})
+
+file(GLOB SHADERS "shaders/*.frag" "shaders/*.vert" "shaders/*.geom")
+source_group("Source Files\\shaders" FILES ${SHADERS})
+
+# Redefine sources ad all the files to display in the IDE.
+file(GLOB SOURCES "*.cpp" "*.h" "*.hpp" "shaders/*.frag" "shaders/*.vert" "shaders/*.geom" "p3d_rasterizer/*.cu" "p3d_rasterizer/*.cuh" "p3d_rasterizer/*.h", "p3d_rasterizer/*.cpp")
+
+find_package(torch REQUIRED)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
+set(CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/extlibs/win64-msvc2017/libtorch/")
+
+# Declare library.
+cuda_add_library(${PROJECT_NAME} SHARED ${SOURCES})
+
+# Define dependencies.
+include_directories(${Boost_INCLUDE_DIRS}
+					${TORCH_INCLUDE_DIRS}
+  					${TORCH_INCLUDE_BASE_DIRS}
+  					${CUDA_INCLUDE_DIRS}
+  					.)
+
+target_link_libraries(${PROJECT_NAME}
+					  ${Boost_LIBRARIES}
+					  ${ASSIMP_LIBRARIES}
+					  ${GLEW_LIBRARIES}
+					  ${OPENGL_LIBRARIES}
+					  ${OpenCV_LIBRARIES}
+					  glfw3
+					  sibr_system
+					  sibr_view
+					  sibr_assets
+					  sibr_renderer
+					  sibr_graphics
+					  ${TORCH_LIBRARY}
+					  ${C10_LIBRARY}
+					  sibr_torchgl_interop
+)
+
+
+# Define export/import flag.
+add_definitions( -DSIBR_EXP_GK_SIBR_EXPORTS -DBOOST_ALL_DYN_LINK  )
+
+set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER "projects/${SIBR_PROJECT}/renderer")
+
+## High level macro to install in an homogen way all our ibr targets
+include(install_runtime)
+ibr_install_target(${PROJECT_NAME}
+    INSTALL_PDB                         ## mean install also MSVC IDE *.pdb file (DEST according to target type)
+	SHADERS ${SHADERS}
+	RSC_FOLDER ${SIBR_PROJECT}
+
+    #STANDALONE  ${INSTALL_STANDALONE}   ## mean call install_runtime with bundle dependencies resolution
+    #COMPONENT   ${PROJECT_NAME}_install ## will create custom target to install only this project
+)

renderer/Config.hpp

+#ifndef __SIBR_GK_SIBR_CONFIG_HPP__
+# define __SIBR_GK_SIBR_CONFIG_HPP__
+
+# include <core/system/Config.hpp>
+# include <core/system/CommandLineArgs.hpp>
+
+# ifdef SIBR_OS_WINDOWS
+#  ifdef SIBR_STATIC_GK_SIBR_DEFINE
+#    define SIBR_EXPORT
+#    define SIBR_NO_EXPORT
+#  else
+#    ifndef SIBR_EXP_GK_SIBR_EXPORT
+#      ifdef SIBR_EXP_GK_SIBR_EXPORTS
+/* We are building this library */
+#        define SIBR_EXP_GK_SIBR_EXPORT __declspec(dllexport)
+#      else
+/* We are using this library */
+#        define SIBR_EXP_GK_SIBR_EXPORT __declspec(dllimport)
+#      endif
+#    endif
+#    ifndef SIBR_NO_EXPORT
+#      define SIBR_NO_EXPORT
+#    endif
+#  endif
+# else
+#  define SIBR_EXP_GK_SIBR_EXPORT
+# endif
+
+namespace sibr {
+
+	/// Arguments for all ULR applications.
+	struct GkIBRAppArgs :
+		virtual BasicIBRAppArgs {
+		Arg<std::string> tensorboard_path = { "tensorboard_path", "" };
+		Arg<std::string> scene_name = { "scene_name", "" };
+		Arg<std::string> iteration = { "iteration" , "" };
+
+		Arg<std::string> images_path = { "images_path" , "" };
+		Arg<std::string> exp_coef_path = { "exp_coef_path" , "" };
+		Arg<std::string> normal_map_path = { "normal_map_path" , "" };
+		Arg<std::string> depth_delta_path = { "depth_delta_path" , "" };
+		Arg<std::string> model_path = { "model_path", "" };
+		Arg<std::string> uncertainties_path = { "uncertainties_path", "" };
+		Arg<std::string> features_path = { "features_path", "" };
+		Arg<std::string> ogl_data_path = { "ogl_data_path", "" };
+		ArgSwitch preprocess_mode = { "preprocess_mode", false, "Render depth maps and normal maps for scene" };
+		ArgSwitch debug_mode = { "debug_mode", false, "Render intermediate results for debugging." };
+		ArgSwitch leave_one_out = { "leave_one_out", false, "Leave one out rendering." };
+		Arg<int> splatLayers = { "splat_layers", 10, "Number of layers for layered rendering." };
+	};
+
+}
+
+#endif  //__SIBR_EXP_GK_SIBR_CONFIG_HPP__

renderer/DeepLearningPointView.cpp

+#include "DeepLearningPointView.hpp"
+#include "projects/torchgl_interop/renderer/torchgl_interop.h"
+#include "./p3d_rasterizer/rasterize_points.h"
+#include "./p3d_rasterizer/soft_depth_test.h"
+#include <torch/script.h>
+#include <math.h>
+#include <c10/cuda/CUDACachingAllocator.h>
+#include <core/graphics/GUI.hpp>
+#include "RenderingUtilities.hpp"
+
+
+namespace sibr
+{
+	DeepLearningPointView::DeepLearningPointView(const GkIBRScene::Ptr scene,
+		const Vector2u& neuralRenderResolution,
+		const Vector2u& totalResolution,
+		const InteractiveCameraHandler::Ptr& camHandle,
+		const std::string& model_path,
+		const int debug_mode) :
+		_scene(scene), _camHandle(camHandle), _debug_mode(debug_mode),
+		_totalResolution(totalResolution), _neuralRenderResolution(neuralRenderResolution)
+	{
+
+		try {
+			// Deserialize the ScriptModule from a file using torch::jit::load().
+			_neural_renderer = torch::jit::load(model_path);
+			_neural_renderer.to(torch::kCUDA);
+			std::cout << "Successfuly loaded model" << std::endl;
+		}
+		catch (const c10::Error& e) {
+			std::cerr << e.msg() << std::endl;
+			std::cerr << "error loading the model\n";
+			SIBR_ERR;
+		}
+
+		//_outTex.reset(new Texture2DRGB32F(sibr::ImageRGB32F(_totalResolution[0], _totalResolution[1]), SIBR_GPU_LINEAR_SAMPLING));
+		_outTex = std::make_shared<sibr::Texture2DRGB32F>(sibr::ImageRGB32F(_totalResolution[0], _totalResolution[1], sibr::Vector3f(1.0f, 0.7f, 0.7f)), SIBR_GPU_LINEAR_SAMPLING);
+		_copyToOutTex = std::make_shared<CopyToTextureOp>(_outTex->handle());
+
+		GLShader::Define::List defines;
+		defines.emplace_back("VERTICAL_FLIP", 1);
+
+		_textureShader.init("TextureShader",
+			sibr::loadFile(sibr::getShadersDirectory() + "/core/texture.vert", defines),
+			sibr::loadFile(sibr::getShadersDirectory() + "/core/texture.frag"));
+
+	}
+
+
+
+
+	torch::Tensor DeepLearningPointView::geom_transform_points(torch::Tensor& points,
+		torch::Tensor& transf_matrix)
+	{
+		torch::Tensor points_hom;
+		if (points.sizes()[1] == 3) {
+			torch::Tensor ones = torch::ones({ points.sizes()[0], 1 }, torch::TensorOptions().device(torch::kCUDA, 0));
+			points_hom = torch::cat(torch::TensorList({ points,  ones }), 1);
+		}
+		if (points.sizes()[1] == 4) {
+			points_hom = points;
+		}
+		torch::Tensor projected_points = torch::matmul(points_hom, transf_matrix);
+		projected_points = projected_points.slice(1, 0, 3) / projected_points.slice(1, 3, 4);
+
+
+		return projected_points;
+	}
+
+	torch::Tensor DeepLearningPointView::geom_transform_vectors(torch::Tensor& vectors,
+		torch::Tensor& transf_matrix)
+	{
+		torch::Tensor vectors_hom;
+		if (vectors.sizes()[1] == 3) {
+			torch::Tensor zeros = torch::zeros({ vectors.sizes()[0], 1 }, torch::TensorOptions().device(torch::kCUDA, 0));
+			vectors_hom = torch::cat(torch::TensorList({ vectors,  zeros }), 1);
+		}
+		if (vectors.sizes()[1] == 4) {
+			vectors_hom = vectors;
+		}
+		torch::Tensor projected_vectors = torch::matmul(vectors_hom, transf_matrix);
+
+		return projected_vectors.slice(1, 0, 3);
+	}
+
+	torch::Tensor DeepLearningPointView::computeJacobian(torch::Tensor& point_cloud,
+		torch::Tensor& normal,
+		const sibr::Camera::Ptr camera)
+	{
+		torch::Tensor s_world, t_world;
+		torch::Tensor o, s, t;
+
+		s_world = torch::ones_like(normal).cuda();
+		s_world.slice(1, 1, 2) = 0.0;
+		s_world.slice(1, 2, 3) = -normal.slice(1, 0, 1) / normal.slice(1, 2, 3);
+		s_world = s_world / s_world.norm(2, 1, true);
+
+		t_world = normal.cross(s_world);
+		t_world = t_world / t_world.norm(2, 1, true);
+
+		torch::Tensor world2view = getP3DWorld2ViewMat(camera).cuda();
+		torch::Tensor proj = getP3DProjMat(camera).cuda();
+
+		o = geom_transform_points(point_cloud, world2view);
+		s = geom_transform_vectors(s_world, world2view);
+		t = geom_transform_vectors(t_world, world2view);
+
+		torch::Tensor jacobian = torch::zeros({ o.sizes()[0], 2, 2 }).cuda();
+		jacobian.slice(1, 0, 1).slice(2, 0, 1) = (s.slice(1, 0, 1) * o.slice(1, 2, 3) - s.slice(1, 2, 3) * o.slice(1, 0, 1)).unsqueeze(1);
+		jacobian.slice(1, 0, 1).slice(2, 1, 2) = (t.slice(1, 0, 1) * o.slice(1, 2, 3) - t.slice(1, 2, 3) * o.slice(1, 0, 1)).unsqueeze(1);
+		jacobian.slice(1, 1, 2).slice(2, 0, 1) = (s.slice(1, 2, 3) * o.slice(1, 1, 2) - s.slice(1, 1, 2) * o.slice(1, 2, 3)).unsqueeze(1);
+		jacobian.slice(1, 1, 2).slice(2, 1, 2) = (t.slice(1, 2, 3) * o.slice(1, 1, 2) - t.slice(1, 1, 2) * o.slice(1, 2, 3)).unsqueeze(1);
+
+		jacobian = (torch::matmul(jacobian, proj.slice(0, 0, 2).slice(1, 0, 2))) / (o.slice(1, 2, 3) * o.slice(1, 2, 3)).unsqueeze(1);
+
+		return jacobian;
+	}
+
+	torch::Tensor DeepLearningPointView::computeInvertedCovariance(torch::Tensor& point_cloud,
+		torch::Tensor& normal,
+		torch::Tensor& uncertainty,
+		const sibr::Camera::Ptr in_camera,
+		const sibr::Camera::Ptr out_camera)
+	{
+		torch::Tensor out_view_jacobian = computeJacobian(point_cloud, normal, out_camera);
+		torch::Tensor in_view_jacobian = computeJacobian(point_cloud, normal, in_camera);
+
+		torch::Tensor full_jacobian = torch::bmm(in_view_jacobian.inverse(), out_view_jacobian);
+		torch::Tensor covariance = torch::bmm(full_jacobian, full_jacobian.transpose(1, 2));
+
+		torch::Tensor uncertainty_matrix = uncertainty.unsqueeze(1) *
+			torch::eye(2, torch::TensorOptions().device(torch::kCUDA, 0)).unsqueeze(0).repeat({ uncertainty.sizes()[0], 1, 1 });
+
+		torch::Tensor scaled_covariance = torch::bmm(covariance, uncertainty_matrix);
+
+		torch::Tensor inverted_covariance = scaled_covariance.inverse();
+		return inverted_covariance;
+	}
+
+	void DeepLearningPointView::onUpdate(Input& input, const Viewport& viewport)
+	{
+		ViewBase::onUpdate(input, viewport);
+	}
+
+	void DeepLearningPointView::projectRGBDCamera(int src_cam_idx, const sibr::Camera& eye, const Vector2u resolution,
+		torch::Tensor& image, torch::Tensor& depth_gmms, torch::Tensor& num_gmms) {
+
+		//src_cam_idx = 17;
+		//sibr::Camera::Ptr eye_override = _scene->cameras()->inputCameras()[1];
+		sibr::Camera::Ptr eye_override = std::make_shared<sibr::Camera>(eye);
+
+		std::tuple<torch::Tensor, torch::Tensor> packed_tensors = _scene->_points3d_color_tuple[src_cam_idx];
+		torch::Tensor point_cloud = std::get<0>(packed_tensors);
+		torch::Tensor point_colors = std::get<1>(packed_tensors) * _scene->_exp_coefs[src_cam_idx];
+		torch::Tensor normals = _scene->_normals[src_cam_idx] / _scene->_normals[src_cam_idx].norm(2, 1, true);
+
+		torch::Tensor pts_transformed = transformPointsToCamera(eye_override, point_cloud);
+
+		/*
+		filter_pos_x = torch.logical_and(viewspace_points[:, 0] < 1.0, viewspace_points[:, 0] > -1.0)
+		filter_pos_y = torch.logical_and(viewspace_points[:, 1] < 1.0, viewspace_points[:, 1] > -1.0)
+		filter_pos = torch.logical_and(filter_pos_x, filter_pos_y)*/
+
+		torch::Tensor in_cam_center = torch::tensor({ { _scene->cameras()->inputCameras()[src_cam_idx]->position().x(),
+														_scene->cameras()->inputCameras()[src_cam_idx]->position().y(),
+														_scene->cameras()->inputCameras()[src_cam_idx]->position().z() } }).cuda();
+		torch::Tensor out_cam_center = torch::tensor({ { eye_override->position().x(),
+														eye_override->position().y(),
+														eye_override->position().z() } }).cuda();
+
+
+		torch::Tensor view_in_ray = (point_cloud.slice(1, 0, 3) - in_cam_center) / (point_cloud.slice(1, 0, 3) - in_cam_center).norm(2, 1, true);
+		torch::Tensor cos_in_view = torch::abs(torch::bmm(normals.view({ -1, 1, 3 }), view_in_ray.view({ -1, 3, 1 })));
+
+		torch::Tensor view_out_ray = (point_cloud.slice(1, 0, 3) - out_cam_center) / (point_cloud.slice(1, 0, 3) - out_cam_center).norm(2, 1, true);
+		torch::Tensor cos_out_view = torch::abs(torch::bmm(normals.view({ -1, 1, 3 }), view_out_ray.view({ -1, 3, 1 })));
+
+		torch::Tensor filter_slanted = torch::logical_and(cos_in_view > 0.1, cos_out_view > 0.1).squeeze();
+
+
+		torch::Tensor inverted_covariance = computeInvertedCovariance(point_cloud.index({ { filter_slanted } }),
+			normals.index({ { filter_slanted } }),
+			_scene->_uncertainties[src_cam_idx].index({ { filter_slanted } }),
+			_scene->cameras()->inputCameras()[src_cam_idx],
+			eye_override);
+
+
+		torch::Tensor pts_features = torch::cat({ point_colors, _scene->_features[src_cam_idx] }, 1);
+		std::tuple<torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor>
+			rasterize_return_pack = RasterizePoints(pts_transformed.index({ { filter_slanted } }),
+				pts_features.index({ { filter_slanted } }),
+				inverted_covariance,
+				8, resolution[1], resolution[0], 50,
+				_scene->cameras()->inputCameras()[7]->zfar(), _scene->cameras()->inputCameras()[7]->znear(),
+				1.0);
+
+		image = std::get<1>(rasterize_return_pack);
+		depth_gmms = std::get<3>(rasterize_return_pack);
+		num_gmms = std::get<4>(rasterize_return_pack);
+	}
+
+	void DeepLearningPointView::onRenderIBR(sibr::IRenderTarget& dst, const sibr::Camera& eye)
+	{
+
+		std::chrono::steady_clock::time_point start, stop;
+		std::chrono::milliseconds duration;
+
+		torch::Tensor image, depth_gmms, num_gmms, final;
+		torch::Tensor w = torch::tensor({}).cuda();
+		std::vector<torch::Tensor> views_to_render;
+
+		cudaDeviceSynchronize();
+		start = std::chrono::high_resolution_clock::now();
+		std::vector<int> closest4cam;
+
+		int num_cameras = 8 + 1;
+		//closest4cam = getNClosestCameras(eye, 8);
+		//closest4cam = _scene->getNBestCoverageCamerasGPU(eye, _neuralRenderResolution, num_cameras, 1.0 / 4.0, true, -1);
+		//std::cout << closest4cam << std::endl;
+		closest4cam = _scene->getNBestCoverageCameras(eye, _neuralRenderResolution, num_cameras, 1.0/16.0, true, -1);
+		//std::cout << closest4cam << std::endl;
+
+		std::sort(closest4cam.begin(), closest4cam.end());
+
+		for (int idx = 0; idx < _scene->cameras()->inputCameras().size(); idx++) {
+			float alpha = 0.05;
+			float w = 0.0;
+			if (std::find(closest4cam.begin(), closest4cam.end(), idx) != closest4cam.end()) {
+				w = 1.0;
+			}
+			float new_w = alpha * w + (1.0 - alpha) * std::get<0>(_scene->_cam_weights_idx[idx]);
+			_scene->_cam_weights_idx[idx] = std::make_tuple(new_w, idx);
+		}
+
+		std::vector<std::tuple<float, int>> best_cams_sorted(num_cameras);
+		std::partial_sort_copy(std::begin(_scene->_cam_weights_idx), std::end(_scene->_cam_weights_idx), std::begin(best_cams_sorted), std::end(best_cams_sorted),
+			std::greater<std::tuple<float, int>>());
+
+		cudaDeviceSynchronize();
+		stop = std::chrono::high_resolution_clock::now();
+		duration = std::chrono::duration_cast<std::chrono::milliseconds>(stop - start);
+		std::cout << "Time taken by choosing cameras: " << duration.count() << " milliseconds" << std::endl;
+
+
+		cudaDeviceSynchronize();
+		start = std::chrono::high_resolution_clock::now();
+		torch::Tensor features_stack = torch::tensor({}).cuda();
+		torch::Tensor depth_gmms_stack = torch::tensor({}).cuda();
+		torch::Tensor num_gmms_stack = torch::tensor({}, torch::TensorOptions().device(torch::kCUDA, 0).dtype(torch::kInt32));
+
+		for (std::tuple<float, int> cam_w_idx : best_cams_sorted) {
+			int dist_cam = std::get<1>(cam_w_idx);
+			float w_cam = std::get<0>(cam_w_idx);
+			projectRGBDCamera(dist_cam, eye, _neuralRenderResolution, image, depth_gmms, num_gmms);
+			image = torch::clamp(image, 0.0, 1.0);
+			//features_stack = torch.cat((features_stack, torch.cat((rendered_point_cloud, depth, mask), dim = 1)), dim = 0)
+			features_stack = torch::cat(torch::TensorList({ features_stack, image }), 0);
+			depth_gmms_stack = torch::cat(torch::TensorList({ depth_gmms_stack, depth_gmms }), 0);
+			num_gmms_stack = torch::cat(torch::TensorList({ num_gmms_stack, num_gmms }), 0);
+			w = torch::cat(torch::TensorList({ w, torch::tensor({w_cam}).unsqueeze(0).cuda() }), 0);
+
+			if (_debug_mode) views_to_render.push_back(image.index({ torch::indexing::Slice(),
+																	 torch::indexing::Slice(torch::indexing::None, 3),
+																	 torch::indexing::Slice(), torch::indexing::Slice() }));
+		}
+		w = w - (w.min() - 0.05);
+		w = w / w.sum();
+
+		cudaDeviceSynchronize();
+		stop = std::chrono::high_resolution_clock::now();
+		duration = std::chrono::duration_cast<std::chrono::milliseconds>(stop - start);
+		std::cout << "Time taken by projecting: " << duration.count() << " milliseconds" << std::endl;
+
+
+		cudaDeviceSynchronize();
+		start = std::chrono::high_resolution_clock::now();
+
+
+		torch::Tensor prob_map;
+		prob_map = SoftDepthTest(depth_gmms_stack, num_gmms_stack);
+		//prob_map = torch::ones({ 9, 1, 600, 900 }, torch::TensorOptions().device(torch::kCUDA, 0));
+
+		/*
+		std::cout << prob_map.sizes() << std::endl;
+		for (int i = 0; i < best_cams_sorted.size(); i++) {
+			ImageRGB32F fake_image;
+			torch::Tensor tmp = prob_map.index({ i }).unsqueeze(0);
+			fake_image = tensorToIm(torch::cat({ tmp, tmp, tmp }, 0).unsqueeze(0));
+			std::stringstream stream;
+			stream << "F:/prob_" << i << ".exr";
+			cv::imwrite(stream.str(), fake_image.toOpenCV());
+
+			ImageRGB32F image;
+			std::cout << features_stack.sizes() << std::endl;
+			tmp = features_stack.index({ i }).slice(0, 0, 3).unsqueeze(0);
+			std::cout << tmp.sizes() << std::endl;
+			image = tensorToIm(tmp);
+			std::stringstream stream1;
+			stream1 << "F:/img_" << i << ".png";
+			image.save(stream1.str());
+
+		}
+
+
+		ImageRGB32F fake_image;
+		fake_image = tensorToIm(torch::cat({ prob_map.sum(0).unsqueeze(0).unsqueeze(0), prob_map.sum(0).unsqueeze(0).unsqueeze(0), prob_map.sum(0).unsqueeze(0).unsqueeze(0) }, 1));
+		std::stringstream stream;
+		stream << "F:/prob_sum.exr";
+		cv::imwrite(stream.str(), fake_image.toOpenCV()); 
+		
+		*/
+
+		w = prob_map * w.view({ -1, 1, 1, 1 });
+
+		cudaDeviceSynchronize();
+		stop = std::chrono::high_resolution_clock::now();
+		duration = std::chrono::duration_cast<std::chrono::milliseconds>(stop - start);
+		std::cout << "Time taken by depth test: " << duration.count() << " milliseconds" << std::endl;
+
+
+		cudaDeviceSynchronize();
+		start = std::chrono::high_resolution_clock::now();
+		std::vector<torch::jit::IValue> run_inputNet;
+
+		run_inputNet.push_back(features_stack);
+		run_inputNet.push_back(w);
+
+		final = _neural_renderer.forward(run_inputNet).toTensor();
+		final += final * (1.0 - _scene->exp_coef_mean);
+		final = torch::clamp(final, 0.0, 1.0);
+
+		//at::cuda::THCCachingAllocator_emptyCache();
+		//c10::cuda::CUDACachingAllocator::emptyCache();
+		views_to_render.insert(views_to_render.begin(), final);
+		cudaDeviceSynchronize();
+		stop = std::chrono::high_resolution_clock::now();
+		duration = std::chrono::duration_cast<std::chrono::milliseconds>(stop - start);
+		std::cout << "Time taken by model: " << duration.count() << " milliseconds" << std::endl;
+
+		cudaDeviceSynchronize();
+		start = std::chrono::high_resolution_clock::now();
+		_copyToOutTex->Compute(makeGrid(torch::TensorList(views_to_render), 3));
+		//_copyToOutTex->Compute(torch::ones({ 1,3,300,1350 }).to(torch::kCUDA));
+		renderTextureRGB(_outTex, dst);
+		stop = std::chrono::high_resolution_clock::now();
+		duration = std::chrono::duration_cast<std::chrono::milliseconds>(stop - start);
+		std::cout << "Time taken by copy and render: " << duration.count() << " milliseconds" << std::endl;
+		std::cout << "=========================================" << std::endl;
+	}
+
+	std::vector<int> DeepLearningPointView::getNBestScoreCameras(const sibr::Camera& ref_cam, int N) {
+		std::vector<int> score_cam_vector;
+
+		const Vector3f ref_pos = ref_cam.position();
+		const Vector3f ref_lookAt = ref_cam.dir() + ref_cam.position();
+		float max_dist = _scene->getMaxDistanceCamera(ref_cam);
+		for (int i = 0; i < _scene->cameras()->inputCameras().size(); i++) {
+			sibr::InputCamera::Ptr other_cam = _scene->cameras()->inputCameras()[i];
+			const Vector3f other_pos = other_cam->position();
+			const Vector3f other_lookAt = other_cam->dir() + other_cam->position();
+			float angle = acos(ref_lookAt.dot(other_lookAt) / (ref_lookAt.norm() * other_lookAt.norm()));
+			float dist = (ref_pos - other_pos).norm();
+			float norm_dist = (dist * M_PI) / max_dist;
+			float penalty = angle + norm_dist;
+			score_cam_vector.push_back(i);
+		}
+		std::sort(score_cam_vector.begin(), score_cam_vector.end());
+		std::vector<int> topN(score_cam_vector.begin(), score_cam_vector.begin() + N);
+		return topN;
+	}
+
+	std::vector<int> DeepLearningPointView::getNClosestCameras(const sibr::Camera& ref_cam, int N) {
+		const Vector3f pos = ref_cam.position();
+		std::vector<std::tuple<float, int>> dist_cam_vector;
+		for (int i = 0; i < _scene->cameras()->inputCameras().size(); i++) {
+			sibr::InputCamera::Ptr cam = _scene->cameras()->inputCameras()[i];
+			dist_cam_vector.push_back(std::tuple<float, int>((pos - cam->position()).norm(), i));
+		}
+		std::sort(dist_cam_vector.begin(), dist_cam_vector.end());
+		std::vector<int> topN;
+		for (int i = 0; i < N; i++) {
+			topN.push_back(std::get<1>(dist_cam_vector[i]));
+		}
+		return topN;
+	}
+
+
+
+	void DeepLearningPointView::renderTextureRGB(sibr::Texture2DRGB32F::Ptr tex, IRenderTarget& dst)
+	{
+		// Bind and clear RT.
+		dst.bind();
+		glViewport(0, 0, dst.w(), dst.h());
+		glClearColor(0, 0, 0, 1);
+		glClearDepth(1.0);
+		glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+
+		// Render the mesh from the current viewpoint, output positions.
+		_textureShader.begin();
+		glActiveTexture(GL_TEXTURE0);
+		glBindTexture(GL_TEXTURE_2D, tex->handle());
+		RenderUtility::renderScreenQuad();
+
+		_textureShader.end();
+		dst.unbind();
+	}
+
+	void DeepLearningPointView::onGUI()
+	{
+		std::string test = "GK_ULR_DL Settings";
+		if (ImGui::Begin(test.c_str())) {
+			if (ImGui::InputInt("Weights Mode", (int*)(&camera_selection_algo), 1, 1)) {
+				camera_selection_algo = std::max(0, std::min(2, camera_selection_algo));
+			}
+			//ImGui::Combo("Weights Mode", (int*)(&camera_selection_algo), "Distance\0Binary MC\0Weighted MC\0\0");
+		}
+	}
+}

renderer/DeepLearningPointView.hpp

+#pragma once
+#include "Config.hpp"
+#include <core/view/MultiMeshManager.hpp>
+#include "GkIBRScene.hpp"
+#include <core/graphics/Shader.hpp>
+#include <core/graphics/Texture.hpp>
+#include "torch/torch.h"
+#include "projects/torchgl_interop/renderer/CopyToTextureOp.h"
+
+namespace sibr
+{
+	class SIBR_EXP_GK_SIBR_EXPORT DeepLearningPointView : public ViewBase
+	{
+	public:
+		DeepLearningPointView(const GkIBRScene::Ptr scene,
+			const Vector2u& neuralRenderResolution,
+			const Vector2u& totalResolution, 
+			const InteractiveCameraHandler::Ptr& camHandle,
+			const std::string& model_path,
+			const int debug_mode);
+
+		virtual void onUpdate(Input& input, const Viewport& viewport);
+		virtual void onRenderIBR(sibr::IRenderTarget& dst, const sibr::Camera& eye);
+		void renderTextureRGB(sibr::Texture2DRGB32F::Ptr tex, IRenderTarget& dst);
+		void renderTextureRGB2(sibr::Texture2DRGB32F::Ptr tex, const sibr::Camera& eye,
+			IRenderTarget& dst);
+	
+		void onGUI() override;
+		int camera_selection_algo = 0;
+	public:
+
+
+	private:
+		torch::Tensor geom_transform_points(torch::Tensor& points,
+			                                torch::Tensor& transf_matrix);
+		torch::Tensor geom_transform_vectors(torch::Tensor& vectors,
+		                                    torch::Tensor& transf_matrix);
+		torch::Tensor computeJacobian(torch::Tensor& point_cloud,
+		                              torch::Tensor& normal,
+		                              const sibr::Camera::Ptr camera);
+
+		torch::Tensor computeInvertedCovariance(torch::Tensor& point_cloud,
+		                                      	torch::Tensor& normal,
+			                                    torch::Tensor& uncertainty,
+			                                    const sibr::Camera::Ptr in_camera,
+			                                    const sibr::Camera::Ptr out_camera);
+
+		std::vector<int> getNBestScoreCameras(const sibr::Camera& ref_cam, int N);
+		std::vector<int> getNClosestCameras(const sibr::Camera& ref_cam, int N);
+
+		void projectRGBDCamera(int src_cam_idx, const sibr::Camera& eye, const Vector2u resolution,
+			torch::Tensor& image, torch::Tensor& depth, torch::Tensor& mask);
+
+		torch::jit::script::Module _neural_renderer;
+		sibr::GkIBRScene::Ptr _scene;
+		sibr::ImageRGB32F image_test;
+		at::Tensor _output;
+		std::shared_ptr<CopyToTextureOp> _copyToOutTex;
+		Texture2DRGB32F::Ptr _outTex;
+		
+		GLShader _textureShader;
+
+		InteractiveCameraHandler::Ptr _camHandle;
+
+		int _debug_mode;
+		Vector2u _totalResolution;
+		Vector2u _neuralRenderResolution;
+	};
+}
\ No newline at end of file

renderer/DeepLearningPointViewOGL.hpp

+#pragma once
+#include "Config.hpp"
+#include <core/view/MultiMeshManager.hpp>
+#include "GkIBRScene.hpp"
+#include <core/graphics/Shader.hpp>
+#include <core/graphics/Texture.hpp>
+#include "torch/torch.h"
+#include "projects/torchgl_interop/renderer/CopyToTextureOp.h"
+#include "projects/torchgl_interop/renderer/torchgl_interop.h"
+#include "projects/torchgl_interop/renderer/TextureArrayInputOp.h"
+#include "./p3d_rasterizer/soft_depth_test.h"
+
+namespace sibr
+{
+	class SIBR_EXP_GK_SIBR_EXPORT DeepLearningPointViewOGL : public ViewBase
+	{
+	public:
+		DeepLearningPointViewOGL(sibr::Window::Ptr window,
+								 const GkIBRScene::Ptr scene,
+		                         const Vector2u& neuralRenderResolution,
+		                         const Vector2u& totalResolution,
+		                         const InteractiveCameraHandler::Ptr& camHandle,
+								 const std::string& ogl_data_path,
+								 const int splatLayers,
+		                         const std::string& model_path,
+		                         const int debug_mode);
+		virtual void onUpdate(Input& input, const Viewport& viewport);
+		virtual void onRenderIBR(sibr::IRenderTarget& dst, const sibr::Camera& eye);
+		int exclude_view = -1;
+	private:
+
+		void loadShaders();
+		void prepareLayeredTexture(sibr::Texture2DArrayRGBA32F::Ptr& tex, const Vector2u& res, int layerCount);
+		void prepareFramebuffer(GLuint& fb_handle, int numAttachments);
+		void buildFeatureBuffer(const std::string& ogl_data_path);
+
+		std::vector<int> getNClosestCameras(const sibr::Camera& ref_cam, int N);
+
+		sibr::Window::Ptr _window;
+
+		torch::jit::script::Module _neural_renderer;
+		sibr::GkIBRScene::Ptr _scene;
+		InteractiveCameraHandler::Ptr _camHandle;
+		Vector2u _totalResolution;
+		Vector2u _neuralRenderResolution;
+		int _layerCount;
+		int _compositingLayerCount;
+		int _totalVertexCount;
+		int _activeCamCount;
+		int _featureCount;
+		int _featureTextureCount;
+		std::vector<int> _vertexCounts;
+		std::vector<int> _cumVertexCount;
+
+		GLShader _depth_shader;
+		GLShader _minmax_mip_shader;
+		GLShader _ewa_point_shader;
+		GLShader _compositing_shader;
+		GLShader _summary_shader;
+
+		GLuniform<sibr::Matrix4f> _depthViewMatrix;
+		GLuniform<sibr::Matrix4f> _depthProjMatrix;
+		GLuniform<sibr::Vector3f> _depthCamPos;
+
+		GLuniform<sibr::Matrix4f> _viewMatrix;
+		GLuniform<sibr::Matrix4f> _projMatrix;
+		
+		struct CameraUBOInfos
+		{
+			Matrix4f viewMatrix;
+			Matrix4f projMatrix;
+			Vector4f position; // 4D to avoid buffer packing problems
+		};
+
+		GLuniform<int> _uniform_ewa_splatLayerCount;
+		GLuniform<sibr::Vector3f> _uniform_ewa_outCamPosition;
+		GLuniform<sibr::Vector2f> _uniform_ewa_depthBounds;
+		std::vector<CameraUBOInfos> _inCameras; 
+		GLuint _inCamBuffer;
+		GLuniform<int> _uniform_compositing_splatLayerCount;
+		GLuniform<int> _uniform_compositing_featureCount;
+		GLuniform<int> _uniform_minmax_level;
+
+		GLuint _proxyFramebuffer;
+		GLuint _splatFramebuffer;
+		GLuint _compositingFramebuffer;
+
+		sibr::RenderTargetRGBA32F::Ptr _proxyDepthTex;
+		sibr::Texture2DArrayRGBA32F::Ptr _splatColorTex;
+		sibr::Texture2DArrayRGBA32F::Ptr _splatAlphaDepthTex;
+		sibr::Texture2DArrayRGBA32F::Ptr _splatDistortionTex;
+		std::vector<sibr::Texture2DArrayRGBA32F::Ptr> _splatFeatsTexVec;
+		sibr::Texture2DArrayRGBA32F::Ptr _compositingTex;
+
+		GLuint _vaoFeatsID;
+		GLuint _bufferFeatsID;
+
+		std::shared_ptr<TextureArrayInputOp> _compositing_texInputOp;
+		std::shared_ptr<TextureArrayInputOp> _alphaDepth_texInputOp;
+
+		torch::Tensor _compositingTensor;
+		torch::Tensor _depth_gmmsTensor;
+		torch::Tensor _num_gmmsTensor;
+
+		std::shared_ptr<CopyToTextureOp> _copyToOutTex;
+		Texture2DRGB32F::Ptr _outTex;
+
+	};
+}
\ No newline at end of file

renderer/GkIBRScene.hpp

+#pragma once
+
+#include "Config.hpp"
+#include <core/scene/BasicIBRScene.hpp>
+#include "torch/torch.h"
+
+namespace sibr
+{
+	class SIBR_EXP_GK_SIBR_EXPORT GkIBRScene : public BasicIBRScene
+	{
+	public:
+		SIBR_CLASS_PTR(GkIBRScene);
+
+		GkIBRScene();
+		GkIBRScene(const GkIBRAppArgs& myArgs, bool preprocess = false);
+
+	public:
+		float getMaxDistanceCamera(const sibr::Camera& ref_cam);
+		void getNormalAndDepthMap(const sibr::Camera& ref_cam, const sibr::Mesh& mesh,
+			const int w, const int h,
+			torch::Tensor& out_renderedTensor);
+		void get3DPositionAndDepthMap(const sibr::Camera& ref_cam, const int w, const int h,
+			torch::Tensor& out_renderedTensor);
+
+		std::vector<int> getNBestCoverageCameras(const sibr::Camera& ref_cam, Vector2u resolution,
+			const int N, const float scale, const bool weighted, const int skip_id);
+
+		std::vector<int> getNBestCoverageCamerasGPU(const sibr::Camera& ref_cam, Vector2u resolution,
+			const int N, const float scale, const bool weighted, const int skip_id);
+
+		std::vector<std::tuple<int, int>> getNBestCoverageCameras_2(const sibr::Camera& ref_cam, Vector2u resolution,
+			const int N, const float scale, const bool weighted, const int skip_id);
+
+	protected:
+		std::tuple < torch::Tensor, torch::Tensor >
+			get3DPointCloudFromCamera(static sibr::Camera::Ptr source_cam,
+				static torch::Tensor color,
+				static torch::Tensor depth,
+				static torch::Tensor pixelsNDC,
+				static torch::Tensor filter);
+		torch::Tensor readTensorFromDisk(std::string path);
+
+		void readDepthMaps(std::string depth_map_path, std::string depth_delta_path);
+		void readNormalMaps(std::string normal_maps_path);
+		void readImages(std::string images_path);
+		void readFeatureMaps(std::string features_path);
+		void readUncertainties(std::string uncertainties_path);
+		void readExpCoef(std::string exp_coef_path);
+
+		void renderPixelPositionsNDC(void);
+		float getNonZeroMin(const torch::Tensor t);
+
+		void renderNormalDepthAtRT(const sibr::Mesh& mesh,
+			const sibr::Camera& eye,
+			sibr::RenderTargetRGBA32F& rt);
+
+		void renderDepthAtRT(const sibr::Mesh& mesh,
+			const sibr::Camera& eye,
+			sibr::RenderTargetRGBA32F& rt);
+
+		void renderScoreAtRT(sibr::RenderTargetLum32F& score,
+			sibr::RenderTargetRGBA32F& current_pos3D,
+			const sibr::Camera& current_cam,
+			sibr::RenderTargetRGBA32F& sampled_pos3D,
+			const sibr::Camera& sampled_cam);
+
+		int renderBestImprovementAtRT(sibr::RenderTargetLum32F& currentScore,
+			sibr::RenderTargetLum32F& dropOutput,
+			sibr::Texture2DArrayLum32F& individualScores,
+			float &total_score,
+			int criterion,
+			std::vector<int> best_cams,
+			int skip_id);
+
+		sibr::GLShader position_depth_shader;
+		GLuniform<Matrix4f> position_depth_proj;
+		GLuniform<Matrix4f> position_depth_world2view;
+
+		sibr::GLShader normal_depth_shader;
+		GLuniform<Matrix4f> normal_depth_proj;
+		GLuniform<Matrix4f> normal_depth_world2view;
+
+		//Camera selection data
+		sibr::GLShader score_shader;
+		GLuniform<Vector3f> score_shader_current_pos;
+		GLuniform<Matrix4f> score_shader_sampled_proj;
+		GLuniform<Vector3f> score_shader_sampled_pos;
+		GLuniform<Vector3f> score_shader_sampled_dir;
+
+		sibr::GLShader compare_shader;
+		GLuniform<int> skid_id_uniform;
+
+		sibr::GLShader maxPool_shader;
+		sibr::GLuniform<int> maxPool_shader_bestId;
+		std::vector<sibr::RenderTargetRGBA32F::Ptr> pos3DTextures;
+
+	public:
+		std::vector <std::tuple <float, int>> _cam_weights_idx;
+		std::vector <torch::Tensor> _images;
+		std::vector <torch::Tensor> _depths;
+		std::vector <torch::Tensor> _features;
+		std::vector <torch::Tensor> _uncertainties;
+		std::vector <torch::Tensor> _exp_coefs;
+		std::vector <torch::Tensor> _normals;
+		std::vector <torch::Tensor> _pixelsNDC;
+		std::vector < std::tuple < torch::Tensor, torch::Tensor >> _points3d_color_tuple;
+		torch::Tensor exp_coef_mean;
+	};
+
+}
\ No newline at end of file

renderer/RenderingUtilities.cpp

+#include "RenderingUtilities.hpp"
+#include "projects/torchgl_interop/renderer/torchgl_interop.h"
+
+torch::Tensor transformPointsToCamera(const sibr::Camera::Ptr dst_cam,
+	const torch::Tensor pointCloud) {
+	torch::Tensor world2proj = getWorld2ProjMat(dst_cam).cuda();
+	torch::Tensor pts_projected = torch::matmul(pointCloud, world2proj);
+	pts_projected = pts_projected.slice(1, 0, 3) / pts_projected.slice(1, 3, 4);
+
+	torch::Tensor world2view = getP3DWorld2ViewMat(dst_cam).cuda();
+	torch::Tensor pts_viewspace = torch::matmul(pointCloud, world2view);
+	pts_viewspace = pts_viewspace.slice(1, 0, 3) / pts_viewspace.slice(1, 3, 4);
+
+	pts_projected.slice(1, 2, 3) = pts_viewspace.slice(1, 2, 3);
+
+	return pts_projected;
+}
\ No newline at end of file

renderer/RenderingUtilities.hpp

+#include "torch/torch.h"
+#include "core/graphics/Camera.hpp"
+
+torch::Tensor transformPointsToCamera(const sibr::Camera::Ptr dst_cam, const torch::Tensor pointCloud);

renderer/p3d_rasterizer/bitmask.cuh

+// Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
+
+#pragma once
+#define BINMASK_H
+
+// A BitMask represents a bool array of shape (H, W, N). We pack values into
+// the bits of unsigned ints; a single unsigned int has B = 32 bits, so to hold
+// all values we use H * W * (N / B) = H * W * D values. We want to store
+// BitMasks in shared memory, so we assume that the memory has already been
+// allocated for it elsewhere.
+class BitMask {
+ public:
+  __device__ BitMask(unsigned int* data, int H, int W, int N)
+      : data(data), H(H), W(W), B(8 * sizeof(unsigned int)), D(N / B) {
+    // TODO: check if the data is null.
+    N = ceilf(N % 32); // take ceil incase N % 32 != 0
+    block_clear(); // clear the data
+  }
+
+  // Use all threads in the current block to clear all bits of this BitMask
+  __device__ void block_clear() {
+    for (int i = threadIdx.x; i < H * W * D; i += blockDim.x) {
+      data[i] = 0;
+    }
+    __syncthreads();
+  }
+
+  __device__ int _get_elem_idx(int y, int x, int d) {
+    return y * W * D + x * D + d / B;
+  }
+
+  __device__ int _get_bit_idx(int d) {
+    return d % B;
+  }
+
+  // Turn on a single bit (y, x, d)
+  __device__ void set(int y, int x, int d) {
+    int elem_idx = _get_elem_idx(y, x, d);
+    int bit_idx = _get_bit_idx(d);
+    const unsigned int mask = 1U << bit_idx;
+    atomicOr(data + elem_idx, mask);
+  }
+
+  // Turn off a single bit (y, x, d)
+  __device__ void unset(int y, int x, int d) {
+    int elem_idx = _get_elem_idx(y, x, d);
+    int bit_idx = _get_bit_idx(d);
+    const unsigned int mask = ~(1U << bit_idx);
+    atomicAnd(data + elem_idx, mask);
+  }
+
+  // Check whether the bit (y, x, d) is on or off
+  __device__ bool get(int y, int x, int d) {
+    int elem_idx = _get_elem_idx(y, x, d);
+    int bit_idx = _get_bit_idx(d);
+    return (data[elem_idx] >> bit_idx) & 1U;
+  }
+
+  // Compute the number of bits set in the row (y, x, :)
+  __device__ int count(int y, int x) {
+    int total = 0;
+    for (int i = 0; i < D; ++i) {
+      int elem_idx = y * W * D + x * D + i;
+      unsigned int elem = data[elem_idx];
+      total += __popc(elem);
+    }
+    return total;
+  }
+
+ private:
+  unsigned int* data;
+  int H, W, B, D;
+};

renderer/p3d_rasterizer/rasterization_utils.cuh

+// Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
+
+#pragma once
+
+// Given a pixel coordinate 0 <= i < S, convert it to a normalized device
+// coordinate in the range [-1, 1]. We divide the NDC range into S evenly-sized
+// pixels, and assume that each pixel falls in the *center* of its range.
+__device__ inline float PixToNdc(int i, int S) {
+  // NDC x-offset + (i * pixel_width + half_pixel_width)
+  return -1 + (2 * i + 1.0f) / S;
+}
+
+__device__ inline float NdcToPix(float i, int S) {
+  return ((i + 1.0)*S - 1.0)/2.0;
+}
+
+// The maximum number of points per pixel that we can return. Since we use
+// thread-local arrays to hold and sort points, the maximum size of the array
+// needs to be known at compile time. There might be some fancy template magic
+// we could use to make this more dynamic, but for now just fix a constant.
+// TODO: is 8 enough? Would increasing have performance considerations?
+const int32_t kMaxPointsPerPixel = 101;
+const int32_t kMaxPointPerPixelLocal = 101;
+
+template <typename T>
+__device__ inline void BubbleSort(T* arr, int n) {
+  bool already_sorted;
+  // Bubble sort. We only use it for tiny thread-local arrays (n < 8); in this
+  // regime we care more about warp divergence than computational complexity.
+  for (int i = 0; i < n - 1; ++i) {
+    already_sorted=true;
+    for (int j = 0; j < n - i - 1; ++j) {
+      if (arr[j + 1] < arr[j]) {
+        already_sorted = false;
+        T temp = arr[j];
+        arr[j] = arr[j + 1];
+        arr[j + 1] = temp;
+      }
+    }
+    if (already_sorted)
+        break;
+  }
+}
+
+__device__ inline void BubbleSort2(int32_t* arr, const float* points, int n) {
+  bool already_sorted;
+  // Bubble sort. We only use it for tiny thread-local arrays (n < 8); in this
+  // regime we care more about warp divergence than computational complexity.
+  for (int i = 0; i < n - 1; ++i) {
+    already_sorted=true;
+    for (int j = 0; j < n - i - 1; ++j) {
+      float p_j0_z = points[arr[j]*3 + 2];
+      float p_j1_z = points[arr[j+1]*3 + 2];
+      if (p_j1_z < p_j0_z) {
+        already_sorted = false;
+        int32_t temp = arr[j];
+        arr[j] = arr[j + 1];
+        arr[j + 1] = temp;
+      }
+    }
+    if (already_sorted)
+        break;
+  }
+}