for comparisons

pbnr_pytorch/diff_rasterization/rasterize_points.cu

@@ -177,7 +177,7 @@ __global__ void BlendPointsPbnrCudaKernel(
 
                     float power = (-1.0/2.0)*(inv_cov00*dx*dx + (inv_cov01+inv_cov10)*dx*dy + inv_cov11*dy*dy);
                     if (power>0.0) {
-                        printf("%f || %f %f %f %f?\n", power, inv_cov00, inv_cov01, inv_cov10, inv_cov11);
+                        //printf("%f || %f %f %f %f?\n", power, inv_cov00, inv_cov01, inv_cov10, inv_cov11);
                         continue;
                     }
                     float g_w = exp(power);
@@ -277,7 +277,7 @@ __global__ void RasterizePointsPbnrCudaKernel(
 
         int k_idx = atomicInc(&(k_idxs[0*H*W + py_rounded*W + px_rounded]), K + 1);
         if (k_idx == K) {
-            printf("Pixel y:%d x:%d exceeded point projection limit\n", py_rounded, px_rounded);
+            //printf("Pixel y:%d x:%d exceeded point projection limit\n", py_rounded, px_rounded);
             //assert(0);
         }
 

pbnr_pytorch/scene_loaders/ibr_scene.py

@@ -135,7 +135,7 @@ class Camera(nn.Module):
     def computeJacobian(self, point_cloud, normal, camera):
         s_world = torch.ones_like(normal, device="cuda")
         s_world[:, 1] = 0.0
-        s_world[:, 2] = -normal[:, 0] / normal[:, 2]
+        s_world[:, 2] = -normal[:, 0] / (normal[:, 2]+0.0000001)
         s_world = s_world / s_world.norm(dim=1, keepdim=True)
 
         t_world = normal.cross(s_world)
@@ -456,6 +456,7 @@ class Scene():
 
     def shuffleCameras(self):
         random.shuffle(self.cameras)
+        random.shuffle(self.blacklist_cameras)
 
     def getAllCameras(self):
         return self.cameras
@@ -580,9 +581,7 @@ class MultiScene:
     def getPCloudCamsForScore(self, viewpoint_cam, sample, N=9):
         max_coverage_neighbors = viewpoint_cam.neighbors
         if sample:
-            filtered_neighbors = randomizeNeighbors(max_coverage_neighbors, N)
-            while len(filtered_neighbors) < N:
-                filtered_neighbors = randomizeNeighbors(max_coverage_neighbors, N)
+            filtered_neighbors = random.sample(max_coverage_neighbors, N)
         else:
             filtered_neighbors = max_coverage_neighbors[:N]
         pcloud_cams = [self.scenes[0].getAllCameras()[cam_idx] for cam_idx in filtered_neighbors]

pbnr_pytorch/test_render_and_save.py

@@ -13,12 +13,13 @@ from utils.system_utils import searchForMaxIteration
 import argparse
 import os
 from utils.image_utils import crop_image
+from os import makedirs
 
 def mse(img1, img2):
-    return (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
+    return (((img1 - img2)) ** 2).mean()
 
 def psnr(img1, img2):
-    mse = (((img1 - img2)) ** 2).view(img1.shape[0], -1).mean(1, keepdim=True)
+    mse = (((img1 - img2)) ** 2).mean()
     return 20 * torch.log10(1.0 / torch.sqrt(mse))
 
 def render_viewpoint(viewpoint_camera, pcloud_cameras, patch=None, gamma=1.0):
@@ -41,15 +42,13 @@ def render_viewpoint(viewpoint_camera, pcloud_cameras, patch=None, gamma=1.0):
         features_stack = torch.cat((features_stack, rendered_point_cloud), dim=0)
         depth_gmms_stack = torch.cat((depth_gmms_stack, depth_gmms), dim=0)
         num_gmms_stack = torch.cat((num_gmms_stack, num_gmms.int()), dim=0)
-        l2_stack = torch.cat((l2_stack, torch.nn.functional.mse_loss(rendered_point_cloud[:,:3,:,:], crop_image(viewpoint_camera.image, patch)*blend_scores).unsqueeze(0)), dim=0)
     color_stack = color_stack.view(color_stack.shape[0], -1, 3, color_stack.shape[2], color_stack.shape[3])
 
     with torch.no_grad():
         prob_map = _SoftDepthTest.apply(depth_gmms_stack, num_gmms_stack)
 
     image = neural_renderer(features_stack, prob_map * blend_scores_stack)
-    return image, color_stack, l2_stack.mean()
-
+    return image, color_stack, None
 
 old_f = sys.stdout
 class F:
@@ -72,10 +71,13 @@ random.seed(0)
 parser = argparse.ArgumentParser(description='Train your network sailor.')
 parser.add_argument('-i', '--input_path', required=False)
 
+parser.add_argument('--scene_name', required=False, default=None)
 parser.add_argument('--load_iter', required=False, type=int, default=None)
 parser.add_argument('--max_radius', required=False, type=int, default=8)
 parser.add_argument('--test_cameras', required=False, default=3)
 parser.add_argument('--extra_features', type=int, default=6)
+parser.add_argument('--w', type=int)
+parser.add_argument('--h', type=int)
 
 args = parser.parse_args()
 
@@ -85,37 +87,44 @@ with torch.no_grad():
     torch.manual_seed(0)
     random.seed(0)
 
-    args.input_path = "F:/gkopanas/pointbasedIBR/scenes/peter/gardenvase.json"
+    dataset_name = args.scene_name
+    args.input_path = "/data/graphdeco/user/gkopanas/scenes/peter/" + dataset_name + "_perview.json"
 
     with open(args.input_path) as json_file:
         input_json = json.load(json_file)
-    input_json["neural_weights_folder"] = r"F:\gkopanas\pointbasedIBR\tensorboard_3d_cluster\peter\gardenvase\neural_renderer"
-    input_json["scenes"][0]["scene_representation_folder"] = r"F:\gkopanas\pointbasedIBR\tensorboard_3d_cluster\peter\gardenvase\gardenvase"
+    
+    input_json["neural_weights_folder"] = "/data/graphdeco/user/gkopanas/pointbased_neural_rendering/pbnr_pytorch/tensorboard_3d/" + dataset_name + "_final/neural_renderer"
+    input_json["scenes"][0]["scene_representation_folder"] = "/data/graphdeco/user/gkopanas/pointbased_neural_rendering/pbnr_pytorch/tensorboard_3d/" + dataset_name + "_final/" + dataset_name
 
-    neural_renderer = None
-    if input_json.get("neural_weights_folder"):
-        if args.load_iter:
-            load_iter = args.load_iter
-        else:
-            load_iter = searchForMaxIteration(input_json.get("neural_weights_folder"))
+    for load_iter in [20000, 40000, 60000, 80000, 100000]:
+        print("Load Iter {}:".format(load_iter))
         neural_renderer = torch.jit.load(os.path.join(input_json.get("neural_weights_folder"), "model_" + str(load_iter)))
 
-    scene = MultiScene(input_json,
-                       args.max_radius, args.extra_features,
-                       int(args.test_cameras), load_iter)
-
-    imgs = torch.tensor([]).to(device)
-    gts = torch.tensor([]).int().to(device)
-
-    for view_cam in scene.scenes[0].blacklist_cameras:
-        view_list = view_cam.neighbors
-        image, image_stack, _ = render_viewpoint(view_cam, scene.getPCloudCamsForScore(view_cam, sample=False, N=13), patch=None)
-        torchvision.utils.save_image(image,
-                                     "F:/out_{}.png".format(view_cam.image_name))
-        torchvision.utils.save_image(view_cam.image,
-                                     "F:/gt_{}.png".format(view_cam.image_name))
-        imgs = torch.cat((imgs, image), dim=0)
-        gts = torch.cat((gts, view_cam.image), dim=0)
-
-    print(psnr(imgs, gts))
+        scene = MultiScene(input_json,
+                           args.max_radius, args.extra_features,
+                           int(args.test_cameras), load_iter)
+
+        output = "/data/graphdeco/user/gkopanas/pointbased_neural_rendering/pbnr_pytorch/tensorboard_3d/" + dataset_name + "_final/backlisted_renders_{}_{}_".format(args.w, args.h) + str(load_iter)
+        makedirs(output, exist_ok=True)
+
+        imgs = torch.tensor([])
+        gts = torch.tensor([])
+
+        for view_cam in scene.scenes[0].blacklist_cameras:
+            view_list = view_cam.neighbors
+            view_cam.image_width = args.w
+            view_cam.image_height = args.h
+            image, image_stack, _ = render_viewpoint(view_cam, scene.getPCloudCamsForScore(view_cam, sample=False, N=10), patch=None)
+            torchvision.utils.save_image(image,
+                                         os.path.join(output, "out_{}.png".format(view_cam.image_name)))
+            torchvision.utils.save_image(view_cam.image,
+                                         os.path.join(output, "gt_{}.png".format(view_cam.image_name)))
+            imgs = torch.cat((imgs, image.cpu()), dim=0)
+            gts = torch.cat((gts, view_cam.image), dim=0)
+
+        num_psnr = psnr(imgs, gts)
+        print(num_psnr)
+        f = open(os.path.join(output, "psnr.txt"), "w")
+        f.write("PSNR: {}".format(num_psnr))
+        f.close()
 

pbnr_pytorch/train_full_pipeline.py

@@ -154,12 +154,15 @@ gamma = 1.0
 
 viewpoint_stack = scene.getAllTrainCameras().copy()
 while True:
+    if iteration%500==0:
+        print("[ITER {}]".format(iteration))
     if iteration==100000:
         break
     viewpoint_idx = randint(0, len(viewpoint_stack)-1)
     viewpoint_cam = viewpoint_stack.pop(viewpoint_idx)
 
-    pcloud_cams = scene.getPCloudCamsForScore(viewpoint_cam, sample=True)
+    n_num = randint(5, 12)
+    pcloud_cams = scene.getPCloudCamsForScore(viewpoint_cam, sample=True, N=n_num)
     random.shuffle(pcloud_cams)
 
     # Zero out the gradients
@@ -204,7 +207,7 @@ while True:
     with torch.no_grad():
         if not viewpoint_stack:
             viewpoint_stack = scene.getAllTrainCameras().copy()
-        if iteration%20000==0:
+        if iteration%10000==0:
             print("[ITER {}]Saving Model...".format(iteration))
             mkdir_p("./{}/neural_renderer/".format(tensorboard_folder))
             neural_renderer.save("./{}/neural_renderer/model_{}".format(tensorboard_folder, iteration))

pbnr_pytorch/utils/graphics_utils.py

@@ -35,7 +35,7 @@ def convert_Zworld_to_Zndc(Zworld, cam):
 def projectToCamera(hom_cloud, cam):
     full_proj_transform = cam.full_proj_transform
     pts_projected = hom_cloud.bmm(full_proj_transform)
-    pts_projected_normalised = pts_projected / pts_projected[..., 3:]
+    pts_projected_normalised = pts_projected / (pts_projected[..., 3:] + 0.00001)
 
     view_transform = cam.world_view_transform
     points_viewspace = hom_cloud.bmm(view_transform)