Merge remote-tracking branch 'github/master'

README.md

@@ -11,6 +11,12 @@ Official repository for the paper [Real-Time High-Resolution Background Matting]
 
  
 
+## New Paper is Out!
+
+Check out [Robust Video Matting](https://peterl1n.github.io/RobustVideoMatting/)! Our new method does not require pre-captured backgrounds, and can inference at even faster speed!
+
+ 
+
 ## Overview
 * [Updates](#updates)
 * [Download](#download)
@@ -24,12 +30,12 @@ Official repository for the paper [Real-Time High-Resolution Background Matting]
 * [Training](#training)
 * [Project members](#project-members)
 * [License](#license)
-* [Community Projects](#community-projects)
 
  
 
 ## Updates
 
+* [Jun 21 2021] Paper received CVPR 2021 Best Student Paper Honorable Mention.
 * [Apr 21 2021] VideoMatte240K dataset is now published.
 * [Mar 06 2021] Training script is published.
 * [Feb 28 2021] Paper is accepted to CVPR 2021.
@@ -75,9 +81,6 @@ Additionally, you can try our notebooks in Google Colab for performing matting o
 We provide a demo application that pipes webcam video through our model and outputs to a virtual camera. The script only works on Linux system and can be used in Zoom meetings. For more information, checkout:
 * [Webcam plugin](https://github.com/andreyryabtsev/BGMv2-webcam-plugin-linux)
 
-#### Web Demo
-Developers in the community has helped us build a web demo. See [Community Projects](#community-projects) section below. 
-
  
 
 ## Usage / Documentation
@@ -107,10 +110,7 @@ Configure `data_path.pth` to point to your dataset. The original paper uses `tra
 ## License ##
 This work is licensed under the [MIT License](LICENSE). If you use our work in your project, we would love you to include an acknowledgement and fill out our [survey](https://docs.google.com/forms/d/e/1FAIpQLSdR9Yhu9V1QE3pN_LvZJJyDaEpJD2cscOOqMz8N732eLDf42A/viewform?usp=sf_link).
 
- 
-
 ## Community Projects
+Projects developed by third-party developers.
 
-A list of projects built by third-party developers in the community. If you have a project to share, fill out this [survey]([survey](https://docs.google.com/forms/d/e/1FAIpQLSdR9Yhu9V1QE3pN_LvZJJyDaEpJD2cscOOqMz8N732eLDf42A/viewform?usp=sf_link)).
-
-* [Web Demo by Gradio](https://gradio.app/g/BackgroundMattingV2): Matting your own images from your browser.
+* [After Effects Plug-In](https://aescripts.com/goodbye-greenscreen/)

eval/benchmark.m

@@ -0,0 +1,62 @@
+#!/usr/bin/octave
+arg_list = argv ();
+bench_path = arg_list{1};
+result_path = arg_list{2};
+
+
+gt_files = dir(fullfile(bench_path, 'pha', '*.png'));
+
+total_loss_mse = 0;
+total_loss_sad = 0;
+total_loss_gradient = 0;
+total_loss_connectivity = 0;
+
+total_fg_mse = 0;
+total_premult_mse = 0;
+
+for i = 1:length(gt_files)
+    filename = gt_files(i).name;
+
+    gt_fullname = fullfile(bench_path, 'pha', filename);
+    gt_alpha = imread(gt_fullname);
+    trimap = imread(fullfile(bench_path, 'trimap', filename));
+    crop_edge = idivide(size(gt_alpha), 4) * 4;
+    gt_alpha = gt_alpha(1:crop_edge(1), 1:crop_edge(2));
+    trimap = trimap(1:crop_edge(1), 1:crop_edge(2));
+    
+    result_fullname = fullfile(result_path, 'pha', filename);%strrep(filename, '.png', '.jpg'));
+    hat_alpha = imread(result_fullname)(1:crop_edge(1), 1:crop_edge(2));
+    
+
+    fg_hat_fullname = fullfile(result_path, 'fgr', filename);%strrep(filename, '.png', '.jpg'));
+    fg_gt_fullname = fullfile(bench_path, 'fgr', filename);
+    hat_fgr = imread(fg_hat_fullname)(1:crop_edge(1), 1:crop_edge(2), :);
+    gt_fgr = imread(fg_gt_fullname)(1:crop_edge(1), 1:crop_edge(2), :);
+    nonzero_alpha = gt_alpha > 0;
+
+
+    % fprintf('size(gt_fgr) is %s\n', mat2str(size(gt_fgr)))
+    fg_mse = mean(compute_mse_loss(hat_fgr .* nonzero_alpha, gt_fgr .* nonzero_alpha, trimap));
+    mse = compute_mse_loss(hat_alpha, gt_alpha, trimap);
+    sad = compute_sad_loss(hat_alpha, gt_alpha, trimap);
+    grad = compute_gradient_loss(hat_alpha, gt_alpha, trimap);
+    conn = compute_connectivity_error(hat_alpha, gt_alpha, trimap, 0.1);
+
+
+    fprintf(2, strcat(filename, ',%.6f,%.3f,%.0f,%.0f,%.6f\n'), mse, sad, grad, conn, fg_mse);
+    fflush(stderr);
+
+    total_loss_mse += mse;
+    total_loss_sad += sad;
+    total_loss_gradient += grad;
+    total_loss_connectivity += conn;
+    total_fg_mse += fg_mse;
+end
+
+avg_loss_mse = total_loss_mse / length(gt_files);
+avg_loss_sad = total_loss_sad / length(gt_files);
+avg_loss_gradient = total_loss_gradient / length(gt_files);
+avg_loss_connectivity = total_loss_connectivity / length(gt_files);
+avg_loss_fg_mse = total_fg_mse / length(gt_files);
+
+fprintf('mse:%.6f,sad:%.3f,grad:%.0f,conn:%.0f,fg_mse:%.6f\n', avg_loss_mse, avg_loss_sad, avg_loss_gradient, avg_loss_connectivity, avg_loss_fg_mse);

eval/compute_connectivity_error.m

@@ -0,0 +1,46 @@
+% compute the connectivity error given a prediction, a ground truth and a trimap.
+% author Ning Xu
+% date 2018-1-1
+
+% pred: the predicted alpha matte
+% target: the ground truth alpha matte
+% trimap: the given trimap
+% step = 0.1
+
+function loss = compute_connectivity_error(pred,target,trimap,step)
+pred = single(pred)/255;
+target = single(target)/255;
+
+[dimy,dimx] = size(pred);
+
+thresh_steps = 0:step:1;
+l_map = ones(size(pred))*(-1);
+dist_maps = zeros([dimy,dimx,numel(thresh_steps)]);
+for ii = 2:numel(thresh_steps)
+    pred_alpha_thresh = pred>=thresh_steps(ii);
+    target_alpha_thresh = target>=thresh_steps(ii);
+    
+    cc = bwconncomp(pred_alpha_thresh & target_alpha_thresh,4);    
+    size_vec = cellfun(@numel,cc.PixelIdxList);
+    [~,max_id] = max(size_vec);
+    
+    omega = zeros([dimy,dimx]);
+    omega(cc.PixelIdxList{max_id}) = 1;
+            
+    flag = l_map==-1 & omega==0;    
+    l_map(flag==1) = thresh_steps(ii-1);
+    
+    dist_maps(:,:,ii) = bwdist(omega);  
+    dist_maps(:,:,ii) = dist_maps(:,:,ii) / max(max(dist_maps(:,:,ii)));
+end
+l_map(l_map==-1) = 1;
+
+pred_d = pred - l_map;
+target_d = target - l_map;
+
+pred_phi = 1 -  pred_d .* single(pred_d>=0.15);
+
+target_phi = 1 -  target_d .* single(target_d>=0.15);
+
+loss = sum(sum(abs(pred_phi - target_phi).*single(trimap==128)));
+

eval/compute_gradient_loss.m

@@ -0,0 +1,19 @@
+% compute the gradient error given a prediction, a ground truth and a trimap.
+% author Ning Xu
+% date 2018-1-1
+
+% pred: the predicted alpha matte
+% target: the ground truth alpha matte
+% trimap: the given trimap
+% step = 0.1
+
+function loss = compute_gradient_loss(pred,target,trimap)
+pred = mat2gray(pred);
+target = mat2gray(target);
+[pred_x,pred_y] = gaussgradient(pred,1.4);
+[target_x,target_y] = gaussgradient(target,1.4);
+pred_amp = sqrt(pred_x.^2 + pred_y.^2);
+target_amp = sqrt(target_x.^2 + target_y.^2);
+
+error_map = (single(pred_amp) - single(target_amp)).^2;
+loss = sum(sum(error_map.*single(trimap==128))) ;

eval/compute_mse_loss.m

@@ -0,0 +1,13 @@
+% compute the MSE error given a prediction, a ground truth and a trimap.
+% author Ning Xu
+% date 2018-1-1
+
+% pred: the predicted alpha matte
+% target: the ground truth alpha matte
+% trimap: the given trimap
+
+function loss = compute_mse_loss(pred,target,trimap)
+error_map = (single(pred)-single(target))/255;
+
+% fprintf('size(error_map) is %s\n', mat2str(size(error_map)))
+loss = sum(sum(error_map.^2.*single(trimap==128))) / sum(sum(single(trimap==128)));

eval/compute_sad_loss.m

@@ -0,0 +1,11 @@
+% compute the SAD error given a prediction, a ground truth and a trimap.
+% author Ning Xu
+% date 2018-1-1
+
+function loss = compute_sad_loss(pred,target,trimap)
+error_map = abs(single(pred)-single(target))/255;
+loss = sum(sum(error_map.*single(trimap==128))) ;
+
+% the loss is scaled by 1000 due to the large images used in our experiment.
+% Please check the result table in our paper to make sure the result is correct. 
+loss = loss / 1000 ;

eval/gaussgradient.m

@@ -0,0 +1,35 @@
+function [gx,gy]=gaussgradient(IM,sigma)
+%GAUSSGRADIENT Gradient using first order derivative of Gaussian.
+%  [gx,gy]=gaussgradient(IM,sigma) outputs the gradient image gx and gy of
+%  image IM using a 2-D Gaussian kernel. Sigma is the standard deviation of
+%  this kernel along both directions.
+%
+%  Contributed by Guanglei Xiong (xgl99@mails.tsinghua.edu.cn)
+%  at Tsinghua University, Beijing, China.
+
+%determine the appropriate size of kernel. The smaller epsilon, the larger
+%size.
+epsilon=1e-2;
+halfsize=ceil(sigma*sqrt(-2*log(sqrt(2*pi)*sigma*epsilon)));
+size=2*halfsize+1;
+%generate a 2-D Gaussian kernel along x direction
+for i=1:size
+    for j=1:size
+        u=[i-halfsize-1 j-halfsize-1];
+        hx(i,j)=gauss(u(1),sigma)*dgauss(u(2),sigma);
+    end
+end
+hx=hx/sqrt(sum(sum(abs(hx).*abs(hx))));
+%generate a 2-D Gaussian kernel along y direction
+hy=hx';
+%2-D filtering
+gx=imfilter(IM,hx,'replicate','conv');
+gy=imfilter(IM,hy,'replicate','conv');
+
+function y = gauss(x,sigma)
+%Gaussian
+y = exp(-x^2/(2*sigma^2)) / (sigma*sqrt(2*pi));
+
+function y = dgauss(x,sigma)
+%first order derivative of Gaussian
+y = -x * gauss(x,sigma) / sigma^2;