#### NOTE
New to DeepInverse? Get started with the basics with the
[5 minute quickstart tutorial](https://deepinv.org/auto_examples/basics/demo_quickstart.html.md#sphx-glr-auto-examples-basics-demo-quickstart-py)..
# Regularization by Denoising (RED) for Super-Resolution.
Implementation of Romano *et al.*[1](#footcite-romano2017little) using as plug-in denoiser the Gradient-Step Denoiser (GSPnP) Hurault *et al.*[2](#footcite-hurault2021gradient) which provides an explicit prior.
```Python
import deepinv as dinv
from pathlib import Path
import torch
from torch.utils.data import DataLoader
from deepinv.optim.data_fidelity import L2
from deepinv.optim.prior import RED
from deepinv.optim import PGD
from deepinv.training import test
from torchvision import transforms
from deepinv.utils.parameters import get_GSPnP_params
from deepinv.utils import load_dataset, load_degradation
```
## Setup paths for data loading and results.
```Python
BASE_DIR = Path(".")
DATA_DIR = BASE_DIR / "measurements"
RESULTS_DIR = BASE_DIR / "results"
DEG_DIR = BASE_DIR / "degradations"
# Set the global random seed from pytorch to ensure
# the reproducibility of the example.
torch.manual_seed(0)
device = dinv.utils.get_device()
```
```none
Selected GPU 0 with 24371.25 MiB free memory
```
## Load base image datasets and degradation operators.
In this example, we use the Set3C dataset and a motion blur kernel from Levin *et al.*[3](#footcite-levin2009understanding).
```Python
dataset_name = "set3c"
img_size = 256 if torch.cuda.is_available() else 32
operation = "super-resolution"
val_transform = transforms.Compose(
[transforms.CenterCrop(img_size), transforms.ToTensor()]
)
dataset = load_dataset(dataset_name, transform=val_transform)
# Generate the degradation operator.
kernel_index = 1
kernel_torch = load_degradation(
"kernels_12.npy", DEG_DIR / "kernels", index=kernel_index
)
kernel_torch = kernel_torch.unsqueeze(0).unsqueeze(
0
) # add batch and channel dimensions
# Use parallel dataloader if using a GPU to speed up training, otherwise, as all computes are on CPU, use synchronous dataloading.
num_workers = 4 if torch.cuda.is_available() else 0
factor = 2 # down-sampling factor
n_channels = 3 # 3 for color images, 1 for gray-scale images
n_images_max = 3 # Maximal number of images to restore from the input dataset
noise_level_img = 0.03 # Gaussian Noise standard deviation for the degradation
p = dinv.physics.Downsampling(
img_size=(n_channels, img_size, img_size),
factor=factor,
filter=kernel_torch,
device=device,
noise_model=dinv.physics.GaussianNoise(sigma=noise_level_img),
)
# Generate a dataset in a HDF5 folder in "{dir}/dinv_dataset0.h5'" and load it.
measurement_dir = DATA_DIR / dataset_name / operation
dinv_dataset_path = dinv.datasets.generate_dataset(
train_dataset=dataset,
test_dataset=None,
physics=p,
device=device,
save_dir=measurement_dir,
train_datapoints=n_images_max,
num_workers=num_workers,
)
dataset = dinv.datasets.HDF5Dataset(
path=dinv_dataset_path, train=True, use_dict_output=True
)
```
```none
/local/jtachell/deepinv/deepinv/deepinv/datasets/base.py:495: DeprecationWarning: The tuple format for dataset outputs is deprecated and will be removed in a future version.It is recommended to set `use_dict_output=True` for better readability and flexibility in returned outputs.The default is currently `False` for backward compatibility, but will be switched to `True` in a future version.
super().__init__(use_dict_output=use_dict_output)
kernels_12.npy degradation downloaded in degradations/kernels
/local/jtachell/deepinv/deepinv/.pixi/envs/docs/lib/python3.12/multiprocessing/popen_fork.py:66: DeprecationWarning: This process (pid=4088353) is multi-threaded, use of fork() may lead to deadlocks in the child.
self.pid = os.fork()
Dataset has been saved at measurements/set3c/super-resolution/dinv_dataset0.h5
```
## Setup the PnP algorithm. This involves in particular the definition of a custom prior class.
We use the proximal gradient algorithm to solve the super-resolution problem with GSPnP.
```Python
# Parameters of the algorithm to solve the inverse problem
early_stop = True # Stop algorithm when convergence criteria is reached
crit_conv = "cost" # Convergence is reached when the difference of cost function between consecutive iterates is
# smaller than thres_conv
thres_conv = 1e-5
backtracking = True
batch_size = 1 # batch size for evaluation is necessarily 1 for early stopping and backtracking to work.
# load specific parameters for GSPnP
lambda_reg, sigma_denoiser, stepsize, max_iter = get_GSPnP_params(
operation, noise_level_img
)
# Select the data fidelity term
data_fidelity = L2()
# The GSPnP prior corresponds to a RED prior with an explicit `g`.
# We thus write a class that inherits from RED for this custom prior.
class GSPnP(RED):
r"""
Gradient-Step Denoiser prior.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.explicit_prior = True
def forward(self, x, *args, **kwargs):
r"""
Computes the prior :math:`g(x)`.
:param torch.Tensor x: Variable :math:`x` at which the prior is computed.
:return: (torch.Tensor) prior :math:`g(x)`.
"""
return self.denoiser.potential(x, *args, **kwargs)
method = "GSPnP"
denoiser_name = "gsdrunet"
# Specify the Denoising prior
prior = GSPnP(denoiser=dinv.models.GSDRUNet(pretrained="download").to(device))
# we want to output the intermediate PGD update to finish with a denoising step.
def custom_output(X):
return X["est"][1]
# instantiate the algorithm class to solve the IP problem.
model = PGD(
prior=prior,
g_first=True,
data_fidelity=data_fidelity,
sigma_denoiser=sigma_denoiser,
lambda_reg=lambda_reg,
stepsize=stepsize,
early_stop=early_stop,
max_iter=max_iter,
crit_conv=crit_conv,
thres_conv=thres_conv,
backtracking=backtracking,
get_output=custom_output,
verbose=False,
)
# Set the model to evaluation mode. We do not require training here.
model.eval()
```
```none
PGD(
(fixed_point): FixedPoint(
(iterator): PGDIteration(
(f_step): fStepPGD()
(g_step): gStepPGD()
)
)
(psnr): PSNR()
)
```
## Evaluate the model on the problem.
We evaluate the PnP algorithm on the test dataset, compute the PSNR metrics and plot reconstruction results.
```Python
save_folder = RESULTS_DIR / method / operation / dataset_name
plot_convergence_metrics = True # plot metrics. Metrics are saved in save_folder.
plot_images = True # plot images. Images are saved in save_folder.
dataloader = DataLoader(
dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False
)
test(
model=model,
test_dataloader=dataloader,
physics=p,
device=device,
plot_images=plot_images,
no_learning_method="A_dagger",
save_folder=RESULTS_DIR / method / operation / dataset_name,
plot_convergence_metrics=plot_convergence_metrics,
verbose=True,
)
```
```none
/local/jtachell/deepinv/deepinv/deepinv/training/trainer.py:593: UserWarning: Update progress bar frequency of 1 may slow down training on GPU. Consider setting freq_update_progress_bar > 1.
warnings.warn(
0%| | 0/3 [00:00
* **References:**
* Yaniv Romano, Michael Elad, and Peyman Milanfar. The little engine that could: regularization by denoising (red). *SIAM Journal on Imaging Sciences*, 10(4):1804–1844, 2017.
* Samuel Hurault, Arthur Leclaire, and Nicolas Papadakis. Gradient step denoiser for convergent plug-and-play. In *International Conference on Learning Representations*. 2021.
* **[3]** Anat Levin, Yair Weiss, Fredo Durand, and William T Freeman. Understanding and evaluating blind deconvolution algorithms. In *2009 IEEE conference on computer vision and pattern recognition*, 1964–1971. IEEE, 2009.
**Total running time of the script:** (0 minutes 4.817 seconds)
🏷 Tags: Super-resolution
[`Download Jupyter notebook: demo_RED_GSPnP_SR.ipynb`](https://deepinv.org/auto_examples/plug-and-play/_downloads/56f7f4c8c920bdca526033f267b517e2/demo_RED_GSPnP_SR.ipynb)
[`Download Python source code: demo_RED_GSPnP_SR.py`](https://deepinv.org/auto_examples/plug-and-play/_downloads/9ff3979619af845b99f959adb1747d90/demo_RED_GSPnP_SR.py)
[`Download zipped: demo_RED_GSPnP_SR.zip`](https://deepinv.org/auto_examples/plug-and-play/_downloads/2927b42c4ba7813fe1cb03aa3a36ec74/demo_RED_GSPnP_SR.zip)
[Gallery generated by Sphinx-Gallery](https://sphinx-gallery.github.io)