Env

import torch

  Automatic differentiation e.g.

x = torch.arange(4.0)
x.requires_grad_(True)
// x=torch.arange(4.0, requires_grad=True)

tensor([0., 1., 2., 3.], requires_grad=True)

\( y = 2\mathbf{x}^\top\mathbf{x} \)

y = 2 * torch.dot(x, x)

tensor(28., grad_fn=<MulBackward0>)

\( \nabla_{\mathbf{x}} \mathbf{x}^\top\mathbf{x} = 2\mathbf{x} \)
\( \nabla_{\mathbf{x}} y = \nabla_{\mathbf{x}} (2\mathbf{x}^\top\mathbf{x}) = 2 \times \nabla_{\mathbf{x}} (\mathbf{x}^\top\mathbf{x}) = 2 \times 2\mathbf{x} = 4\mathbf{x} \)
\( \frac{\partial y}{\partial x_i} = 4x_i \)

y.backward()

tensor([ 0.,  4.,  8., 12.])

Clean grad

x.grad.zero_()

tensor([0., 0., 0., 0.])

tensor([0., 1., 2., 3.], requires_grad=True)