Loading content

← Back to home

Total Differential. Gradient. Linearization in N Variables. Part 2

Total Differential. Gradient. Linearization in N Variables. Part 2

Almaz Khusnutdinov

Almaz Khusnutdinov•Math enthusiast, love coding and mathematics

Published 10/3/2025

Gradient Vector. Linear Approximation for N-variables

Proof Based Problems

Problem 1.1

\text{Let }f:\mathbb{R}^n\to\mathbb{R}^m.\ \text{(a) Give the Fr\'echet definition of differentiability at }a\in\mathbb{R}^n.\\ \text{(b) Prove uniqueness of the derivative.}\\ \text{(c) Show that if all partial derivatives of }f\text{ exist in a neighborhood of }a\text{ and are continuous at }a,\\ \text{then }f\text{ is Fr\'echet differentiable at }a\text{ and }Df(a)\text{ is the Jacobian at }a.\\[8pt] \textbf{Answer and detailed explanation: }\\ \text{(a) Definition. }f\text{ is Fr\'echet differentiable at }a\text{ if there exists a linear map }L:\mathbb{R}^n\to\mathbb{R}^m\\ \text{such that } \displaystyle \lim_{h\to 0}\frac{\|f(a+h)-f(a)-Lh\|}{\|h\|}=0.\ \text{This }L\text{ is denoted }Df(a).\\[6pt] \text{(b) Uniqueness. Suppose }L_1,L_2\text{ both satisfy the definition. For }h\neq 0,\\ \frac{\|(L_1-L_2)h\|}{\|h\|}\le \frac{\|f(a+h)-f(a)-L_1h\|}{\|h\|}+\frac{\|f(a+h)-f(a)-L_2h\|}{\|h\|}\xrightarrow[h\to0]{}0.\\ \text{Thus }\|(L_1-L_2)h\|=o(\|h\|).\ \text{Setting }h=t\,v\ (t\to0,\ \|v\|=1)\text{ gives }\|(L_1-L_2)v\|=o(1)\to0.\\ \text{Hence }(L_1-L_2)v=0\ \forall v,\ \text{so }L_1=L_2.\\[6pt] \text{(c) Continuity of partials }\Rightarrow\text{ Fr\'echet differentiability.}\\ \text{Write }f=(f^{(1)},\dots,f^{(m)}).\ \text{Fix }a\text{ and }h=(h_1,\dots,h_n).\ \text{Define the chain }a^{(0)}=a,\ a^{(k)}=a+ (h_1,\dots,h_k,0,\dots,0).\\ \text{For each component }f^{(j)}\text{, apply the 1D mean value theorem along the }k\text{-th coordinate segment:}\\ f^{(j)}(a^{(k)})-f^{(j)}(a^{(k-1)})= \partial_{x_k}f^{(j)}(\xi^{(j)}_k)\,h_k,\ \ \xi^{(j)}_k\ \text{lies on }[a^{(k-1)},a^{(k)}].\\ \text{Summing over }k\text{ yields } f^{(j)}(a+h)-f^{(j)}(a)=\sum_{k=1}^n \partial_{x_k}f^{(j)}(a)\,h_k+\sum_{k=1}^n\big(\partial_{x_k}f^{(j)}(\xi^{(j)}_k)-\partial_{x_k}f^{(j)}(a)\big)h_k.\\ \text{Let }J\in\mathbb{R}^{m\times n}\text{ be the Jacobian at }a,\ J_{jk}=\partial_{x_k}f^{(j)}(a),\ \text{and set }Lh:=Jh.\\ \text{Vector form: } f(a+h)-f(a)=Lh+R(h),\quad R(h):=\Big(\sum_k\delta^{(1)}_k h_k,\dots,\sum_k\delta^{(m)}_k h_k\Big),\\ \text{with }\delta^{(j)}_k=\partial_{x_k}f^{(j)}(\xi^{(j)}_k)-\partial_{x_k}f^{(j)}(a).\\ \text{Continuity of all partials at }a\text{ gives }\delta^{(j)}_k\to0\ \text{as }h\to0.\ \text{Hence } \|R(h)\|\le \sum_{j,k}|\delta^{(j)}_k|\,|h_k|.\\ \text{Normalize: } \frac{\|R(h)\|}{\|h\|}\le \sum_{j,k}|\delta^{(j)}_k|\,\frac{|h_k|}{\|h\|}\le \sum_{j,k}|\delta^{(j)}_k|\xrightarrow[h\to0]{}0.\\ \text{Therefore } \displaystyle \lim_{h\to0}\frac{\|f(a+h)-f(a)-Lh\|}{\|h\|}=0,\ \text{so }f\text{ is Fr\'echet differentiable at }a\text{ with }Df(a)=L=J.

Problem 1.2

\text{Let }f:\mathbb{R}^n\to\mathbb{R}\text{ have all partial derivatives continuous at }a=(a_1,\dots,a_n).\\ \text{Set }h=(h_1,\dots,h_n),\ a+h=(a_1+h_1,\dots,a_n+h_n),\ \Delta f:=f(a+h)-f(a).\\[6pt] \text{Prove }f\text{ is differentiable at }a\text{ and } \Delta f=\sum_{i=1}^n f_{x_i}(a)\,h_i+o(\|h\|).\\[6pt] \textbf{Answer (outline with detailed explanation): }\\ \text{Write a coordinatewise telescoping sum } \Delta f=\sum_{i=1}^n \Big[f(a^{(i)})-f(a^{(i-1)})\Big],\\ \text{where }a^{(0)}=a,\ a^{(i)}=(a_1+h_1,\dots,a_i+h_i,\ a_{i+1},\dots,a_n).\\ \text{Each bracket is a one-variable change with other variables frozen:}\\ f(a^{(i)})-f(a^{(i-1)})=f_{x_i}(\xi_i)\,h_i,\ \text{for some }\xi_i\ \text{on the segment between }a^{(i-1)}\text{ and }a^{(i)}.\\ \Rightarrow\ \Delta f=\sum_{i=1}^n f_{x_i}(\xi_i)\,h_i=\sum_{i=1}^n f_{x_i}(a)\,h_i+\sum_{i=1}^n\big(f_{x_i}(\xi_i)-f_{x_i}(a)\big)h_i.\\ \text{Continuity of }f_{x_i}\text{ at }a\text{ gives }f_{x_i}(\xi_i)\to f_{x_i}(a)\ \text{as }h\to0.\\ \text{Normalize and bound: } \dfrac{|\Delta f-\sum f_{x_i}(a)h_i|}{\|h\|}\le \sum_{i=1}^n |f_{x_i}(\xi_i)-f_{x_i}(a)|\,\dfrac{|h_i|}{\|h\|}\le \sum_{i=1}^n |f_{x_i}(\xi_i)-f_{x_i}(a)|\to0.\\ \text{Hence } \Delta f=\sum_{i=1}^n f_{x_i}(a)\,h_i+o(\|h\|),\ \text{so }dz=\sum_{i=1}^n f_{x_i}(a)\,dx_i.

Problem 1.3

\text{Let }g:\mathbb{R}^m\to\mathbb{R}^n\text{ be differentiable at }a,\ \ f:\mathbb{R}^n\to\mathbb{R}\text{ be differentiable at }g(a).\\ \text{Prove that }f\circ g\text{ is differentiable at }a\text{ and }D(f\circ g)(a)=Df(g(a))\circ Dg(a).\\[6pt] \textbf{Answer (outline): }\\ \text{Write }g(a+k)=g(a)+Dg(a)[k]+\rho(k),\ \ \frac{\|\rho(k)\|}{\|k\|}\to0.\\ \text{Then }f(g(a+k))=f\big(g(a)+Dg(a)[k]+\rho(k)\big)=f(g(a))+Df(g(a))\big[Dg(a)[k]+\rho(k)\big]+r(k),\\ \text{with }\ \frac{|r(k)|}{\|Dg(a)[k]+\rho(k)\|}\to0.\\ \text{Hence }f\circ g(a+k)-f\circ g(a)=\big(Df(g(a))\circ Dg(a)\big)[k]+\underbrace{Df(g(a))[\rho(k)]+r(k)}_{o(\|k\|)}.\\ \text{Therefore }D(f\circ g)(a)=Df(g(a))\circ Dg(a).

Problem 1.4

\text{Let }f:\mathbb{R}^n\to\mathbb{R}\text{ be differentiable at }a\text{ with differential }Df(a).\\ \text{For any unit vector }u\in\mathbb{R}^n,\ \text{prove the directional derivative exists and }D_u f(a)=\nabla f(a)\cdot u.\\[6pt] \textbf{Answer (outline): }\\ \text{Set }h=tu,\ t\to0.\ \ f(a+tu)-f(a)=Df(a)[tu]+r(t),\ \ \frac{r(t)}{|t|}\to0.\\ \text{Divide by }t:\ \frac{f(a+tu)-f(a)}{t}=Df(a)[u]+\frac{r(t)}{t}.\\ \text{Limit }t\to0\text{ gives }D_u f(a)=Df(a)[u]=\nabla f(a)\cdot u.

Problem 1.5

\text{Let }f:\mathbb{R}^n\to\mathbb{R},\ f(x)=\|x\|^2=\sum_{i=1}^n x_i^2.\ \text{Prove that }f\text{ is Fr\'echet differentiable at every }a\in\mathbb{R}^n,\\ \text{find }Df(a)[h]\text{ and the linearization }T_a(x).\\[6pt] \textbf{Detailed explanation: }\\ f(a+h)-f(a)=\|a+h\|^2-\|a\|^2=2\,a\cdot h+\|h\|^2.\\ \text{Set }Lh:=2\,a\cdot h.\ \text{Then } \dfrac{|f(a+h)-f(a)-Lh|}{\|h\|}=\dfrac{\|h\|^2}{\|h\|}=\|h\|\to0.\\ \text{Hence }Df(a)=L,\ \ Df(a)[h]=2\,a\cdot h,\ \ T_a(x)=f(a)+Df(a)[x-a]=\|a\|^2+2\,a\cdot(x-a).\\[4pt] \textbf{Answer: }Df(a)[h]=2\,a\cdot h,\quad T_a(x)=\|a\|^2+2\,a\cdot(x-a).

Problem 1.6

\text{Let }f:\mathbb{R}^n\to\mathbb{R}\ \text{be differentiable at }a.\ \text{Show that the Fr\'echet derivative is unique.}\\[6pt] \textbf{Answer (outline): } \text{Assume }L_1,L_2\text{ both satisfy }\|f(a+h)-f(a)-L_i h\|/\|h\|\to0.\\ \text{Then }\|(L_1-L_2)h\|/\|h\|\le \sum_{i=1}^2 \|f(a+h)-f(a)-L_i h\|/\|h\|\to0.\\ \text{Fix unit }v\text{ and set }h=t v:\ \|(L_1-L_2)v\|=\lim_{t\to0}\|(L_1-L_2)tv\|/|t|=0\ \Rightarrow\ L_1=L_2.

Problem 1.7

\text{Let }g:\mathbb{R}^n\to\mathbb{R}^m\text{ be Fr\'echet differentiable at }a,\ \ f:\mathbb{R}^m\to\mathbb{R}^p\text{ be Fr\'echet differentiable at }g(a).\\ \text{Prove rigorously that }f\circ g\text{ is Fr\'echet differentiable at }a\text{ and }D(f\circ g)(a)=Df(g(a))\,Dg(a).\\[6pt] \textbf{Answer (outline):}\\ \text{Write }g(a+h)=g(a)+Lh+\rho(h),\ \ L:=Dg(a),\ \ \|\rho(h)\|/\|h\|\to0.\\ \text{Write }f(y+k)=f(y)+Mk+r_y(k),\ \ M:=Df(y),\ \ \|r_y(k)\|/\|k\|\to0,\ \text{with }y=g(a).\\ \text{Set }k=Lh+\rho(h).\ \ \text{Then }\\ f(g(a+h))-f(g(a))=f\big(y+k\big)-f(y)=M\big(Lh+\rho(h)\big)+r_y\big(Lh+\rho(h)\big).\\ \text{Thus }f\circ g(a+h)-f\circ g(a)=\underbrace{(ML)h}_{Df(g(a))\,Dg(a)[h]}+\underbrace{M\rho(h)+r_y(Lh+\rho(h))}_{=:R(h)}.\\ \text{Estimate the remainder:}\\ \frac{\|M\rho(h)\|}{\|h\|}\le \|M\|\,\frac{\|\rho(h)\|}{\|h\|}\xrightarrow[h\to0]{}0,\ \ \ \frac{\|r_y(Lh+\rho(h))\|}{\|h\|}=\frac{\|r_y(\cdot)\|}{\|Lh+\rho(h)\|}\cdot\frac{\|Lh+\rho(h)\|}{\|h\|}\xrightarrow[h\to0]{}0.\\ \text{Hence }\displaystyle \lim_{h\to0}\frac{\|R(h)\|}{\|h\|}=0,\ \text{so }f\circ g\text{ is Fr\'echet differentiable at }a\text{ with }D(f\circ g)(a)=Df(g(a))\,Dg(a).

Problem 1.8

\text{Let }f:\mathbb{R}^n\to\mathbb{R},\quad f(x)=\|Mx\|^2=x^{\!\top}M^{\!\top}Mx,\ \text{where }M\in\mathbb{R}^{k\times n}\text{ is fixed}.\\ \text{Prove that }f\text{ is Fr\'echet differentiable at every }a\in\mathbb{R}^n,\ \text{find }Df(a)[h]\text{ and the linearization }T_a(x).\\[6pt] \textbf{Detailed explanation: }\\ f(a+h)-f(a)=\|M(a+h)\|^2-\|Ma\|^2=2\,(Ma)\cdot(Mh)+\|Mh\|^2.\\ \text{Set }Lh:=2\,(M^{\!\top}Ma)\cdot h.\ \text{Then } \dfrac{|f(a+h)-f(a)-Lh|}{\|h\|}=\dfrac{\|Mh\|^2}{\|h\|}\le \|M\|^2\,\|h\|\to 0.\\ \text{Hence }Df(a)=L,\ \ Df(a)[h]=2\,(M^{\!\top}Ma)\cdot h,\ \ T_a(x)=f(a)+Df(a)[x-a]=\|Ma\|^2+2\,(M^{\!\top}Ma)\cdot(x-a).\\[4pt] \textbf{Answer: }Df(a)[h]=2\,(M^{\!\top}Ma)\cdot h,\quad T_a(x)=\|Ma\|^2+2\,(M^{\!\top}Ma)\cdot(x-a).

Problem 1.9

\text{Let }f:\mathbb{R}^n\to\mathbb{R},\ f(x)=\sum_{i=1}^n w_i x_i^2\ \text{with fixed }w_i\in\mathbb{R}.\ \text{Prove }f\text{ is Fr\'echet differentiable at every }a\in\mathbb{R}^n,\ \text{find }Df(a)[h]\text{ and }T_a(x).\\[6pt] \textbf{Answer: }Df(a)[h]=2\sum_{i=1}^n w_i a_i h_i,\quad T_a(x)=f(a)+\sum_{i=1}^n 2 w_i a_i (x_i-a_i).

Problem 1.10

\text{Let }f:\mathbb{R}^n\to\mathbb{R},\ f(x)=\|x\|^2+b\cdot x\ \text{with fixed }b\in\mathbb{R}^n.\ \text{Prove }f\text{ is Fr\'echet differentiable at every }a\in\mathbb{R}^n,\ \text{find }Df(a)[h]\text{ and }T_a(x).\\[6pt] \textbf{Answer: }Df(a)[h]=(2a+b)\cdot h,\quad T_a(x)=\|a\|^2+b\cdot a+(2a+b)\cdot(x-a).

Computational Problems

Problem 2.1

\text{Let }f(x,y,z)=\sin x+xy-2z^3.\ \text{At }a=(0,1,-1)\text{ find }dw,\ \text{the linearization }T(x,y,z),\ \text{and estimate }f(0.02,1.00,-1.01).\\[6pt] \textbf{Answer: }f_x=\cos x+y,\ f_y=x,\ f_z=-6z^2.\ \text{At }a:\ f(a)=2,\ \nabla f(a)=(2,0,-6).\\ dw=2\,dx+0\,dy-6\,dz,\quad T(x,y,z)=2+2(x-0)-6(z+1),\quad f(0.02,1.00,-1.01)\approx 2.10.

Problem 2.2

\text{Let }g:\mathbb{R}^2\to\mathbb{R}^2,\ g(x,y)=(e^x\cos y,\ e^x\sin y),\ \ f(u,v)=u^2+v^2.\\ \text{Compute }D(f\circ g)(a)[h]\ \text{for arbitrary }a=(a_1,a_2),\ h=(h_1,h_2).\\[6pt] \textbf{Answer: }(f\circ g)(x,y)=e^{2x}\ \Rightarrow\ \nabla(f\circ g)(a)=(2e^{2a_1},\,0),\ \ D(f\circ g)(a)[h]=2e^{2a_1}\,h_1.

Problem 2.3

\text{Let }f:\mathbb{R}^3\to\mathbb{R},\ f(x)=\ln(1+x_1+2x_2+3x_3).\ \text{At }a=0,\ \text{find the directional derivative along }u=\frac{(1,-1,2)}{\sqrt{6}}.\\[6pt] \textbf{Answer: }\nabla f(0)=(1,2,3)\ \Rightarrow\ D_u f(0)=\nabla f(0)\cdot u=\dfrac{1-2+6}{\sqrt{6}}=\dfrac{5}{\sqrt{6}}.

Problem 2.4

\text{Let }g:\mathbb{R}^2\to\mathbb{R}^3,\ g(x,y)=\big(xe^{y},\ y^{2},\ \sin x\big),\ \ f:\mathbb{R}^3\to\mathbb{R},\ f(u,v,w)=u^{2}+v\,w.\\ \text{Compute }D(f\circ g)(a)[h]\ \text{for arbitrary }a=(a_1,a_2),\ h=(h_1,h_2).\\[6pt] \textbf{Answer: }D(f\circ g)(a)[h]=\big(2a_1 e^{2a_2}+a_2^{2}\cos a_1\big)\,h_1+\big(2a_1^{2} e^{2a_2}+2a_2\sin a_1\big)\,h_2.

Problem 2.5

\text{Let }g:\mathbb{R}^2\to\mathbb{R}^2,\ g(x,y)=(x+y,\ e^{x-y}),\ \ f(u,v)=\dfrac{u}{v}.\\ \text{Compute }D(f\circ g)(a)[h]\ \text{for arbitrary }a=(a_1,a_2),\ h=(h_1,h_2).\\[6pt] \textbf{Answer: }D(f\circ g)(a)[h]=e^{-(a_1-a_2)}\Big(\,(1-(a_1+a_2))\,h_1+(1+(a_1+a_2))\,h_2\Big).

Problem 2.6

\text{Let }g:\mathbb{R}^2\to\mathbb{R}^2,\ g(r,\theta)=(r\cos\theta,\ r\sin\theta),\ \ f(u,v)=(u^{2}-v^{2},\ 2uv).\\ \text{Find the Jacobian }J_{f\circ g}(r,\theta).\\[6pt] \textbf{Answer: }J_{f\circ g}(r,\theta)=\begin{bmatrix} 2r\cos 2\theta & -2r^{2}\sin 2\theta\\[4pt] 2r\sin 2\theta & \ \ 2r^{2}\cos 2\theta \end{bmatrix}.

Problem 2.7

\text{Let }g:\mathbb{R}^{n}\to\mathbb{R}^{n},\ g(x)=Ax+b,\ \ A\in\mathbb{R}^{n\times n},\ b\in\mathbb{R}^{n},\ \ c\in\mathbb{R}^{n}.\\ \text{Let }f:\mathbb{R}^{n}\to\mathbb{R},\ f(u)=\varphi(c\cdot u)\ \text{with }\varphi\in C^{1}.\\ \text{Compute }D(f\circ g)(a)[h]\ \text{for arbitrary }a,h\in\mathbb{R}^{n}.\\[6pt] \textbf{Answer: }D(f\circ g)(a)[h]=\varphi'\!\big(c\cdot(Aa+b)\big)\,\big(c\cdot Ah\big).

Problem 2.8

\text{Let }g:\mathbb{R}^2\to\mathbb{R}^3,\ g(x,y)=(x^{2},\ y^{2},\ xy),\ \ f:\mathbb{R}^3\to\mathbb{R},\ f(u,v,w)=uv+e^{w}.\\ \text{Compute }D(f\circ g)(a)[h]\ \text{for arbitrary }a=(a_1,a_2),\ h=(h_1,h_2).\\[6pt] \textbf{Answer: }D(f\circ g)(a)[h]=\Big(2a_1 a_2^{2}+a_2 e^{a_1 a_2}\Big)h_1+\Big(2a_2 a_1^{2}+a_1 e^{a_1 a_2}\Big)h_2.

Problem 2.9

\text{Let }g:\mathbb{R}^3\to\mathbb{R}^2,\ g(x,y,z)=\big(\ln(1+x+y),\ z\,e^{x}\big),\ \ f:\mathbb{R}^2\to\mathbb{R},\ f(u,v)=u^{3}+ \sin v.\\ \text{Find the total differential }d(f\circ g)\text{ at }a=(a_1,a_2,a_3)\text{ applied to }h=(h_1,h_2,h_3).\\[6pt] \textbf{Answer: }d(f\circ g)_a[h]=\ 3\!\left(\frac{h_1+h_2}{1+a_1+a_2}\right)\!\big(\ln(1+a_1+a_2)\big)^{2}\ +\ \cos(a_3 e^{a_1})\,(a_3 e^{a_1}h_1+e^{a_1}h_3).

Decomposition of Total Differential

Problem 3.1

\text{Let }g:\mathbb{R}^{n}\to\mathbb{R}^{m},\ g(x)=Ax+b,\ A\in\mathbb{R}^{m\times n},\ b\in\mathbb{R}^{m}.\ \text{Let }f:\mathbb{R}^{m}\to\mathbb{R},\ f(u)=\varphi\big(d\cdot (Cu)+\alpha\big),\ C\in\mathbb{R}^{k\times m},\ d\in\mathbb{R}^{k},\ \alpha\in\mathbb{R},\ \varphi\in C^{1}.\\ \text{Compute }D(f\circ g)(a)[h]\ \text{for arbitrary }a\in\mathbb{R}^{n},\ h\in\mathbb{R}^{n}.\\[6pt] \textbf{Detailed explanation: }u=Ax+b.\ \text{Inner scalar }s=d\cdot(Cu)+\alpha.\ \nabla_u s=C^{\!\top}d.\ \text{Hence }\nabla_u f(u)=\varphi'(s)\,C^{\!\top}d.\ Dg(a)[h]=Ah.\ \Rightarrow D(f\circ g)(a)[h]=\big(\nabla_u f(u)\big|_{u=Aa+b}\big)\cdot Ah=\varphi'\!\big(d\cdot(C(Aa+b))+\alpha\big)\,(C^{\!\top}d)\cdot(Ah).\\[4pt] \textbf{Answer: }D(f\circ g)(a)[h]=\varphi'\!\big(d\cdot(C(Aa+b))+\alpha\big)\,\big(d\cdot(CAh)\big).

Problem 3.2

\text{Let }g:\mathbb{R}^{n}\to\mathbb{R}^{m},\ g(x)=Ax+b,\ A\in\mathbb{R}^{m\times n},\ b\in\mathbb{R}^{m}.\ \text{Let }f:\mathbb{R}^{m}\to\mathbb{R},\ f(u)=\varphi(u^{\!\top}Su),\ S\in\mathbb{R}^{m\times m}\text{ symmetric},\ \varphi\in C^{1}.\\ \text{Compute }D(f\circ g)(a)[h]\ \text{for arbitrary }a,h.\\[6pt] \textbf{Answer: }u=Aa+b,\ \ D(f\circ g)(a)[h]=\varphi'\!\big(u^{\!\top}Su\big)\,\big(2u^{\!\top}S(Ah)\big).

Problem 3.3

\text{Let }g:\mathbb{R}^{n}\to\mathbb{R}^{m},\ g(x)=Ax+b,\ A\in\mathbb{R}^{m\times n},\ b\in\mathbb{R}^{m},\ c\in\mathbb{R}^{m}.\ \text{Let }f:\mathbb{R}^{m}\to\mathbb{R},\ f(u)=\psi\!\big(r(c\cdot u)\big),\ \psi,r\in C^{1}.\\ \text{Compute }D(f\circ g)(a)[h]\ \text{for arbitrary }a,h.\\[6pt] \textbf{Answer: }u=Aa+b,\ \ D(f\circ g)(a)[h]=\psi'\!\big(r(c\cdot u)\big)\,r'\!\big(c\cdot u\big)\,\big(c\cdot Ah\big).

Table of Contents

Total Differential. Gradient. Linearization in N Variables. Part 2 - nerjik.com