\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\newcommand{\G}{\mathbb{G}}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 15 -- February 28} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Fang Cai}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} % Put the reading covered in the lecture here
Van der Vaart Chapter 18 and 19.
\par
%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%
\paragraph{Outline:}
\begin{itemize}
\item Uniform laws/limits
	\begin{itemize}
	\item Chaining and equicontinuity (entropy integrals)
    \item Uniform CLTs and Brownian Bridge
    \item Donsker classes (uniform CLTs)
	\end{itemize}
\item Moduli of continuity and convergence rate
\end{itemize}
\par

\section{Uniform laws/limits}
Recall from last lecture
\begin{theorem}[Uniform limits via stochastic equicontinuity]
\label{unform limit}
Stochastic process $(X_{n,t})_{t\in T}$ converges in distribution to $(X_t)_{t\in T}$ in $L_\infty(T)$ iff
\begin{enumerate}
\item[(i)] Finite dimensional convergence (FIDI): 
\[
(X_{n,t_1},\ldots, X_{n, t_k}) \cd (X_{t_i}, \ldots, X_{t_k}).
\]
\item[(ii)] Equicontinuity: $\forall \epsilon>0, \eta>0, \exists$ partition
$T_1,\ldots,T_m$ of $T$ s.t.
\[
\limsup_{n\to\infty} 
\P (\max_{i}\sup_{s,t\in T_i} | X_{n,s} - X_{n,t}|\ge \epsilon) 
\le \eta.
\]
\end{enumerate}
\end{theorem}
Usually (i) is free by CLT but game is in showing (ii), i.e. moduli of continuity of $X_n \in L_{\infty}(T)$


\begin{corollary}[Uniform limits via entropy integral]
\label{Uniform limits via entropy integral}
Suppose $(T,d)$ is a totally bounded metric space with
$\lim_{\delta \downarrow 0 }\limsup_{n \to \infty} 
\P (\sup_{d(s,t)\le \delta}|X_{n,s}-X_{n,t}| \ge \epsilon) = 0$
and  FIDI of $X_n$ to $X\in L_{\infty}(T)$. Then 
$(X_{n,t})_{t\in T}\cd (X_t)_{t\in T}$ in $L_\infty(T)$.
\end{corollary}

\begin{proof}
Let $\epsilon > 0$. Choose partition $\{T_i\}_{i=1}^{m}$ of $T$ such that 
$\text{diam}(T_i)\le \delta$. Then 
$$\max_{i\le m}\sup_{s,t \in T_i} |X_{n,s}-X_{n,t}|
\le  \sup_{d(s,t) \le \delta} |X_{n,s}-X_{n,t}|,$$
 so the process is stochastic equicontinuous.
\end{proof}


\begin{definition}[Donster Class] A collection of functions $\mc F$ is $\P$-Donsker if the process $(\sqrt{n}(\P_n-\P)f)_{f\in \mc F}$ converges to a tight limit in $L_{\infty}(\mc F)$.
\end{definition}
This limit is a Gaussian process because 
$\sqrt{n}(\P_n-\P)f \cd \normal (0, \var_p(f))) $ by CLT, so FIDI gives that limit process, denoted by $\G_p \in L_{\infty}(\mc F)$ (i.e $\G_p f \in \R$).
\begin{align*}
\E [\G_pf] &= 0\\
\E [\G_pf \G_p g] &= \cov_p (f,g) = \P fg - \P f \, \P g
\end{align*}
Idea: If $\G_n:=\sqrt{n} (\P_n - \P)$, where $X_i \simiid \P $, 
$\G_n \in L_{\infty}(\mc F)$. Then
$$\G_n f = \sqrt{n} (\P_n - \P)f =
\frac{1}{\sqrt{n}}\sum_{i=1}^n \left(f(X_i)-\E_p[f(X_i)] \right)$$
and
$$\left( 
\G_p f_1, \dots, \G_p f_k
\right)
\sim
\normal \left(0, \, [\cov_p(f_i, f_j)]_{i,j=1}^k \right)
$$
\begin{example}[$\P$-Brownian bridge ]
$F(t) = \P (X \le t)$. Let $\mc F = \{\indic{\cdot \le t},\, t \in \R \}$. Then 
$
\sqrt{n}(F_n(\cdot)-F(\cdot)) \cd \G_p
$
in $L_{\infty}(\R)$. Note that
$${\E [\indic{X \le t} \indic{X \le s}] = F(s \wedge t)}
\implies
{\cov(\indic{X \le t}, \indic{X \le s}) = F(s \wedge t)-F(S)F(t)}.$$

Furthermore, if $\P = \text{Uniform}([0,1])$, get $\G_p$ with 
${\cov(\G_p(t), \G_p(s)) = s \wedge t-st}$ and 
$\G_P(t) \sim \normal (0, t(1-t))$.
\end{example}
\begin{theorem}[Uniform CLT via entropy integral] Let $\mc F$ be a collection of functions. There exists an envelop function $B: X \to \R_+$, with 
$$
\int_{0}^{\infty} \sup_{Q} \sqrt{\log N(\mc F,\, L_2(Q),\, \norm{B}_{L_{2}(Q)} \epsilon )} \; d\epsilon < \infty
$$
where the supremum is take over all discrete probability measure. If  $\P B^2 < \infty$, then $\mc F$ is $\P$-Donsker.
\end{theorem}
\begin{proof-sketch}
Let $\mc F_{\delta}:={f-g : \norm{f-g}_{L_2(\P)} \le \delta, \, f,g\in \mc F }$. Let $\G_n = \sqrt{n}(\P_n-\P)$ and 
$\P_n^o = \frac{1}{n}\sum_{i=1}^n \epsilon_i \indic{X_i}$, then
\begin{align*}
\P\left(\sup_{\norm{f-g} \le \delta} |\G_n(f-g)| \le \epsilon \ge \epsilon\right) \le &
\frac{\E \left[\sup_{\norm{f-g}\le \delta}  |\sqrt{n}(\P_n-\P)(f-g)|\right]} {\epsilon}\\
\le &\frac{2}{\epsilon} \, \E\left[
\sup_{f\in \mc F_{\delta}} \sqrt{n} |\P_n^o f|
\right]
\end{align*}
by symmetrization. 
By Dudley's integral,
$$\E\left[
\sup_{f\in \mc F_{\delta}} \sqrt{n} |\P_n^o f|
\right]
\le  C \E \left[ \int_0^{\infty}
\sqrt{\log N(\mc F_{\delta}, {L_2(\P_n)}, \epsilon)}
\, d\epsilon
\right]
$$

If we set $\theta_n=\sup_{f\in \mc F_{\delta}} \norm{f}_{L_2(\P_n)}$ and $b_n=\norm{B}_{L_2(\P_n)}$, 
then the above
\begin{align*}
\E\left[
\sup_{f\in \mc F_{\delta}} \sqrt{n} |\P_n^o f|
\right]
\le &  C \E \left[ \int_0^{\theta_n}
\sqrt{\log N(\mc F_{\delta}, {L_2(\P_n)}, \epsilon)}
\,d\epsilon
\right]\\
\le & 
 C b_n \E \left[ \int_0^{\theta_n/b_n}
\sqrt{\log N(\mc F_{\delta}, {L_2(\P_n)}, b_n\epsilon)}
\,d\epsilon
\right]
\end{align*}
Note that $N(\mc F_{\delta}, L_2(Q),\epsilon) \le N^2(\mc F, L_2(Q),\epsilon)$. (Consider all pairs $(f-g)$ in $\mc F$)
so 
\begin{align*}
\E\left[
\sup_{f\in \mc F_{\delta}} \sqrt{n} |\P_n^o f|
\right] 
&\le 
C b_n \E 
\left[ \int_0^{\theta_n/b_n}
\sqrt{\log N(\mc F, {L_2(\P_n)}, b_n\epsilon)}
\,d\epsilon
\right]\\
&\le
C b_n\E
\left[ \int_0^{\theta_n/b_n}
\sup_Q
\sqrt{\log N(\mc F, L_2(Q), \norm{B}_Q \epsilon)}
\,d\epsilon
\right]
\end{align*}
By the our entropy integral condition, we have
$$ \int_0^{a}
\sup_Q
\sqrt{\log N(\mc F, L_2(Q), \norm{B}_Q \epsilon)}
\,d\epsilon \to 0 \quad \text{as} \quad a \downarrow 0$$
Knowing that $\mc F$ is Glivenko-Canteli, get $\theta_n \cp 0$ as $\delta \downarrow 0$. Note that $b_n \cp \E \norm{B}_{L_2(P)} > 0$ as $n \to \infty$. Again using our entropy integral condition, we have
\begin{align*}
&\lim_{\delta\downarrow 0} \limsup_{n\to \infty}
\P\left(\sup_{\norm{f-g} \le \delta} |\G_n(f-g)| \le \epsilon \ge \epsilon\right) \\
\le &
\lim_{\delta\downarrow 0} \limsup_{n\to \infty}
\frac{Cb_n}{\epsilon}\E
\left[ \int_0^{\theta_n/b_n}
\sup_Q
\sqrt{\log N(\mc F, L_2(Q), \norm{B}_Q \epsilon)}
\,d\epsilon
\right] = 0.
\end{align*}
Then $\mc F$ is $\P$-Donsker follows from Corollary \ref{Uniform limits via entropy integral}.
\end{proof-sketch}

\section{M-Estimator Revisited}
We will use all of our uniform laws, CLTs to get
\begin{enumerate}
\item[(i)] consistency result
\item[(ii)] rate of convergence using moduli of continuity.
\end{enumerate}
\begin{definition}
Criterion function $m_{\theta}: \mc X \to \R$. Let $M_n(\theta) := \P_n m_{\theta}$ and $M(\theta) = \P m_\theta$.  
Then $\what{\theta}_n \in \argmax_{\theta \in \Theta}$ is the \textbf{M-estimator}. 

Criterion function $\psi_{\theta}: \mc X \to \R$. Let $\Psi_n(\theta) := \P_n \psi_{\theta}$ and $\Psi(\theta) = \P \psi_\theta$.  
Then $\what{\theta}_n$ s.t. $\Psi_n(\what{\theta}_n) = 0$ is the \textbf{Z-estimator}. (Often $\psi_{\theta}(x) = \nabla_{\theta} m_{\theta}(x)$ )
\end{definition}
\subsection{Consistency}

\begin{theorem}[Argmax consistency theorem]
Suppose we have
\begin{enumerate}
\item Uniform law of large number(ULLN): 
$$\sup_{\theta \in \Theta} |M_n(\theta) -M(\theta)| \cp 0$$
\item Separation: $$\sup_{\theta: d(\theta, \theta_0) \ge \epsilon} M(\theta) < M(\theta_0)$$ for all $\epsilon > 0$.
\end{enumerate}
Then $\what{\theta}_n \cp \theta_0$.
\end{theorem}

\begin{proof}
\end{proof}
Fix $\epsilon > 0$. Let
$\eta(\epsilon) := M(\theta_0) - \sup_{\theta: d(\theta, \theta_0) \ge \epsilon} M(\theta)$. Then 
$$d(\what{\theta}_n, \theta_0) \ge \epsilon \implies
M(\theta_0) \ge M(\what{\theta}_n) +\eta(\epsilon)
$$ but then
\begin{align*}
M_n(\theta_0) \ge & M(\theta_0) -o_p(1)\\
\ge & M(\what{\theta}_n) + \eta(\epsilon) - o_p(1)\\
\ge & M_n(\what{\theta}_n) + \eta(\epsilon) - o_p(1)
\end{align*}
by ULLN. 
Hence,
$$\P\left(d(\what{\theta}_n, \theta_0) \ge \epsilon\right)
\le \P \left(M_n(\theta_0) \ge  M_n(\what{\theta}_n) + \eta(\epsilon) - o_p(1)\right) \to 0.
$$


\subsection{Rate of convergence via moduli of continuity}
\textbf{Idea:} if $M(\theta))$ shrinks quickly way from $\theta_0$, but
$(M_n(\theta) - M(\theta)) - (M_n(\theta_0)) - M(\theta_0))$ are well behaved (i.e. have small error), then the M-estimator
$\what{\theta}_n=\argmax_\theta M_n(\theta)$ cannot be bad.\\

\begin{proofof}{non-random case}

Suppose $$M(\theta_0) \ge M(\theta) + \norm{\theta -\theta_0}^2,$$ which is reasonable by Taylor expansion $\nabla M(\theta_0)=0,\nabla^2 M(\theta_0)<0 $. Suppose we have local error guarantee,
$$\sup_{\norm{\theta -\theta_0} \le \delta}
|(M_n(\theta) - M(\theta)) - (M_n(\theta_0) - M(\theta_0))|
\le \frac{\phi(\delta)}{\sqrt{n}}
$$
for some $\phi$ with $\phi(\delta) \to 0$ as $\phi \to 0$, i.e. modulus of continuity near $\theta_0$. \\

Let $\what{\theta}_n=\argmax_\theta M_n(\theta) $ and assume consistency $\what{\theta}_n \cp \theta_0$.
Then 
\begin{align*}
M_n(\what{\theta}_n) \ge& M_n(\theta_0) \\
=& M_n(\theta_0) - M(\theta_0) + M(\what{\theta}_n)+M(\theta_0) - M(\what{\theta}_n)\\
\ge & M_n(\theta_0) - M(\theta_0) + M(\what{\theta}_n) 
+\norm{\what{\theta}_n -\theta_0}^2
\end{align*}
i.e.
\begin{align*}
0 \ge & -\left((M_n(\what{\theta}_n) - M(\what{\theta}_n)) - (M_n(\theta_0) - M(\theta_0))\right) + \norm{\what{\theta}_n -\theta_0}^2\\
\ge & \frac{-\phi\left(\norm{\what{\theta}_n -\theta_0}\right)}{\sqrt{n}} + \norm{\what{\theta}_n -\theta_0}^2.
\end{align*}

Rearranging gives 
\begin{equation}
\label{critical}
\norm{\what{\theta}_n -\theta_0}^2 \le
\frac{\phi\left(\norm{\what{\theta}_n -\theta_0}\right)}{\sqrt{n}}.
\end{equation}

Let $r_n^2 \ge \frac{\phi\left(\frac{1}{r_n}\right)}{\sqrt{n}}$. 
If $\norm{\what{\theta}_n - \theta_0} > \frac{1}{r_n}$, then contradicts \eqref{critical} , so we have $$\norm{\what{\theta}_n - \theta_0} \le \frac{1}{r_n}.$$
\end{proofof}

\begin{example*}
Let $\phi(\delta) = \delta.$ Then 
$\norm{\what{\theta}_n - \theta_0}^2
\le \frac{\norm{\what{\theta}_n - \theta_0}}{\sqrt{n}}.$
Then $\norm{\what{\theta}_n - \theta_0} \le \frac{1}{\sqrt{n}}.$
\end{example*}
\end{document}
