\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}
\usepackage{enumerate}
\usepackage{amssymb}
\usepackage{amsbsy}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}



\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 12 -- February  16} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Youngsuk Park, Youngtak Sohn}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Outline:} % Put the reading covered in the lecture here
\begin{itemize}
\item Uniform Laws via Entropy numebrs
\item Classes with finite entropies
\\-Nonparametric classes
\\-VC classes
\end{itemize}
\paragraph{Recap:}
Given $\mathcal{F}$ with distance $d$, $N(\mathcal{F},  d, \epsilon)=min \{N \in \mathbb{N}\ | ~\exists~ \epsilon$-cover of $\mathcal{F}: \{f_i\}_{i=1}^{N}$ in distance $d\}$\\
\underline{Chaining}: If $\{X_{t}\}$ is sub-Gaussian precess,
$\log(\E \exp(\lambda(X_s - X_t))) \leq \frac{\lambda^2d(s,t)^{2}}{2}$
. Then, $\mathbb{E}(\sup \limits_ {t \in T}X_t)\leq C\mathcal{J}(\mathcal{F},d)=C \int_{0}^\infty \sqrt{\log N(\mathcal{F}, d , \epsilon)}d\epsilon$.




%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%
Entropy number $\rightarrow$ uniform laws.
For empirical distirbution $P_n$, let $L_p(P_n)$ be the $L_p$ norm w.r.t. $P_n$, i.e., $\norm{f}_{L_p(P_n)} = (\frac{1}{n} \sum |f(X_i)|^p)^{1/p}$.

\example Often use $L_2(P_n)$ norm in symmetrized processes, i.e., if $Z_f:=\frac{1}{\sqrt{n}} \sum \epsilon_i f(X_i)$ where $\epsilon_i \overset{i.i.d}{\sim} unif(-1. +1)$. 

For fixed $X_1, \ldots, X_n$,

\begin{align*}
\log(\E \exp(\lambda(Z_f - Z_g)))= \log(\E \exp(\lambda\frac{1}{\sqrt{n}}\sum (\epsilon_i(f(X_i) - g(X_i) ) ))\\
\leq \frac{\lambda^2}{2n} \sum (f(X_i) - g(X_i) )^2 = \frac{\lambda^2}{2} \norm{f-g}^2_{L_2(P_n)}.
\end{align*}
i.e., $f \rightarrow \frac{1}{\sqrt{n}} \sum \epsilon_i f(X_i)$ is an $\norm{\cdot}^2_{L_2(P_n)}$  sub-Gaussian process.
so, 
\begin{align*}
\E[\sup_{f \in \mathcal{F}} | \frac{1}{\sqrt{n}}\sum \epsilon_i f(X_i)| \mid X_1\ldots, X_n] \leq C \int_{0}^\infty \sqrt{\log N(\mathcal{F},  L_2(P_n) , \epsilon)} d\epsilon.
\end{align*}

For $M < \infty$ , let $f_M(x) =  f(x)$ if $|f(x)| \leq M$ or 0 otherwise.
Let $\mathcal{F}$ be a collection of functions on $\mathcal{X}$ with envelop $F$, i.e., $|f(x)| \leq F(x)$ for all $x \in \mathcal{X}$ and $F \in L_1(P)$. Define $\mathcal{F}_M := \{f_M\}_{f\in \mathcal{F}}$.
\\
\begin{theorem}
(ULLNs with entropies): If $\sqrt{\log N(\mathcal{F},  L_1(P_n) , \epsilon)} = o_p(n)$  for all $M < \infty, \epsilon >0$, then $\norm{P_n - P}_{\mathcal{F}}\overset{p}{\to}0$, ie. $\mathcal{F}$ is G.C. class.
\end{theorem}

\begin{proof}
Let $P_n^0$($f$)$:=\frac{1}{n} \sum \epsilon_i f(X_i)$ where $\epsilon_i \overset{i.i.d}{\sim} unif(-1. +1)$. Then $|P_n^0f|\leq P_n|f| = \norm{f}_{L_1(P_n)}$.
\begin{align*}
\E[\norm{P_n-P}] \leq 2 \E[\sup\limits_{f \in \mathcal{F}}||P_n^0f|] 
\leq 2 \E[\sup\limits_{f \in \mathcal{F}}||P_n^0(f-f_{M})|]+\mathbb{E}[\sup\limits_{f \in \mathcal{F_{M}}}|P_n^0f|] 
\end{align*}

Note that,~$2 \E[\sup\limits_{f \in \mathcal{F}}||P_n^0(f-f_{M})|]\leq 2 \E[F 1(F \geq M)].$

Now control $E[\sup_{f} |P_n^0 f|]$. Let $\mathcal{G}$ be a minimal $\epsilon$ - cover of $\mathcal{F}_m$ in $L_1(P_n)$ norm. Then, $\mathop{card}(\mathcal{G}) = N(\mathcal{F}_M, L_1(P_n), \epsilon).$\\
Therefore, $\sup\limits_{f \in \mathcal{F}}||P_n^0f|\leq \max\limits_{g \in \mathcal{G}}|P_n^0g|+\epsilon$
 (by triangular inequality).


Note that w.l.o.g. $|g(x)| \leq M$, all $g \in \mathcal{G}$, so $P_n^0g$ is $\frac{M^2}{n}$ sub- Gaussian. So
\begin{align*}
\E[\max \limits_{g \in \mathcal{G}}|P_n^0g|\mid X] \leq 2 \sqrt[]{2 \frac{M^2}{n} \log N(\mathcal{F},  L_1(P_n) , \epsilon)}
\end{align*}
\\LHS is less than $M$, and the RHS is $\sqrt{\frac{M}{n} o_p(n)} = o_p(1)$)
\begin{align*}
\E [\E [\max _{g} |P_n^0g|]] \leq \E[min(M,o_p(1))] \rightarrow 0 \mbox{ as } n \rightarrow \infty.\\
\Rightarrow \E[\norm{P_n - P}_{\mathcal{F}}] \leq 2 \E[F 1(F\geq M)] + o(1) + \epsilon
\end{align*}
Since, $\E[F 1(F \geq M)]\rightarrow 0$~as~$ n\rightarrow \infty$, the proof is done.
\end{proof}
\underline{Understand uniform entropies}: Often random covering numbers such as $N(\mathcal{F}, L_r(P_n), \epsilon)$ are a bit annoying. so try to give conditions such that $\sup\limits_P N(\mathcal{F}, L_r(P_n), \epsilon)$  can be controlled.


Let's look at some examples in non-parametric function classes.

\example Let $\mathcal{F}$ be the collection of $1-$ Lipschitz functions on $[0,1]$ with $f(0)=0$. Fix $\epsilon >0$, consider $\norm{f}_{\infty} := \sup_{x\in [0,1]} |f(x)|$. 
By dividing the unit intervals by intervals with length $\epsilon$ and moving along x axis by epsilon with 3 choice of directions, namely up(45 degree angle), staright, down(45 degree angle)
Packing $\# s \geq 3^{\frac{1}{\epsilon}}$(1-Lipschitz function's height change associated widch change of $\epsilon$ is also at most $\epsilon$). Thus, $\sup_{P: \mathop{supp} P = [0,1]} \log N(\mathcal{F}, L_r(P), \epsilon)\leq \frac{C}{\epsilon}$ where $c < \infty$ is absolute constant.
So 
\begin{align*}
\E [\sup |P_n - Pf|] \leq 2 \E [\sup_{f} |P_n^of|] 
\leq \frac{c}{\sqrt{n}} \E[ \int_0^1
 \sqrt{\log N(\mathcal{F}, L_2(P_n), \epsilon)} d \epsilon]
\leq  \frac{c}{\sqrt{n}} \int_0^1
 \frac{1}{\sqrt{\epsilon}} d \epsilon
\leq \frac{c}{\sqrt{n}}
\end{align*}
In $2 + $ dimensions, divided boxes with length $\epsilon$ has $\frac{1}{\epsilon^2}$ boxes, (or $(\frac{1}{\epsilon})^d$ in $d$ dimensions), so 
\begin{align*}
\log N(\mathcal{F}, \norm{\cdot}_\infty, \epsilon) \geq \frac{c}{\epsilon ^{2}}
\Rightarrow \mathcal{J}(\mathcal{F}, \norm{\cdot}_\infty) = \int_0^1 \frac{1}{\epsilon} = +\infty.
\end{align*}



\underline{Vapnik- Chervonenkis (VC classes)}
Collections of functions or sets with nice combinatorial structure allowing uniform entropy/covering number bounds. 


\begin{definition} Let $\mathcal{C}$ be a collection of sets and $X = \{X_1,\ldots, X_n\}$ be a collection of points . A vector $y \in \{+1, -1\}^n$ is a labeling of X. Say $\mathcal{C}$ \underline{shatters} $X$ if for all labelings $y$ of $X$, $\exists$  a set $A\in \mathcal{C}$, i.e., $X_i \in A$ if $y_i = 1$ and  $X_i  \not\in A$ if $y_i = -1$.
\end{definition}
Equivalently, $\{x_1,\ldots, x_n\}\cap \mathcal{C} = \{A\cap \{x_1,\ldots, x_n \mid A \in \mathcal{C}\}\}=2^{X}$.

\example Let $x_1, x_2, x_3 \in \reals^2$, not collinear. 
$\mathcal{C}$ = $\{$half space in $\reals^2\}$. Than $\mathcal{C}$ shatters $\{x_1, x_2, x_3\}$


\begin{definition}: The VC- dimension VC($\mathcal{C}$) is the size of the largest set $\{x_1,\ldots, x_n\}$ s.t. $\mathcal{C}$ shatters $\{x_1,\ldots, x_n\}$.
\end{definition}


\begin{definition} $\Delta_n(\mathcal{C}, \{ x_1,\ldots, x_n \}) : = $ the number of labelings $\mathcal{C}$ realizes on $\{x_i\}$. Then VC(${\mathcal{C}}$)  $:= \sup \{ n \in \mathbb{N} \mid \max_{x_1,\ldots, x_n} \Delta_n(\mathcal{C},\{x_i\}) = 2^n\}$.
\end{definition}


\example Half-spaces in $\reals^d$ have VC($\mathcal{C}$) $= d+1$, Think of $\reals^2$. Then $VC(\mathcal{C}) \geq 3$.
To do rigorously requires arguing (by geometry) that we would have to have the situation where diagonal labeling does not work.

\lemma (Sauer- Shelah) for any class $\mathcal{C}$, 
\begin{align*}
\max_{x_1,\ldots, x_n} \Delta_n(\mathcal{C}, \{x_i\}) \leq \sum_{k=0}^{VC(\mathcal{C})} \binom{n}{k} = O(n^{VC(\mathcal{C})}).
\end{align*}
\underline{Consequence}: If $\sup\limits_{x_1,\ldots, x_n} \Delta_n(\mathcal{C}, \{x_i\}) < 2^n$, then $\Delta_n(\mathcal{C}, \{x_i\})$ is polynomial in $n$.

Let $L_r(P)$ norm on sets $A \subset \mathcal{X}$ be defined by $\norm{1_A}_{L_r(P)} = \left(\int 1(x\in A)^r dP(x) \right)^{1/r}$

\begin{theorem}: $\exists$ a universal constant $K < \infty$ s.t. $\forall \epsilon >0$, 
\begin{align*}
\sup_{P}N(\mathcal{C},L_r(P),\epsilon) \leq K \cdot VC(\mathcal{C}) \cdot (4e)^{VC(\mathcal{C})}(\frac{1}{\epsilon})^{r \cdot VC(\mathcal{C})}
\end{align*}


$\Rightarrow \log N(\mathcal{C},L_r(P),\epsilon) \leq c \cdot r \cdot VC(\mathcal{C}) \cdot \log(1/\epsilon)$
\end{theorem}
\end{document}
