\documentclass[12pt]{article}

\usepackage{enumerate}
\usepackage{amssymb}
\usepackage{amsbsy}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{latexsym}
\usepackage{graphicx}
\usepackage{xifthen} % Allows us to put in optional arguments to questions
\usepackage{color}
\usepackage{manfnt}
\usepackage[top=2.54cm,bottom=2.54cm,right=2.54cm,left=2.54cm]{geometry}

%%% New version of \caption puts things in smaller type, single-spaced 
%%% and indents them to set them off more from the text.
\makeatletter
\long\def\@makecaption#1#2{
  \vskip 0.8ex
  \setbox\@tempboxa\hbox{\small {\bf #1:} #2}
  \parindent 1.5em  %% How can we use the global value of this???
  \dimen0=\hsize
  \advance\dimen0 by -3em
  \ifdim \wd\@tempboxa >\dimen0
  \hbox to \hsize{
    \parindent 0em
    \hfil 
    \parbox{\dimen0}{\def\baselinestretch{0.96}\small
      {\bf #1.} #2
      %%\unhbox\@tempboxa
    } 
    \hfil}
  \else \hbox to \hsize{\hfil \box\@tempboxa \hfil}
  \fi
}
\makeatother

\newcommand{\mc}[1]{\mathcal{#1}}
\def\mbf#1{\mathbf{#1}}
\def\mrm#1{\mathrm{#1}}
\def\mbi#1{\boldsymbol{#1}} % Bold and italic (math bold italic)
\def\v#1{\mbi{#1}} % Vector notation

\newcommand{\norm}[1]{\left\|{#1}\right\|} % A norm with 1 argument
\newcommand{\lone}[1]{\norm{#1}_1} % l1 norm
\newcommand{\ltwo}[1]{\norm{#1}_2} % l2 norm
\newcommand{\linf}[1]{\norm{#1}_\infty} % l-infinity norm
\newcommand{\dnorm}[1]{\norm{#1}_*} % Dual norm
\newcommand{\lfro}[1]{\left\|{#1}\right\|_{\rm Fr}} % Frobenius norm
\newcommand{\matrixnorm}[1]{\left|\!\left|\!\left|{#1}
  \right|\!\right|\!\right|} % Matrix norm with three bars
\newcommand{\matrixnorms}[1]{|\!|\!|{#1}|\!|\!|} % Small matrix norm
\newcommand{\opnorm}[1]{\matrixnorm{#1}_{\rm op}}
\newcommand{\opnorms}[1]{\matrixnorms{#1}_{\rm op}}
\newcommand{\normbigg}[1]{\bigg\|{#1}\bigg\|} % A norm with 1 argument and bigg
                                              % brackets.
\newcommand{\lonebigg}[1]{\normbigg{#1}_1} % l1 norm
\newcommand{\ltwobigg}[1]{\normbigg{#1}_2} % l2 norm
\newcommand{\linfbigg}[1]{\normbigg{#1}_\infty} % l-infinity norm
\newcommand{\norms}[1]{\|{#1}\|} % A norm with 1 argument and normal (small)
                                 % brackets.
\newcommand{\lones}[1]{\norms{#1}_1} % l1 norm with small brackets
\newcommand{\ltwos}[1]{\norms{#1}_2} % l2 norm with small brackets
\newcommand{\linfs}[1]{\norms{#1}_\infty} % l-infinity norm with small brackets

\newcommand{\hinge}[1]{\left[{#1}\right]_+}

\newcommand{\defeq}{:=}
\newcommand{\eqdef}{=:}
\newcommand{\what}[1]{\widehat{#1}} % Wide hat 
\newcommand{\wt}[1]{\widetilde{#1}} % Wide tilde
\newcommand{\wb}[1]{\overline{#1}} % Wide bar

\newcommand{\half}{\frac{1}{2}}

\newcommand{\reals}{\mathbb{R}} % Real number symbol
\newcommand{\R}{\mathbb{R}}
\newcommand{\C}{\mathbb{C}} % Complex numbers
\newcommand{\integers}{\mathbb{Z}} % Integer symbol
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\rationals}{\mathbb{Q}} % Rational numbers
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\naturals}{\mathbb{N}} % Natural numbers
\newcommand{\N}{\mathbb{N}}

\newcommand{\<}{\left\langle} % Angle brackets
\renewcommand{\>}{\right\rangle} % End angle brackets

\renewcommand{\iff}{\Leftrightarrow}
\renewcommand{\choose}[2]{\binom{#1}{#2}}  % Choose
\newcommand{\chooses}[2]{{}_{#1}C_{#2}}  % Small choose

%%%% Probability symbols and associated distances %%%%

\newcommand{\E}{\mathbb{E}} % Expectation symbol
\renewcommand{\P}{\mathbb{P}} % Probability symbol
\newcommand{\var}{{\rm Var}} % Variance
\newcommand{\cov}{\mathop{\rm Cov}} % Covariance
\newcommand{\simiid}{\stackrel{\rm iid}{\sim}}
\newcommand{\openleft}[2]{\left({#1},{#2}\right]} % Interval open on left
\newcommand{\openright}[2]{\left[{#1},{#2}\right)} % Interval open on right

\newcommand{\indic}[1]{\mbf{1}\left\{#1\right\}} % Indicator function

% Distances between probability measures
\newcommand{\tvnorm}[1]{\norm{#1}_{\rm TV}} % Total variation
\newcommand{\tvnorms}[1]{\norms{#1}_{\rm TV}}
\newcommand{\dkl}[2]{D_{\rm kl}\left({#1} |\!| {#2}\right)} % KL divergence
\newcommand{\dkls}[2]{D_{\rm kl}({#1} |\!| {#2})}  % Small KL-divergence
\newcommand{\dchi}[2]{D_{\chi^2}\left({#1} |\!| {#2}\right)}  % chi^2-divergence
\newcommand{\fdiv}[2]{D_f\left({#1} |\!| {#2}\right)} % f divergence
\newcommand{\kl}[2]{D_{\rm kl}\left({#1} |\!| {#2} \right)}
\newcommand{\dhel}{d_{\rm hel}}  % Hellinger distance
\newcommand{\helaff}{A_{\rm hel}}  % Hellinger affinity

% Convergence of random variables
\newcommand{\cd}{\stackrel{d}{\rightarrow}}
\newcommand{\cas}{\stackrel{a.s.}{\rightarrow}}
\newcommand{\cp}{\stackrel{p}{\rightarrow}}

\newcommand{\normal}{\mathsf{N}}  % Normal distribution
\newcommand{\uniform}{\mathsf{Uni}}  % Uniform distribution

% Simple floor/ceiling stuff
\newcommand{\floor}[1]{\left\lfloor{#1} \right\rfloor}
\newcommand{\ceil}[1]{\left\lceil{#1} \right\rceil}

\providecommand{\argmax}{\mathop{\rm argmax}} % Defining math symbols
\providecommand{\argmin}{\mathop{\rm argmin}}
\providecommand{\dom}{\mathop{\rm dom}}
\providecommand{\diag}{\mathop{\rm diag}}
\providecommand{\tr}{\mathop{\rm tr}}
\providecommand{\abs}{\mathop{\rm abs}}
\providecommand{\card}{\mathop{\rm card}}
\providecommand{\sign}{\mathop{\rm sign}}
\providecommand{\cl}{\mathop{\rm cl}}
\providecommand{\interior}{\mathop{\rm int}}
\providecommand{\conv}{\mathop{\rm Conv}}
\providecommand{\relint}{\mathop{\rm relint}}
\providecommand{\vol}{\mathop{\rm Vol}}
\providecommand{\supp}{\mathop{\rm supp}}

\providecommand{\minimize}{\mathop{\rm minimize}}
\providecommand{\maximize}{\mathop{\rm maximize}}
\providecommand{\subjectto}{\mathop{\rm subject\;to}}

% Proof environments
% The Theorems are numbered consecutively
% Lemmas are numbered by section, and observations, claims, facts, and 
% assumptions take their numbering. Propositions and definitions have their
% own numbering by section.
\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{observation}[theorem]{Observation}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{definition}{Definition}[section]
\newtheorem{claim}[theorem]{Claim}
\newtheorem{fact}[theorem]{Fact}
\newtheorem{assumption}{Assumption}
\newtheorem{conjecture}[theorem]{Conjecture}

\renewenvironment{proof}{\noindent{\bf Proof}\hspace*{1em}}{\qed\bigskip\\}
\newenvironment{proof-sketch}{\noindent{\bf Sketch of Proof}
  \hspace*{1em}}{\qed\bigskip\\}
\newenvironment{proof-idea}{\noindent{\bf Proof Idea}
  \hspace*{1em}}{\qed\bigskip\\}
\newenvironment{proof-of-lemma}[1][{}]{\noindent{\bf Proof of Lemma {#1}}
  \hspace*{1em}}{\qed\bigskip\\}
\newenvironment{proof-of-proposition}[1][{}]{\noindent{\bf
    Proof of Proposition {#1}}
  \hspace*{1em}}{\qed\bigskip\\}
\newenvironment{proof-of-theorem}[1][{}]{\noindent{\bf Proof of Theorem {#1}}
  \hspace*{1em}}{\qed\bigskip\\}
\newenvironment{inner-proof}{\noindent{\bf Proof}\hspace{1em}}{
  $\bigtriangledown$\medskip\\}
%% \newenvironment{proof-of-lemma}[1][{}]{\noindent{\bf Proof of Lemma {#1}}
%%   \hspace*{1em}\renewcommand{\qedsymbol}{$\bigtriangledown$}}{\qed\bigskip\\}
\newenvironment{proof-attempt}{\noindent{\bf Proof Attempt}
  \hspace*{1em}}{\qed\bigskip\\}
\newenvironment{proofof}[1]{\noindent{\bf Proof} of {#1}:
  \hspace*{1em}}{\qed\bigskip\\}
\newenvironment{remark}{\noindent{\bf Remark}
  \hspace*{1em}}{\bigskip}

%% Example environments

\newcounter{example}

\newenvironment{example}[1][]{
  \refstepcounter{example}
  \ifthenelse{\isempty{#1}}{%
    % There is no text to display in the example
    \noindent \textbf{Example \theexample:}\hspace*{.05em}
  }{%
    \noindent \textbf{Example \theexample} ({#1})\textbf{:}\hspace*{.05em}
  }
}{%
  $\clubsuit$ \bigskip
}

\newenvironment{example*}[1][]{
  \ifthenelse{\isempty{#1}}{%
    % There is no text to display in the example
    \noindent \textbf{Example:}\hspace*{.05em}
  }{%
    \noindent \textbf{Example} ({#1})\textbf{:}\hspace*{.05em}
  }
}{%
  $\clubsuit$ \bigskip
}
% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\begin{document}


\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 16 -- March 2, 2017} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Jaime Roquero Gimenez }  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} % Put the reading covered in the lecture here

\paragraph{Outline:}
\begin{enumerate}
\item Finish uniform laws, rates and moduli of continuity.
\item Asymptotic testing: Power, Local alternatives, Efficiency and comparing tests
\end{enumerate}
\section{Recap: M-estimation problem}

We consider the objective function and its empirical form
\begin{align*}
M_{n}: \,& \Theta \rightarrow \mathbb{R}
\\
M : & \, \theta \mapsto \mathbb{E}[M_{n}(\theta)]
\end{align*}

and we consider the case where $M(\theta_0)\geq M(\theta) + d(\theta,\theta_0)^2$ near $\theta_0 = \argmax_{\theta \in \Theta} M(\theta)$.

\medskip

Idea: compare $M(\theta_0) - M(\theta)$ (Gaps in population) vs. $M_{n}(\theta_0)- M_{n}(\theta)$ (Gaps in empirical version).

\bigskip

\begin{example}
Suppose we had
\[
\sup_{d(\theta_0,\theta)\leq \delta} |M_{n}(\theta)-M(\theta) - (M_{n}(\theta_0)-M(\theta_0))| \leq \frac{\sigma \delta}{\sqrt{n}}
\]

Let $\hat{\theta}_{n} = \argmax_{\theta}M_{n}(\theta)$, assume consistency of the estimator: ie. $\hat{\theta}_n \rightarrow \theta_0$ in probability.
Then 
\begin{align*}
M_{n}(\hat{\theta}_n)  \geq & \: M_n(\theta_0)=  M_n(\theta_0)-M(\theta_0)+M(\hat{\theta}_n)+M(\theta_0)-M(\hat{\theta}_n)
\\ \geq & \: M_n(\theta_0)-M(\theta_0)+M(\hat{\theta}_n) + d(\theta_0, \hat{\theta}_n)^2
\end{align*}
\begin{align*}
0 \geq & \: M_n(\theta_0)-M(\theta_0) - (M_n(\hat{\theta}_n)-M(\hat{\theta}_n))
\\0 \geq & \: -\frac{\sigma d(\theta_0, \hat{\theta}_n)}{\sqrt{n}} + d(\theta_0, \hat{\theta}_n)^2
\end{align*}
ie. $d(\theta_0, \hat{\theta}_n)\leq \frac{\sigma}{\sqrt{n}}$

\end{example}

\begin{example}
Let $m_\theta(x)$ be Lipschitz in $\theta$, and $M_n(\theta)=P_n m_\theta (x)$.

Let $P_n^{0} = \frac{1}{n} \sum_{i=1}^{n}\epsilon_i \delta_{X_i}$ the symmetrized empirical measure.

Then
\begin{align*}
& \mathbb{E}\,\Big[\sup_{d(\theta,\theta_0)\leq \delta}\big|M_n(\theta)-M(\theta)-(M_n (\theta_0) - M(\theta_0))\big|\Big]
\\ (\star) & \leq 2 \, \mathbb{E}\,[\sup_{d(\theta,\theta_0)\leq \delta}P_n^{0}(m_\theta - m_{\theta_{0}})(X)] 
\\ (\star\star) & \leq \frac{Cst}{\sqrt{n}}\int_{0}^{\delta}\sqrt{\log N(\delta - ball, d, \epsilon)} \: d\epsilon
\\ & \leq \frac{Cst}{\sqrt{n}}\int_{0}^{\delta}\sqrt{d \log (1+ \frac{2\delta}{\epsilon})} \: d\epsilon
\\ & \leq Cst \: \frac{\delta \sqrt{d}}{\sqrt{n}}
\end{align*}

Where in $(\star)$ we apply symmetrization and in $(\star\star)$ we bound by the entropy integral given that
\[
\mathbb{E}\, \Bigg[ \exp \Big(\lambda \sum_{i=1}^n \epsilon_i \big(m_\theta (X_i)- m_{\theta_{0}}(X_i) \big) \Big) \Bigg] \leq \exp \Big( \frac{\lambda^2}{2}n d(\theta,\theta_{0})^2\Big)
\]
as $m_\theta (x)$ is Lipschitz in $\theta$, so $\sqrt{n}P_{n}^{0}(m_\theta - m_{\theta_{0}})$ is $d^2$ sub-gaussian.

\end{example}


\section{Moduli of continuity and convergence rates}


\begin{theorem}
Suppose $M(\theta_0)\geq M(\theta) + d(\theta,\theta_0)^2$ near $\theta_0$.

Let $\phi$ be such that $\phi(c\delta)\leq c^{\alpha}\phi(\delta)$ for some $\alpha \in (0,2)$.

Assume
\[
\mathbb{E}\,\Big[\sup_{d(\theta,\theta_0)\leq \delta}\big|M_n(\theta)-M(\theta)-(M_n (\theta_0) - M(\theta_0))\big|\Big] \leq \frac{\phi(\delta)}{\sqrt{n}}
\]

Let $r_n \rightarrow +\infty$ such that $r_n^2\phi(\frac{1}{r_n})\leq \sqrt{n}$.

If $\hat{\theta}_n \rightarrow \theta_0$ in probability, then $r_n d(\hat{\theta}_n, \theta_0)=O_{P}(1)$
\end{theorem}

Idea: Estimation errors can scale at most as $\frac{\phi(\delta)}{\sqrt{n}} \sim \frac{d(\theta,\theta_0)^{\alpha}}{\sqrt{n}}$ with $\alpha < 2$.
But the growth of the objective function is quadratic: $d(\theta,\theta_0)^2$.

We solve $\delta^2 = \frac{\delta^\alpha}{\sqrt{n}}$ : it implies $\delta = \big( \frac{1}{n}\big)^{\frac{1}{2(2-\alpha)}}$

\bigskip

\begin{proof}
Let $\eta \leq \delta$. Then $\mathbb{P}\big(d(\hat{\theta}_n,\theta_0)\geq \eta \big) = o(1)$ (consistency of the estimator).

Define the shell: $S_{nj}= \{\theta: 2^{j-1}\leq r_nd(\theta,\theta_0)\leq 2^{j} \}$.

Consider the event 
\[
d(\hat{\theta}_n,\theta_0) \geq \frac{2^t}{r_n}
\]
for some $t \geq 0$.

Then $\exists j \geq t$ such that $\hat{\theta}_n \in S_{jn}$.

Therefore
\begin{align*}
\mathbb{P}(r_n d(\hat{\theta}_n , \theta_0)\geq 2^{t}) & \leq \mathbb{P}(r_n d(\hat{\theta}_n,\theta_0)\geq 2^t, d(\hat{\theta}_n,\theta_0)\leq \eta) + \mathbb{P}(d(\hat{\theta}_n,\theta_0)>\eta)
\\ & = \Big(\sum_{j\leq t, \, 2^{j-1}\leq r_n \eta} \mathbb{P}(\hat{\theta}_n \in S_{jn}) \Big) + \mathbb{P}(d(\hat{\theta}_n,\theta_0)>\eta)
\\ & \leq \Big(\sum_{j\leq t, \, 2^{j-1}\leq r_n \eta} \mathbb{P}(\exists \theta \in S_{jn}, M_n(\theta)\geq M_n(\theta_0)) \Big) + o(1)
\end{align*}

But if $M_n(\theta)\geq M_n(\theta_0)$ for some $\theta \in S_{jn}$ then 
\begin{align*}
M_n(\theta)-M(\theta) \geq &\: M_n(\theta_0) - M(\theta_0) +M(\theta_0)-M(\theta)
\\ \geq & \: M_n(\theta_0) - M(\theta_0) + d(\theta,\theta_0)^2
\end{align*}
So 
\[
\sup_{\theta \in S_{jn}}\big| (M_n(\theta)-M(\theta))-(M_n(\theta_0)-M(\theta_0))  \big| \geq \frac{2^{2j}}{r_n^2}
\]


And 

\begin{align*}
\mathbb{P}(\exists \theta \in S_{jn}, M_n(\theta)\geq & M_n(\theta_0)) 
\\ \leq  \:& \mathbb{P}\big(\sup_{\theta \in S_{jn}}\big|  (M_n(\theta)-M(\theta))-  (M_n(\theta_0)-M(\theta_0))  \big| \geq \frac{4^{j}}{r_n^2}\big)
\\ \leq \: & \frac{r_n^2}{4^j} \: \mathbb{E}\big[\sup_{\theta \in S_{jn}}\big|  (M_n(\theta)-M(\theta))-  (M_n(\theta_0)-M(\theta_0))  \big| \big]
\\ \leq \: & \frac{r_n^2}{4^j\sqrt{n}}\: \phi\big( \frac{2^j}{r_n}\big)\; \leq \; \frac{r_n^2 \, 2^{\alpha j}}{4^j\sqrt{n}}\phi\big( \frac{1}{r_n}\big) \; \leq \; 2^{j(\alpha - 2)}
\end{align*}

So we bound the inequality above with this result and get:

\[
\mathbb{P}(d(\hat{\theta}_n, \theta_0) \geq \frac{2^t}{r_n}) \leq \sum_{j\geq t} 2^{-j(2-\alpha)} + o(1)
\]

So by taking $t$ large enough we get that $r_n d(\hat{\theta}_n,\theta_0)= O_P(1)$

\end{proof}

\begin{observation}
If $\phi(\delta)= \delta^{\alpha}$, solving $r_n^2\phi(\frac{1}{r_n})=\sqrt{n}$ gives $r_n = n^{\frac{1}{2(2-\alpha)}}$ for the convergence rate.

More generally, if $M(\theta_0)\geq M(\theta) + d(\theta,\theta_0)^\beta$ then we could choose $r_n$ such that $r_n^\beta \phi(\frac{1}{r_n})\leq \sqrt{n}$ so that $r_n = n^{\frac{1}{2(\beta-\alpha)}}$. In order to get $r_n \rightarrow + \infty$, we want cases where growth in $d(.,.)^{\beta}$ dominates moduli of continuity $\phi(\delta) \sim \delta^\alpha$ at size $\sqrt{n}$.
\end{observation}

\end{document}