\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 2 -- January 12} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Yi Liu}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading: A.W. van der Vaart. Asymptotic Statistics.  Chapter 2 and Chapter 3} 

\begin{enumerate}
\item Prohorov Theorem
\item Portmanteau Lemma 
\item Delta Method
\end{enumerate}
 \vspace{-.2cm}
 \section{Prohorov Theorem}

\theoremstyle{definition}
\begin{definition}
A collection of random vectors $\{X_{\alpha}\}_{\alpha \in A}$ is uniformly tight if for all $\epsilon>0$, there exists M such that 
$$\sup_{\alpha} \P( \| X_\alpha\| \geq M) \leq \epsilon $$
\end{definition}

\begin{remark}
A single random vector is tight since $\lim_{n \rightarrow \infty} \P (\|X\| >M) = 0 $
\end{remark}
\\
\begin{remark}
If $X_n$ converges in distribution to $X$, then $\{ X_n\}_{n \in \N}$ is uniformly tight\\
\begin{proof}
Fix a number M such that $\P(\|X\| \geq M) < \epsilon$. By the portmanteau lemma $ \P(\| X_n\|\geq M)$ exceeds $\P(\|X\| \geq M)$ arbitrarily little for sufficient large n. Thus there exists N such that $P(\|X\| \geq M) < 2\epsilon$, for all $n\geq N$. Because each of the finitely many variables $X_n$ with $n<N$ is tight, the value fo M can be increased, if necessary, to ensure that $\P(\|X_n\| \geq M) < 2 \epsilon$ for every n. 
\end{proof}
\end{remark}
\begin{theorem}[Prohorov's theorem]
A collection of random vectors $\{X_{\alpha}\}_{\alpha \in A}$ is uniformly tight if and only if it is sequentially compact for weak convergence. i.e. $\forall $ sequences $\{X_{n} \}_{n \in \N} \subset \{X_{\alpha}\}_{\alpha \in A}$, there exist $n_k$, a subsequence and a random vector $X$ such that $X_{nk}$ converges in distribution in $X$.
\end{theorem}
\begin{proof}
d-dimensional analogue of Helly selection 
\end{proof}
\begin{example}
Random variable bounded in expectation\\
Let $\{X_{\alpha}\}_{\alpha \in A}$ satisfy $\E(\|X_{\alpha}\|)<M <\infty
$, for $\alpha \in A$. Then $\{X_{\alpha}\}_{\alpha \in A}$ is uniformly tight.\\
\begin{proof}
By markov inequality, \\ 
$$\P(\|X_\alpha\| \geq C) \leq \frac{\E(\|X_\alpha\|^p))}{C^p}\leq \frac{M^p}{C^p} \rightarrow 0$$
as $C\rightarrow \infty$
\end{proof}
\end{example}
\section{Portmanteau Theorem}
\begin{theorem}{Portmanteau Theorem}
Let $X_n$, $X$ be random vectors, then the following are equivalent. 
\begin{enumerate}
\item $X_n$ converges in distribution $X$ 
\item $\E(f(X_n))$ converges in distribution to $\E(f(X))$ for all bounded and continuous f
\item $\E(f(X_n))$ converges in distribution to $\E(f(X))$ for one-Lipschitz f with $f \in [0,1]$ 
\item $\lim\inf_{n\rightarrow \infty} \E(f(X_n)) \geq E(f(X))$ for non-negative and continuous f.
\item $\lim\inf_{n \rightarrow \infty}\P(X_n\in O) \geq \P(X \in O) $ for all open set O
\item $\lim\sup_{n \rightarrow \infty}\P(X_n\in C) \leq \P(X \in C) $ for all closed set C
\item $\lim_{n \rightarrow \infty}\P(X_n\in B) = \P(X \in B) $ for all set B such that $\P(X \in \delta B) = 0$ 
\end{enumerate}
\end{theorem}
\begin{remark}
We call a collection of function $\mathcal{F}$a determining class if $E(f(X_n)) \rightarrow E(f(X))$ for all $f \in \mathcal{F}$ if and only if $X_n \rightarrow X$ . For example, the characteristic function are determining class\\
\end{remark}
\begin{remark}
The function has to be bounded in this case. Consider the following counter example.\\
Let $g(x) =x$ and 
\[ X_n =
  \begin{cases}
    n^2      & \quad \text{with probability} \frac{1}{n}\\
    0  & \quad \text{otherwise } \\
  \end{cases}
\] 
$X_n$ converges in probability to 0 and therefore converges in distribution to 0. However, $$\E(g(X_n)) = n \rightarrow \infty $$
\end{remark}
\section{Delta Method}
Prompt: Suppose you have a sequence of statistics $T_n$ that estimate a parameter $\theta$ and you know that $r_n(T_n - \theta)$ converges in distribution to T when $r_n \rightarrow \infty$. (Here we understand $r_n$ as the rate ) \\
Suppose $\phi$ is smooth in the neighborhood of $\theta$, it is possible to say anything about $\phi(T_n)$? \\
\begin{theorem}{Delta Method} Let $r_n \rightarrow \infty$ and $\phi:\R^d \rightarrow \R^k$ be differentiable at $\theta$ and assume that $r_n (T_n - \theta)$ in distribution to $T$ for some random vector $T$, then 
\begin{enumerate}
\item $r_n(\phi(T_n) = \phi(\theta))$converges in distribution to $\phi'(\theta)T$
\item $r_n(\phi(T_n) = \phi(\theta)) - r_n\phi'(\theta)(T_n - \theta)$ converges in probability to 0
\end{enumerate}
Here $\phi'(\theta) \in \R^{k \times d}$ is the Jacobian Matrix of derivatives $[\phi'(\theta)]_{ij} = \frac{\partial\phi_i(\theta)}{\partial \theta_j}$
\end{theorem}
\begin{proof}
By taylor theorem, we have that 
$$\phi(t) = \phi(\theta) + \phi'(\theta)(t- \theta) + o(\|t-\theta\|) $$
as $t\rightarrow \theta$\\
This is equivalent to 
\begin{equation}
\phi(t) = \phi(\theta) + \phi'(\theta)(t- \theta) + R(|t-\theta|)
\end{equation}
where $R(t-\theta) = o(\|t-\theta\|)$\\
we know that $r_n(T_n -\theta) = O_p(1)$ because of the convergence in distribution.\\
$$r_n R(T_n -\theta) = r_n O_p(\|T_n -\theta\|) = O_p(\|r_n (T_n -\theta)\|) = o_p(1) $$
which converges in probability to 0. Now put everything together and use equation i
$$r_n(\phi(T_n) -\phi(\theta)) + r_n\phi'(\theta)(T_n- \theta) \cp 0 $$
This concludes the proof for part 2)\\
Now, we note that $ r_n\phi'(\theta)(T_n- \theta) \cd \phi'(\theta)T$ so we apply 
Slutsky lemma to get that 
$$r_n(\phi(T_n) -\phi(\theta)) \cd \phi'(\theta)T $$
\end{proof}
\begin{example}
$X_i \stackrel{iid}{\sim} P$,$\E(X) = \theta \neq 0$, $\E(\|X\|^2) \leq \infty$, $\cov(X) = \Gamma $ and
$\phi(h)= \frac{1}{2}\|h\|^2$
Then 
$$ \sqrt{n}(\frac{1}{n}\sum_{i=1}^k X_i - \theta) \cd \normal(0,\Gamma)$$
Now, assume that 
$\Delta \phi (\theta) =\theta$
So 
$$\sqrt{n}(\frac{1}{2}\| \frac{1}{n} \sum X_i \|^2 - \frac{1}{2} \|\theta\|^2)\cd \normal(0,\theta^T \Gamma \theta) $$
Now if $\| \theta\|^2=0$, all we get is that $$\sqrt{n}(\frac{1}{2}\| \frac{1}{n} \sum X_i \|^2)\cp 0$$
\end{example}
\\

\begin{remark}
A generalization of the delta method to higher order problems can be done with taylor expansion with careful use of $O_p$ and $o_p$ 
\end{remark}\\
\begin{remark}
If $\phi'(\theta) =0$, the higher order expansion allows us to get more powerful results and faster rate fo convergences
\end{remark}\\
\begin{theorem}
Let $r_n \rightarrow 0$ be deterministic. Suppose that $r_n(T_n -\theta) \cd T$, Let $\phi:\R \rightarrow \R$ be twice differentiable at $\theta$ such that $\nabla \phi(\theta)=0$, then we get 
$$r_n^2 (\phi(T_n) - \phi(\theta)) \cd \frac{1}{2}T^T\nabla^2 \phi(\theta)T$$
where $\nabla^2 \phi(\theta)$ is the Hessian Matrix
\end{theorem}
\begin{proof}
As before we will do taylor expansion 
$$\phi(t)  = \phi(\theta) + \nabla \phi(\theta)^T(t-\theta) + \frac{1}{2}(t-\theta)^T \nabla^2 \phi(\theta) (t-\theta) + R(t-\theta)$$
where $R(h) = o(\|h\|^2)$ as $h\rightarrow 0$ 
$$r_n^2 R(T_n - \theta) = r_n^2 O_p(\|T_n- \theta\|^2) \cp 0 $$
So we have 
$$r_n^2(\phi(T_n)-\phi(\theta)) = \frac{1}{2} r_n(T_n-\theta)^T \nabla^2 \phi(\theta) r_n(T_n-\theta)) + o_p(1) $$
So $$r_n^2(\phi(T_n)-\phi(\theta))  \cd T^T \nabla^2 \phi(\theta)T  $$
by continuous mapping
\end{proof}
\begin{example}KL-divergences and likelihood for Bernoulli random variable.\\
$$\dkl{P}{Q} = \int p \log(\frac{p}{q}) d \mu$$
where $p,q$ are density with respect to $\mu$ 
We know that $$ \dkl{P}{Q} \geq 0$$ and 
$$\dkl{P}{Q} =  0$$ if and only if $p=q$ 
Here we consider the Bernoulli: $P_\theta$ where $X\in \{0,1\}$
\[ X_n =
  \begin{cases}
    1      & \quad \text{with probability} \theta\\
    0  & \quad \text{with probability } 1-\theta \\
  \end{cases}
\] 
Consider the estimator 
$$\hat{\theta} = \frac{1}{n}\sum_{i=1}^n X_i$$
By CLT, we know that 
$$\sqrt{n}(\hat{\theta}-\theta) \cd \normal(0.\theta(1-\theta)) $$
Now, we compute the KL-distance between $$\dkl{P_{\hat{\theta}}}{P_{\theta}} $$
$$\phi(t) = \dkl{P_t}{P_\theta} = t\log(\frac{t}{\theta})+(1-t)\log(\frac{1-t}{1-\theta}) $$
$$\phi'(t) = \log(\frac{t}{1-t})-\log(\frac{\theta}{1-\theta}) $$
We know that $$\phi'(\theta) =0 $$ so we try 
$$\phi''(t) = \frac{1}{t(1-t)} $$
$$\phi''(\theta)= \frac{1}{\theta(1-\theta)} $$
now, take $r_n = \sqrt{n}$,$T_n = \hat{\theta}$, we have that 
$$n \dkl{P_{\hat{\theta}}}{P_{\theta}} \cd \frac{1}{2}\frac{1}{\theta(1-\theta)} T^2 = \frac{1}{2}Z^2 = \frac{1}{2}\chi^2_1$$

\end{example}
% Put the reading covered in the lecture here

%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%


\end{document}