\documentclass[11pt]{article}

% This package contains many of the macros you may need.
%
% A useful environment defined in it is the example environment, which you can
% invoke via \begin{example} and \end{example}.
\usepackage{scribe}

\begin{document}

\begin{center}
  \fcolorbox{black}{white}{
    \begin{minipage}{.9\columnwidth}
      \textbf{Stats 300b: Theory of Statistics}
      \hfill
      \textbf{Winter 2017} \\
      \vspace{-.2cm}
      % Instead of Lecture 1, January 10, put the actual lecture number and
      % date of the lecture here
      \begin{center}
        {\LARGE Lecture 7 -- January 31} \\
      \end{center}
      \emph{Lecturer: John Duchi}
      \hfill
      \emph{Scribe: Kenneth Tay}  % You should put your name here if you scribe
    \end{minipage}
  }
\end{center}

\begin{center}
  \dbend ~~  \emph{\textbf{Warning:} these notes may contain factual errors}
\end{center}

\paragraph{Reading:} % Put the reading covered in the lecture here

%%%%%%%%%%%%%%%%% SCRIBE NOTES HERE %%%%%%%%%%%%%%%%%%%%
\paragraph{Outline of the Lecture 7:}
\begin{itemize}
\item U-Statistics
\begin{itemize}
\item Variance computations
\item Projections of random variables and vectors
\item Asymptotic normality of U-statistics
\end{itemize}
\end{itemize}

\section{Variance of U-Statistics}
Recall these definitions that we set up last lecture:
\begin{definition}
Given a symmetric kernel function $h: \mathcal{X}^r \rightarrow \R$, define the associated \textbf{U-statistic} as
\begin{equation*} U_n := \frac{1}{\binom{n}{r}} \sum_{\beta \subseteq [n], |\beta| = r} h(X_\beta). \end{equation*}
\end{definition}

\begin{definition}
For each $c \in \{ 0, \dots, r \}$, define
\begin{equation*} h_c (x_1, \dots, x_c) := \E [h(x_1, \dots, x_c, X_{c+1}, \dots, X_r)]. \end{equation*}

Define $\hat{h}_c$ to be the centered version of $h_c$, i.e.
\begin{equation*} \hat{h}_c := h_c - \E[h_c] = h_c - \theta, \end{equation*}
where $\theta = \E U_n$.
\end{definition}

\begin{definition}
For each $c \in \{ 0, \dots, r \}$, define
\begin{equation*} \zeta_c := \var [h_c(X_1, \dots, X_c)] = \E [h_c(X_1, \dots, X_c)^2]. \end{equation*}

(Note that $\zeta_0 = 0$.)
\end{definition}

\paragraph{Goal:} Write $\var U_n$ as a sum of the $\zeta_c$'s.

\begin{lemma}
If $\alpha, \beta \subseteq [n]$, $S = \alpha \cap \beta$, $c = |S|$, then
\begin{equation*} \E \left[ \hat{h}(X_\alpha) \hat{h}(X_\beta) \right] = \zeta_c. \end{equation*}
\end{lemma}

\begin{proof}
Using the symmetry of $h$,
\begin{align*}
\E \left[ \hat{h}(X_\alpha) \hat{h}(X_\beta) \right] &= \E \left[ \hat{h}(X_{\alpha \setminus S}, X_S) \hat{h}(X_{\beta \setminus S}, X_S) \right] \\ 
&= \E \left[ \E [\hat{h}(X_{\alpha \setminus S}, X_S) \mid X_S] \cdot \E [ \hat{h}(X_{\beta \setminus S}, X_S) \mid X_S] \right] &&\text{(since $X_{\alpha \setminus S}$, $X_{\beta \setminus S}$ indep.)} \\ 
&= \E \left[ \hat{h}_c(X_S) \cdot \hat{h}_c(X_S) \right] \\ 
&= \zeta_c.
\end{align*}
\end{proof}

\begin{theorem}
Let $U_n$ be an $r^{th}$ order U-statistic. Then
\begin{equation*} \var U_n = \frac{r^2}{n}\zeta_1 + O(n^{-2}). \end{equation*}
\end{theorem}

\begin{proof}
There are $\binom{n}{r}\binom{r}{c}\binom{n-r}{r-c}$ ways to select a pair of subsets of $[n]$, each of size $r$, with $c$ common elements. Hence,
\begin{align*}
U_n - \theta &= \binom{n}{r}^{-1} \sum_{|\beta| = r} \hat{h}(X_\beta), \\ 
\var U_n &= \binom{n}{r}^{-2} \sum_{|\alpha| = r}\sum_{|\beta| = r} \E \left[\hat{h}(X_\alpha)\hat{h}(X_\beta)\right] \\ 
&= \binom{n}{r}^{-2} \sum_{c = 1}^r \binom{n}{r}\binom{r}{c}\binom{n-r}{r-c} \zeta_c \\ 
&= \sum_{c = 1}^r \frac{r!^2}{c! (r-c)!^2} \frac{(n-r)(n-r-1)\dots(n-2r+c+1)}{n(n-1)\dots(n-r+1)} \zeta_c.
\end{align*}

For fixed $c$, $\frac{(n-r)(n-r-1)\dots(n-2r+c+1)}{n(n-1)\dots(n-r+1)}$ has $r-c$ terms in the numerator and $r$ terms in the denominator. Hence,

\begin{align*}
\var U_n &= r^2 \frac{(n-r)(n-r-1)\dots(n-2r+2)}{n(n-1)\dots(n-r+1)} \zeta_1 + \sum_{c=2}^r O\left(\frac{n^{r-c}}{n^r} \right) \zeta_c \\ 
&= r^2 \left[\frac{1}{n} + O(n^{-2}) \right] \zeta_1 + O(n^{-2}) \\ 
&= \frac{r^2}{n}\zeta_1 + O(n^{-2}).
\end{align*}
\end{proof}

With this theorem, we know that the variance of U-statistics behaves like the variance of a sample mean plus high-order errors.

\paragraph{New Goal:} Show that $U_n$ is asymptotically normal by projecting out all high-order interactions.

To do this, we need some theory on projections.

\section{Projections}
Let $\mathcal{V}$ be a Hilbert space, i.e. there is an inner product $\< \cdot, \cdot \>$ on $\mathcal{V}$, an associated norm $\ltwos{v}^2 = \< v, v \>$, and $\mathcal{V}$ is complete w.r.t. this norm. (Note that $\ltwos{v} = 0$ iff $v = 0$.)

\begin{definition}
Let $C \subseteq \mathcal{V}$ be a convex and closed set. Define the \textbf{projection of $w$ onto $C$} as
\begin{equation*} \pi_C(w) := \argmin_{v \in C} \{ \ltwos{w-v}^2 \}. \end{equation*}
\end{definition}

\begin{theorem}
$\pi_C(w)$ exists, is unique, and is characterized by the inequality
\begin{equation}\label{eqn:projineq} \< w - \pi_C(w), v - \pi_C(w) \> \leq 0. \end{equation}
\end{theorem}

Loosely speaking, the inequality means that the ``angle'' between $w - \pi_C(w)$ and $v - \pi_C(w)$ is obtuse.

\begin{corollary}\label{cor:linearproj}
Suppose $C$ is a linear subspace of $\mathcal{V}$. Then $\pi_c(w)$ is the projection of $w$ onto $C$ iff for all $v \in C$,
\begin{equation*}
\< w - \pi_C(w), v \> = 0.
\end{equation*}
\end{corollary}

\begin{proof}
If $C$ is linear, then $v \in C \Leftrightarrow -v \in C$. Hence, by Equation \ref{eqn:projineq},
\begin{align*}
\< w - \pi_C(w), v \> &\leq \< w - \pi_C(w), \pi_C(w) \> \text{ and} \\ 
\< w - \pi_C(w), v \> &\leq -\< w - \pi_C(w), \pi_C(w) \>, \\ 
\Rightarrow \< w - \pi_C(w), v \> &= 0.
\end{align*}
\end{proof}

Let's now put these ideas in the random variable setting.
\begin{fact}
Random variables with 2 moments form a Hilbert space iwth inner product $\<X, Y \> = \E [XY]$. We wil call this space $L_2(P)$.
\end{fact}

\begin{corollary}
If $\mathcal{S}$ is a linear subspace of $L_2(P)$, then $\hat{S} \in \mathcal{S}$ is the projection of $T \in L_2(P)$ onto $\mathcal{S}$ iff for all $S \in \mathcal{S}$,
\begin{equation*} \E [(T - \hat{S})S] = 0. \end{equation*}

If this is the case, then
\begin{equation*} \E [T^2] = \E [(T - \hat{S})^2] + \E [\hat{S}^2]. \end{equation*}
\end{corollary}

\begin{proof}
The characterization of $\hat{S}$ follows directly from Corollary \ref{cor:linearproj}.
\begin{align*}
\E [T^2] &= \E [(T - \hat{S} + \hat{S})^2] \\ 
&= \E [(T - \hat{S})^2] + \E [\hat{S}^2] + 2 \cov (T - \hat{S}, S) \\ 
&= \E [(T - \hat{S})^2] + \E [\hat{S}^2].
\end{align*}
\end{proof}

\paragraph{Idea:} Try to understand when $T_n$ and its projections have the same asymptotic behavior.

\begin{theorem}
Let $T_n$ be statistics, and let $\hat{S}_n$ be the projections of $T_n$ onto subspaces $\mathcal{S}_n$ which contain constant random variables.
\begin{equation*}
\text{If } \displaystyle\frac{\var T_n}{\var \hat{S}_n} \rightarrow 1, \text{ then } \frac{T_n - \E T_n}{\sqrt{\var T_n}} - \frac{\hat{S}_n - \E \hat{S}_n}{\sqrt{\var \hat{S}_n}} \cp 0.
\end{equation*}
\end{theorem}

\begin{proof}
Let $A_n = \displaystyle\frac{T_n - \E T_n}{\sqrt{\var T_n}} - \frac{\hat{S}_n - \E \hat{S}_n}{\sqrt{\var \hat{S}_n}}$. Note that $\E A_n = 0$. Thus, if we can show that $\var A_n \rightarrow 0$, we are done.

Note that 
\begin{align*}
\cov (T_n, \hat{S}_n) &= \E [T_n \hat{S}_n] - \E[T_n] \E[\hat{S}_n] \\ 
&= \E[(T_n - \hat{S}_n + \hat{S}_n)\hat{S}_n] - \E[\hat{S}_n]^2 \\ 
&= \E [\hat{S}_n^2] - \E[\hat{S}_n]^2 \\ 
&= \var \hat{S}_n.
\end{align*}

Hence,
\begin{align*}
\var A_n &= \var \frac{T_n - \E T_n}{\sqrt{\var T_n}} + \var \frac{\hat{S}_n - \E \hat{S}_n}{\sqrt{\var \hat{S}_n}} - \frac{2 \cov (T_n, \hat{S}_n)}{\sqrt{\var T_n \var \hat{S}_n}} \\ 
&= 2 - 2 \sqrt{\frac{\var \hat{S}_n}{\var T_n}} \\ 
&\rightarrow 0.
\end{align*}
\end{proof}

\subsection{Conditional Expectations}
Conditional expectations are simply projections.

\begin{definition}
If $X \in L_2(P)$, $Y$ is a random variable, $\mathcal{S} = \{ \text{all measurable functions } g(Y) \text{ with } \E [g^2(Y)] < \infty \}$, we define the \textbf{conditional expectation of $X$ given $Y$}, $\E [X \mid Y]$, as the projection of $X$ onto $\mathcal{S}$, i.e.
\begin{equation*} \E \left[ \left( X - \E [X \mid Y] \right) g(Y) \right] = 0 \end{equation*}
for all $g \in \mathcal{S}$.
\end{definition}

By choosing $g$ appropriately, some nice properties of conditional expectation are immediate:
\begin{itemize}
\item $\E [X - \E[X \mid Y]] = 0$, and
\item $\E [f(Y)X \mid Y] = f(Y) \E [X \mid Y]$.
\end{itemize}

\subsection{H\'{a}jek Projections}
\paragraph{Idea:} Apply these ideas to U-statistics, i.e. project them onto spaces of the form $\displaystyle\sum_{i = 1}^n g_i(X_i)$.

\begin{lemma}[11.10 in VDV]
Let $X_1, \dots, X_n$ be independent. Let $\mathcal{S} = \left\{ \displaystyle\sum_{i=1}^n g_i(X_i): g_i \in L_2(P) \right\}$.

If $\E T^2 < \infty$, then the projection $\hat{S}$ of $T$ onto $\mathcal{S}$ is given by
\begin{equation}\label{eqn:hajek} \hat{S} = \sum_{i = 1}^n \E [T \mid X_i] - (n-1) \E T. \end{equation}
\end{lemma}

\begin{proof}
Note that
\begin{equation*} \E \left[ \E [T \mid X_i] \mid X_j \right] = \begin{cases} \E[T \mid X_i] &\text{if } i = j, \\ \E T &\text{if } i \neq j. \end{cases} \end{equation*}

If $\hat{S}$ is as stated in Equation \ref{eqn:hajek}, then
\begin{align*} \E [\hat{S} \mid X_j] &= (n-1)\E T + \E[T \mid X_j] - (n-1)\E T = \E[T \mid X_j], \\ 
\E [(T - \hat{S})g_j(X_j)] &= \E [\E[T - \hat{S} \mid X_j] g_j(X_j)] \\ 
&= 0, \\ 
\E \left[ (T - \hat{S})\sum_{j=1}^n g_j(X_j) \right] &= 0.
\end{align*}
Thus, $\hat{S}$ must be the projection of $T$ onto $\mathcal{S}$.
\end{proof}

\paragraph{Next move:} Project the U-statistic $U_n$ onto the space $\left\{ \displaystyle\sum_{i=1}^n g_i(X_i): g_i \in L_2(P) \right\}$. We will show that $\var \hat{U}_n = \var U_n + O(n^{-2})$ so that $\displaystyle\frac{\var \hat{U}_n}{\var U_n} \rightarrow 1$, and then use it to show that $\hat{U}_n \cd \text{Normal}$.

\end{document}
