\documentclass[10pt]{beamer}

\usetheme[progressbar=frametitle]{metropolis}
\usepackage{appendixnumberbeamer}

\usepackage{booktabs}
\usepackage[scale=2]{ccicons}

\usepackage{pgfplots}
\usepgfplotslibrary{dateplot}

\usepackage{xspace}
\newcommand{\themename}{\textbf{\textsc{metropolis}}\xspace}

\usepackage{amsmath}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\argmin}{argmin}


\title{Information Theory meets Machine Learning}
%\subtitle{}
% \date{\today}
\date{}
\author{Kedar Tatwawadi}
\institute{EE376a Course}
% \titlegraphic{\hfill\includegraphics[height=1.5cm]{logo.pdf}}

\begin{document}

\maketitle

\begin{frame}{Table of contents}
  \setbeamertemplate{section in toc}[sections numbered]
  \tableofcontents[hideallsubsections]
\end{frame}

\section{Introduction}

\begin{frame}[fragile]{ML and IT}
ML/Statistics \& Information theory are two sides of the same coin!
\begin{figure}
\includegraphics[width=\textwidth]{img1.png}
\caption{ML and IT}
\end{figure}
\end{frame}

\begin{frame}[fragile]{A short (very) intro to ML}

\begin{figure}
\includegraphics[width=\textwidth]{map1.png}
\caption{ML zoo}
\end{figure}
\end{frame}


\begin{frame}[fragile]{A short (very) intro to ML}
\begin{figure}
\includegraphics[width=\textwidth]{map2.png}
\caption{ML Zoo}
\end{figure}
\end{frame}


\begin{frame}[fragile]{A short (very) intro to ML}
\begin{figure}
\includegraphics[width=\textwidth]{unsup.png}
\caption{ML Zoo}
\end{figure}
\end{frame}



\begin{frame}[fragile]{Supervised Learning}
Given data tuples $(X_1,y_1), (X_2,y_2), \ldots, (X_N,y_N)$, find a function $F$ such that:
\pause
\begin{equation*}
F(X) = y
\end{equation*}
\pause
\begin{figure}
\includegraphics[width=\textwidth]{classificatn.png}
%\caption{}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Supervised Learning}
\begin{enumerate}
\item What is the function $F$? 
\item SVM, ConvNet, Recurrent Neural Network, Decision Tree ...
\end{enumerate}

\pause
{\color{blue} Take CS229, CS231n courses!}
\end{frame}


\begin{frame}[fragile]{A short (very) intro to ML}
\begin{figure}
\includegraphics[width=\textwidth]{map3.png}
\caption{ML Zoo}
\end{figure}
\end{frame}


\begin{frame}[fragile]{A short (very) intro to ML}
\begin{figure}
\includegraphics[width=\textwidth]{map4.png}
\caption{ML Zoo}
\end{figure}
\end{frame}

\section{Unsupervised Learning}
\begin{frame}[fragile]{Unsupervised Learning}
Given data: $X_1,X_2,X_3,\ldots, X_N$ \\
"Learn" something useful about $X$ 
\begin{enumerate}
\item Clustering
\item Data Representation
\item Distribution of the data
\end{enumerate}
\end{frame}

\begin{frame}[fragile]{Clustering}
\begin{figure}
\includegraphics[width=0.8\textwidth]{cluster.png}
\caption{Clustering}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Data Representation}
\begin{figure}
\includegraphics[width=\textwidth]{word2vec.png}
\caption{Word2Vec Representation}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Data Representation}
\begin{figure}
\includegraphics[width=\textwidth]{imgadd.png}
\caption{DCGAN Image Latent Representation}
\end{figure}
\end{frame}

\section{Learning Data Distribution}
\begin{frame}[fragile]{Learning the distribution}
”Learn” the underlying \textbf{Distribution of the data} \\
Given data: $X^{(1)}, X^{(2)}, \ldots, X^{(N)}$ with distribution $p_X(X)$, \textbf{How do we learn $p_X(X)$?}
\pause

\textbf{Use cases}
\begin{enumerate}
\item Sampling
\item Prediction
\item De-noising
\item Compression
\end{enumerate}
\end{frame}

\begin{frame}[fragile]{Sampling}
\begin{figure}
\includegraphics[width=\textwidth]{sampling.png}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Prediction}
\begin{figure}
\includegraphics[width=\textwidth]{prediction.png}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Denoising}
\begin{figure}
\includegraphics[width=\textwidth]{denoising.png}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Learning the distribution}
”Learn” the underlying \textbf{Distribution of the data}
\begin{enumerate}
\item Sampling
\item Prediction
\item De-noising
\item {\color{red} Compression}
\end{enumerate}
\end{frame}

\begin{frame}[fragile]{Learning the distribution}
Data: $X^{(1)}, X^{(2)}, \ldots, X^{(N)}$ i.i.d (independent and identically distributed) with distribution $p_X(X)$
\begin{itemize}
%\item $X^{(i)}$ are high dimentional \\
%$X^{(i)} = (Y_1^{(i)}, Y_2^{(i)},  \ldots, Y_d^{(i)})$
\item $X_i \in \mathcal{X}$
\item Potentially $|\mathcal{X}|$ can be high
\end{itemize}
\pause
\textbf{How do we learn $p_X(X)$?}
\begin{itemize}
\pause
\item We can use the Log-loss (Cross-entropy loss) to learn $p_X(X)$

\begin{equation}
p_X = \argmin_{q(X)} \mathrm{E}_{p_X} \log \frac{1}{q(X)}
\end{equation}
\end{itemize}
\end{frame}


\begin{frame}[fragile]{Learning the distribution}
Data: $X^{(1)}, X^{(2)}, \ldots, X^{(N)}$ with distribution $p_X(X)$
\begin{align*}
\mathrm{E}_{p_X} \log \frac{1}{q(X)} &= \sum_{x \in \mathcal{X}} p(x) \log \frac{1}{q(x)} \\
                                                              &=  \sum_{x \in \mathcal{X}} p(x) \log \frac{1}{p(x)} \frac{p(x)}{q(x)}\\
                                                              &= \sum_{x \in \mathcal{X}} p(x) \log \frac{1}{p(x)} +  \sum_{x \in \mathcal{X}} p(x) \log \frac{p(x)}{q(x)}\\
                                                              &= H_p(X) + D_{KL}(p_X||q) 
\end{align*}
\pause

\textbf{
\begin{equation*}
p_X = \argmin_{q(X)} \mathrm{E}_{p_X} \log \frac{1}{q(X)}
\end{equation*}
}
\end{frame}



\begin{frame}[fragile]{Learning the distribution}
\textbf{
\begin{equation*}
p_X = \argmin_{q(X)} \mathrm{E}_{p_X} \log \frac{1}{q(X)}
\end{equation*}
}

\begin{itemize}
\item In practice we consider empirical expectation instead:

\begin{align*}
\argmin_{q(X)} \mathrm{E}_{p_X} \log \frac{1}{q(X)} &\approx \argmin_{q(X)} \frac{1}{N} \sum_{i=1}^N \log \frac{1}{q(X^{(i)})} \\
\end{align*}
\end{itemize}

\end{frame}


\begin{frame}[fragile]{Learning the distribution}
\begin{itemize}
\item In practice we consider empirical expectation instead:

\begin{align*}
\argmin_{q(X)} \frac{1}{N} \sum_{i=1}^N \log \frac{1}{q(X^{(i)})} &= \argmin_{q(X)} \frac{1}{N}  \log \frac{1}{q(X_1)q(X_2)\ldots q(X_N)} \\
&= \argmin_{q(X)} \sum_{x\in \mathcal{X}} \frac{n_x}{N} \log \frac{1}{q(x)} \\
&= \argmin_{q(X)} \mathrm{E}_{\hat{p}_X}   \log \frac{1}{q(x)} 
\end{align*}
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Learning the distribution}
\begin{align*}
\argmin_{q(X)} \frac{1}{N} \sum_{i=1}^N \log \frac{1}{q(X^{(i)})}  &= \argmin_{q(X)} \mathrm{E}_{\hat{p}_X}   \log \frac{1}{q(x)} \\ 
&=  \hat{p}_X(x) = \frac{n_x}{N}
\end{align*}
\pause
\begin{itemize}
\item When $X = (Y_1,Y_2,\ldots,Y_d)$, $|\mathcal{X}| = k^d$
\item For high $|\mathcal{X}|$, $\hat{p}_X$ is not useful!  
\pause
\item We need more data, or ... some regularization.
\end{itemize}
\end{frame}


\begin{frame}[fragile]{Data Example}
\begin{figure}
\includegraphics[width=\textwidth]{table.png}
\end{figure}

\begin{itemize}
\item $X = (Y_1,Y_2,\ldots,Y_d)$, $|\mathcal{X}| = k^d$, $N \approx$ number of dimensions. 
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Regularization}
\begin{align*}
\argmin_{q(X)} \mathrm{E}_{p_X}   \log \frac{1}{q(x)}  &= \argmin_{q(X)} \mathrm{E}_{\hat{p}_X}   \log \frac{1}{q(x)} \\
&\approx  \argmin_{q(X) \in Q} \mathrm{E}_{\hat{p}_X}   \log \frac{1}{q(x)}
\end{align*}

\begin{itemize}
\item $q(X) = q(Y_1,Y_2,\ldots,Y_d) = q_1(Y_1)q_2(Y_2|Y_1)q_3(Y_3|Y_2,Y_1)\ldots q_d(Y_d|Y_1,\ldots,Y_{d-1})$
\item $Q$ restricts some distributions \\
e.g.: $q(Y_1,Y_2,\ldots,Y_d) = q_1(Y_1)q_2(Y_2)q_3(Y_3)\ldots q_d(Y_d)$
\end{itemize}

\end{frame}

\begin{frame}[fragile]{$Q_I$ independent distributions}

\begin{itemize}
\item $Q_I$ restricts the distribution over the $d$ dimensions to be independent \\
e.g.: $q(Y_1,Y_2,\ldots,Y_d) = q_1(Y_1)q_2(Y_2)q_3(Y_3)\ldots q_d(Y_d)$

\begin{align*}
\argmin_{q(X) \in Q_I} \mathrm{E}_{\hat{p}_X}   \log \frac{1}{q(x)} = (\hat{q}_1(y_1),\ldots,\hat{q}_d(y_d))
\end{align*}

\item $q(Y_1,Y_2,\ldots,Y_d) = \hat{q}_1(y_1)\hat{q}_2(y_2)\ldots \hat{q}_d(y_d)$ \\
is not very useful for the tabular dataset
\end{itemize}
\end{frame}



\begin{frame}[fragile]{Tabular Example}

\begin{figure}
\includegraphics[width=\textwidth]{table.png}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Tree-based distributions}
\begin{itemize}
\item  We restrict distributions to $\mathcal{T}$: \\
e.g.: $\mathcal{T} = \{ q| q(Y_1,Y_2,\ldots,Y_d) = q_1(Y_1)q_2(Y_2|Y_{i_2})q_3(Y_3|Y_{i_3}) \ldots q_d(Y_d|Y_{i_d}) \} $
\pause
\item For every $Y_i$, we allow dependence on one of the other variables $Y_{i_j}, i_j < i$
\pause
\item  This exactly corresponds to a "tree distribution"
\end{itemize}

\end{frame}






\begin{frame}[fragile]{Tree-based distributions | Examples}
\begin{itemize}
\item  Example tree distribution: \\
$q(Y_1,Y_2,\ldots,Y_5) = q_1(Y_1)q_2(Y_2|Y_1)q_3(Y_3|Y_1) q_4(Y_4|Y_2) q_5(Y_5|Y_2)$
\end{itemize}

\begin{figure}
\includegraphics[width=0.7\textwidth]{graph.png}
\caption{Graph example}
\end{figure}

\end{frame}



\begin{frame}[fragile]{Tree-based distributions | Examples}
\begin{itemize}
\item  Example tree distribution: \\
$q(Y_1,Y_2,Y_3) = q_1(Y_1)q_2(Y_2|Y_1)q_3(Y_3|Y_2)$
\
\end{itemize}

\begin{figure}
\includegraphics[width=\textwidth]{graph2.png}
\caption{Graph example}
\end{figure}

\end{frame}


\begin{frame}[fragile]{Tree-based distributions }

\begin{figure}
\includegraphics[width=0.5\textwidth]{graph.png}
\caption{Graph example}
\end{figure}

\begin{itemize}
%\item  Example tree distribution: \\
%$q(Y_1,Y_2,\ldots,Y_5) = q_1(Y_1)q_2(Y_2|Y_1)q_3(Y_3|Y_1) q_4(Y_4|Y_2) q_5(Y_5|Y_2)$
\item Tree distributions are practical! No of parameters $= dk^2$
\pause
\item Sampling is easy (in a breadth-first search order): \\
$Y_1 \rightarrow Y_2 \rightarrow Y_3 \rightarrow Y_4 \rightarrow Y_5$ 
\end{itemize}



\end{frame}





\begin{frame}[fragile]{Tree-based distributions }
\begin{itemize}
\item  Can be used for compression, using Arithmetic coding: 
\item $q(Y_1,Y_2,\ldots,Y_5) = q_1(Y_1)q_2(Y_2|Y_1)q_3(Y_3|Y_1) q_4(Y_4|Y_2) q_5(Y_5|Y_2)$
\end{itemize}

\begin{figure}
\includegraphics[width=\textwidth]{arithmetic.png}
\caption{HW3 Q3(f)}
\end{figure}

%\begin{figure}
%\includegraphics[width=0.5\textwidth]{graph.png}
%\caption{Graph example}
%\end{figure}

\end{frame}


\begin{frame}[fragile]{Chow-Liu Tree Algorithm }
\begin{itemize}
\item Let $\hat{I}(Y_i;Y_j)$ be the mutual information computed using the "empirical" distribution: $\hat{p}_X(X) = \hat{p}_X(Y_1,Y_2,\ldots,Y_d)$ 

The best tree graph representing the data can be found by:
\begin{equation}
G = \argmax \sum_{edges(i,j)} \hat{I}(Y_i;Y_j)
\end{equation}
\pause
\item {\color{red} Intuition:}: Add edges which have "high' correlation. 

\end{itemize}



%\begin{figure}
%\includegraphics[width=0.5\textwidth]{graph.png}
%\caption{Graph example}
%\end{figure}

\end{frame}




\begin{frame}[fragile]{Data Example}
\begin{figure}
\includegraphics[width=\textwidth]{table.png}
\end{figure}

\begin{itemize}
\item $X = (Y_1,Y_2,\ldots,Y_d)$, $|\mathcal{X}| = k^d$, $N \approx$ number of dimensions. 
\end{itemize}
\end{frame}

\begin{frame}[fragile]{Data Example}
\begin{figure}
\includegraphics[width=0.49\textwidth]{it.png}
\includegraphics[width=0.49\textwidth]{mst.png}
\end{figure}
\end{frame}


\begin{frame}[fragile]{Chow-Liu Tree Algorithm }
\begin{itemize}
\item Let $\hat{I}(Y_i;Y_j)$ be the mutual information computed using the "empirical" distribution: $\hat{p}_X(X) = \hat{p}_X(Y_1,Y_2,\ldots,Y_d)$ 

The best tree graph representing the data can be found by:
\begin{equation}
G = \argmax \sum_{edges(i,j)} \hat{I}(Y_i;Y_j)
\end{equation}
\item {\color{red} Intuition:}: Add edges which have "high' correlation. 
\item {\color{red} We will proove this in the class!}

\end{itemize}
\end{frame}



\begin{frame}[fragile]{Practical Considerations}
\begin{figure}
\includegraphics[width=0.4\textwidth]{it.png}
\includegraphics[width=0.4\textwidth]{mst.png}
\end{figure}

\begin{itemize}
\item Exaustive search over all trees is not possible, use $O(d\log d)$ algorithm such as Kruskal's or Prim's algorithm
\pause
\item Need to compute $O(d^2)$ mutual informations, which is the more costly part
\end{itemize}
\end{frame}


\begin{frame}[fragile]{Practical Considerations}

\begin{itemize}

\item 
 \begin{equation}
G = \argmax \sum_{edges(i,j)} \hat{I}(Y_i;Y_j)
\end{equation}
 $G$ is a solution to the problem: 
 \begin{equation*}
 \argmin_{q(X)} \mathrm{E}_{\hat{p}_X}   \log \frac{1}{q(x)} \approx \argmin_{q(X)} \mathrm{E}_{p_X}   \log \frac{1}{q(x)}
 \end{equation*}
 
 and thus "approximates" $\hat{p}_X(X)$. 
 
 \pause
 \item We really want to solve the problem: 
\begin{equation*}
\argmax \sum_{edges(i,j)} I(Y_i;Y_j)
\end{equation*}

\pause
\item Using $N$ samples we can have better estimators for $I(Y_i,Y_j)$ than the empirical plug-in estimator $\hat{I}(Y_i,Y_j)$ 

\item Information theory helps us get better estimators!

\end{itemize}
\end{frame}


\begin{frame}[fragile]{Practical Considerations}

\begin{itemize}

\item Using $N$ samples we can have better estimators for $I(Y_i,Y_j)$ than the empirical plug-in estimator $\hat{I}(Y_i,Y_j)$ 

\item Information theory helps us get better estimators!

\end{itemize}
\begin{figure}
\includegraphics[width=0.8\textwidth]{JVHW}
\end{figure}

\end{frame}


\begin{frame}[fragile]{Practical Considerations}

\begin{itemize}

\item Using $N$ samples we can have better estimators for $I(Y_i,Y_j)$ than the empirical plug-in estimator $\hat{I}(Y_i,Y_j)$ 

\item Information theory helps us get better estimators!

\end{itemize}
\begin{figure}
\includegraphics[width=0.8\textwidth]{orig}
\end{figure}

\end{frame}





\begin{frame}[fragile]{Practical Considerations}

\begin{itemize}

\item Using $N$ samples we can have better estimators for $I(Y_i,Y_j)$ than the empirical plug-in estimator $\hat{I}(Y_i,Y_j)$ 

\item Information theory helps us get better estimators!

\end{itemize}
\begin{figure}
\includegraphics[width=0.8\textwidth]{new}
\end{figure}

\end{frame}





\begin{frame}[fragile]{Practical Considerations}

\begin{itemize}

\item Using $N$ samples we can have better estimators for $I(Y_i,Y_j)$ than the empirical plug-in estimator $\hat{I}(Y_i,Y_j)$ 

\item Information theory helps us get better estimators!

\end{itemize}
\begin{figure}
\includegraphics[width=0.8\textwidth]{large}
\end{figure}
\end{frame}


\begin{frame}[fragile]{Practical Considerations}
\begin{itemize}
\item When we solve the optimization problem, we are not penalizing model complexity: 
 \begin{equation}
G = \argmax \sum_{edges(i,j)} \hat{I}(Y_i;Y_j)
\end{equation}
 $G$ is a solution to the problem: 
 \begin{equation*}
 \end{equation*}
\item Practically this is important. For example in compression, we also need space to store the distributions $\hat{p}(Y_i|J_j)$ themselves! (along with arithmetic coding). 
 \pause
 \item The \textbf{BIC Criteria} (Bayesian Information Criteria), alters the optimization by adding a penalty function for model complexity
\begin{equation*}
\argmax \sum_{edges(i,j)} \left(  \hat{I}(Y_i;Y_j) + \frac{1}{2} \log N |\mathcal{Y}_i| |\mathcal{Y}_j| \right) 
\end{equation*}

\end{itemize}
\end{frame}

\begin{frame}[fragile]{General Bayesian Networks}
\begin{figure}
\includegraphics[width=0.7\textwidth]{grade-model}
\end{figure}
\end{frame}




\begin{frame}[fragile]{Chow-Liu Algorithm for Bayesian Networks}
\begin{itemize}
\item Let $\hat{I}(Y_i;Y_j)$ be the mutual information computed using the "empirical" distributions. 
\end{itemize}

For general bayesian networks: 
\begin{equation}
G = \argmax \sum_{edges(i,j)} \hat{I}(Y_i;Y_j)
\end{equation}

\begin{itemize}
\item {\color{blue} Chow-liu algorithm for Bayesian networks is an approximation based on the intuition for tree-based algorithms}
\item Exact solutions no more possible. Apply heuristic greedy schemes 
\end{itemize}
\end{frame}



\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{1nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}


\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{2nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{3nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{4nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{5nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{6nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{7nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{8nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}


\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{9nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}


\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{10nlp}
\caption{Slides borrowed from CS224n lecture, Jan 22}
\end{figure}
\end{frame}

\begin{frame}[fragile]{Learning Distributions | Language model}
\begin{figure}
\includegraphics[width=0.9\textwidth]{11nlp}
\includegraphics[width=0.9\textwidth]{11anlp}
\caption{DeepZip Language compressor based on Language models}
\end{figure}
\end{frame}


\section{ML for Lossy Compression}

\begin{frame}{Representation learning for Compression}
\begin{itemize}
\item More recently, ML is being used for lossy compression of data
\item \textbf{Why use ML?}
\begin{enumerate}
\item Unclear, high-dimensional data models. Eg: natural images. 
\pause
\item Unclear Loss functions for lossy compression
\item EG: Image Compression $\rightarrow$ "Human perception loss"
\end{enumerate} 
\end{itemize}
\end{frame}


\begin{frame}[fragile]{ML for Compressionl}
\begin{figure}
\includegraphics[width=0.9\textwidth]{waveone1}
\caption{Waveone image compression using neural networks}
\end{figure}
\end{frame}

\begin{frame}[fragile]{ML for Compressionl}
\begin{figure}
\includegraphics[width=0.9\textwidth]{waveone2}
\caption{Waveone image compression using neural networks}
\end{figure}
\end{frame}


\begin{frame}[fragile]{ICLR Compression Contest}
\begin{figure}
\includegraphics[width=0.9\textwidth]{clic}
\caption{More details: https://www.compression.cc}
\end{figure}
\end{frame}


\begin{frame}[fragile]{ML for Compression}
\begin{itemize}
\item Lots of very different types of techniques used. The core idea is: \\
\begin{enumerate}
\item Learn a "smooth" representation for the Image
\item Quantize the representation
\item Entropy coding of the representation
\end{enumerate}
\pause
\item The "smoothness" of the representation is the key to good lossy compression. 
\item  Different techniques used for compression: Autoencoders, VAE, GANs 
\end{itemize}
\end{frame}



\begin{frame}[fragile]{ML for Compression}
Some notable papers:
\begin{itemize}
\item Toderici, George, et al. "Full resolution image compression with recurrent neural networks." Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2017.
\end{itemize}

\begin{figure}
\includegraphics[width=0.45\textwidth]{g1}
\includegraphics[width=0.45\textwidth]{g2}
\caption{Compression over multiple iterations}
\end{figure}

\end{frame}



\begin{frame}[fragile]{ML for Compression}
Some notable papers:
\begin{itemize}
\item Rippel et.al, "Real-time adaptive Image compression" ICML'17 Proceedings 
\end{itemize}

\begin{figure}
\includegraphics[width=0.9\textwidth]{waveone}
\caption{Using a GAN based loss}
\end{figure}
\end{frame}


\begin{frame}[fragile]{ML for Channel Coding}
\begin{figure}
\includegraphics[width=0.9\textwidth]{kim1}
\caption{Using a RNN for channel coding}
\end{figure}
\end{frame}


\begin{frame}[fragile]{ML for Channel Coding}

\begin{figure}
\includegraphics[width=0.9\textwidth]{kim2}
\caption{Using a RNN for channel coding}
\end{figure}
\end{frame}


\begin{frame}[fragile]{ML for Channel Coding}

\begin{figure}
\includegraphics[width=0.9\textwidth]{kim3}
\caption{Using a RNN for channel coding}
\end{figure}

\end{frame}




\begin{frame}[fragile]{ML for Joint Source-Channel Coding}

\begin{figure}
\includegraphics[width=0.9\textwidth]{choi}
\caption{NECST: Neural Joint Source-Channel Code, Choi et.al. arxiv}
\end{figure}

\end{frame}


\begin{frame}[fragile]{Conclusion?}
ML/Statistics \& Information theory are two sides of the same coin!
\begin{figure}
\includegraphics[width=\textwidth]{img1.png}
\caption{ML and IT}
\end{figure}
\end{frame}


%{\color{red} Take CS228 course if interested!}



%\begin{itemize}
%\item $X = (Y_1,Y_2,\ldots,Y_d)$, $|\mathcal{X}| = k^d$, $N \approx$ number %of dimensions. 
%\end{itemize}



{\setbeamercolor{palette primary}{fg=black, bg=yellow}
\begin{frame}[standout]
  Thank You!
\end{frame}
}

\appendix


\end{document}
