%------------------------------------------------------------
% Tony Hyun Kim
% CS229: PS#1, Question 2d
%------------------------------------------------------------
clear all; close all;

fSize = 16;

% Load and display data
X = load('q2x.dat');
Y = load('q2y.dat');
plot(X,Y,'.'); hold on;
set(gca,'FontSize',fSize);
xlabel('x');
ylabel('y');
title('PS#1, Q2, Part d');

% Numerical parameters for regression
N = 100;
x = linspace(-5,12,N);
xlim([x(1) x(end)]);
x = [ones(N,1) x']; % Append x0=1

% Linear (unweighted) regression
m = length(X);
X = [ones(m,1) X];
A = X'*X;
theta_unweighted = A\X'*Y;
h_unweighted     = x*theta_unweighted;
plot(x(:,2)',h_unweighted');

% Weighted regression
%   Basically, we have to solve a different regression problem
%   for each point of x (since the weights are functions of x)
tau = 0.8;
h_weighted = zeros(size(h_unweighted));
for i = 1:N
    xi = x(i,:); % This is the point we are currently solving for
    e  = X-kron(xi,ones(m,1)); % Offset of training set sample from the
                               %    the point xi of evaluation
    w  = exp(-sum(e.^2,2)/(2*tau^2));
    W  = diag(w);
    
    A = X'*W*X;
    theta_weighted = A\X'*W*Y;
    h_weighted(i) = xi*theta_weighted;
end
plot(x(:,2)',h_weighted','r');
legend('Training set','Linear (unweighted)','Weighted (\tau=0.8)',...
       'Location','Best');