\documentclass{article}[12pt]

% useful packages
\usepackage{titlesec}
\usepackage{fullpage}
\usepackage{amsmath,amssymb,amsthm,amsfonts}
\usepackage{graphicx}
\usepackage{enumerate}
\usepackage{algorithm,algorithmic}
\usepackage{xcolor}
\usepackage{bbm}
\usepackage{url}
\usepackage{hyperref}
\usepackage{caption,subcaption}

% theorem type environments
\newtheorem{thm}{Theorem}
\newtheorem{prop}{Proposition}
\newtheorem{lemma}{Lemma}
\newtheorem{cor}{Corollary}
\newtheorem{defn}{Definition}
\newtheorem{assump}{Assumption}
\newtheorem{example}{Example}
\newtheorem{conjecture}{Conjecture}

% frequently used symbols
\newcommand{\bE}{\mathbb{E}}
\newcommand{\bP}{\mathbb{P}}
\newcommand{\bQ}{\mathbb{Q}}
\newcommand{\bR}{\mathbb{R}}
\newcommand{\bS}{\mathbb{S}}
\newcommand{\bN}{\mathbb{N}}
\newcommand{\bZ}{\mathbb{Z}}
\newcommand{\sC}{{\mathcal C}} 
\newcommand{\sD}{{\mathcal D}} 
\newcommand{\sE}{{\mathcal E}} 
\newcommand{\sF}{{\mathcal F}} 
\newcommand{\sL}{{\mathcal L}} 
\newcommand{\sH}{{\mathcal H}} 
\newcommand{\sN}{{\mathcal N}} 
\newcommand{\sO}{{\mathcal O}} 
\newcommand{\sP}{{\mathcal P}} 
\newcommand{\sR}{{\mathcal R}} 
\newcommand{\sS}{{\mathcal S}}
\newcommand{\sU}{{\mathcal U}} 
\newcommand{\sX}{{\mathcal X}} 
\newcommand{\sY}{{\mathcal Y}} 
\newcommand{\sZ}{{\mathcal Z}}

% operators
\newcommand{\sign}{\mathop{\mathrm{sign}}}
\newcommand{\supp}{\mathop{\mathrm{supp}}} % support
\newcommand{\argmin}{\operatornamewithlimits{arg\ min}}
\newcommand{\argmax}{\operatornamewithlimits{arg\ max}}
\newcommand{\dist}{\operatorname{dist}}
\newcommand{\tr}{\text{tr}}
\newcommand{\vecop}{\text{vec}}
\newcommand{\st}{\operatorname{s.t.}}
\newcommand{\cut}{\setminus}
\newcommand{\ind}[1]{\mathbbm{1}\left\{#1\right\}} 
\newcommand{\given}{\ | \ }

% grouping operators
\newcommand{\brac}[1]{\left[#1\right]}
\newcommand{\set}[1]{\left\{#1\right\}}
\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}
\newcommand{\paren}[1]{\left(#1\right)}
\newcommand{\norm}[1]{\left\|#1\right\|}
\newcommand{\ip}[2]{\left\langle #1,#2 \right\rangle}

% code commands
\newcommand{\matlab}{\textsc{Matlab }}
\newcommand{\python}{\textsc{Python }}

% header command
\newcommand{\homework}[3]{
    \pagestyle{myheadings}
    \thispagestyle{plain}
    \newpage
    \setcounter{page}{1}
    \setlength{\headsep}{10mm}
    \noindent
    \begin{center}
    \framebox{
        \vbox{\vspace{2mm}
            \hbox to 6.28in { {\bf EE 516: Mathematical Foundations of Machine Learning
            \hfill Winter \the\year} }
        \vspace{4mm}
        \hbox to 6.28in { {\Large \hfill Homework #1 \hfill} }
        \vspace{2mm}
        \hbox to 6.28in { \Large \hfill Due: #2, 11:59PM PT \hfill }
        \vspace{2mm}
        \hbox to 6.28in { {\it Student Name: #3} \hfill {\it Instructor Name: John Lipor}}
        \vspace{2mm}}
   }
   \end{center}
   \markboth{Homework #1}{Homework #1}
   \vspace*{4mm}
}

% For problem titles
\titleformat{\section}{\normalfont\bf}{Problem \thesection}{1em}{}

\begin{document}

% PUT YOUR NAME BELOW WHERE I HAVE \X
\homework{3}{February 3, \the\year}{\X}
% PUT YOUR NAME BELOW WHERE I HAVE \X

\section{\normalfont{(5 pts each)}}

Let $A \in \bR^{m \times n}$ and suppose $W \in \bR^{m \times m}$ and $Q \in \bR^{n \times n}$ are each orthogonal matrices.

\begin{enumerate}[(a)]
    \item Show that $A$ and $WAQ$ have the same singular values. Consequently, $A$ and $WAQ$ have the same rank, Frobenius norm, and operator norm. This is why the Frobenius and operator norms are called \emph{unitarily invariant}; their value does not change when the matrix is multiplied from the left and/or right by a unitary (or orthogonal) matrix. Any norm that depends on the singular values of $A$ will, by definition, be unitarily invariant.
    \item Suppose $W$ and $Q$ are nonsingular but not necessarily orthogonal matrices. Do $A$ and $WAQ$ still have the same rank? Prove or give a counterexample.
    \item In the setting of part (b), do $A$ and $WAQ$ have the same singular values? Prove or give a counterexample.
\end{enumerate}

\section{\normalfont{(5 pts each)}}

Let $A = \begin{bmatrix} 1 & 0 \\ 0 & 0 \end{bmatrix}$.

\begin{enumerate}[(a)]
    \item Determine the nullspace of $A$, denoted by $\sN(A)$, and the range or column space of $A$, denoted by $\sR(A)$.
    \item Are $\sN(A)$ and $\sR(A)$ equal? Is this true in general? If not, provide a counter-example.
\end{enumerate}

\section{\normalfont{(7 pts, 3 pts)}}

\begin{enumerate}[(a)]
    \item Determine how the eigenvalues of
        \begin{equation*}
            B = \begin{bmatrix} 0 & A \\ A^{T} & 0 \end{bmatrix}
        \end{equation*}
        are related to the singular values of $A \in \bR^{n \times n}$.
    \item Verify that the eigenvectors of $B$ are the normalized versions of
        \begin{equation*}
            \begin{bmatrix}
                u_{i} \\
                v_{i}
            \end{bmatrix}
            \vspace{5em}
            \text{and}
            \vspace{5em}
            \begin{bmatrix}
                u_{i} \\
                -v_{i}
            \end{bmatrix},
        \end{equation*}
\end{enumerate}
where $u_{i}$ and $v_{i}$ are arbitrary left and right (respectively) singular vectors of $A$.
\noindent \textbf{Hint:} To determine the eigenvalues, start off with some numerical experiments on self-generated $A$ matrices and form a conjecture. The commands \texttt{eig} and \texttt{svd} will be helpful. You should also write $A$ in terms of its SVD, i.e., $A = U\Sigma V^{T}$.

\section{\normalfont{(7 pts)}}

The \emph{polar factorization} of a matrix $A \in \bR^{n \times n}$ is given by $A = QS$, where $Q$ is orthogonal and $S = S^{T} \succeq 0$, i.e., $S$ is positive semi-definite. Express $Q$ and $S$ in terms of $U$, $\Sigma$, and $V$, where $A = U \Sigma V^{T}$.

\section{\normalfont{(3 pts each)}}

Let $A = xy^{T}$, where neither $x$ nor $y$ is 0.
\begin{enumerate}[(a)]
    \item How many linearly-independent columns does $A$ have? \textbf{Explain why!}
    \item What is the rank of $A$? \textbf{Explain why!}
    \item Generate $A$ in \matlab or \python using random (\texttt{randn}) $x, y \in \bR^{100}$. Plot the singular values of $A$ and include the plot here, as well as your code. The \texttt{stem} function works best for this.
\end{enumerate}

\section{\normalfont{(5 pts, 2 pts)}}

When $D$ is an $m \times n$ rectangular diagonal matrix, its pseudo-inverse $D^{+}$ is an $n \times m$ rectangular diagonal matrix whose nonzero entries are the reciprocals $1/d_{k}$ of the diagonal entries of $D$. Thus, a matrix $A$ having SVD $A = U \Sigma V^{T}$ has $A^{+} = V \Sigma^{+} U^{T} = V_{r} \Sigma_{r}^{-1} U_{r}^{T}$. \textbf{Working with just this definition}, determine by hand the pseudo-inverse of
\begin{enumerate}[(a)]
    \item $A = xy^{T}$, where neither $x$ nor $y$ is 0
    \item $A = xx^{T}$, where $x \neq 0$.
\end{enumerate}
You may again wish to from a conjecture using computational experiments.

\section{\normalfont{(7 pts)}}

Prove that the orthogonal complement of the range of a matrix $A$ equals the nullspace of $A^{T}$, i.e.,
\begin{equation*}
    \sR(A)^{\perp} = \sN(A^{T}).
\end{equation*}
\textbf{Hint:} A common strategy for proving that two sets $X, Y$ are equal is to prove that $X \subset Y$ and $Y \subset X$.

\section{\normalfont{(5 pts)}}

In this problem, you will implement another tool for use in the computer vision algorithm \emph{photometric stereo}, which allows us to reconstruct a 3D object's surface from 2D images of it under different lighting conditions.

Suppose we are in a dark room with an object on a dark table, a camera fixed above it, and a moveable light source. We model the object surface as $z = f(x,y)$, where $(x,y)$ denotes the coordinates on the table and $z$ is the height above the table. Assume that the $m \times n$ sized image $I$ is a representation of $f(x,y)$ for each $(x,y)$ tuple. (Given one light source, $f(x,y)$ is the $z$ coordinate of the position where a ray of light hits the surface at $(x,y,z)$.) The pixel
intensity $I(x,y)$ indicates how much light reflects off the surface $f(x,y)$. If our object is diffuse (also called matte or Lambertian), one can derive the relationship
\begin{equation}
    I(x,y) = \alpha(x,y)(\ell^{T}n(x,y)),
    \label{eq:intensityRelation}
\end{equation}
where $\ell \in \bR^{3}$ is a unit-norm vector describing the orientation of the incident light rays on the surface, $n(x,y) \in \bR^{3}$ is the unit-norm surface normal vector of $f$ at $(x,y,f(x,y))$, and $\alpha(x,y) > 0$ is a scaling constant called the surface albedo. Note that \eqref{eq:intensityRelation} is a \emph{scalar} value.

Now suppose that we take $d$ images $I_{1},\dots,I_{d}$ of our object, with lighting directions $\ell_{1},\dots,\ell_{d}$. For any $(x,y)$, we can stack \eqref{eq:intensityRelation} into an overdetermined system of equations
\begin{equation}
    \underbrace{\begin{bmatrix} I_{1}(x,y) \\ \vdots \\ I_{d}(x,y) \end{bmatrix}}_{=: I_{xy}} \approx \underbrace{\begin{bmatrix} \ell_{1} & \dots & \ell_{d} \end{bmatrix}^{T}}_{=: L^{T}} \underbrace{\left( \alpha(x,y) n(x,y) \right)}_{=: \rho(x,y)}.
    \label{eq:approxForm}
\end{equation}
We could solve \eqref{eq:approxForm} for $\rho(x,y)$ when $d = 3$, but in practice when there is noise or our assumptions do not hold exactly (they never do), a more robust approach is to take $d > 3$ images and solve the least-squares problem
\begin{equation}
    \rho(x,y) = \argmin_{\rho \in \bR^{3}} \norm{I_{xy} - L^{T}\rho}_{2}^{2}
    \label{eq:lsForm}
\end{equation}
and approximate the surface norm $n$ by
\begin{equation}
    n^{*}(x,y) = \frac{\rho(x,y)}{\norm{\rho(x,y)}}_{2}.
    \label{eq:nStar}
\end{equation}

\textbf{Your task} for this problem is to complete the function called \texttt{compute\_normals} (in files for download) that computes the unit-norm surface normal vectors for each pixel in a scene by solving \eqref{eq:lsForm} and \eqref{eq:nStar}. You may use the \texttt{$\backslash$} or \texttt{linalg.lstsq} command, where $A \backslash b$ computes the solution $x_{\text{LS}} = \argmin_{x} \norm{Ax - b}_{2}^{2}$. \textbf{Turn in} your code for the implemented function and the resulting images generated by the
\texttt{catdemo} script, which should match Fig.~\ref{fig:catFigs} below.

\textbf{Hint:} You can solve this for all $mn$ points simultaneously by vectorizing $I$.

% Below is a way to include figures in latex---you can use this if you call your figure `catFigs.eps'. This will not work until you have such a figure though.
\begin{figure}[h]
    \centering
    \includegraphics[width=0.8\textwidth]{./catFigs.eps}
    \caption{Output images from Problem 8.}
    \label{fig:catFigs}
\end{figure}

\section{Reflection \normalfont{(5 pts per problem)}}

In this problem, your task is to look back through Homeworks 1-2 and identify the \textbf{two} problems you found most difficult to solve and reflect on the problem solving strategies that got you through them. Complete the below tasks.

\begin{itemize}
    \item For each problem, identify the top 1-3 tools/definitions from the course material you used.
    \item For each problem, write down the problem solving advice you would give to a peer attempting to solve the problem at the same point in the course. \textbf{Do not} simply give away the answer---place yourself in the role of an instructor during office hours.
\end{itemize}


\end{document}
