\documentclass{article}[12pt]

% useful packages
\usepackage{titlesec}
\usepackage{fullpage}
\usepackage{amsmath,amssymb,amsthm,amsfonts}
\usepackage{graphicx}
\usepackage{enumerate}
\usepackage{algorithm,algorithmic}
\usepackage{xcolor}
\usepackage{bbm}
\usepackage{url}
\usepackage{hyperref}

% theorem type environments
\newtheorem{thm}{Theorem}
\newtheorem{prop}{Proposition}
\newtheorem{lemma}{Lemma}
\newtheorem{cor}{Corollary}
\newtheorem{defn}{Definition}
\newtheorem{assump}{Assumption}
\newtheorem{example}{Example}
\newtheorem{conjecture}{Conjecture}

% frequently used symbols
\newcommand{\bE}{\mathbb{E}}
\newcommand{\bP}{\mathbb{P}}
\newcommand{\bQ}{\mathbb{Q}}
\newcommand{\bR}{\mathbb{R}}
\newcommand{\bS}{\mathbb{S}}
\newcommand{\bN}{\mathbb{N}}
\newcommand{\bZ}{\mathbb{Z}}
\newcommand{\sC}{{\mathcal C}} 
\newcommand{\sD}{{\mathcal D}} 
\newcommand{\sE}{{\mathcal E}} 
\newcommand{\sF}{{\mathcal F}} 
\newcommand{\sL}{{\mathcal L}} 
\newcommand{\sH}{{\mathcal H}} 
\newcommand{\sN}{{\mathcal N}} 
\newcommand{\sO}{{\mathcal O}} 
\newcommand{\sP}{{\mathcal P}} 
\newcommand{\sR}{{\mathcal R}} 
\newcommand{\sS}{{\mathcal S}}
\newcommand{\sU}{{\mathcal U}} 
\newcommand{\sX}{{\mathcal X}} 
\newcommand{\sY}{{\mathcal Y}} 
\newcommand{\sZ}{{\mathcal Z}}

% operators
\newcommand{\sign}{\mathop{\mathrm{sign}}}
\newcommand{\supp}{\mathop{\mathrm{supp}}} % support
\newcommand{\argmin}{\operatornamewithlimits{arg\ min}}
\newcommand{\argmax}{\operatornamewithlimits{arg\ max}}
\newcommand{\dist}{\operatorname{dist}}
\newcommand{\tr}{\text{tr}}
\newcommand{\vecop}{\text{vec}}
\newcommand{\st}{\operatorname{s.t.}}
\newcommand{\cut}{\setminus}
\newcommand{\ind}[1]{\mathbbm{1}\left\{#1\right\}} 
\newcommand{\given}{\ | \ }

% grouping operators
\newcommand{\brac}[1]{\left[#1\right]}
\newcommand{\set}[1]{\left\{#1\right\}}
\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}
\newcommand{\paren}[1]{\left(#1\right)}
\newcommand{\norm}[1]{\left\|#1\right\|}
\newcommand{\ip}[2]{\left\langle #1,#2 \right\rangle}

% code commands
\newcommand{\matlab}{\textsc{Matlab }}
\newcommand{\python}{\textsc{Python }}

% header command
\newcommand{\homework}[3]{
    \pagestyle{myheadings}
    \thispagestyle{plain}
    \newpage
    \setcounter{page}{1}
    \setlength{\headsep}{10mm}
    \noindent
    \begin{center}
    \framebox{
        \vbox{\vspace{2mm}
            \hbox to 6.28in { {\bf EE 516: Mathematical Foundations of Machine Learning
            \hfill Winter \the\year} }
        \vspace{4mm}
        \hbox to 6.28in { {\Large \hfill Homework #1 \hfill} }
        \vspace{2mm}
        \hbox to 6.28in { \Large \hfill Due: #2, 11:59PM PT \hfill }
        \vspace{2mm}
        \hbox to 6.28in { {\it Student Name: #3} \hfill {\it Instructor Name: John Lipor}}
        \vspace{2mm}}
   }
   \end{center}
   \markboth{Homework #1}{Homework #1}
   \vspace*{4mm}
}

% For problem titles
\titleformat{\section}{\normalfont\bf}{Problem \thesection}{1em}{}

\begin{document}

% PUT YOUR NAME BELOW WHERE I HAVE \X
\homework{1}{January 20, \the\year}{\X}
% PUT YOUR NAME BELOW WHERE I HAVE \X

\noindent \textbf{Note:} For all questions asked in this course, unless the question is simply asking about definitions, please justify all answers. Stating the correct answer without justification will earn no credit. For example, if you write down a string of equations, each successive equation should be accompanied by some justification such as ``by orthogonality'' or ``which is a property of the determinant.'' \textbf{Include} your code whenever you are asked to write some. This is
easily done by attaching the pdf using the \texttt{pdfpages} package (see link on course website).

\section{\normalfont{(5 pts, 5 pts, 1 pt)}}

Let $A$ be an $m \times n$ matrix defined such that $A_{i,j} = i$.

\begin{enumerate}[(a)]
    \item Express $A$ mathematically as the outer product of two appropriately-defined vectors.
    \item Write (on paper) a one-line \matlab or \python expression for expressing $A$ when $m = 3$ and $n = 2$.
    \item Test your expression and include your code and the output.
\end{enumerate}

\section{\normalfont{(10 pts)}}

Using the property that $\det(A) = \det(A^{T})$ and that $A, B \in \bR^{n \times n} \implies \det(AB) = \det(A)\det(B)$, solve the following problem:
\\ \\
If $A$ is orthogonal, what possible values are there for $\det(A)$?

\section{\normalfont{(10 pts, 10 pts, 5 pts, 5 pts)}}

\begin{enumerate}[(a)]
    \item For $x, y \in \bR^{n}$, prove that $\det(I - xy^{T}) = 1 - y^{T}x$. Use the properties below, and \textbf{do not} use Sylvester's determinant identity/the matrix determinant lemma.
    \item Express $\det(\lambda I - xy^{T})$ in terms of $\lambda$, $y^{T}x$, and $n$.
    \item What are the solutions of the equation $\det(\lambda I - xy^{T}) = 0$? Note that these are exactly the eigenvalues of the matrix $xy^{T}$.
    \item When are the eigenvalues of the matrix $xy^{T}$ all equal to 0 even when the matrix is not equal to zero?
\end{enumerate}

Hint: Use the following properties.
\begin{itemize}
    \item If $A, B \in \bR^{n \times n}$ then $\det(AB) = \det(A)\det(B)$.
    \item If $A \in \bR^{n \times n}$ is invertible then $\det(A^{-1}) = \frac{1}{\det(A)}$.
    \item If $A \in \bR^{n \times n}$ is invertible and $D \in \bR^{m \times m}$, then $\det \begin{bmatrix} A & B \\ C & D \end{bmatrix} = \det(A)\det(D - CA^{-1}B)$.
    \item If $A \in \bR^{n \times n}$ and $D \in \bR^{m \times m}$ is invertible, then $\det \begin{bmatrix} A & B \\ C & D \end{bmatrix} = \det(D)\det(A - BD^{-1}C)$.
\end{itemize}

\section{\normalfont{(5 pts, 10 pts, 7 pts, 3 pts)}}

For $A \in \bR^{n \times n}$, the \emph{trace} of $A$, denoted by $\tr(A)$, is defined as the sum of its diagonal elements: $\tr(A) = \sum_{i = 1}^{n} a_{ii}$.

\begin{enumerate}[(a)]
    \item Show that the trace is a linear function, i.e., if $A, B \in \bR^{n \times n}$ and $\alpha, \beta \in \bR$, then $\tr(\alpha A + \beta B) = \alpha \tr(A) + \beta \tr(B)$.
    \item Show that $\tr(AB) = \tr(BA)$ when both $AB$ and $BA$ are square, even though in general $AB \neq BA$.
    \item Let $S \in \bR^{n \times n}$ be \emph{skew-symmetric}, i.e., $S^{T} = -S$. Show that $\tr(S) = 0$. 
    \item Either prove the converse of part (c)---that all matrices with trace zero are skew symmetric---or provide a counterexample.
\end{enumerate}

\section{\normalfont{(7 pts)}}

Let $U_{1}, U_{2}, \dots, U_{k} \in \bR^{n \times n}$ be unitary matrices. Show that the product $U_{1}U_{2}\dots U_{k}$ is a unitary matrix (formal proof not required---simply state your reasoning).

\section{\normalfont{(3 pts, 3 pts, 1 pt)}}

\begin{enumerate}[(a)]
    \item Determine the eigenvalues (show your hand calculations) of $A = \begin{bmatrix} 6 & 16 \\ -1 & -4 \end{bmatrix}$.
    \item For the matrix $A$ above, compute the determinant and trace of $A$. Compute the product of the eigenvalues of $A$ and the sum of the eigenvalues. How do these compare to the determinant and trace?
    \item Check your answers using the \texttt{eig} or \texttt{linalg.eig} command and include your output result.
\end{enumerate}

\section{\normalfont{(5 pts)}}

Rewrite the expression
\begin{equation*}
    y = \sum_{i = 1}^{n} \sum_{j = 1}^{n} x_{i} A_{ij} x_{j}
\end{equation*}
as a matrix-vector operation in terms of the $n \times n$ matrix $A$ and the $n \times 1$ vector $x$ whose $i$th entry is $x_{i}$. Check your answer in \matlab or \python by trying some random examples (no need to include code or output).

\section{\normalfont{(2 pts, 5 pts, 2 pts)}}

Discrete time convolution (a DSP term) of an input signal $x[n]$ and a filter $h[n]$ is defined in textbooks as:
\begin{equation*}
    y = h * x = x * h \implies y[m] = \sum_{k = -\infty}^{\infty} h[k] x[m-k] = \sum_{n = -\infty}^{\infty} x[n] h[m-n],
\end{equation*}
where $*$ denotes convolution, not ordinary multiplication. In practice, we often use finite-length signals and filters, and not all textbooks define this case clearly. The convolution of an input signal $(x[0],\dots,x[N-1])$ with a finite impulse response (FIR) filter $(h[0],\dots,h[K-1])$ becomes:
\begin{equation*}
    y[m] = \sum_{k = 0}^{\min(K-1,m)} h[k] x[m-k] = \sum_{n = 0}^{\min(N-1,m)} x[n] h[m-n], \qquad m = 0,\dots,M-1.
\end{equation*}
This is exactly the type of convolution used in ``convolutional neural networks'' that are a hot topic in machine learning.

\begin{enumerate}[(a)]
    \item Determine $M$, the length of the (possibly) non-zero part of $y$, in terms of $N$ and $K$.
    \item We will use the notation $y, h, x$ to denote the finite vectors of length $M, K$ and $N$, respectively. Determine (write down) the matrix $H$ such that $y = h * x = Hx$.
    \item Implement a function called \texttt{convolution} that takes $h, x$ as inputs and outputs $y = h * x$ as a matrix operation (WITHOUT calling \texttt{conv} or a similar function). Compare the output of your function to the built-in \texttt{conv} or \texttt{np.convolve} function. Turn in your code demonstrating that your code works (or doesn't).
\end{enumerate}

\section{\normalfont{(5 pts)}}

The Kronecker product of two matrices $A \in \bR^{m_{1} \times n_{1}}, B \in \bR^{m_{2} \times n_{2}}$ is defined as
\begin{equation*}
    A \otimes B = \begin{bmatrix} a_{11}B & a_{12}B & \dots & a_{1n_{1}}B \\
                    & & \ddots \\
                    a_{m_{1}1}B & a_{m_{1}2}B & \dots & a_{m_{1}n_{1}}B
                \end{bmatrix} \in \bR^{(m_{1}m_{2}) \times (n_{1}n_{2})}.
\end{equation*}
Let $\vecop(C)$ be the ``vectorize'' operation that returns as its output a $mn \times 1$ vector formed by stacking the $n$ columns of $C \in \bR^{m \times n}$ on top of each other.

For arbitrary vectors $x \in \bR^{m}$ and $y \in \bR^{n}$, prove that
\begin{equation*}
    \vecop(xy^{T}) = y \otimes x \in \bR^{mn}.
\end{equation*}
Verify in \matlab using $\vecop(C) = \texttt{C(:)}$ or \python using \texttt{reshape} or \texttt{flatten}.

\section{\normalfont{(5 pts)}}

\textbf{Note:} The purpose of this problem is to help you gauge how much you learn throughout this course and to provide motivation for the time we will spend on heavily theoretical topics. You are not expected to understand the below paper in great detail. You may benefit from guides on reading a research paper, such as the ones \href{http://blizzard.cs.uwaterloo.ca/keshav/home/Papers/data/07/paper-reading.pdf}{here} and \href{https://github.com/jtleek/readingpapers}{here}.

\vspace{1em}
\noindent Spend at most one hour reading the paper, ``K-SVD: An Algorithm for Designing Overcomplete Dictionaries for Sparse Representation'' by Aharon, Elad, and Bruckstein. Comment on your ability to understand the following:
\begin{itemize}
    \item the problem the authors are trying to solve
    \item why this problem is interesting or important
    \item the mathematical formulation of the problem
    \item the mathematical/algorithmic solution to the problem
    \item why the mathematical/algorithmic solution works.
\end{itemize}
\textbf{Do not} summarize the problem, why it is important, etc. Instead, comment on \textit{your ability to understand these things} given your current background. The goal of this course is to provide you with the tools needed to understand papers like the one above in enough detail to implement them and explain them to others.

\end{document}
