\documentclass[11pt]{article}
\usepackage{geometry}                % See geometry.pdf to learn the layout options. There are lots.
\geometry{letterpaper}                   % ... or a4paper or a5paper or ... 
%\geometry{landscape}                % Activate for for rotated page geometry
\usepackage[parfill]{parskip}    % Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}
%\usepackage{amssymb}
\usepackage{amsmath}    % need for subequations
\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}
%\usepackage{hyperref}
\usepackage[ae,hyper]{Rd}

%\usepackage{amssymb}
\usepackage{epstopdf}
\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}
\title{Appendix--2: More on Matrices}
\author{\href{http://personality-project.org/revelle.html}{William Revelle}\\
Northwestern University}
\date{}                                           % Activate to display a given date or no date

\begin{document}
\maketitle
\section{Multiple regression as a system of simultaneous equations}
Many problems in data analysis require solving a system of simultaneous equations. For instance, in multiple regression with two predictors and one criterion with a set of correlations of: 


\begin{equation}
 \left\{ 
 \begin{array}{rrr}
 r_{x1x1} &   r_{x1x2} &   r_{x1y}\\
 r_{x1x2} &   r_{x2x2}  &    r_{x2y}\\
 r_{x1y}   & r_{x2y}  &    r_{yy}
       \end{array} 
   \right\}
\end{equation}
we want to find the find weights, $\beta_i$,  that when multiplied by $x_1$ and $x_2$ maximize the correlations with y.  That is, we want to  solve the two simultaneous equations
\begin{equation}
\label{eq:2}
 \left\{ \begin{array}{rl}
 r_{x1x1}\beta_1  +     r_{x1x2} \beta_2  = r_{x1y} \\
 r_{x1x2} \beta_1  +  r_{x2x2} \beta_2  = r_{x2y} \\
       \end{array}  \right\}.
\end{equation}

We can directly solve these two equations by adding and subtracting terms to the two such that we end up with a solution to the first in terms of $\beta_1$ and to the second in terms of $\beta_2$:

\begin{equation}
\nonumber
 \left\{ \begin{array}{rl}
\beta_1  +     r_{x1x2} \beta_2/ r_{x1x1}  = r_{x1y}/ r_{x1x1} \\
 r_{x1x2} \beta_1/   r_{x2x2} +  \beta_2  = r_{x2y} /  r_{x2x2}\\
       \end{array}  \right\}
\end{equation}
%
which becomes
\begin{equation}
\label{eq:4}
 \left\{ \begin{array}{rl}
\beta_1  =      (r_{x1y}  -   r_{x1x2} \beta_2)/ r_{x1x1}\\
 \beta_2  = (r_{x2y} -  r_{x1x2} \beta_1)/  r_{x2x2}  \\
       \end{array}  \right\}
\end{equation}
%
Substituting the second row of \eqref{eq:4} into the first row, and vice versa we find
\begin{equation}
\nonumber
 \left\{ \begin{array}{rl}
\beta_1  =      (r_{x1y}  -   r_{x1x2} (r_{x2y} -  r_{x1x2} \beta_1)/  r_{x2x2} )/ r_{x1x1}\\
 \beta_2  = (r_{x2y} -  r_{x1x2} (r_{x1y}  -   r_{x1x2} \beta_2)/ r_{x1x1})/  r_{x2x2}  \\
       \end{array}  \right\}
\end{equation}
%
Collecting terms, we find:
\begin{equation}
\nonumber
 \left\{ \begin{array}{rl}
\beta_1 r_{x1x1} r_{x2x2}  =      (r_{x1y}  r_{x2x2}  -   r_{x1x2} (r_{x2y} -  r_{x1x2} \beta_1)  )\\
 \beta_2  r_{x2x2}r_{x1x1}  = (r_{x2y} r_{x1x1} -  r_{x1x2} (r_{x1y}  -   r_{x1x2} \beta_2)  \\
       \end{array}  \right\}
\end{equation}

and rearranging once again:
\begin{equation}
\nonumber
 \left\{ \begin{array}{rl}
\beta_1 r_{x1x1} r_{x2x2} -  r_{x1x2}^2 \beta_1 =      (r_{x1y}  r_{x2x2 }-   r_{x1x2} (r_{x2y}   )\\
 \beta_2 r_{x1x1}  r_{x2x2}-   r_{x1x2}^2 \beta_2 = (r_{x2y} r_{x1x1} -  r_{x1x2} (r_{x1y}    \\
       \end{array}  \right\}
\end{equation}

Struggling on:

\begin{equation}
\nonumber
 \left\{ 
 \begin{array}{rl}
\beta_1( r_{x1x1} r_{x2x2} -  r_{x1x2}^2)  =      r_{x1y} r_{x2x2 }  -   r_{x1x2} r_{x2y}   \\
 \beta_2( r_{x1x1}  r_{x2x2} -   r_{x1x2}^2 ) = r_{x2y} r_{x1x1} -  r_{x1x2} r_{x1y}    \\
 \end{array} \right\}
\end{equation}


 
 And finally:
 \begin{equation}
\nonumber
 \left\{ \begin{array}{rl}
\beta_1 =     ( r_{x1y} r_{x2x2 }  -   r_{x1x2} r_{x2y} )/ (r_{x1x1} r_{x2x2} -  r_{x1x2}^2)   \\
 \beta_2 = (r_{x2y} r_{x1x1} -  r_{x1x2} r_{x1y} )/( r_{x1x1}  r_{x2x2}-   r_{x1x2}^2 )   \\       \end{array} 
 \right\}
\end{equation}


\section{Matrix representation of simultaneous equation}
Alternatively, these two equations  \eqref{eq:2} may be represented as the product of a vector of unknowns (the $\beta$s ) and  a matrix of coefficients of the predictors (the $r_{xi}$'s) and a matrix of coefficients for the criterion ($rx_iy$):
\footnote{See \href{personality-project.org/r/sem.appendix.1.pdf}{Appendix -1} for a detailed discussion of how this is done in practice with some ``real" data using the statistical program, R. In R, the inverse of a square matrix, X, is found by the solve function: X.inv <- solve(X)}

\begin{equation}
\label{matreg}
\left( \beta_1       \beta_2   \right)
\left(\begin{array}{rrr} 
 r_{x1x1}  &    r_{x1x2}   \\
r_{x1x2}  &   r_{x2x2}   \\      
	 \end{array} 
   \right)
   = \left(r_{x1y} \\
      \quad  r_{x2x2} \right)
\end{equation}\\

 If we let $\beta =  \left( \beta_1       \beta_2   \right)$,  R =  $\left(\begin{array}{rrr} 
 r_{x1x1}  &    r_{x1x2}   \\
r_{x1x2}  &   r_{x2x2}   \\      
	 \end{array} 
   \right)
$
and $ r_{xy} = \left( r_{x1y} \\
      \quad  r_{x2x2} \right)$ then equation \eqref{matreg} becomes
\begin{equation}
\label{bry}
\beta R = r_{xy}
\end{equation}
and we can solve \eqref{bry} for $\beta$ by multiplying both sides by the inverse of R.
\begin{equation}
\nonumber
\beta = \beta R R^{-1} = r_{xy} R^{-1}
\end{equation}
%
%
%
\subsection{Finding the inverse of a 2 x 2 matrix}
But, how do we find the inverse ($R^{-1}$)?  As an example we solve the inverse of a 2 x2 matrix, but the technique may be applied to a matrix of any size.  First, define the identity matrix, I, as 
\begin{equation}
\nonumber
I =  \left(\begin{array}{rrr} 
 1  &   0   \\
0&   1   \\      
	 \end{array} 
   \right)
\end{equation}
and then the equation 
\begin{equation}
\nonumber
R  = IR
\end{equation} may be represented as
%
\begin{equation}
\nonumber
\left(\begin{array}{rrr} 
 r_{x1x1}  &    r_{x1x2}   \\
r_{x1x2}  &   r_{x2x2}   \\      
	 \end{array} 
   \right)
   =
 \left(\begin{array}{rrr} 
 1  &   0   \\
0&   1   \\      
	 \end{array} 
   \right)
\left(\begin{array}{rrr} 
 r_{x1x1}  &    r_{x1x2}   \\
r_{x1x2}  &   r_{x2x2}   \\      
	 \end{array} 
   \right)
\end{equation}\\
%
Dropping the $x$ subscript (for notational simplicity) we have
\begin{equation}
\label{RIR}
\left(\begin{array}{rrr} 
 r_{11}  &    r_{12}   \\
r_{12}  &   r_{22}   \\      
	 \end{array} 
   \right)
   =
 \left(\begin{array}{rrr} 
 1  &   0   \\
0&   1   \\      
	 \end{array} 
   \right)
\left(\begin{array}{rrr} 
 r_{11}  &    r_{12}   \\
r_{12}  &   r_{22}   \\      
	 \end{array} 
   \right)
\end{equation}
We may multiply both sides of equation \eqref{RIR} by a simple transformation matrix (T) without changing the equality.  If we do this repeatedly until the left hand side of equation \eqref{RIR} is the identity matrix, then the first matrix on the right hand side will be the inverse of R.  We do this in several steps to show the process.


Let 
\begin{equation}
\nonumber
 T_1 =  \left( \begin{array}{rrr} 
 \frac{ 1}{r_{11}}  &   0   \\
0&  \frac{1}{r_{22}}  \\      
	 \end{array} 
   \right)
\end{equation} then we multiply both sides of equation \eqref{RIR} by $T_1$ and we have 
\begin{equation}
T_1 R =  T_1 I R
\end{equation}
\begin{equation}
\label{RIR2}
\left(\begin{array}{ccc} 
1 &   \frac{ r_{12}}{r_{11}}  \\
\frac{r_{12}}{r_{22}} &   1   \\      
	 \end{array} 
   \right)
   =
 \left(\begin{array}{ccc} 
 \frac{1}{ r_{11}}  &   0   \\
0&   \frac{1}{r_{22}}   \\      
	 \end{array} 
   \right)
   \left( \begin{array}{rrr} 
 r_{11}  &    r_{12}   \\
r_{12}  &   r_{22}   \\      
	 \end{array} 
   \right)
\end{equation}
%
Then, by letting 
\begin{equation}
\nonumber 
T_2 =  \left(\begin{array}{ccc} 
1  &   0   \\
- \frac{r_{12}}{r_{22}}&1  \\      
	 \end{array} 
   \right)
    \end{equation} 
 and multiplying $T_2$ times both sides of equation \eqref{RIR2} we have
 
\begin{equation}
\label{RIR3}
%R
\left(\begin{array}{ccc} 
1 &   \frac{ r_{12}}{r_{11}}  \\
0 &    - \frac{ r_{12}^2}{r_{11}r_{22}} \\      
	 \end{array} 
   \right)
  =
 \left( \begin{array}{ccc} 
 \frac{1}{ r_{11}}  &   0   \\
-\frac{r_{12}}{r_{11}r_{22}}&   \frac{1}{r_{22}}   \\      
	 \end{array} 
   \right)
     \left(\begin{array}{rrr} 
 r_{11}  &    r_{12}   \\
r_{12}  &   r_{22}   \\      
	 \end{array} 
   \right)
\end{equation}
%
Then, we let 
\begin{equation}
\nonumber 
T_3 =  \left(\begin{array}{ccc} 
1  &  - \frac{r_{12}}{r_{11}}   \\
0&1  \\      
	 \end{array} 
   \right)
    \end{equation} 
 and multiplying $T_3$ times both sides of equation \eqref{RIR3} we have
 


\begin{equation}
%R
\left(\begin{array}{ccc} 
1 &  0  \\
0 &     - \frac{ r_{12}^2}{r_{11}r_{22}}  \\      
	 \end{array} 
   \right)
  %
 =
 \left(\begin{array}{ccc} 
 \frac{1}{ r_{11}}  &   0   \\
-\frac{r_{12}}{r_{11}r_{22}}&   \frac{1}{r_{22}}   \\      
\end{array} 
   \right)
     \left(\begin{array}{rrr} 
 r_{11}  &    r_{12}   \\
r_{12}  &   r_{22}   \\      
	 \end{array} 
   \right)
\end{equation}

The previous example was drawn out to be easier to follow, but it would be possible to combine several steps together and define the $T_1$ matrix as the product of $T_1 T_2 T_3$ =
\begin{equation}
T_1 = 
\left(\begin{array}{ccc}
\frac{1}{r_{11}} & 0 \\
0 & \frac{1}{r_{11}}
\end{array}\right)
\end{equation}

\begin{equation}
T_3 T_2 T_1 R = T_3 T_2 T_1 I R
\end{equation} or, in other words
\begin{equation}
T_3 T_2 T_1 R  =  I =  R^{-1} R
\end{equation}
\begin{equation}
T_3 T_2 T_1 I = R^{-1}
\end{equation}


\end{document}  