\renewcommand{\coddate}{{\small \today}}
\renewcommand{\sheweek}{3. Multivariate Distributions}


%-------------------------------------------------------------
%
\section{Multivariate Distributions}
%

%**********
\subsection{Joint Distributions}
%**********

\begin{enumerate}
\item \textbf{Joint distribution function}: If $X_1,\ldots,X_n$
are random variables, the joint (cumulative) distribution function
is
$$
F(x_1,x_2,\ldots,x_n) = P(X_1 \leq x_1, X_2 \leq x_2, \ldots, X_n
\leq x_n).
$$
We sometimes use the notation $F_{X_1,\ldots,X_n}(x_1,\ldots,x_n)$
when we need to distinguish the joint distribution function of
$X_1,\ldots,X_n$ from other distribution functions.

\item \textbf{Bivariate distribution function}: For two random
variable $X$ and $Y$ the joint distribution functions is
$$
F(x,y) = P(X \leq x, Y \leq y).
$$
We sometimes use the notation $F_{X,Y}(x,y)$ for the joint
distribution function of $X$ and $Y$.
\begin{enumerate}
\item \begin{tabular}[t]{l}
$F(-\infty,y) = \lim_{x \rightarrow -\infty} F(x,y) = 0$, \\
$F(x,-\infty) = \lim_{y \rightarrow -\infty} F(x,y) = 0$, \\
$F(\infty,\infty) = \lim_{x \rightarrow \infty,y \rightarrow
\infty} F(x,y) = 1$.
\end{tabular}
\item $P(x_1 < X \leq x_2, y_1 < Y \leq y_2) = F(x_2, y_2) - F(x_1,
y_2) - [F(x_2,y_1) - F(x_1,y_1)]$. \item \begin{tabular}[t]{l} Right
continuous in $x$: $\lim_{h \downarrow 0} F(x+h,y) =
F(x,y)$, \\
Right continuous in $y$: $\lim_{h \downarrow 0} F(x,y+h) =
F(x,y)$.
\end{tabular}
\end{enumerate}

\item \textbf{Marginal distribution functions}: If $F_{X,Y}$ is the
joint distribution function of $X$ and $Y$ then the marginal
distribution functions are given by
\begin{align*}
F_X(x) =& \lim_{y \rightarrow \infty} F_{X,Y}(x,y) =
F_{X,Y}(x,\infty), \\
F_Y(y) =& \lim_{x \rightarrow \infty} F_{X,Y}(x,y) =
F_{X,Y}(\infty,y).
\end{align*}

\item \textbf{Discrete bivariate distributions}: for discrete
random variables $X$ and $Y$.
\begin{enumerate}
\item Joint mass function: $f(x,y) = f_{X,Y}(x,y) = P(X=x, Y=y)$.
\item Probabilities: $P(x_1 < X \leq x_2, y_1 < Y \leq y_2) =
\sum_{x_1 < x \leq x_2} \sum_{y_1 < y \leq y_2} f(x,y)$.
\item Marginal mass functions:
\begin{tabular}[t]{l}
$f_X(x) = \sum_y f_{X,Y}(x,y)$, \\
$f_Y(y) = \sum_x f_{X,Y}(x,y)$.
\end{tabular}
\end{enumerate}

\item \textbf{Continuous bivariate distributions}: for jointly
continuous random variables $X$ and $Y$.
\begin{enumerate}
\item Joint density function: is a positive real-valued function
$f$ such that
$$
F(x,y) = \int_{-\infty}^y \int_{-\infty}^x f(u,v) du dv \quad {\rm
for} \ {\rm each} \ x,y \in \R.
$$
\item Probabilities: $P(x_1 < X \leq x_2, y_1 < Y \leq y_2) =
\int_{y_1}^{y_2} \int_{x_1}^{x_2} f(x,y) dx dy$.
\item Marginal density functions:
\begin{tabular}[t]{l}
$f_X(x) = \int_{-\infty}^{\infty} f_{X,Y}(x,y)dy$, \\
$f_Y(y) = \int_{-\infty}^{\infty} f_{X,Y}(x,y)dx$,
\end{tabular}
\end{enumerate}

\item \textbf{Expectation of a function of two variables}:
If $g$ is a well-behaved, real-valued function of two variables
($g: \R^2 \longrightarrow \R$) and $X$ and $Y$ are random variable
with joint mass/density function $f_{X,Y}$ then
$$
E[g(X,Y)] =
 \left\{\begin{array}{ll}
\sum_y \sum_x g(x,y)
f_{X,Y}(x,y), & \text{discrete case}, \\
\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} g(x,y)
f_{X,Y}(x,y) dx dy, & \text{continuous case}.
\end{array} \right.
$$

\item \textbf{Multivariate generalisations}: for random variables
$X_1,\ldots,X_n$ and for $j = 1,\ldots,n$.
\begin{enumerate}
\item Marginal distribution:
$ F_{X_j}(x_j) = F_{X_1,\ldots,X_n}(\infty,\ldots,\infty,x_j,
\infty,\ldots,\infty) $
\item Marginal mass/density
$$
f_{X_j}(x_j) =
 \left\{\begin{array}{ll}
\sum_{x_1} \ldots \sum_{x_{j-1}} \sum_{x_{j+1}} \ldots \sum_{x_n}
f_{X_1,\ldots,X_n}(x_1,\ldots,x_n), & \text{discrete case}, \\
\int_{-\infty}^{\infty} \ldots \int_{-\infty}^{\infty}
f_{X_1,\ldots,X_n}(x_1,\ldots,x_n) dx_1 \ldots dx_{j-1} dx_{j+1}
\ldots dx_n, & \text{continuous case}.
\end{array} \right.
$$
\item Expectation of a function of $n$ variables:
$$
E[g(X_1,\ldots,X_n)] =
 \left\{\begin{array}{ll}
\sum_{x_1} \ldots \sum_{x_n} g(x_1,\ldots,x_n)
f_{X_1,\ldots,X_n}(x_1,\ldots,x_n), & \text{discrete}, \\
\int_{-\infty}^{\infty} \ldots \int_{-\infty}^{\infty}
g(x_1,\ldots,x_n) f_{X_1,\ldots,X_n}(x_1,\ldots,x_n) dx_1 \ldots
dx_n, & \text{continuous}.
\end{array} \right.
$$

\end{enumerate}
\end{enumerate}


%**********
\subsection{Conditional Distributions}
%**********
\begin{enumerate}
\item \textbf{Discrete conditional distributions}:  $X$ and $Y$
discrete random variables with $P(X=x)>0$.
\begin{enumerate}
\item Conditional distribution function of $Y$ given $X=x$:
$F_{Y|X}(y|x) = P(Y \leq y|X=x)$.
\item Conditional mass function of $Y$ given $X=x$:
$ f_{Y|X}(y|x) = P(Y=y|X=x) = f_{X,Y}(x,y)/f_X(x)$.
\item Relationship between distribution and mass functions:
$F_{Y|X}(y|x) = \sum_{y_i \leq y} f_{Y|X}(y_i|x)$.
\end{enumerate}

\item \textbf{Continuous conditional distributions}:
$X$ and $Y$ jointly continuous random variables with $f_X(x) > 0$.
\begin{enumerate}
\item Conditional distribution function of $Y$ given $X=x$:
$F_{Y|X}(y|x) = \int_{-\infty}^y (f_{X,Y}(x,v)/f_X(x))dv$.
\item Conditional density function of $Y$ given $X=x$:
$ f_{Y|X}(y|x) = f_{X,Y}(x,y)/f_X(x)$.
\end{enumerate}

\item \textbf{Conditional, joint and marginal densities}: given
$f_X(x) > 0$ and (for (d)) $f_{Y}(y) > 0$.
\begin{enumerate}
\item Conditional mass/density
$$
f_{Y|X}(y|x) = \frac{f_{X,Y}(x,y)}{f_{X}(x)} =
 \left\{\begin{array}{ll}
\frac{f_{X,Y}(x,y)}{\sum_y f_{X,Y}(x,y)}, & \text{discrete case}, \\
\frac{f_{X,Y}(x,y)}{\int_{-\infty}^{\infty} f_{X,Y}(x,y)dy}, &
\text{continuous case}.
\end{array} \right.
$$
\item Joint mass/density: $f_{X,Y}(x,y) = f_{Y|X}(y|x) f_X(x)$.
\item Marginal mass/density
$$
f_{Y}(y) =
 \left\{\begin{array}{ll}
\sum_x f_{Y|X}(y|x) f_X(x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} f_{Y|X}(y|x) f_X(x) dx, & \text{continuous
case}.
\end{array} \right.
$$
\item Reverse conditioning
$$
f_{X|Y}(x|y) = \frac{f_{X,Y}(x,y)}{f_Y(y)} = \frac{f_X(x)}{f_Y(y)}
f_{Y|X}(y|x).
$$
\end{enumerate}
\newpage

\item \textbf{Conditional expectation}: $X$ and $Y$ random
variables.
\begin{enumerate}
\item Condition expectation of $Y$ given $X$: define
$$
\psi(x) = E(Y|X=x) = \left\{ \begin{array}{ll}
\sum_y y f_{Y|X}(y|x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} y f_{Y|X}(y|x)dy, & \text{continuous
case}.
\end{array}\right.
$$
The conditional expectation of $Y$ given $X$ is $E(Y|X) = \psi(X)$
(a random variable).

\item Condition expectation of $g(Y)$ given $X$: If $g$ is a well-behaved,
real-valued function, define
$$
h(x) = E(g(Y)|X=x) = \left\{ \begin{array}{ll}
\sum_y g(y) f_{Y|X}(y|x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} g(y) f_{Y|X}(y|x)dy, & \text{continuous
case}.
\end{array}\right.
$$
The conditional expectation of $g(Y)$ given $X$ is
$E(g(Y)|X)=h(X)$ (a random variable).

\item Law of iterated expectations:
$$
E[\psi(X)] = E[E(Y|X)] = E(Y).
$$
Useful consequence,
$$
E(Y) = \left\{\begin{array}{ll}
\sum_x E(Y|X=x) f_X(x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} E(Y|X=x) f_X(x) dx, & \text{continuous
case}.
\end{array} \right.
$$
\end{enumerate}


\item \textbf{Conditional variance}: for random variables $X$ and
$Y$, define
$$
\omega(x) = \var(Y|X=x) = \left\{\begin{array}{ll}
\sum_y [y-E(Y|X=x)]^2 f_{Y|X}(y|x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} [y-E(Y|X=x)]^2 f_{Y|X}(y|x)dy, &
\text{continuous case}.
\end{array}\right.
$$
The conditional variance of $Y$ given $X$ is $\var(Y|X) =
\omega(X) = E(Y^2|X) - [E(Y|X)]^2$. The conditional variance is a
random variable and, using the law of iterated expectations, we
can show that $E[\var(Y|X)] = \var(Y) - \var[E(Y|X)]$.

\end{enumerate}

%**********
\subsection{Dependence}
%**********
\begin{enumerate}
\item \textbf{Independence of random variables}
The random variables $X_1, \ X_2,\ldots, \ X_n$ are (mutually)
independent if and only if the events $\{X_1 \leq x_1\}, \ \{X_2
\leq x_2\}, \ldots, \ \{X_n \leq x_n\}$ are independent for all
choices of $x_1,x_2,\ldots,x_n$.

If $X_1,X_2,\ldots,X_n$ independent, then for all $x_1,x_2,
\ldots,x_n$:
\begin{enumerate}
\item $F_{X_1,\ldots,X_n}(x_1,\ldots,x_n) = F_{X_1}(x_1)
F_{X_2}(x_2) \ldots F_{X_n}(x_n)$,
\item $f_{X_1,\ldots,X_n}(x_1,\ldots,x_n) = f_{X_1}(x_1)
f_{X_2}(x_2) \ldots f_{X_n}(x_n)$,
\item $E(X_1 X_2 \ldots X_n) = E(X_1) E(X_2)\ldots E(X_n)$,
\item $g_1(X_1), g_2(X_2), \ldots, g_n(X_n)$ are independent
for real-valued functions $g_1,  g_2, \ldots,  g_n$.
\end{enumerate}


\item \textbf{Covariance function}: for random variables $X$ and
$Y$,
$$
\cov(X,Y) = E[(X-E(X))(Y-E(Y))] = E(XY) - E(X)E(Y).
$$
\begin{enumerate}
\item Symmetry: $\cov(X,Y) = \cov(Y,X)$.
\item With constant multipliers: $\cov(aX, bY) = ab \cov(X,Y)$.
\item Bilinearity: $\cov(X_1+X_2,Y_1+Y_2) = \cov(X_1,Y_1) +
\cov(X_1,Y_2) + \cov(X_2,Y_1) + \cov(X_2,Y_2)$.
\item Variance: \begin{tabular}[t]{l}
$\var(X) = \cov(X,X)$,\\ $\var(X+Y) = \var(X) + \var(Y) + 2
\cov(X,Y)$, \\ $\var(X-Y) = \var(X) + \var(Y) - 2 \cov(X,Y)$.
\end{tabular}
\item If $X$ and $Y$ are independent, $\cov(X,Y) = 0$.
\end{enumerate}

\item \textbf{Correlation coefficient}: for random variables $X$ and
$Y$,
$$
\corr(X,Y) = \frac{\cov(X,Y)}{\sqrt{\var(X) \var(Y)}}.
$$
Correlation is scaled covariance, $|\corr(X,Y)| \leq 1$.
Independent implies uncorrelated but the reverse implication does
not hold.
\end{enumerate}

%**********
\subsection{Sums of Independent Random Variables}
%**********
\begin{enumerate}
\item \textbf{Bivariate case}: for independent random variables $X$ and $Y$,
let $Z=X+Y$ be the sum.
\begin{enumerate}
\item Density of sum: is given by
$$
f_Z(z) = \left\{\begin{array}{ll}
\sum_x f_X(x) f_Y(z-x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} f_X(x) f_Y(z-x)dx, & \text{continuous
case}.
\end{array} \right.
$$
This type of function is known as a \textit{convolution}, so $f_Z$
is the convolution of $f_X$ and $f_Y$ denoted $f_Z = f_X * f_Y$.
Note that the operation is commutative so $f_X * f_Y = f_Y * f_X$.
\item Moment generating function of sum: $M_Z(t) = M_X(t) M_Y(t)$.
\item Cumulant generating function of sum: $K_Z(t) = K_X(t) +
K_Y(t)$.
\end{enumerate}
\item \textbf{Multivariate identically distributed case}: for
independent identically distributed random variables
$X_1,\ldots,X_n$ with common density function $f$, let $S =
\sum_{j=1}^n X_j$.
\begin{enumerate}
\item Density of sum: $f_S(x) = (f*f*\ldots*f)(x)$, $f$ convolved with itself $n$ times.
\item Moment generating function of sum: $M_S(t) = [M_X(t)]^n$.
\item Cumulant generating function of sum: $K_S(t) = n K_X(t)$.
\end{enumerate}

\item \textbf{Examples}
\begin{enumerate}
\item Normal: \begin{tabular}[t]{l} $X_j \sim iN(\mu,\sigma^2) \Rightarrow S \sim N(n
\mu, n \sigma^2)$. \end{tabular}
\item Poisson: \begin{tabular}[t]{l}
$X \sim \pois(\lambda_1), \ Y \sim \pois(\lambda_2)
\Rightarrow Z \sim \pois(\lambda_1 + \lambda_2)$, \\
$X_j \sim i\pois(\lambda) \Rightarrow S \sim \pois(n \lambda)$.
\end{tabular}
\item Gamma: \begin{tabular}[t]{l}
$X \sim \gam(r_1, \theta), \ Y \sim \gam(r_2, \theta)
\Rightarrow Z \sim \gam(r_1 + r_2, \theta)$, \\
$X_j \sim i\Exp(\lambda) \Rightarrow S \sim \gam(n,\lambda)$.
\end{tabular}
\end{enumerate}
\end{enumerate}

%*****************
\subsection{Joint Moments}
%*****************
%
If $X$ and $Y$ are random variables with joint mass/density
function $f_{X,Y}$ then the $(r,s)^{{\rm th}}$ joint moment is
$$
m_{r,s} = E(X^r Y^s) = \left\{\begin{array}{ll}
\sum_y \sum_x x^r y^s f_{X,Y}(x,y), & \text{discrete case}, \\
\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} x^r y^s
f_{X,Y}(x,y)dxdy, & \text{continuous case}.
\end{array} \right.
$$
The $(r,s)^{{\rm th}}$ joint central moment is
$$
 \mu_{r,s} =
E[(X-E(X))^r (Y-E(Y))^s] =   \left\{\begin{array}{ll}
\sum_y \sum_x [(x-\mu_X)^r (y-\mu_Y)^s] f_{X,Y}(x,y), & \text{discrete case}, \\
\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} [(x-\mu_X)^r
(y-\mu_Y)^s] f_{X,Y}(x,y)dxdy, & \text{continuous case}.
\end{array} \right.
$$

\begin{enumerate}
\item \textbf{Properties of joint moments}
\begin{enumerate}
\item $r^{{\rm th}}$ moment for $X$: $m_{r,0} = E(X^r)$.
\item $r^{{\rm th}}$ central moment for $X$: $\mu_{r,0} = E[(X-\mu_X)^r]$.
\item Covariance: $\mu_{1,1} = E[(X-E(X))(Y-E(Y))] = \cov(X,Y)$.
\end{enumerate}

\item \textbf{Joint moment generating function}
$$
M_{X,Y}(t,u) = E(e^{tX + uY}) =  \left\{\begin{array}{ll}
\sum_y \sum_x e^{tx+uy} f_{X,Y}(x,y), & \text{discrete case}, \\
\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} e^{tx+uy}
f_{X,Y}(x,y)dxdy, & \text{continuous case}.
\end{array} \right.
$$

\begin{enumerate}
\item Coefficient of $t^r u^s$: $$\frac{1}{r!s!} E(X^r Y^s)
=\frac{1}{r!s!} m_{r,s}$$.
\item Moment generating function for marginal:
\begin{tabular}[t]{l}
$M_X(t) = E(e^{tX}) = M_{X,Y}(t,0)$, \\
$M_Y(t) = E(e^{uY}) = M_{X,Y}(0,u)$.
\end{tabular}
\item Derivatives at zero:
$$
M_{X,Y}^{(r,s)}(0,0) = \left. \frac{d^{r+s}}{dt^r du^s}
M_{X,Y}(t,u) \right|_{t=0,u=0} = E(X^r Y^s) = m_{r,s}.
$$
\item If $X$ and $Y$ independent: $M_{X,Y}(t,u) = M_X(t) M_Y(u)$.
\end{enumerate}

\item \textbf{Conditional moment generating function}: define
$$
\nu(u,x) = M_{Y|X}(u|x) = E(e^{uY}|X=x) = \left\{\begin{array}{ll}
\sum_y e^{uy} f_{Y|X}(y|x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} e^{uy} f_{Y|X}(y|x)dy, & \text{continuous
case}.
\end{array} \right.
$$
The conditional moment generating function of $Y$ given $X$ is
$M_{Y|X}(u|X) = \nu(u,X) = E(e^{uY}|X)$. This is a conditional
expectation so it is a random variable. We can calculate joint
moment generating function and moment generating function for
marginal $Y$ from the conditional moment generating function,
\begin{align*}
& M_{X,Y}(t,u) = E(e^{tX+uY}) = E[e^{tX} M_{Y|X}(u|X)], \\
& M_Y(u) = M_{X,Y}(0,u) = E[M_{Y|X}(u|X)].
\end{align*}

\item \textbf{Joint cumulants}: let $K_{X,Y}(t,u) = \log
M_{X,Y}(t,u)$, then we define the $(r,s)^{{\rm th}}$ joint
cumulant $\kappa_{r,s}$ as the coefficient of $(t^r u^s)/(r! s!)$
in the expansion of $K_{X,Y}$. Thus, $\kappa_{1,1} = \cov(X,Y)$.

\item \textbf{Multivariate generalisation}: for random variables
$X_1,\ldots,X_n$ with joint mass/density function $f_{X_1,\ldots,X_n}$.
\begin{enumerate}
\item Joint moments:
\begin{align*}
m_{r_1,\ldots,r_n} = & E(X_1^{r_1} \ldots X_n^{r_n}) \\
= & \left\{\begin{array}{ll} \sum_{x_1} \ldots \sum_{x_n}
x_1^{r_1} \ldots x_n^{r_n} f_{X_1,\ldots,X_n}(x_1,\ldots,x_n),
& \text{discrete case}, \\
\int_{-\infty}^{\infty} \ldots \int_{-\infty}^{\infty} x_1^{r_1}
\ldots x_n^{r_n} f_{X_1,\ldots,X_n}(x_1,\ldots,x_n)dx_1 \ldots
dx_n, & \text{continuous case}.
\end{array} \right.
\end{align*}

\item Joint central moments: $\mu_{r_1,\ldots,r_n} =
E[(X_1-E(X_1))^{r_1}\ldots (X_n - E(X_n))^{r_n}]$


\item Joint moment generating function:
$M_{X_1,\ldots,X_n}(t_1,\ldots,t_n) = E(e^{t_1 X_1+\ldots+t_n
X_n})$. The coefficient of $t_1^{r_1}\ldots t_n^{r_n}$ in the
expansion of $M_{X_1,\ldots,X_n}$ is $E(X_1^{r_1} \ldots
X_n^{r_n})/(r_1! \ldots r_n!)$.

\item Joint cumulant generating function:
$K_{X_1,\ldots,X_n}(t_1,\ldots,t_n) = \log(
M_{X_1,\ldots,X_n}(t_1,\ldots,t_n))$. The $(r_1,\ldots,r_n)^{{\rm
th }}$ joint cumulant is defined as the coefficient of
$(t_1^{r_1}\ldots t_n^{r_n})/(r_1! \ldots r_n!)$ in the expansion
of $K_{X_1,\ldots,X_n}$.

\item Independence: if $X_1,\ldots,X_n$ are independent then
$M_{X_1,\ldots,X_n}(t_1,\ldots,t_n) = M_{X_1}(t_1)\ldots
M_{X_n}(t_n)$.
\end{enumerate}
\end{enumerate}

%**********
\subsection{Compounding and Random Sums}
%**********
\begin{enumerate}
\item \textbf{From conditional to marginal}: for random variables
$X$ and $Y$ suppose that we know the conditional distribution of
$Y$ given $X$ and the marginal distribution of $X$.
\begin{enumerate}
\item Joint density: $f_{X,Y}(x,y) = f_{Y|X}(y|x) f(x)$.
\item Marginal density for $Y$:
$$
f_Y(y) = \left\{\begin{array}{ll}
\sum_x f_{Y|X}(y|x) f(x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} f_{Y|X}(y|x) f(x) dx, & \text{discrete
case}.
\end{array} \right.
$$
\item Moment generating function for $Y$:
$$
M_Y(t) = E[M_{Y|X}(t|X)] \left\{\begin{array}{ll}
\sum_x M_{Y|X}(t|x) f(x), & \text{discrete case}, \\
\int_{-\infty}^{\infty} M_{Y|X}(t|x) f(x) dx, & \text{discrete
case}.
\end{array} \right.
$$
\end{enumerate}

\item \textbf{Random sums}: suppose that $X_1, X_2, \ldots$ is a sequence of
independent identically distributed random variables and $S = X_1
+ \ldots + X_N$ where $N$ is a random variable.
\begin{enumerate}
\item Conditional \begin{tabular}[t]{ll}
 expectation: & $E(S|N) = N E(X)$, \\
 variance:    & $\var(S|N) = N \var(X)$, \\
 moment generating function: & $M_{S|N}(t|N) = [M_X(t)]^N$.
 \end{tabular}
 \item Marginal \begin{tabular}[t]{ll}
  expectation: & $E(S) = E(N) E(X)$, \\
  variance: & $\var(S) = E(N) \var(X) + \var(N) [E(X)]^2$, \\
  moment generating function: & $M_S(t) = M_N(\log M_X(t))$, \\
  cumulant generating function: & $K_S(t) = K_N(K_X(t))$.
 \end{tabular}
 \end{enumerate}

\item \textbf{Examples}: let $Y|X=x$ denote the random variable
with density $f_{Y|X}(y|x)$.

\begin{enumerate}
\item \begin{tabular}[t]{rl} Binomial and Poisson: &
$Y|N=n \sim \bin(n,p)$ and $\ N \sim \pois(\lambda) \Rightarrow Y
\sim \pois(\lambda p)$, \\
& (equivalently $Y = \sum_{j=1}^N X_j$ where $X_j \sim {\rm
Bernoulli}(p)$).
\end{tabular}

\item \begin{tabular}[t]{rl} Binomial and beta: &
$Y| \Theta = \theta \sim \bin(m,\theta)$ and $\Theta \sim
\bet(\alpha,\beta)$ \\
$ \Rightarrow$ & $ f_Y(y) = \binom{m}{y}
\frac{\Gamma(\alpha+\beta)}{\Gamma(\alpha)\Gamma(\beta)} /
\frac{\Gamma(\alpha+\beta+m)}{\Gamma(\beta+m-y)\Gamma(\alpha+y)}$.
\end{tabular}

\item \begin{tabular}[t]{rl} Poisson and gamma: & $Y|\Lambda = \lambda \sim \pois(\lambda)$
and $\Lambda \sim \gam(r,\alpha)$ \\
$ \Rightarrow$ & $f_Y(y) = \frac{\Gamma(r+y)}{\Gamma(r) y!}
(\frac{\alpha}{\alpha+1})^r (\frac{1}{\alpha+1})^y.
$ \\
  & $Y$ has a negative binomial distribution \\
  & (in which $r$ is not
required to be an integer).
\end{tabular}

\item \begin{tabular}[t]{rl} Poisson mixing:
 & $Y|\Lambda = \lambda \sim \pois(\lambda)$ and $\Lambda$
 distributed on $[0,\infty)$ \\
 $\Rightarrow$ & $E(Y) = E(\Lambda), \ \var(Y) = E(\Lambda) +
 \var(\Lambda)$.
 \end{tabular}

\item \begin{tabular}[t]{rl} Poisson limit random sum:
 & $S = \sum_{j=1}^N X_j$ where $N \sim
\pois(\lambda)$ \\
$\Rightarrow$ & $E(S) = \lambda E(X), \ \var(S) = \lambda E(X^2)$, \\
& $M_S(t) = M_N(\log M_X(t)) = e^{\lambda(M_X(t)-1)}$, \\
& $K_S(t) = \lambda(M_X(t)-1)$, $\kappa_r = \lambda E(X^r)$.
\end{tabular}

\item \begin{tabular}[t]{rl} Random sum of exponentials:
& $S = \sum_{j=1}^N X_j$ where $X_j \sim \Exp(\theta)$ and $N \sim
{\rm Geometric}(p)$ \\
$\Rightarrow$ & $S \sim \Exp(\theta p)$.
\end{tabular}

\end{enumerate}
\end{enumerate}


%
%**********
\subsection{Multivariate Normal Distribution}
%**********
\begin{enumerate}
\item \textbf{Bivariate case}
\begin{enumerate}
\item Independent normals: if $U$ and $V$ are $iN(0,1)$ then
\begin{align*}
f_{U,V}(u,v) = & \tfrac{1}{2 \pi} e^{-(u^2+v^2)/2}, \\
M_{U,V}(s,t) = & e^{(s^2 + t^2)/2}.
\end{align*}

\item Standard bivariate normal: if $X=U$ and $Y = \rho U +
\sqrt{1- \rho^2} V$ then $Y \sim N(0,1)$ and $\corr(X,Y) = \rho$.
The joint distribution is a standard bivariate normal;
$$
f_{X,Y}(x,y) = \tfrac{1}{2 \pi \sqrt{1 - \rho^2}} e^{-(x^2 - 2
\rho x y + y^2)/(2(1-\rho^2))}.
$$

\item General bivariate normal: if $X^* = \mu_X + \sigma_X X$ and
$Y^* = \mu_Y + \sigma_Y Y$ then $X^* \sim N(\mu_X, \sigma_X^2)$
and $Y^* \sim N(\mu_Y, \sigma_Y^2)$ with $\corr(X^*, Y^*) = \rho$
and
$$
f_{X^*, Y^*}(x,y) = \frac{1}{\sigma_X \sigma_Y} f_{X,Y}
(\frac{x-\mu_X}{\sigma_X}, \frac{y - \mu_Y}{\sigma_Y}).
$$

\item Conditional distribution: of $Y^*$ given $X^*$ is
$$
Y^*|X^* = x \ \sim \ N(\mu_y + \rho
\frac{\sigma_Y}{\sigma_X}(x-\mu_X), \ \sigma_Y^2 (1 - \rho^2)).$$
\end{enumerate}

\item \textbf{Multivariate case}
\begin{enumerate}
\item Multivariate normal density: let $X_1,\ldots,X_n$ be random
variables and define a random vector $\mX = (X_1,\ldots,X_n)'$,
where $'$ denotes transpose, so $\mX$ is an $n \times 1$ vector.
If $X_1,\ldots,X_n$ are jointly normal then $\mX \sim N(\mmu,
\mSig)$, where the mean $\mmu = E(\mX)$ is an $n \times 1$ vector
and the covariance matrix $\mSig = \var(\mX)$ is an $n \times n$
matrix whose $(i,j)^{{\rm th}}$ entry is $\cov(X_i,X_j)$. The
joint density functions is
$$
f_{X_1,\ldots,X_n}(x_1,\ldots,x_n) = f_{\mX}(\mx) = (2 \pi)^{-n/2}
| \mSig|^{-1/2} e^{-(\mx - \mmu)' \mSig^{-1} (\mx-\mmu)/2}
$$
\item Conditional expectation for multivariate normal: suppose
that $\mX = (X_1,\ldots,X_n)'$ and $\mY = (Y_1,\ldots,Y_m)'$, for
some integers $n$ and $m$, and $X \sim N(\mmu_X, \mSig_X)$ and $Y
\sim N(\mmu_Y, \mSig_Y)$ . If, in addition, $\cov(\mX,\mY) =
\mSig_{XY} = \mSig_{YX}'$, then
\begin{align*}
E(\mY|\mX) = & \mmu_Y + \mSig_{YX} \mSig_X^{-1} (\mX - \mmu_X),\\
\var(\mY|\mX) = & \mSig_Y - \mSig_{YX} \mSig_X^{-1} \mSig_{XY}.
\end{align*}
\end{enumerate}
\end{enumerate}



%**********
\subsection{Laws of Large Numbers}
%**********

\begin{enumerate}
\item \textbf{Types of convergence}
Let $Y,Y_1,Y_2,\ldots,$ be a collection of random variables.
\begin{enumerate}
\item Convergence in distribution: $Y_n \xra{\  D \ } Y$ if $P(Y_n \leq
y) \lra P(Y \leq y)$ as $n \lra \infty$.
\item Convergence in probability: $Y_n \xra{\  P \ } Y$ if $P(|Y_n
- Y| > \eps) \lra 0$ as $n \lra \infty$ for all $\eps >0$.
\item Convergence almost surely: $Y_n \xra{\  a.s. \ } Y$ if $A = \{\omega \in \Omega: Y_n(\omega) \lra Y(\omega)
\text{ as } n \lra \infty\} \Rightarrow P(A) = 1$.
\end{enumerate}

\item \textbf{Convergence of sample mean}
Let $X_1,X_2,\ldots,$ be a sequence of independent identically
distributed random variables with distribution function $F_X$,
finite mean $\mu$ and, for part (d), finite non-zero variance
$\sigma^2$. We definite $S_n = \sum_{j=1}^n X_j$ as the partial
sum of $n$ independent identically distributed random variables
and $\frac{1}{n} S_n$ is the sample mean (a random variable).
\begin{enumerate}
\item The law of large numbers: $\frac{1}{n} S_n \xra{\ D \ } \mu$
as $n \lra \infty$.
\item  Weak law of large numbers: $\frac{1}{n} S_n \xra{\ P \ } \mu$
as $n \lra \infty$.
\item  Strong law of large numbers: $\frac{1}{n} S_n \xra{\ a.s. \ } \mu$
as $n \lra \infty$.
\item  Central limit theorem:
$$
\frac{S_n - n \mu}{\sqrt{n \sigma^2}} \xra{\ D \ } N(0,1) \ \text{
as } \ n \lra \infty .
$$
\end{enumerate}
Note: convergence to $\mu$ is convergence to the degenerate random
variable with all of the mass concentrated at the single value
$\mu$.
\end{enumerate}


%**********
\subsection{Transformations of Random Variables}
%**********

\begin{enumerate}

\item \textbf{Bivariate transformations}: Suppose that continuous
random variables $X_1$ and $X_2$ are transformed into new random
variables $Y_1$ and $Y_2$ by a one-to-one function $T: \R^2 \lra
\R^2$ so $(Y_1,Y_2)' = T(X_1,X_2)'$. The joint density of $Y_1$
and $Y_n$ is given by
\begin{equation} \label{transf:eq}
f_{Y_1,Y_2}(y_1,y_2) = \left\{\begin{array}{ll}
f_{X_1,X_2}(x_1(y_1,y_2), x_2(y_1,y_2)) |J(y_1,y_2)|, & \text{if }
(y_1,y_2) \text{ is in the range of } $T$, \\ 0, &
\text{otherwise},
\end{array} \right.
\end{equation}
where $(x_1(y_1,y_2), x_2(y_1,y_2))' = T^{-1}(y_1,y_2)'$ and
\begin{equation} \label{Jac:eq}
J(y_1,y_2) = \left| \begin{array}{ll}
\dd{x_1}{y_1} &  \dd{x_2}{y_1} \\
\dd{x_1}{y_2} &  \dd{x_2}{y_2}
\end{array} \right| = \dd{x_1}{y_1}   \dd{x_2}{y_2} - \dd{x_2}{y_1}
\dd{x_1}{y_2}.
\end{equation}
This is the Jacobian of the inverse
transformation.

\item \textbf{Ratios of random variables}: To illustrate the
bivariate transformation, consider the following question. What is
the probability density function of the ratio $X_1 / X_2$. Let
$$
Y_1 = X_1/ X_2 \quad \text{and} \quad Y_2 = X_2.
$$
Simple rearrangement gives us $X_1 = Y_1 Y_2$ and $X_2 = Y_2$.
Thus in term of coordinates the inverse transformation is
$$
x_1(y_1,y_2) = y_1 y_2 \quad \text{and} \quad x_2(y_1,y_2) = y_2.
$$
Taking partial derivatives yields
$$
\dd{x_1}{y_1} = y_2, \ \  \dd{x_1}{y_2} = y_1, \ \ \dd{x_2}{y_1} =
0, \ \ \dd{x_2}{y_2} = 1.
$$
From equation \eqref{Jac:eq} above, we see that the Jacobian of
the inverse transformation is $|J(y_1,y_2)| = |y_2|$. Substitution
in equation \eqref{transf:eq} gives us the joint density,
$$
f_{Y_1,Y_2} (y_1,y_2) = f_{X_1,X_2}(y_1y_2,y_2)|y_2|.
$$
Integrating over $y_2$ yields the marginal for the ratio $Y_1$,
$$
f_{Y_1}(y_1) = \int_{-\infty}^{\infty} f_{X_1,X_2}(y_1y_2, y_2)
|y_2| d y_2.
$$

\item \textbf{Multivariate generalisation}
Consider the $n$-dimensional random vector $\mX =
(X_1,\ldots,X_n)'$ and a one-to-one transformation $T: \R^n \lra
\R^n$ to give another random vector $\mY = T \mX$. The joint
density function for $\mY$ is given by
$$
f_{\mY} (\my) = \left\{\begin{array}{ll} f_{\mX}(\mx(\my))
|J(\my)|, & \text{if } \my \text{ is in the range of } $T$,
\\ 0, & \text{otherwise},
\end{array} \right.
$$
where $\mx(\my) = T^{-1}\my$ and
$$
J(\my) = \left| \begin{array}{llll}
\dd{x_1}{y_1} &  \dd{x_2}{y_1} & \cdots & \dd{x_n}{y_1} \\
\dd{x_1}{y_2} &  \dd{x_2}{y_2} &        &  \vdots   \\
\vdots        &                & \ddots & \vdots    \\
\dd{x_1}{y_n} & \cdots         & \cdots & \dd{x_n}{y_n}
\end{array} \right|,
$$
is the Jacobian of the inverse transformation.
\end{enumerate}
