\subsection{The exponential family}

The \emph{natural exponential family} with \emph{canonical
parameterisation} has p.d.f of the form
\[
f(y)=\exp\left\{  \frac{y\theta+c\left(  \theta\right)
}{\phi}+d\left(
y,\phi\right)  \right\}  \text{.}%
\]


$\theta$ is the \emph{canonical} (or \emph{natural}) parameter for
the distribution. The parameter $\phi$ is called the \emph{scale}
parameter. Most times we shall consider that this is known, and does
not need to be estimated.

\subsubsection{Mean and variance}

The loglikelihood is
\[
l\left(  \theta,\phi;y\right)  =\frac{y\theta+c\left(  \theta\right)
}{\phi
}+d\left(  y,\phi\right)  \text{.}%
\]
The score function is
\[
s(\theta)=\frac{\partial l}{\partial\theta}=\frac{y+c^{\prime}\left(
\theta\right)  }{\phi}%
\]
and this has mean zero, so
\[
\operatorname*{E}\left[  Y\right]  =-c^{\prime}\left(  \theta\right)  \text{.}%
\]
The leading element of the Hessian is
\[
\frac{\partial^{2}l}{\partial\theta^{2}}=\frac{c^{\prime\prime}\left(
\theta\right)  }{\phi}\text{.}%
\]
This does not involve $y$, so the leading element of the information
matrix $\mathcal{I}$ is $-c^{\prime\prime}\left(  \theta\right)
/\phi$. However this is equal to the variance of $s(\theta)$ which
is
\[
\operatorname*{E}\left[  s(\theta)^{2}\right]
=\operatorname*{E}\left[ \left( \frac{\partial
L}{\partial\theta}\right)  ^{2}\right]
\]
because $\operatorname*{E}\left[  s(\theta)\right]  =0$. So
\begin{align*}
\operatorname*{E}\left[  \left(  \frac{Y-\operatorname*{E}\left[
Y\right] }{\phi}\right)  ^{2}\right]   &  =-c^{\prime\prime}\left(
\theta\right)
/\phi\\
\frac{1}{\phi^{2}}\operatorname*{Var}\left[  Y\right]   &
=-c^{\prime\prime
}\left(  \theta\right)  /\phi\\
\operatorname*{Var}\left[  Y\right]   &  =-\phi
c^{\prime\prime}\left(
\theta\right)  \text{.}%
\end{align*}


We have therefore shown that
\[
\mu=\operatorname*{E}\left[  Y\right]  =-c^{\prime}\left(
\theta\right)
\text{,}%
\]
and that
\[
\sigma^{2}=\operatorname*{Var}\left[  Y\right]  =-\phi
c^{\prime\prime}\left(
\theta\right)  \text{.}%
\]


Note that
\[
\operatorname*{Var}\left[  Y\right]  =\phi\frac{d\mu}{d\theta}=\phi
V\left( \mu\right)
\]
where $V\left(  \mu\right)  $ is the \emph{variance function}. Here
are the variance functions and the \emph{dispersion parameter}
$\phi$ for some
standard distributions.%
\[%
\begin{tabular}
[c]{rrr}%
Distribution & Variance function & Dispersion parameter\\
Normal & $1$ & $\sigma^{2}$\\
Poisson & $\mu$ & $1$\\
Binomial & $\mu\left(  1-\mu\right)  $ & $1$\\
Gamma & $\mu^{2}$ & $1/\alpha$%
\end{tabular}
\]

\subsubsection{Moment Generating function}
\begin{align*}
M_Y(t)&=\int e^{yt}e^{\frac{y\theta+c(\theta)}{\phi}+d(y,\phi)}dy\\
&= \int e^{\frac{y(\theta+t\phi)+c(\theta)}{\phi}+d(y,\phi}dy\\
&=e^{\frac{c(\theta)-c(\theta+t\phi)}{\phi}}\int
e^{\frac{y(\theta+t\phi)+c(\theta+t\phi)}{\phi}+d(y,\phi)}dy\\
&=e^{\frac{c(\theta)-c(\theta+t\phi)}{\phi}}.
\end{align*}
The cumulant generating function is
$\frac{c(\theta)-c(\theta+t\phi}{\phi}$.


\subsubsection{Normal distribution}
The Normal distribution has
\begin{align*}
\ln\left(  f\right)   &  =-\frac{1}{2}\frac{\left(  y-\mu\right)  ^{2}}%
{\sigma^{2}}-\frac{1}{2}\ln\left(  2\pi\sigma^{2}\right) \\
&  =\frac{y\mu-\frac{1}{2}\mu^{2}}{\sigma^{2}}-\frac{y^{2}}{2\sigma^{2}}%
-\frac{1}{2}\ln\left(  2\pi\sigma^{2}\right)
\end{align*}


So we take
\begin{align*}
\theta &  =\mu\text{,}\\
\phi &  =\sigma^{2}\text{,}\\
c\left(  \theta\right)   &  =-\frac{1}{2}\theta^{2}\text{, and}\\
d\left(  y,\phi\right)   &
=-\frac{y^{2}}{2\phi}-\frac{1}{2}\ln\left(
2\pi\phi\right)  \text{.}%
\end{align*}


We check the mean
\begin{align*}
\operatorname*{E}\left[  Y\right]   &  =-c^{\prime}\left(  \theta\right) \\
&  =\theta=\mu
\end{align*}
and the variance
\begin{align*}
\operatorname*{Var}\left[  Y\right]   &  =-\phi
c^{\prime\prime}\left(
\theta\right) \\
&  =\phi=\sigma^{2}\text{.}%
\end{align*}
The cumulant generating function is
\[
-\frac{1}{2}\frac{\mu^2}{\sigma^2}
+\frac{1}{2}\frac{(\mu+\sigma^2t)^2}{\sigma^2}
\]
which is
\[
\mu t+\frac{1}{2}\sigma^2t^2.
\]


\subsubsection{The Gamma distribution}
\[
f(y)=\frac{1}{\Gamma(\nu)}\frac{1}{y}\left(\frac{\nu
y}{\mu}\right)^\nu e^{-\frac{\nu y}{\mu}}
\]
so
\[
ln(f)=\nu\left(y\left(-\frac{1}{\mu}\right)-\ln\mu\right)+(\nu-1)\ln
y -\ln\Gamma (\nu) +\nu\ln\nu.
\]



So we take
\begin{align*}
\theta &  =-\frac{1}{\mu}\text{,}\\
\phi &  = \frac{1}{\nu}\text{,}\\
c\left(  \theta\right)   &  =-\ln\mu=\ln(-\theta) \text{, and}\\
d\left(  y,\phi\right)   & =-\ln y-\ln\Gamma (\nu) +\nu\ln\nu  \text{.}%
\end{align*}

We check the mean
\begin{align*}
\operatorname*{E}\left[  Y\right]   &  =-c^{\prime}\left(  \theta\right) \\
&  =-1/\theta=\mu
\end{align*}
and the variance
\begin{align*}
\operatorname*{Var}\left[  Y\right]   &  =-\phi
c^{\prime\prime}\left(
\theta\right) \\
&  =\phi\frac{1}{\theta^2}=\frac{\mu^2}{\nu}\text{.}%
\end{align*}

The cumulant generating function is
\[
K_Y(t)=\frac{\ln(1\mu)+\ln(1/\mu-t/\nu)}{1/\nu}=\frac{1}{\nu}\ln
\frac{1}{1-\frac{\mu t}{\nu}}
\]


\subsubsection{The Poisson distribution}
The probability mass function is
\[
f(y)=e^{-\lambda}\lambda^y/y!
\]
so
\[
ln(f)=\frac{y\ln (\lambda)-\lambda}{1}-\ln y!.
\]
So we take
\begin{align*}
\theta &  =\ln\lambda\\
\phi &  = 1\\
c\left(  \theta\right)   &  =-\lambda = -e^\theta\text{, and}\\
d\left(  y,\phi\right)   & =-\ln y!.
\end{align*}

We check the mean
\begin{align*}
\operatorname*{E}\left[  Y\right]   &  =-c^{\prime}\left(  \theta\right) \\
&  = e^\theta=\lambda.
\end{align*}
and the variance
\begin{align*}
\operatorname*{Var}\left[  Y\right]   &  =-\phi
c^{\prime\prime}\left(
\theta\right) \\
&  =-1\times (-e^\theta)=\lambda.
\end{align*}

The cumulant generating function is
\begin{align*}
K_X(t)&=-e^{\ln\lambda} +e^{\ln\lambda+t}\\
&=\lambda(e^t-1).
\end{align*}
\subsubsection{Binomial distribution}
The probability mass function is
\[
f(y)=\binom{n}{y}\pi^y(1-\pi)^{n-y}.
\]
so
\[
ln(f)=\frac{y\ln \frac{\pi}{1-\pi}+n\ln (1-\pi)}{1}+\ln\binom{n}{y}.
\]
So we take
\begin{align*}
\theta &=\ln \frac{\pi}{1-\pi}\\
\phi &= 1\\
c\left(  \theta\right)   &= n\ln(1-\pi) = -n\ln (1+e^\theta)\text{, and}\\
d\left(  y,\phi\right)   &= \ln\binom{n}{y}.
\end{align*}

We check the mean
\begin{align*}
\operatorname*{E}\left[  Y\right]   &  =-c^{\prime}\left(  \theta\right) \\
&  = n\frac{e^\theta}{1+e^\theta}=n\pi.
\end{align*}
and the variance
\begin{align*}
\operatorname*{Var}\left[  Y\right]   &  =-\phi
c^{\prime\prime}\left(
\theta\right) \\
&  =n\frac{e^\theta}{(1+e^\theta)^2}=n\pi(1-\pi).
\end{align*}
The cumulant generating function is
\begin{align*}
K_X(t)&=n\ln(1-\pi)+n\ln (1+e^{\ln \frac{\pi}{1-\pi}+t})\\
&=n\ln(1-\pi)+n\ln (1+\frac{\pi}{1-\pi}e^t)\\
&=n\ln(1-\pi)+n\ln (\frac{1-\pi+\pi e^t}{1-\pi})\\
&=n\ln (1-\pi+\pi e^t).
\end{align*}

\subsubsection{Application to a sample}

The loglikelihood of $n$ \emph{independent} and \emph{identically
distributed} observations from the distribution is
\[
l\left(  \bm{\beta},\phi;y\right)  =\sum_{i=1}^{n}\left\{  \frac
{y_{i}\theta+c\left(  \theta\right)  }{\phi}+d\left(  y_{i}%
,\phi\right)  \right\}
\].
With $\phi$ considered as fixed, it is immediately clear from the
factorisation criterion that $\sum_i y_i$ is a sufficient statistic
for $\theta$.
\subsection{Exponential Family: General case}
The exponential family with canonical parameters are distributions
with density functions or probability mass functions of the form
\[
f(\bm{y})=\exp\left[\sum_{i=1}^s\theta_i
T_i(\bm{y})+c(\bm{\theta})+d(y)\right].
\]
Here we don't explicitly put in the scale parameter that we
considered before as a known parameter, but it may be absorbed into
the functions $d(y),\theta_i$ so this family includes the previous
one. The support of the density function or probability mass
function is assumed not to depend on the parameters $\theta_i$.  To
make for simplicity we assume that there is no linear dependence in
the set of $T_i$s or in the set of $\theta_i$s, and also that there
is an s-dimensional rectangle of values $\theta_i$ for which the
density or probability mass function is properly defined.

\subsubsection{sufficiency}
The factorisation criterion gives immediately that
$(T_1(\bm{y}),\dots ,T_s(\bm{y}))$ is a sufficient statistic for
$(\theta_1,\dots ,\theta_s)$.

Essentially, the only nice regular cases of distributions with
simple sufficient statistics are those in the exponential family.

Note that $\bm{y}$ can be a vector of values in the definition, so
that the joint density for a random sample of observations from a
natural exponential family with canonical parameters is a member of
the general exponential family.

\subsubsection{Mean and Covariance}
The loglikelihood is
\[
l\left(  \bm{\theta};\bm{T}(\bm{y})\right)
=\bm{T}(\bm{y})^T\bm{\theta}+c\left( \bm{\theta}\right) +d\left(
\bm{y}\right).
\]
The score function is
\[
s(\bm{\theta})=\frac{\partial
l}{\partial\bm{\theta}}=\bm{T}(\bm{y})+c^{\prime}\left(
\bm{\theta}\right)
\]
and this has mean zero, so
\[
\operatorname*{E}\left[ \bm{T}(y)\right]  =-c^{\prime}\left(
\bm{\theta}\right).
\]
The Hessian is
\[
\frac{\partial^{2}l}{\partial\bm{\theta}\partial\bm{\theta}^T}=c^{\prime\prime}\left(
\theta\right).
\]
This does not involve $y$, so the information matrix $\mathcal{I}$
is $-c^{\prime\prime}\left( \bm{\theta}\right)$. However this is
equal to the variance-covariance matrix of $s(\bm{\theta})$ which is
\[
\operatorname*{E}\left[  s(\bm{\theta})s(\bm{\theta})^T\right]
=\operatorname*{E}\left[ \left( \frac{\partial
L}{\partial\bm{\theta}}\right)\left( \frac{\partial
L}{\partial\bm{\theta}}\right)^T\right]
\]
because $\operatorname*{E}\left[  s(\bm{\theta})\right]  =0$. So
\begin{align*}
\operatorname*{E}\left[  \left(
\bm{T}(\bm{y})-\operatorname*{E}\left[
\bm{T}(\bm{y})\right]\right)\left(
\bm{T}(\bm{y})-\operatorname*{E}\left[
\bm{T}(\bm{y})\right]\right)^T \right]   & =-c^{\prime\prime}\left(
\bm{\theta}\right).\\
\end{align*}


We have therefore shown that
\[
\mu=\operatorname*{E}\left[ \bm{T}(\bm{y})\right] =-c^{\prime}\left(
\bm{\theta}\right),
\]
and that
\[
\operatorname*{Var}\left[  \bm{T}(\bm{y})\right]
=-c^{\prime\prime}\left( \bm{\theta}\right).
\]

\subsubsection{Joint Cumulant Generating Function}
It is easy to show that the joint cumulant generating function of
$T_1(\bm{y}),T_2(\bm{y}),\dots ,T_k(\bm{y})$ is
\[
c(\theta_1,\theta_2,\dots
,\theta_k)-c(t_1+\theta_1,t_2+\theta_2,\dots ,t_k+\theta_k)
\]

\subsubsection{Multinomial Distribution}
For the multinomial distribution we have probability mass function
\[
f(\bm{y})=\frac{n!}{y_1!y_2!\dots y_k!}\pi_1^{y_1}\pi_2^{y_2}\dots
\pi_k^{y_k}
\]
where the $\{y_i\}$ are non-negative integers with $\sum_i y_i=n$,
and the $\{\pi_i\}$ are probabilities satisfying $\sum_i \pi_i=1$.
The probability mass function is
\begin{align*}
f(\bm{y})&=\exp[y_1\ln\pi_1+y_2\ln\pi_2+\dots
y_k\ln\pi_k+\ln\frac{n!}{y_1!y_2!\dots y_k!}]\\
&=\exp[y_1\ln\pi_1+y_2\ln\pi_2+\dots
(n-\sum_i y_i)\ln\pi_k+\ln\frac{n!}{y_1!y_2!\dots y_k!}]\\
&=\exp[y_1\ln(\pi_1/ \pi_k)+y_2\ln(\pi_2/\pi_k)+\dots
+y_{k-1}\ln(\pi_{k-1}/\pi_k)+n\ln\pi_k+\ln\frac{n!}{y_1!y_2!\dots
y_k!}].
\end{align*}

So this is in the general exponential family with
\begin{align*}
T_i(\bm{y})&=y_i\text{ for }i=1,2\dots ,(k-1)\\
\theta_i&=\ln (\pi_i/\pi_k)\\
c(\bm{\theta})&=n\ln\pi_k=n\ln(1/(1+\sum_1^{k-1}\exp\theta_i)\\
d(\bm{y})&=\ln\frac{n!}{y_1!y_2!\dots y_k!}.
\end{align*}
We use in the above $\pi_k=\sum_{i=1}^{k-1}\pi_i$.

A sufficient statistic for $\pi_1,\pi_2,\dots ,\pi_{k-1}$ is
$y_1,y_2,\dots ,y_{k-1}$.

The mean for $T_1(\bm{y})$ is
\begin{align*}
-\frac{\partial}{\partial \theta_1}c\left(
\bm{\theta}\right)&=\frac{\partial}{\partial\theta_1}n\ln(1/(1+\sum_1^{k-1}\exp\theta_i)\\
&=n\frac{\exp\theta_1}{1+\sum_1^{k-1}\exp\theta_i}\\
&=n\frac{\frac{\pi_1}{\pi_k}}{1+\sum_1^{k-1}\frac{\pi_i}{\pi_k}}\\
&=n\pi_1.
\end{align*}
The variances and covariances follow on further differentiation.
\subsubsection{Normal Distribution}
With both mean and variance unknown, the normal distribution has
density function
\begin{align*}
f(y)&=\frac{1}{\sqrt{2\pi}\sigma}\exp\left[-\frac{1}{2}\frac{(y-\mu)^2}{\sigma^2}\right]\\
&=\exp\left[y\frac{\mu}{\sigma^2}+y^2\frac{-1}{2\sigma^2}-\frac{\mu^2}{2\sigma^2}-\ln(\sqrt{2\pi}\sigma)\right].
\end{align*}
So we could take
\begin{align*}
T_1(y)&=y\\
T_2(y)&=y^2\\
\theta_1&=\frac{\mu}{\sigma^2}\\
\theta_2&=\frac{-1}{2\sigma^2}\\
c(\bm{\theta})&=-\frac{\mu^2}{2\sigma^2}-\ln\sigma=\frac{\theta_1^2}{4\theta_2}+\frac{1}{2}\ln(-\theta_2).
\end{align*}
The means are
\begin{align*}
E(y)&=-\frac{\partial}{\partial
\theta_1}c(\bm{\theta})=-\frac{2\theta_1}{4\theta_2}=\mu\\
E(y^2)&=-\frac{\partial}{\partial
\theta_2}c(\bm{\theta})\\
&=\frac{\theta_1^2}{4\theta_2^2}-\frac{1}{2\theta_2}\\
&=\mu^2+\sigma^2.
\end{align*}
\newpage
\subsubsection{Minimum Variance Unbiased Estimators}
In the simple case when there is just one parameter, the exponential
family has density or probability mass function
\[
f(\bm{y})=\exp\left[\theta T(\bm{y})+c(\theta)+d(\bm{y})\right].
\]
The score function is
\[
s(\theta)=T(\bm{y})+c^{\prime}\left( \theta\right)
\]
and so the Fisher information for $\theta$, which is the variance of
the score function is
\[
I(\theta)=\var T(\bm{y}).
\]
The Cram\'{e}r-Rao lower bound on the variance of an estimator
$S(\bm{y})$ which is unbiased for $-c^{\prime}\left( \theta\right)$
is
\[
\var S \ge \frac{(-c^{\prime\prime})^2}{I(\theta)}.
\]

Since the Fisher information  for $\theta$ can also be written as
$-c^{\prime\prime}$, we have
\[
\var S \ge \frac{I(\theta)^2}{I(\theta)}=I(\theta)=\var T.
\]
So, $T(\bm{y})$ is a minimum variance unbiased estimator of
$-c^{\prime}\left( \theta\right)$. A little more work will show that
only an unbiased estimator of $-c^{\prime}\left( \theta\right)$ can
achieve the Cram\'{e}r-Rao lower bound. For other functions of
$\theta$, the CR bound can't be used to show a minimum variance
property.
