\documentclass[addpoints, 12pt]{exam}
\usepackage{graphics}
\usepackage{amsfonts}
\usepackage{hyperref}
\usepackage{color}
\pagestyle{myheadings}
%\printanswers
\setlength{\parindent}{0ex}
\setlength{\textheight}{9in}
\setlength{\textwidth}{6in}
\setlength{\oddsidemargin}{0in}
\setlength{\evensidemargin}{0in}
\setlength{\topmargin}{-0.5cm}
\setlength{\parskip}{0.1in}
\newenvironment{refs}{\begin{list}{}{\setlength{\itemsep}{0cm}\setlength{\topsep}{0cm} \setlength{\leftmargin}{0.5cm}\setlength{\itemindent}{-0.5cm}\setlength{\parsep}{0.3cm}}
}{\end{list}}
\begin{document}
\vspace{5cm}
\begin{center}
\Large
\textbf{EDGE stats lab}
\normalsize
\fbox{\fbox{\parbox{5.5in}{\centering
The goal of this assignment is to increase your understanding of the key statistical concepts of population, sample, standard deviation, significance, power and effect size. It does this in the context of the two sample t-test. Through completion of this lab, it is expected you will become familiar with the EDGE (Excel Dataset GEnerator) spreadsheet (available at the EDGE website linked via Learn). The spreadsheet allows you to carry out experiments that will help you increase your understanding of these key statistical concepts. }}}
\end{center}
\begin{questions}
\question
\textbf{Generating a sample under a given model, carrying out t-tests and calculating confidence intervals.}
\\
\\\fbox{\parbox{5.5in}{\textbf{Scenario:} \centering Exam scores for all Loughborough University graduates who studied Psychology or Geography were gathered. For these populations, it was found that the scores were normally distributed with a mean score for Psychology students of 62\%, and a mean score for Geography students of 56\%. The standard deviation for Psychology students was 4\%, and the standard deviation for Geography students was 5\%. In this question, we will examine how to use data collected from a SRS from each of these populations to make inferences about these population parameters if we did not know them.}}
\vspace{0.5cm}
\begin{parts}
\part{Open the spreadsheet entitled `Two sample t-test'. Use this tool to generate exam scores for a SRS of size 15 from each population, Psychology students and Geography students, using the instructions at the top of the spreadsheet. List the values you generated on your answer sheet.}
\begin{solution}
The two sample t-test is a hypothesis test used to compare the means of two populations. In the spreadsheet enter $\mu_{1}$=62, $\mu_{2}$=56, $\sigma_{1}$=4 and $\sigma_{2}$=5. Then follow the instructions on the spreadsheet to generate a sample of size 15 in sample 1 and another sample of size 15 in sample 2.
\end{solution}
\part{We are interested in determining from our sample if exam scores for Psychology and Geography students are different. Determine the null and alternative hypotheses relevant to this question. Which one is true given the way you have chosen your data?}
\begin{solution}
The null hypothesis is that there is no difference in the mean scores of Psychology students and Geography students at Loughborough University, i.e. H$_{0}: \mu_{1} - \mu_{2} = 0$. This can also be written as H$_{0}: \mu_{1}=\mu_{2}$.
\\
\\The alternative hypothesis states that there is a difference in the mean scores of Psychology students and Geography students at Loughborough University, i.e. H$_{a}: \mu_{1} - \mu_{2} \ne 0$. This can also be written as H$_{a}: \mu_{1} \ne \mu_{2}$.
\\
In this setting, the two population means are different, so the alternative hypothesis is true.
\end{solution}
\part{Calculate the mean values of sample 1 and sample 2, $\hat{y}_{1}$ and $\hat{y}_{2}$, by hand. Check these values with those calculated by the spreadsheet in the green cells labelled $\hat{y}$.}
\begin{solution}
The mean values for sample 1 and 2 are calculated using the formulae:
\large
$$\hat{y}_{1}=\frac{\sum_{i=1}^{n}{y_{1i}}}{n_{1}}$$
$$\hat{y}_{2}=\frac{\sum_{i=1}^{n}{y_{2i}}}{n_{2}}$$
\normalsize
Your sample sizes are $n_{1}$=15 and $n_{2}$=15. The term $y_{1i}$ denotes the data points for sample 1 and $y_{2i}$ denotes the data points for sample 2.
\end{solution}
\part{Calculate the t-statistic by hand using the sample statistics produced in the green box. Check this against the result that is produced in the spreadsheet labelled `t-statistic'.}
\begin{solution}
The t-statistic is calculated using the formula:
\large
$$t=\frac{\hat{y}_{1}-\hat{y}_{2}}{\sqrt{\frac{s_{1}^{2}}{n_{1}}+\frac{s_{2}^{2}}{n_{2}}}}$$
\normalsize
Your sample sizes are $n_{1}$=15 and $n_{2}$=15. The values for $\hat{y}_{1}$, $\hat{y}_{2}$, $s_{1}$ and $s_{2}$ are produced in the green boxes in the spreadsheet, and these differ depending on the samples that you generated in part (a). These values are the sample means and standard deviations for the two samples.
\end{solution}
\part{Calculate the degrees of freedom by hand using the sample statistics produced in the green box. Check this against the result that is produced in the spreadsheet labelled `df by hand'.}
\begin{solution}
The degrees of freedom are calculated using the formulae below.
\begin{itemize}
\item{If $n_{1} = n_{2}$, $df = (n_{1}+n_{2})-2$}
\item{If $n_{1} \ne n_{2}$, $df = min(n_{1}-1, n{_2}-1)$}
\end{itemize}
For this scenario, as both $n_{1}$ and $n_{2}$ equal 15, $df = 15+15-2 = 28$.
\end{solution}
\part{Find the critical t-value using your degrees of freedom at the 0.05 significance level. There is a t-distribution table located in the spreadsheet for you to obtain your critical t-value. The value that you find may differ slightly from the one produced using the software.}
\begin{solution}
Use $\alpha$=0.05 when looking up the critical t-value in the t-table. In part (e) you should have found the degrees of freedom to be 28 and so using the t-table in sheet 6 you obtain the critical t-value of 2.048407.
\end{solution}
\begin{subparts}
\subpart{What can you conclude about the significance of your t-statistic?}
\subpart{What does this mean in terms of your hypotheses?}
\begin{solution}
If your t-statistic calculated in part (d) is $\geq$ the critical t-value then you can reject the null hypothesis and conclude that there is a significant difference between the mean scores for Psychology and Geography students.
\\
\\If your t-statistic calculated in part (d) is $<$ the critical t-value then you cannot reject the null hypothesis and so you can conclude that there is not a significant difference between the means scores for Psychology and Geography students.
\end{solution}
\end{subparts}
\part{Use your degrees of freedom, t-statistic and the t-distribution table to approximate the p-value.}
\begin{solution}
Using the t-distribution table in the spreadsheet find the row corresponding to 28 degrees of freedom. Using your t-statistic, look along this row and find which values your t-statistic lies between. Your p-value estimate then lies between the two corresponding $\alpha$ values.
\end{solution}
\begin{subparts}
\subpart{Compare your p-value estimate with the precise p-value calculated in the green cell labelled `p-value by hand'.}
\subpart{Comment on the significance of this precise p-value at a 95\% confidence level.}
\begin{solution}
If the p-value is $<$ 0.001 then it is extremely significant.
\\If the p-value is between 0.001 and 0.01 then it is very significant.
\\If the p-value is between 0.01 and 0.05 then it is significant.
\\If the p-value is $>$ 0.05 then it is not significant.
\\
\\If the p-value is $<$ 0.05 you can reject the null hypothesis and accept the alternative hypothesis that there is a significant difference between the mean scores for Psychology and Geography students.
\\If the p-value is $>$ 0.05 you cannot reject the null hypothesis. Therefore, you can conclude that there is not a significant difference between the means scores for Psychology and Geography students.
\end{solution}
\end{subparts}
\part{Calculate a 95\% confidence interval around the difference in means for your sample by hand. Check your answer with the confidence interval produced in the spreadsheet labelled `95\% C.I. around difference in means by hand'.}
\begin{solution}
Formula for 100(1-$\alpha$)\% confidence interval:
\large
$$\hat{y}_{1}-\hat{y}_{2} \pm t_{\alpha/2,n-1}\sqrt{\frac{s_{1}^{2}}{n_{1}}+\frac{s_{2}^{2}}{n_{2}}}$$
\normalsize
The values for $\hat{y}_{1}$, $\hat{y}_{2}$, $s_{1}$ and $s_{2}$ are produced in the green boxes in the spreadsheet. In part (f) you should have obtained the critical t-value of 2.048407. Inputing all these values will produce a 95\% confidence interval.
\end{solution}
\begin{subparts}
\subpart{Is 0 included in the interval?}
\begin{solution}
If the confidence interval includes 0 then it can be said that there is no significant difference between the mean scores for Psychology students and Geography students. If the confidence interval does not include 0 then it can be said there is a significant difference between the mean scores for Psychology students and Geography students.
\end{solution}
\subpart{How is this related to the p-value that you estimated?}
\end{subparts}
\end{parts}
\question
\textbf{Interpreting p-values and confidence intervals.}
\\
\\\fbox{\parbox{5.5in}{\centering Set the population mean for Psychology students, $\mu_{1}$, to equal the population mean for Geography students, $\mu_{2}$, i.e. H$_{0}: \mu_{1} - \mu_{2} = 0$. Delete your sample from question 1 by highlighting the first purple column, right clicking and selecting `Clear Contents'. Set $\mu_{1}$ and $\mu_{2}$ both equal to 60 and set $\sigma_{1}$ and $\sigma_{2}$ both equal to 5. Using the instructions in the spreadsheet, generate 20 datasets in the purple columns consisting of two samples, each of size 15. The p-values for each dataset will automatically be recorded in the yellow column labelled `P-value' near the bottom of the spreadsheet. The goal of this question is to investigate the relationship between p-values, significance level and confidence intervals.}}
\vspace{0.5cm}
\begin{parts}
\part{Insert a screenshot of the spreadsheet page here.}
\part{Is the null hypothesis of the two-sample t-test true or false for these samples?}
\part{Define p-value}
\part{Define significance level}
\part{How many of your p-values are significant at a 95\% confidence level? How many do you expect to be significant from the definition of confidence level?}
\begin{solution}
As we are using a 95\% confidence level under the null hypothesis, it is expected that 95\% of the samples will be non-significant (i.e. p$>$0.05) and 5\% will be significant (i.e. p$<$0.05).
\\For 20 samples, you would expect 19 to be non-significant and one to be significant.
\end{solution}
\part{How many p-values would you expect to be significant at the 95\% confidence level from 100 samples? How
many would you expect to be significant at the 99\% confidence level from 100 samples?}
\begin{solution}
You would expect to have 5 significant p-values at the 95\% level, and 1 at the 99\% level.
\end{solution}
\part{The spreadsheet will automatically plot a histogram of p-values. Explain how this compares to your expectations for the histogram from the definition of confidence level.}
\begin{solution}
From the definition, you expect the proportion of p-values in a particular bar to be the length of the interval
the bar represents. So you would expect 50\% of p-values to be between 0.5 and 1, 25\% between 0.25 and 0.5, etc.
\end{solution}
\part{The spreadsheet will automatically plot the p-values in a scatter plot. Is there any pattern? Discuss what you observe.}
\begin{solution}
Your scatter plot should be a random plot with no obvious pattern.
\end{solution}
\part{Define confidence interval.}
\part{The spreadsheet will automatically record the value for $\hat{y}_{1}-\hat{y}_{2}$, the lower bound of the confidence interval and the upper bound of the confidence interval in the yellow box at the bottom of the spreadsheet. Similarly, the spreadsheet will automatically plot the confidence intervals on a graph in the spreadsheet. How many of the confidence intervals include 0? How does this relate to the p-values? Discuss what you observe.}
\begin{solution}
\end{solution}
\end{parts}
\newpage
\question
\textbf{Experimenting with significance, sample size, standard deviation, effect size and power.}
\begin{center}
\fbox{\parbox{5.5in}{
Recall from the lecture notes:
\\When we make decisions in statistics, there are four possible scenarios. We can arrange these in a table:
\begin{center}
\resizebox{0.7\textwidth}{!}
{\includegraphics{table.png}}
\end{center}
}}
\end{center}
\begin{parts}
\part{In Question 2, which part of the table above was relevant?}
\begin{solution}
The first column was relevant because the null hypothesis was true.
\end{solution}
\part{Clear your datasets from question 2 by highlighting the purple columns, right clicking and selecting `Clear Contents'. Enter your values from question 1 into the blue cells: $\mu_{1}=62$, $\mu_{2}=56$, $\sigma_{1}=4$ and $\sigma_{2}=5$ and generate 20 datasets in the purple columns consisting of two samples, each of size 15. For each dataset, the p-value, the value for $\hat{y}_{1}-\hat{y}_{2}$ and the lower and upper bounds of the 95\% confidence interval will automatically be recorded in the yellow boxes. Take screenshots of the yellow boxes and graphs, and paste these into a Word document as you will need to compare these values with those produced in the following parts. }
\part{For this data, which part of the table above is relevant? Why?}
\begin{solution}
The second column is now relevant because the alternative hypothesis is true as we have set the two population
means equal to different numbers.
\end{solution}
\part{Define the power of a statistical test.}
\part{Now using the results in the spreadsheet, estimate the power of the t-test here against the effect size that we have: $\epsilon = \mu_1 - \mu_2 = 6$.
Also estimate the width of the confidence intervals as the average width of the confidence intervals you have obtained.}
\part{Clear the contents of the purple columns. Increase the \textbf{sample size} to 50 and generate 20 new datasets consisting of two samples, each of size 25. The p-values, the value for $\hat{y}_{1}-\hat{y}_{2}$ and the lower and upper bounds of the 95\% confidence interval will be recorded in the yellow boxes. Again, take screenshots of the yellow boxes and graphs, and paste these into the Word document. Now from the new results, estimate the power and the width of the confidence intervals. In the pink boxes on the spreadsheet state the change from part (a) in the following.}
\begin{subparts}
\subpart{Power estimate.}
\begin{solution}
Increase in power.
\end{solution}
\subpart{Width of the confidence interval.}
\begin{solution}
Decrease.
\end{solution}
\end{subparts}
\part{Clear the contents of the purple columns. Reset the sample size to 30, increase the \textbf{standard deviation} by setting $\sigma_{1}=10$ and $\sigma_{2}=12$, and generate 20 new datasets consisting of two samples, each of size 15. The spreadsheet will record the p-values, the value for $\hat{y}_{1}-\hat{y}_{2}$ and the lower and upper bounds of the 95\% confidence interval in the yellow boxes. Again, take screenshots of the yellow boxes and graphs, and paste these into the Word document. Estimate the power and width of confidence intervals in this situation. In the pink boxes on the spreadsheet state the change from part (a) in the following.}
\begin{subparts}
\subpart{Power estimate.}
\begin{solution}
Decrease in power.
\end{solution}
\subpart{Width of the confidence interval.}
\begin{solution}
Increase.
\end{solution}
\end{subparts}
\part{Clear the contents of the purple columns. Reset $\sigma_{1}$ to 4 and $\sigma_{2}$ to 5, increase the \textbf{effect size} to 12 by keeping $\mu_{1}=62$ and setting $\mu_{2}=50$, and generate 20 new datasets consisting of two samples, each of size 15. The p-values, the value for $\hat{y}_{1}-\hat{y}_{2}$ and the lower and upper bounds of the 95\% confidence interval will be recorded in the yellow boxes. Again, take a screenshot of the yellow boxes and paste into the Word document. Estimate the power and width of confidence intervals in this situation. In the pink boxes on the spreadsheet state the change from part (a) in the following.}
\begin{subparts}
\subpart{Power estimate.}
\begin{solution}
Increase in power.
\end{solution}
\subpart{Width of the confidence interval.}
\begin{solution}
No change.
\end{solution}
\end{subparts}
\part{For each of the cases in (a), (b), (c) and (d) use the Russ Lenth applet on the link below to calculate the power. Record the power you obtain from the applet for each case.
\\ \url{http://homepage.stat.uiowa.edu/~rlenth/Power/}
\\Do these results agree with your answers in the pink boxes for the change in power?}
\end{parts}
\question
Watch the `dance of the p-values' video:
\\ \url{http://www.youtube.com/watch?v=ez4DgdurRPg}
\begin{parts}
\part{What does it mean to say that a measurement is reliable?}
\part{Are p-values on a sample a reliable measure of the difference between two populations? Why or why not?}
\part{What advantages do confidence intervals have in this regard over p-values?}
\end{parts}
\end{questions}
\end{document}