\documentclass[compress, violet]{beamer}

\usepackage[utf8x]{inputenc}
%\usepackage{default}
\usepackage{verbatim}
\usepackage{graphicx}
\usepackage{mathrsfs}


\usepackage{geometry}
\usepackage{amsthm}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{tikz}
\usepackage{stmaryrd}
\usepackage{mathtools}

\usepackage{multicol} % Required for multiple columns
\setlength{\columnsep}{1.5em} % Slightly increase the space between columns
\setlength{\columnseprule}{0mm} % No horizontal rule between columns

\usepackage{verbatim}
\usepackage{wrapfig}
\usepackage{tikz} % Required for flow chart
\usepackage{etex}
\usepackage[all]{xy}

\usepackage{float}
\usepackage{array,multirow}
\usepackage{blkarray}
\usepackage{wasysym}
\usepackage{float}




\newcommand{\tab}{\hspace*{2em}}


\usepackage{beamerthemesplit}
\usetheme{Boadilla}
\usecolortheme[RGB={190,20,20}]{structure}

\xdefinecolor{darkred}{RGB}{190,20,20}

\newcommand{\compresslist}{ % Define a command to reduce spacing within itemize/enumerate environments, this is used right after \begin{itemize} or \begin{enumerate}
\setlength{\itemsep}{1pt}
\setlength{\parskip}{0pt}
\setlength{\parsep}{0pt}
}

\setcounter{MaxMatrixCols}{20}

\newcommand{\KK}{\mathbb{K}}
\newcommand{\ZZ}{\mathbb{Z}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\AF}{\mathbb{A}}
\newcommand{\GG}{\mathbb{G}}
\newcommand{\OO}{\mathcal{O}}
\newcommand{\SC}{\mathcal{C}}
\newcommand{\CS}{\mathcal{S}}
\newcommand{\LL}{\mathcal{L}}
\newcommand{\UU}{\mathcal{U}}
\newcommand{\SG}{\mathcal{G}}
\newcommand{\SF}{\mathcal{F}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\FQ}{\mathbb{F}_q}
\newcommand{\FP}{\mathbb{F}_p}
\newcommand{\BQ}{\overline{\mathbb{F}}_q}
\newcommand{\BP}{\overline{\mathbb{F}}_p}
\newcommand{\BF}{\overline{\mathbb{F}}}
\newcommand{\BK}{\overline{K}}

\newcommand{\Aut}{\operatorname{Aut}}
\newcommand{\Gal}{{\operatorname{Gal}}}
\newcommand{\Spec}[1]{{\operatorname{Spec}(#1)}}
\newcommand{\Hom}{{\operatorname{Hom}}}
\newcommand{\invlim}[1]{\lim\limits_{\overleftarrow{#1}}}
\newcommand{\dirlim}[1]{\lim\limits_{\overrightarrow{#1}}}
\newcommand{\Frob}{{\operatorname{F\hskip-1pt rob}}}
\newcommand{\frob}{{\operatorname{frob}}}
\newcommand{\GLtwo}{{\operatorname{GL_2}}}
\newcommand{\trace}{{\operatorname{Trace}}}
\newcommand{\End}{\operatorname{End}}
\newcommand{\car}{{\operatorname{char}}}
\newcommand{\Int}{\operatorname{Int}}

\newtheorem{proposition}{Proposition}
\newtheorem{Th}{Theorem}
\newtheorem{Cor}{Corollary}
\newtheorem{Def}{Definition}
\newtheorem{t1}{Theorem 1}
\newtheorem{t2}{Theorem 2}

\beamertemplatenavigationsymbolsempty
%\newtheorem{corollary}{Corollary}

%\definecolor{lightblue}{rgb}{0.145,0.6666,1} 
\definecolor{lightblue}{RGB}{0,153,153} % Defines the color used for content box headers
\definecolor{darkred}{RGB}{190,20,20}

\title{Writing Good Questions}
\author[CALM]{Matthew Eichhorn}
\institute{Cornell University}
\date{February 24, 2022}
\titlegraphic{\hspace*{0cm}\vspace*{0.5cm}\includegraphics[width=2cm]{Cornell-seal}}

\begin{document}
\maketitle

\begin{frame} \frametitle{Workshop Norms}
    \begin{itemize}
        \item Everyone has something to contribute to the discussion. 
        \begin{itemize}
            \item Give everyone an equal opportunity to participate.
            \item Be respectful of others’ ideas.
        \end{itemize}
        \vspace{10pt}
        \item We are all here to learn.
        \begin{itemize}
            \item Listen attentively to understand others' thoughts.
            \item Acknowledge that no one here has all of the answers. 
        \end{itemize}
        \vspace{10pt}
    \end{itemize}
    
    \vspace{10pt}
    \hspace{10pt} Is there anything else that we should add to this list?
    \vspace{20pt}
    
    \hspace{10pt} Also, feel free to interrupt at any time with questions. 
\end{frame}

\begin{frame} \frametitle{Why do we ask questions?}
\pause
    Questions are used as a tool of assessment. 
    \vspace{10pt}
    
    Two types of assessment:
    \vspace{10pt}
    
    \textbf{Formative:} Student responses guide learning process
    \vspace{5pt}
    
    \begin{quote}
        \small
        ``...\;instructors elicit, interpret, and use evidence about student achievement to make decisions about the next steps in instruction with the goal of improving instruction and student learning.'' [Black \& William, 2009]
    \end{quote}
    
    \vspace{10pt}

    \textbf{Summative:} Evaluates student proficiency with regard to the learning outcomes. 
\end{frame}

\begin{frame}{Formative Assessments}
    Questions are used to help \textbf{inform} instruction.
    
    \vspace{10pt}
    
    Some examples of formative assessment:
    \begin{itemize}
        \item Reading ``Quizzes''
        \item Polling Questions (clicker, letter cards)
        \item Misconception checking (spot the error)
        \item Post-lesson feedback (muddiest point, minute papers, exit tickets)
    \end{itemize}
    
    \vspace{20pt}
    
    \textbf{Do you have formative assessment strategies that you have found helpful in your classes?}
\end{frame}

\begin{frame}{Summative Assessments}
    What does a ``good'' exam question look like?
    
    \vspace{200pt}
\end{frame}

\begin{frame}{Activity (Part 1)}
    \textbf{Let's write a ``good question.''}
    
    \vspace{10pt} 
    \begin{itemize}
        \item Get together in groups of 2-3. It may be helpful to pair up with people who are teaching similar courses.
        
        \vspace{10pt} 
        \item Decide on a topic that you would like to assess. Write an exam question that focuses on this topic and incorporates ideas from the previous slide. 
        
        \vspace{10pt}
        \item Later in the workshop, you'll swap questions with another group for the second part of the activity.
    \end{itemize}
\end{frame}

\begin{frame}
    \vspace{30pt}
    \centering
    {\color{darkred} \Large{How can we describe the questions that we write?}}
\end{frame}

\begin{frame}{Bloom's Taxonomy}
    The (revised) Bloom's Taxonomy breaks down learning into a hierarchy of cognitive processes. 
    
    \begin{center}
        \includegraphics[scale=0.5]{taxonomy.jpg}
    \end{center}
\end{frame}

\begin{frame}{Types of Knowledge}
    Anderson et.al. (2001) introduce a second dimension for classifying questions, the knowledge dimension. They suggest that there are 4 types of knowledge:
    
    \vspace{10pt}
    \begin{itemize}
        \item \textbf{Factual:} Having an acquaintance with terminology and specific facts of a discipline.
        \item \textbf{Conceptual:} Understanding the interrelationships among the basic elements within a theory. 
        \item \textbf{Procedural:} Being able to appropriately apply subject-specific methods, techniques and algorithms. 
        \item \textbf{Metacognitive:} The ability to reason about how one understands and reasons about the subject.
    \end{itemize}
\end{frame}

\begin{frame}{Categorizing a Question}
   We can visualize these dimensions with the following chart. 
   \vspace{20pt} 
   
   \begin{minipage}{0.05\textwidth}
        \hspace{20pt}
        \scriptsize
        \rotatebox{90}{\textbf{Knowledge Dimension} \hspace{20pt}}
   \end{minipage}
   \begin{minipage}{0.94\textwidth}
   \renewcommand{\arraystretch}{2}
   {\scriptsize
   \hspace{120pt}\textbf{Cognitive Process Dimension}
   \vspace{10pt}
   
   \begin{tabular}{|c|c|c|c|c|c|c|}
        \hline & Remember & Understand & Apply & Analyze & Evaluate & Create \\ \hline 
        Factual & & & & & & \\ \hline 
        Conceptual & & & & & & \\ \hline 
        Procedural & & & & & & \\ \hline 
        Metacognitive & & & & & & \\ \hline 
   \end{tabular}}
   \end{minipage}
   
   \vspace{20pt}
   
   Such a chart can help ensure that the questions on an exam cover a broad spectrum of cognitive processes and assess student's attainment of various types of knowledge.  
\end{frame}

\begin{frame}{An Example}
    Consider the following linear algebra exam question:
    \begin{block}{}
        Let $A$ be an $n \times n$ matrix. Here, you'll reason about the the relationship between eigenvectors of $A$ and $A^2$.
        
        \begin{enumerate}
            \item Suppose that $v$ is an eigenvector of $A$ with eigenvalue $\lambda$. Argue that $v$ is also an eigenvector of $A^2$. What is its associated eigenvalue?
            \item Give an example of a matrix $A$ and a vector $v$ such that $v$ is an eigenvector of $A^2$ but not $A$. 
        \end{enumerate}
    \end{block}
    Where would you place this question in the chart?
    \vspace{6pt}
    
    \textbf{Cognitive Process:} 
    
    \begin{center}
        Remember \quad Understand \quad Apply \quad Analyze \quad Evaluate \quad Create
    \end{center}
    
    \textbf{Knowledge:} 
    
    \begin{center}
        Factual \qquad Conceptual \qquad Procedural \qquad Metacognitive
    \end{center}
\end{frame}


\begin{frame}{Activity (Part 2)}
    \textbf{Let's analyze a ``good question.''}

    \vspace{6pt}
    \begin{itemize} 
        \item Swap questions with another nearby group.
        
        \vspace{10pt} 
        \item Read through the other group's question, and think about how you would solve it during an exam.
        
        \vspace{10pt}
        \item Decide where you would place the question in the chart. 
    \end{itemize}
    
    \vspace{30pt}
    \textbf{If you have time:}
    
    \vspace{10pt}
    Think about how you could adjust the question to move it to different boxes in the chart. 
\end{frame}

\begin{frame}{Discussion Questions}
    \begin{itemize}
        \item Which cognitive processes/types of knowledge are the easiest to incorporate into an exam? The hardest? 
        \vspace{10pt}
        \item Were there any rows/columns in the chart that you felt were not appropriate to include on your exam? Why?
        \vspace{10pt}
        \item Do you think that filling out a diagram like this will be useful to you when writing exams in the future? Why or why not?
        \vspace{10pt}
        \item Are there other strategies that you employ when writing questions for an exam?
        \vspace{20pt}
    \end{itemize}
\end{frame}

\begin{frame}{References}
    \begin{itemize}
        \small 
        \item Abell, M.L., et.al., MAA Instructional Practices Guide: Project Leadership Team.. Mathematical Assn America, 2018.
        \vspace{5pt}
        \item Black, P., Wiliam, D. Developing the theory of formative assessment. Educ Asse Eval Acc 21, 5 (2009). https://doi.org/10.1007/s11092-008-9068-5
        \vspace{5pt}
        \item Anderson, L. W., \& Krathwohl, D. R. (2001). A Taxonomy for Learning, Teaching and Assessing: A Revision of Bloom’s Taxonomy of Educational Objectives: Complete Edition. New York: Longman.
    \end{itemize}
    
    \vspace{20pt}

    Credit also to Prairie Wentworth-Nice, who led a similar teaching seminar a couple years ago that served as inspiration for this workshop.
\end{frame}

\begin{frame}{Bonus: Giving Good Examples}

By nature, math courses are definition-heavy. Examples are how students digest these definitions. When writing examples:

\vspace{10pt}

\begin{itemize}
    \item \textbf{Start simple:} The first examples should be very basic (trivial) and familiar to students.
    \item \textbf{Eliminate redundancy:} Each example should give students something new to think about, or highlight another aspect of the definition.
    \item \textbf{Include non-examples:} These are important to help delineate the ``boundaries'' of definitions.
    \item \textbf{Make connections:} Having running examples throughout the course can help students recall previous material and see how topics fit together. 
\end{itemize}
    
\end{frame}




\end{document}