%\VignetteDepends{}
%\VignetteIndexEntry{Slides for using SQL from R}
%\VignettePackage{AdvancedR2011}

\documentclass{beamer}

\usepackage{BioconductorSlides}
\SweaveOpts{keep.source=TRUE}

\title{Using Databases in R}
\author{Marc Carlson \\ Fred Hutchinson Cancer Research Center}
\date{17-18 February, 2011}

\begin{document}

<<setup, echo=FALSE>>=
options(width = 40)
@ 

\frame{\titlepage}

\section[Outline]{}
\frame{\tableofcontents}

% \section{Introduction}
% \begin{frame}{frame title}
% \end{frame}

% \begin{frame}[fragile]{Subsetting and non-specific filtering}
% <<code-block>>=
% ## use [fragile] in frames with Sweave
% x <- 1:10
% @ 
% \end{frame}

% \section{Resources}

% \begin{frame}{Resources}
%   %% Include key resources -- urls, books, ...
% \end{frame}




%-----------------------------------------------
%-----------------------------------------------
\section{Introduction}



%-----------------------------------------------
\begin{frame}{Relational Databases}
\begin{block}{Relational database basics}
  \begin{itemize}
    \item Data stored in \textit{tables}
    \item Tables related through \textit{keys}
    \item Relational model called a \textit{schema}
    \item Tables designed to avoid redundancy
  \end{itemize}    
\end{block}
\begin{block}{Beneficial uses by R packages}
  \begin{itemize}
    \item Out-of-memory data storage
    \item Fast access to data subsets
    \item Databases accessible by other software
  \end{itemize}
\end{block}    
\end{frame}



%-----------------------------------------------
\begin{frame}{Uses of Relational Databases in Bioconductor}
\begin{block}{Annotation packages}
  \begin{itemize}
    \item Organism, genome (e.g. \Rpackage{org.Hs.eg.db})
    \item Microarray platforms (e.g. \Rpackage{hgu95av2.db})
    \item Homology (e.g. \Rpackage{hom.Hs.inp.db})
  \end{itemize}    
\end{block}
\begin{block}{Software packages}
  \begin{itemize}
    \item Transcript annotations (e.g. \Rpackage{GenomicFeatures})
    \item NGS experiments (e.g. \Rpackage{Genominator})
    \item Annotation infrastructure (e.g. \Rpackage{AnnotationDbi})
  \end{itemize}
\end{block}    
\end{frame}



%-----------------------------------------------
%-----------------------------------------------

%% Slides to introduce GenomicFeatures and discuss what it does. (and
%% why we store DBs on the disk) (10-15 slides)

% \section{Example Database: The \Rpackage{GenomicFeatures} Package}
%% Slides to introduce GenomicFeatures and discuss what it does. (and

% \begin{frame}{The \Rpackage{GenomicFeatures} package}
%   What is does
%   \begin{itemize}
%     \item Builds databases on the fly as needed.
%     \item Supported sources: UCSC tracks, biomaRt, or custom.
%     \item Has accessor methods to make it easy to use this data.
%   \end{itemize}
%   \vskip12pt
%   Why it was built
%   \begin{itemize}
%     \item Needed a generic way to support access to multiple resources
%     \item Users needed reproducible data for research
%     \item Data needed to be formatted in a convenient way
%   \end{itemize}
%   \vskip12pt
%   Why it uses a database
%   \begin{itemize}
%     \item Annotation data is highly relational
%     \item Other annnotation data is also stored in SQLite databases
%     \item Allows access from disc (which can free up memory)
%   \end{itemize}
% \end{frame}



%-----------------------------------------------
\begin{frame}{What do I mean by relational?}
  \begin{figure}
    \includegraphics[width=4.25in]{database/transcriptschema.png}
  \end{figure}
\end{frame}



% %-----------------------------------------------
% \begin{frame}[fragile]{From R it looks pretty simple}
% \begin{block}{Making a TranscriptDb object}
% <<makeUCSC, eval=FALSE>>=
% library(GenomicFeatures)
% mm9KG <-
%   makeTranscriptDbFromUCSC(genome = "mm9",
%                            tablename = "knownGene")
% @
% \end{block}
% <<preload, echo=FALSE>>=
% library(GenomicFeatures)
% if(!exists("mm9KG",mode="TranscriptDb")) {
%     mm9KG <-
%       loadFeatures(system.file("extdata","mm9KG.sqlite",
%                                 package = "AdvancedR2011"))
% }
% @   
% \begin{block}{Saving and Loading}
% <<saveAndLoad, eval=FALSE>>=
% saveFeatures(mm9KG, file="mm9KG.sqlite")
% mm9KG <-
%   loadFeatures(system.file("extdata", "mm9KG.sqlite",
%                            package = "AdvancedR2011"))
% @
% \end{block}
% \end{frame}


% %-----------------------------------------------
% \begin{frame}[fragile, shrink=15]{There are even accessors etc.}
% <<transcripts>>=
% head(transcripts(mm9KG), 3)
% @ 
% \end{frame}


% %-----------------------------------------------
% \begin{frame}[fragile]{But what actually happens when we do this?}
% <<transcriptsDetails,eval=FALSE>>=
% options(verbose=TRUE)
% txs <- transcripts(mm9KG)
% @ 
% \begin{verbatim}
% SQL QUERY: SELECT tx_chrom, tx_start, tx_end, tx_strand,
% transcript._tx_id AS tx_id , tx_name FROM transcript
% ORDER BY tx_chrom, tx_strand, tx_start, tx_end
% \end{verbatim}

% \begin{block}{}
%   Notice how the database query is pretty simple?
%   \begin{itemize}
%     \item DB joins promote flexible access
%     \item BUT: there can be a cost if using A LOT of joins
%     \item Therefore (in this case) a hybrid approach:
%       Retrieve relevant records and subset in R
%   \end{itemize}      
% \end{block}
% \end{frame}


% %-----------------------------------------------
% \begin{frame}{Remember: our DB schema has more than 1 table}
%   \begin{figure}
%     \includegraphics[width=4.25in]{database/transcriptschema.png}
%   \end{figure}
% \end{frame}



%-----------------------------------------------
%-----------------------------------------------
%% I should probably show people how to see the schemas before I get
%% to this next part...  OR I can show them this next.

\section{Basic SQL}
%% Slides to teach some basic SQL so that people can follow along (10 slides)

\begin{frame}{SQL in 3 slides}
  Structured Query Language (SQL) is the most common language
  for interacting with relational databases.
\end{frame}

%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{Database Retrieval}
\begin{block}{Single table selections}
  \begin{verbatim}
    SELECT * FROM gene;
    SELECT gene_id, gene._tx_id FROM gene;

    SELECT * FROM gene WHERE _tx_id=49245;
    SELECT * FROM transcript WHERE tx_name LIKE '%oap.1'; 
  \end{verbatim}
\end{block}
\begin{block}{Inner joins}
  \begin{verbatim}
    SELECT gene.gene_id,transcript._tx_id
      FROM gene, transcript
      WHERE gene._tx_id=transcript._tx_id;

    SELECT g.gene_id,t._tx_id 
      FROM gene AS g, transcript AS t
      WHERE g._tx_id=t._tx_id
      AND t._tx_id > 10;
  \end{verbatim}
\end{block}
\end{frame}

%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{Database Modifications}
\begin{block}{CREATE TABLE}
  \begin{verbatim}
    CREATE TABLE foo (
      id INTEGER,
      string TEXT
    );
  \end{verbatim}
\end{block}
\begin{block}{INSERT}
  \begin{verbatim}
    INSERT INTO foo (id, string) VALUES (1,"bar");
  \end{verbatim}
\end{block}
\begin{block}{CREATE INDEX}
  \begin{verbatim}
    CREATE INDEX fooInd1 ON foo(id);
  \end{verbatim}
\end{block}
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{Some extra tricks}
\begin{block}{You can use AS to rename parts of a query}
  \begin{verbatim}
    SELECT * FROM gene AS g;
  \end{verbatim}
\end{block}
\begin{block}{You can use ORDER BY to ensure that things are sorted}
  \begin{verbatim}
    SELECT * FROM gene ORDER BY gene_id;
  \end{verbatim}
\end{block}
\begin{block}{You can use () to indicate sub-queries. This can be useful when you have more complex queries to express or if you need to combine different kinds of joins into a single query.}
  \begin{verbatim}
    SELECT * FROM transcript as t, (SELECT * FROM gene) AS g 
     WHERE g._tx_id = t._tx_id;
  \end{verbatim}
\end{block}
\end{frame}


% DEMO: demonstrate the SQL statements on the test DB.

%-----------------------------------------------
\begin{frame}[fragile]{It's sometimes a good idea to use a stand-alone client}
  \begin{itemize}
    \item SQLite3 demo
  \end{itemize}  
\end{frame}




%-----------------------------------------------
%-----------------------------------------------
\section{Using SQL from within R}
%% Slides to introduce DBI, RSQLite and how these are used to: 1)
%% build DBs, and 2) access the details of this database.(10-15 slides)


%-----------------------------------------------
\begin{frame}[fragile]{The \Rpackage{DBI} package}
  \begin{itemize}
    \item Provides a nice generic access to databases in R
    \item Many of the functions are convenient and simple to use
  \end{itemize}  
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{Some popular DBI functions}
<<driverUsageAndListTables>>=
library(RSQLite) #loads DBI too, (but we need both)
drv <- SQLite()
con <- dbConnect(drv, dbname=system.file("extdata", 
                 "mm9KG.sqlite", package="AdvancedR2011"))
dbListTables(con)
dbListFields(con,"transcript")
@ 
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{The dbGetQuery approach}
<<dbGetQuery>>=
dbGetQuery(con, "SELECT * FROM transcript LIMIT 3")
@ 
\end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{The dbSendQuery approach}
If you use result sets, you also need to put them away
<<dbSendQuery>>=
res <- dbSendQuery(con, "SELECT * FROM transcript")
fetch(res, n= 3)
dbClearResult(res)
@ 
Calling fetch again will get the next three results.
This allows for simple iteration.
\end{frame}


% %-----------------------------------------------
% \begin{frame}[fragile]{Hypothetically:}
% Select the exons from the minus strand of chromosome 9.
% <<exercise1,echo=FALSE>>=
% sql <- "SELECT * FROM exon WHERE exon.exon_strand='-' 
%         AND exon.exon_chrom='chr9'"
% res <- dbGetQuery(con, sql)
% @ 
% \end{frame}


%-----------------------------------------------
\begin{frame}[fragile,shrink=15]{Setting up a new DB}
  First, lets close the connection to our other DB:
<<dbDisconnect>>=
   dbDisconnect(con)
@ 
  Then lets make a new database.
  Notice that we specify the database name with "dbname" 
  This allows it to be written to disc instead of just memory.
<<makeNewDb>>=
drv <- SQLite()
con <- dbConnect(drv, dbname="myNewDb.sqlite")
@ 
Once you have this, you may want to make a new table
<<CreateNewTable>>=
dbGetQuery(con, "CREATE Table foo (id INTEGER, string TEXT)")
@ 

\end{frame}


% % -----------------------------------------------
% \begin{frame}[fragile]{Hypothetically:}
%   Create a database and then put a table in it called genePheno to store the genes mutated and a phenotypes associated with each.  Plan for genePheno to hold the following gene IDs and phenotypes (as a toy example):
% \begin{verbatim}
% data = data.frame(id=c(69773,20586,258822,18315),
%                   string=c("Dead",
%                            "Alive",
%                            "Dead",
%                            "Alive"),
%                   stringsAsFactors=FALSE)
% \end{verbatim}
% <<exercise2,echo=FALSE>>=
% drv <- dbDriver("SQLite")
% dbcon <- dbConnect(drv, dbname="myExDb.sqlite")
% cval <- dbGetQuery(dbcon, 
%                    "CREATE Table genePheno 
%                     (id INTEGER, string TEXT)")
% @ 
% \end{frame}


%-----------------------------------------------
\begin{frame}[fragile]{The \Rpackage{RSQLite} package}
  \begin{itemize}
    \item Provides SQLite access for R
    \item Much better support for complex inserts
  \end{itemize}  
\end{frame}




%-----------------------------------------------
\begin{frame}[fragile]{Prepared queries}
<<LabelledPreparedQueries>>=
data <- data.frame(c(226089,66745),
                  c("C030046E11Rik","Trpd52l3"), 
                  stringsAsFactors=FALSE)
names(data) <- c("id","string")
sql <- "INSERT INTO foo VALUES ($id, $string)"
dbBeginTransaction(con)
dbGetPreparedQuery(con, sql, bind.data = data)
dbCommit(con)
@ 
Notice that we want strings instead of factors in our data.frame
\end{frame}




% % -----------------------------------------------
% \begin{frame}[fragile]{Hypothetically:}
%   Now take a moment to insert that data into your database.
% <<exercise3,echo=FALSE>>=
% data <- data.frame(id=c(69773,20586,258822,18315),
%                    string=c("Dead",
%                             "Alive",
%                             "Dead",
%                             "Alive"),
%                    stringsAsFactors=FALSE)
% sql <- "INSERT INTO genePheno VALUES ($id,$string)"
% bval <- dbBeginTransaction(dbcon)
% gval <- dbGetPreparedQuery(dbcon, sql, bind.data = data)
% cval <- dbCommit(dbcon)
% @ 
% \end{frame}





%-----------------------------------------------
\begin{frame}[fragile]{in SQLite, you can ATTACH Dbs}
  The (SQLite specific) SQL that we want looks quite simple:
  \begin{verbatim}
    ATTACH "mm9KG.sqlite" AS db;
  \end{verbatim}  
  So we just need to do something like this:
<<ATTACH>>=
db <- system.file("extdata", "mm9KG.sqlite", 
                  package="AdvancedR2011")
dbGetQuery(con, sprintf("ATTACH '%s' AS db",db))
@ 
\end{frame}



%-----------------------------------------------
\begin{frame}[fragile, shrink=15]{You can join across attached Dbs}
  The SQL this looks like:
  \begin{verbatim}
    SELECT * FROM db.gene AS dbg, foo AS f 
    WHERE dbg.gene_id=f.id;
  \end{verbatim}  
  Then in R:
<<ATTACHJoin>>=
  sql <- "SELECT * FROM db.gene AS dbg, 
          foo AS f WHERE dbg.gene_id=f.id"
  res <- dbGetQuery(con, sql)
  res
@ 
\end{frame}



%% take out the final exercises (for now)

% % -----------------------------------------------
% \begin{frame}[fragile, shrink=15]{Your turn part 4}
%   Now create a cross join to your database and extract the \_tx\_id 's from the gene table there using your gene IDs as a foreign key.
% <<exercise4,echo=FALSE>>=
%   #1st attach
%   db <- system.file("extdata", "mm9KG.sqlite", 
%                     package="AdvancedR2011")
%   att <- dbGetQuery(dbcon, sprintf("ATTACH '%s' AS db",db))
%   #then select
%   sql <- "SELECT * FROM db.gene AS dbg, 
%           genePheno AS gp WHERE dbg.gene_id=gp.id"
%   res <- dbGetQuery(dbcon, sql)
% @ 
% \end{frame}





% %-----------------------------------------------
% \begin{frame}[fragile, shrink=15]{Your turn part 5}
%  Now conect your cross join to the transcript table in the database and extract the fields from that table while still using your gene IDs as a foreign key.
% <<exercise5,echo=FALSE>>=
%   sql <- "SELECT * FROM db.gene AS dbg, 
%           genePheno AS gp, transcript AS t 
%           WHERE dbg.gene_id=gp.id
%           AND dbg._tx_id=t._tx_id"
%   res <- dbGetQuery(dbcon, sql)
% @ 
% \end{frame}




%-----------------------------------------------
\begin{frame}
The End.
\end{frame}



% %-----------------------------------------------
% \begin{frame}
% No seriously, that was it.  

% There is nothing "back there" that you need to see...

% That was the end.
% \end{frame}




% %-----------------------------------------------
% \begin{frame}[fragile, shrink=15]{XML}
%  Using XML in R: the XML package (using xpath)
 
% <<XML>>=
%   ## 1st assemble a URL to use NCBIs web services
%   entrezGenes = c(1,100008564)
%   library(XML)
%   baseUrl <- "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?"
%   xsep <- paste(entrezGenes, collapse=",")
%   url <- paste(baseUrl,"db=gene&id=",xsep,"&retmode=xml", sep="")
%   url

%   ## Then parse this XML for use by xpathApply
%   EGSet <- xmlParse(url)
%   ## Then you can use xpathApply() to parse using xpath expressions
%   ## xmlValue() returns the contents of that tag
%   speciesNames <- unlist(xpathApply(EGSet, "//Org-ref_taxname", xmlValue))
%   speciesNames
% @ 
% \end{frame}




<<cleanup,echo=FALSE,results=hide>>=
 file.remove("myNewDb.sqlite")
@ 




\end{document}
