├── R ├── .gitignore ├── README.md └── getting-started.R ├── RMarkdown ├── .gitignore ├── figure │ └── viz-1.png ├── README.md ├── literate-programming-demo.R ├── literate-programming-demo.Rmd └── literate-programming-demo.md ├── .gitignore ├── figure └── result.png ├── sql ├── record-and-call-counts-by-reference.sql ├── variant-level-data-for-brca1.sql ├── sample-variant-counts-for-brca1.sql └── sample-level-data-for-brca1.sql ├── javascript ├── README.md └── index.html ├── CONTRIBUTING.rst ├── README.md └── LICENSE /R/.gitignore: -------------------------------------------------------------------------------- 1 | .httr-oauth 2 | -------------------------------------------------------------------------------- /RMarkdown/.gitignore: -------------------------------------------------------------------------------- 1 | .httr-oauth 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.Rproj 2 | RMarkdown/*.html 3 | .Rproj.user 4 | .Rhistory 5 | .RData 6 | .Rapp.history 7 | .idea 8 | -------------------------------------------------------------------------------- /figure/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlegenomics/getting-started-bigquery/master/figure/result.png -------------------------------------------------------------------------------- /RMarkdown/figure/viz-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/googlegenomics/getting-started-bigquery/master/RMarkdown/figure/viz-1.png -------------------------------------------------------------------------------- /sql/record-and-call-counts-by-reference.sql: -------------------------------------------------------------------------------- 1 | #standardSQL 2 | -- 3 | -- Count the number of records and nested calls for each reference. 4 | -- 5 | SELECT 6 | reference_name, 7 | COUNT(reference_name) AS num_records, 8 | COUNT(call.call_set_name) AS num_calls 9 | FROM 10 | `@THE_TABLE` v, v.call call 11 | GROUP BY 12 | reference_name 13 | ORDER BY 14 | reference_name 15 | -------------------------------------------------------------------------------- /sql/variant-level-data-for-brca1.sql: -------------------------------------------------------------------------------- 1 | #standardSQL 2 | -- 3 | -- Retrieve variant-level information for BRCA1 variants. 4 | -- 5 | SELECT 6 | reference_name, 7 | start, 8 | `end`, 9 | reference_bases, 10 | ARRAY_TO_STRING(v.alternate_bases, ',') AS alts, 11 | quality, 12 | ARRAY_TO_STRING(v.filter, ',') AS filter, 13 | ARRAY_TO_STRING(v.names, ',') AS names, 14 | ARRAY_LENGTH(v.call) AS num_samples 15 | FROM 16 | `@THE_TABLE` v 17 | WHERE 18 | reference_name IN ('17', 'chr17') 19 | AND start BETWEEN 41196311 AND 41277499 # per GRCh37 20 | -- Skip non-variant segments. 21 | AND EXISTS (SELECT alt FROM UNNEST(v.alternate_bases) alt WHERE alt NOT IN ("", "<*>")) 22 | ORDER BY 23 | start, 24 | alts 25 | -------------------------------------------------------------------------------- /sql/sample-variant-counts-for-brca1.sql: -------------------------------------------------------------------------------- 1 | #standardSQL 2 | -- 3 | -- Sample variant counts within BRCA1. 4 | -- 5 | WITH brca1_calls AS ( 6 | SELECT 7 | reference_name, 8 | start, 9 | `end`, 10 | reference_bases, 11 | ARRAY_TO_STRING(v.alternate_bases, ',') AS alts, 12 | call.call_set_name, 13 | call.genotype[SAFE_ORDINAL(1)] AS first_allele, 14 | call.genotype[SAFE_ORDINAL(2)] AS second_allele 15 | FROM 16 | `@THE_TABLE` v, v.call call 17 | WHERE 18 | reference_name IN ('chr17', '17') 19 | AND start BETWEEN 41196311 AND 41277499 # per GRCh37 20 | ) 21 | 22 | SELECT 23 | call_set_name, 24 | COUNT(call_set_name) AS variant_count 25 | FROM brca1_calls 26 | WHERE 27 | first_allele > 0 OR second_allele > 0 28 | GROUP BY 29 | call_set_name 30 | ORDER BY 31 | call_set_name 32 | -------------------------------------------------------------------------------- /sql/sample-level-data-for-brca1.sql: -------------------------------------------------------------------------------- 1 | #standardSQL 2 | -- 3 | -- Retrieve sample-level information for BRCA1 variants. 4 | -- 5 | SELECT 6 | reference_name, 7 | start, 8 | `end`, 9 | reference_bases, 10 | ARRAY_TO_STRING(v.alternate_bases, ',') AS alts, 11 | quality, 12 | ARRAY_TO_STRING(v.filter, ',') AS filters, 13 | ARRAY_TO_STRING(v.names, ',') AS names, 14 | call.call_set_name, 15 | (SELECT STRING_AGG(CAST(gt AS STRING)) from UNNEST(call.genotype) gt) AS genotype 16 | FROM 17 | `@THE_TABLE` v, v.call call 18 | WHERE 19 | reference_name IN ('17', 'chr17') 20 | AND start BETWEEN 41196311 AND 41277499 # per GRCh37 21 | -- Skip non-variant segments. 22 | AND EXISTS (SELECT alt FROM UNNEST(v.alternate_bases) alt WHERE alt NOT IN ("", "<*>")) 23 | ORDER BY 24 | start, 25 | alts, 26 | call_set_name 27 | -------------------------------------------------------------------------------- /RMarkdown/README.md: -------------------------------------------------------------------------------- 1 | # Getting started in RMarkdown 2 | 3 | 1. Read the rendered version of the RMarkdown [literate-programming-demo.md](./literate-programming-demo.md) 4 | 5 | 2. Execute the code 6 | chunk by chunk in RStudio [literate-programming-demo.Rmd](./literate-programming-demo.Rmd) 7 | or 8 | line by line in R [literate-programming-demo.R](./literate-programming-demo.R) 9 | 10 | ### Notes 11 | 12 | [literate-programming-demo.md](./literate-programming-demo.md) was created from [literate-programming-demo.Rmd](./literate-programming-demo.Rmd) via 13 | ``` 14 | library(knitr) 15 | knit("./literate-programming-demo.Rmd") 16 | ``` 17 | 18 | [literate-programming-demo.R](./literate-programming-demo.R) was created from [literate-programming-demo.Rmd](./literate-programming-demo.Rmd) via 19 | ``` 20 | library(knitr) 21 | purl("./literate-programming-demo.Rmd", documentation=1) 22 | ``` 23 | -------------------------------------------------------------------------------- /R/README.md: -------------------------------------------------------------------------------- 1 | # Getting started in R 2 | 3 | 1. If needed, install the [bigrquery](https://github.com/hadley/bigrquery#authentication) package. 4 | 5 | ``` 6 | # The currently released version 0.3.0 does not yet have the parameter 7 | # to use Standard SQL instead of Legacy SQL, so we install from github. 8 | library(devtools) 9 | install_github('rstats-db/bigrquery') 10 | ``` 11 | 12 | 2. From the R prompt set the working directory and run the script: 13 | 14 | ``` 15 | setwd("path/to/getting-started-bigquery/R") 16 | source("getting-started.R") 17 | GettingStarted("your-project-id") 18 | ``` 19 | 20 | # Troubleshooting 21 | 22 | * If you see an `Error: Access Denied` response from the `GettingStarted` function, 23 | that means your Project ID is invalid. Follow the instructions in the top level 24 | [README](https://github.com/googlegenomics/getting-started-bigquery) 25 | to get a valid Project ID. 26 | -------------------------------------------------------------------------------- /javascript/README.md: -------------------------------------------------------------------------------- 1 | # Getting started in javascript 2 | 3 | | Deprecated | 4 | |--------------------| 5 | | This example is deprecated. The Google Genomics v1beta2 API is scheduled for turndown on August 15, 2016. When the API is turned down, this example will cease to function. | 6 | | For an example of accessing the Google Genomics v1 API from Javascript, see the [Getting Started with the API](https://github.com/googlegenomics/getting-started-with-the-api/) repository. | 7 | 8 | 1. Javascript requires a client ID in addition to the Project ID you already set up: 9 | 10 | 1. First select your [BigQuery enabled project](https://console.developers.google.com/project/_/apiui/credential) 11 | in the Google Developers Console. 12 | 13 | 2. Once you are redirected to the **Credentials** tab, click **Create new Client ID** under 14 | the OAuth section. 15 | 16 | 3. Set **Application type** to **Web application**, and change 17 | the **Authorized javascript origins** to `http://localhost:8000` 18 | 19 | 4. Click the **Create Client ID** button 20 | 21 | 5. From the newly created **Client ID for web application**, save the `Client ID` 22 | value. 23 | 24 | 25 | 2. Run a http server (this requires [python](https://www.python.org/download/)): 26 | ``` 27 | cd getting-started-bigquery/javascript 28 | python -m SimpleHTTPServer 8000 29 | ``` 30 | 31 | 3. Using the Client ID value that you made in step 1, view the code at: 32 | http://localhost:8000#your-client-id-goes-here 33 | -------------------------------------------------------------------------------- /R/getting-started.R: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the 'License'); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an 'AS IS' BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # For more information see https://github.com/hadley/bigrquery 16 | library(bigrquery) 17 | 18 | # Provide a table if you want to query your own data 19 | DisplayAndDispatchQuery <- function(project, queryUri, 20 | table="genomics-public-data.platinum_genomes.variants") { 21 | # Read in the SQL from a file or URL. 22 | querySql <- readChar(queryUri, nchars=1e6) 23 | # Find and replace the table name placeholder with the table name. 24 | querySql <- sub("@THE_TABLE", table, querySql, fixed=TRUE) 25 | # Display the updated SQL. 26 | cat(querySql) 27 | # Dispatch the query to BigQuery for execution. 28 | query_exec(querySql, project, useLegacySql = FALSE) 29 | } 30 | 31 | GettingStarted <- function(project) { 32 | result <- DisplayAndDispatchQuery(project, "../sql/record-and-call-counts-by-reference.sql") 33 | 34 | cat("\nSummary:\n") 35 | print(summary(result)) 36 | 37 | cat("\nResult:\n") 38 | print(result) 39 | } 40 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | How to contribute 2 | =================================== 3 | 4 | First of all, thank you for contributing! 5 | 6 | The mailing list 7 | ---------------- 8 | 9 | For general questions or if you are having trouble getting started, try the 10 | `Google Genomics Discuss mailing list `_. 11 | It's a good way to sync up with other people who use googlegenomics including the core developers. You can subscribe 12 | by sending an email to ``google-genomics-discuss+subscribe@googlegroups.com`` or just post using 13 | the `web forum page `_. 14 | 15 | 16 | Submitting issues 17 | ----------------- 18 | 19 | If you are encountering a bug in the code or have a feature request in mind - file away! 20 | 21 | 22 | Submitting a pull request 23 | ------------------------- 24 | 25 | If you are ready to contribute code, Github provides a nice `overview on how to create a pull request 26 | `_. 27 | 28 | Some general rules to follow: 29 | 30 | * Do your work in `a fork `_ of this repo. 31 | * Create a branch for each update that you're working on. 32 | These branches are often called "feature" or "topic" branches. Any changes 33 | that you push to your feature branch will automatically be shown in the pull request. 34 | * Keep your pull requests as small as possible. Large pull requests are hard to review. 35 | Try to break up your changes into self-contained and incremental pull requests. 36 | * The first line of commit messages should be a short (<80 character) summary, 37 | followed by an empty line and then any details that you want to share about the commit. 38 | * Please try to follow the existing syntax style 39 | 40 | When you submit or change your pull request, the Travis build system will automatically run tests. 41 | If your pull request fails to pass tests, review the test log, make changes and 42 | then push them to your feature branch to be tested again. 43 | 44 | 45 | Contributor License Agreements 46 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 47 | 48 | All pull requests are welcome. Before we can submit them though, there is a legal hurdle we have to jump. 49 | You'll need to fill out either the individual or corporate Contributor License Agreement 50 | (CLA). 51 | 52 | * If you are an individual writing original source code and you're sure you 53 | own the intellectual property, then you'll need to sign an `individual CLA 54 | `_. 55 | * If you work for a company that wants to allow you to contribute your work, 56 | then you'll need to sign a `corporate CLA 57 | `_. 58 | 59 | Follow either of the two links above to access the appropriate CLA and 60 | instructions for how to sign and return it. Once we receive it, we'll be able to 61 | accept your pull requests. 62 | -------------------------------------------------------------------------------- /RMarkdown/literate-programming-demo.R: -------------------------------------------------------------------------------- 1 | ## ----one time setup, eval=FALSE------------------------------------------ 2 | ## ### To install the bigrquery package. The currently released version 0.3.0 does not yet 3 | ## ### have the parameter to use Standard SQL instead of Legacy SQL, so we install from github. 4 | ## library(devtools) 5 | ## install_github('rstats-db/bigrquery') 6 | 7 | ## ----initialize, message=FALSE, warning=FALSE---------------------------- 8 | library(bigrquery) 9 | library(ggplot2) 10 | library(xtable) 11 | 12 | ## ----eval=FALSE---------------------------------------------------------- 13 | ## ######################[ CHANGE ME ]################################## 14 | ## # This codelab assumes that the current working directory is where the Rmd file resides. 15 | ## setwd("/YOUR/PATH/TO/getting-started-bigquery/RMarkdown") 16 | ## 17 | ## # Set the Google Cloud Platform project id under which these queries will run. 18 | ## project <- "YOUR-PROJECT-ID" 19 | ## ##################################################################### 20 | 21 | ## ------------------------------------------------------------------------ 22 | # By default this codelab runs upon the Illumina Platinum Genomes Variants. 23 | # Change the table here if you wish to run these queries against a different table. 24 | theTable <- "genomics-public-data.platinum_genomes.variants" 25 | 26 | ## ------------------------------------------------------------------------ 27 | DisplayAndDispatchQuery <- function(queryUri) { 28 | # Read in the SQL from a file or URL. 29 | querySql <- readChar(queryUri, nchars=1e6) 30 | # Find and replace the table name placeholder with our table name. 31 | querySql <- sub("@THE_TABLE", theTable, querySql, fixed=TRUE) 32 | # Display the updated SQL. 33 | cat(querySql) 34 | # Dispatch the query to BigQuery for execution. 35 | query_exec(querySql, project, use_legacy_sql = FALSE) 36 | } 37 | 38 | ## ----comment=NA---------------------------------------------------------- 39 | result <- DisplayAndDispatchQuery("../sql/sample-variant-counts-for-brca1.sql") 40 | 41 | ## ----result, comment=NA-------------------------------------------------- 42 | head(result) 43 | summary(result) 44 | str(result) 45 | 46 | ## ----viz, fig.align="center", fig.width=10------------------------------- 47 | ggplot(result, aes(x=call_set_name, y=variant_count)) + 48 | geom_bar(stat="identity") + coord_flip() + 49 | ggtitle("Count of Variants Per Sample") 50 | 51 | ## ----comment=NA---------------------------------------------------------- 52 | result <- DisplayAndDispatchQuery("../sql/variant-level-data-for-brca1.sql") 53 | 54 | ## ----echo=FALSE, message=FALSE, warning=FALSE, comment=NA, results="asis"---- 55 | print(xtable(head(result)), type="html", include.rownames=F) 56 | 57 | ## ----comment=NA---------------------------------------------------------- 58 | result <- DisplayAndDispatchQuery("../sql/sample-level-data-for-brca1.sql") 59 | 60 | ## ----echo=FALSE, message=FALSE, warning=FALSE, comment=NA, results="asis"---- 61 | print(xtable(head(result)), type="html", include.rownames=F) 62 | 63 | ## ----provenance, comment=NA---------------------------------------------- 64 | sessionInfo() 65 | 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # getting-started-bigquery 2 | 3 | The repository contains examples of using BigQuery with 4 | genomics data. The code within each language-specific folder demonstrates the 5 | same set of queries upon the Platinum Genomes dataset. For more detail about 6 | this data see [Google Genomics Public Data](https://cloud.google.com/genomics/data/platinum-genomes). 7 | 8 | ## Using the BigQuery browser tool 9 | 10 | 1. Go to the BigQuery [Browser Tool](https://bigquery.cloud.google.com/table/genomics-public-data:platinum_genomes.variants). 11 | 1. Click on **"Compose Query"**. 12 | 1. Copy and paste the following query into the dialog box and click on **"Run Query"**: 13 | ``` 14 | #standardSQL 15 | -- Count the number of records (variant and reference segments) we have in 16 | -- the dataset and the total number of calls nested within those records. 17 | -- 18 | -- The source data for table genomics-public-data:platinum_genomes.variants 19 | -- was gVCF so a record can be a particular variant or a non-variant segment. 20 | -- https://sites.google.com/site/gvcftools/home/about-gvcf 21 | -- 22 | SELECT 23 | reference_name, 24 | COUNT(reference_name) AS num_records, 25 | SUM(ARRAY_LENGTH(call)) AS num_calls 26 | FROM 27 | `genomics-public-data.platinum_genomes.variants` v 28 | GROUP BY 29 | reference_name 30 | ORDER BY 31 | reference_name 32 | ``` 33 | 34 | View the results! 35 | 36 | Query Results 37 | 38 | ## What next? 39 | ---------- 40 | * Work through the [Analyze variants using Google BigQuery](http://googlegenomics.readthedocs.org/en/latest/use_cases/analyze_variants/analyze_variants_with_bigquery.html) codelab. The purpose of this codelab is to help you: 41 | * learn how to use the Google BigQuery query tool 42 | * learn valuable BigQuery SQL syntax 43 | * become familiar with the variants table created by a Google Genomics variant export 44 | * Try a few more queries in the [sql](./sql) subdirectory. 45 | + [variant-level-data-for-brca1.sql](./sql/variant-level-data-for-brca1.sql) 46 | + [sample-level-data-for-brca1.sql](./sql/sample-level-data-for-brca1.sql) 47 | + [sample-variant-counts-for-brca1.sql](./sql/sample-variant-counts-for-brca1.sql) 48 | + Replace `_THE_TABLE_` with `genomics-public-data:platinum_genomes.variants` or your own table if you have exported variants from Google Genomics to BigQuery. 49 | * New to BigQuery? 50 | + See the [query reference](https://cloud.google.com/bigquery/query-reference). 51 | + See the BigQuery book [Google BigQuery Analytics](http://www.wiley.com/WileyCDA/WileyTitle/productCd-1118824822.html) 52 | * New to working with variants? 53 | + See an overview of the [VCF data format](http://vcftools.sourceforge.net/VCF-poster.pdf). 54 | * Looking for more advanced sample queries? 55 | + See [BigQuery Examples](https://github.com/googlegenomics/bigquery-examples). 56 | 57 | Alternate ways to work with BigQuery 58 | -------------------------------------- 59 | 60 | Instead of using the browser tool to send queries to BigQuery, you can use code in many languages to call the BigQuery API. 61 | 62 | * Try the "getting started" samples in one or more languages by navigating to the subdirectory in this repository for the desired language: 63 | + [RMarkdown](./RMarkdown) 64 | + [R](./R) 65 | * All languages will require a Project ID from a project that has the BigQuery API enabled. 66 | + Follow the [BigQuery sign up instructions](https://cloud.google.com/bigquery/sign-up) if you do not yet have a valid project. (Note: you do not need to enable billing for the small examples in this repository) 67 | + You can find the Project ID for your new project in the 68 | [Google Cloud Console](https://console.cloud.google.com). 69 | 70 | * For more information on accessing BigQuery from other languages, see: 71 | [Create A Simple Application With the API](https://cloud.google.com/bigquery/create-simple-app-api) 72 | -------------------------------------------------------------------------------- /javascript/index.html: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 |
29 |
30 |

BigQuery integration example

31 | 32 |
33 | 34 | 46 |
47 | 48 |
49 | 50 | 51 |
52 | 53 |
54 | 55 | 56 |
57 | 58 | 60 |
61 | 62 |
63 |
64 | 65 |
66 |
67 |
68 | 69 | 125 | 126 | 127 | -------------------------------------------------------------------------------- /RMarkdown/literate-programming-demo.Rmd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | # Literate Programming with R and BigQuery 18 | 19 | ## R Markdown Introduction 20 | 21 | This is an [R Markdown](http://rmarkdown.rstudio.com/) document. By using RMarkdown, we can write R code in a [literate programming](http://en.wikipedia.org/wiki/Literate_programming) style interleaving snippets of code within narrative content. This document can be read, but it can also be executed. Most importantly though, it can be rendered so that the results of an R analysis at a point in time are captured. 22 | 23 | It is written in [Markdown](http://daringfireball.net/projects/markdown/syntax), a simple formatting syntax for authoring web pages. See the [`rmarkdown` package](http://cran.r-project.org/web/packages/rmarkdown/index.html) for more detail about how to use RMarkdown from R. [RStudio](http://www.rstudio.com/) has support for [R Markdown](http://rmarkdown.rstudio.com/) from its user interface. 24 | 25 | Now let's proceed with a specific example of [literate programming](http://en.wikipedia.org/wiki/Literate_programming) for [BigQuery](https://cloud.google.com/bigquery/). 26 | 27 | ## Setup 28 | 29 | If you have not used the [bigrquery](https://github.com/hadley/bigrquery) package previously, you will likely need to do something like the following to get it installed: 30 | 31 | ```{r one time setup, eval=FALSE} 32 | ### To install the bigrquery package. The currently released version 0.3.0 does not yet 33 | ### have the parameter to use Standard SQL instead of Legacy SQL, so we install from github. 34 | library(devtools) 35 | install_github('rstats-db/bigrquery') 36 | ``` 37 | 38 | Next we will load our needed packages into our session: 39 | ```{r initialize, message=FALSE, warning=FALSE} 40 | library(bigrquery) 41 | library(ggplot2) 42 | ``` 43 | 44 | And set a few variables: 45 | ```{r eval=FALSE} 46 | ######################[ CHANGE ME ]################################## 47 | # This codelab assumes that the current working directory is where the Rmd file resides. 48 | setwd("/YOUR/PATH/TO/getting-started-bigquery/RMarkdown") 49 | 50 | # Set the Google Cloud Platform project id under which these queries will run. 51 | project <- "YOUR-PROJECT-ID" 52 | ##################################################################### 53 | ``` 54 | 55 | ```{r} 56 | # By default this codelab runs upon the Illumina Platinum Genomes Variants. 57 | # Change the table here if you wish to run these queries against a different table. 58 | theTable <- "genomics-public-data.platinum_genomes.variants" 59 | ``` 60 | 61 | 62 | And write a little convenience function: 63 | ```{r} 64 | DisplayAndDispatchQuery <- function(queryUri) { 65 | # Read in the SQL from a file or URL. 66 | querySql <- readChar(queryUri, nchars=1e6) 67 | # Find and replace the table name placeholder with our table name. 68 | querySql <- sub("@THE_TABLE", theTable, querySql, fixed=TRUE) 69 | # Display the updated SQL. 70 | cat(querySql) 71 | # Dispatch the query to BigQuery for execution. 72 | query_exec(querySql, project, use_legacy_sql = FALSE) 73 | } 74 | ``` 75 | 76 | ## Running a Query in R 77 | 78 | Now we're ready to execute our query, bringing the results down to our R session for further examination: 79 | ```{r comment=NA} 80 | result <- DisplayAndDispatchQuery("../sql/sample-variant-counts-for-brca1.sql") 81 | ``` 82 | 83 | Let us examine our query result: 84 | ```{r result, comment=NA} 85 | head(result) 86 | summary(result) 87 | str(result) 88 | ``` 89 | We can see that what we get back from bigrquery is an R dataframe holding our query results. 90 | 91 | ## Data Visualization of Query Results 92 | 93 | Now that our results are in a dataframe, we can easily apply data visualization to our results: 94 | ```{r viz, fig.align="center", fig.width=10} 95 | ggplot(result, aes(x=call_set_name, y=variant_count)) + 96 | geom_bar(stat="identity") + coord_flip() + 97 | ggtitle("Count of Variants Per Sample") 98 | ``` 99 | 100 | Its clear to see that number of variants within BRCA1 for each sample corresponds roughly to two levels. 101 | 102 | We can then examine the variant level data more closely: 103 | ```{r comment=NA} 104 | result <- DisplayAndDispatchQuery("../sql/variant-level-data-for-brca1.sql") 105 | ``` 106 | Number of rows returned by this query: `r nrow(result)`. 107 | 108 | Displaying the first few rows of the dataframe of results: 109 | ```{r echo=FALSE, message=FALSE, warning=FALSE, comment=NA, results="asis"} 110 | knitr::kable(head(result)) 111 | ``` 112 | 113 | 114 | And also work with the sample level data: 115 | ```{r comment=NA} 116 | result <- DisplayAndDispatchQuery("../sql/sample-level-data-for-brca1.sql") 117 | ``` 118 | Number of rows returned by this query: `r nrow(result)`. 119 | 120 | 121 | Displaying the first few rows of the dataframe of results: 122 | ```{r echo=FALSE, message=FALSE, warning=FALSE, comment=NA, results="asis"} 123 | knitr::kable(head(result)) 124 | ``` 125 | 126 | ## Provenance 127 | 128 | Lastly, let us capture version information about R and loaded packages for the sake of provenance. 129 | ```{r provenance, comment=NA} 130 | sessionInfo() 131 | ``` 132 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /RMarkdown/literate-programming-demo.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | # Literate Programming with R and BigQuery 18 | 19 | ## R Markdown Introduction 20 | 21 | This is an [R Markdown](http://rmarkdown.rstudio.com/) document. By using RMarkdown, we can write R code in a [literate programming](http://en.wikipedia.org/wiki/Literate_programming) style interleaving snippets of code within narrative content. This document can be read, but it can also be executed. Most importantly though, it can be rendered so that the results of an R analysis at a point in time are captured. 22 | 23 | It is written in [Markdown](http://daringfireball.net/projects/markdown/syntax), a simple formatting syntax for authoring web pages. See the [`rmarkdown` package](http://cran.r-project.org/web/packages/rmarkdown/index.html) for more detail about how to use RMarkdown from R. [RStudio](http://www.rstudio.com/) has support for [R Markdown](http://rmarkdown.rstudio.com/) from its user interface. 24 | 25 | Now let's proceed with a specific example of [literate programming](http://en.wikipedia.org/wiki/Literate_programming) for [BigQuery](https://cloud.google.com/bigquery/). 26 | 27 | ## Setup 28 | 29 | If you have not used the [bigrquery](https://github.com/hadley/bigrquery) package previously, you will likely need to do something like the following to get it installed: 30 | 31 | 32 | ```r 33 | ### To install the bigrquery package. The currently released version 0.3.0 does not yet 34 | ### have the parameter to use Standard SQL instead of Legacy SQL, so we install from github. 35 | library(devtools) 36 | install_github('rstats-db/bigrquery') 37 | ``` 38 | 39 | Next we will load our needed packages into our session: 40 | 41 | ```r 42 | library(bigrquery) 43 | library(ggplot2) 44 | ``` 45 | 46 | And set a few variables: 47 | 48 | ```r 49 | ######################[ CHANGE ME ]################################## 50 | # This codelab assumes that the current working directory is where the Rmd file resides. 51 | setwd("/YOUR/PATH/TO/getting-started-bigquery/RMarkdown") 52 | 53 | # Set the Google Cloud Platform project id under which these queries will run. 54 | project <- "YOUR-PROJECT-ID" 55 | ##################################################################### 56 | ``` 57 | 58 | 59 | ```r 60 | # By default this codelab runs upon the Illumina Platinum Genomes Variants. 61 | # Change the table here if you wish to run these queries against a different table. 62 | theTable <- "genomics-public-data.platinum_genomes.variants" 63 | ``` 64 | 65 | 66 | And write a little convenience function: 67 | 68 | ```r 69 | DisplayAndDispatchQuery <- function(queryUri) { 70 | # Read in the SQL from a file or URL. 71 | querySql <- readChar(queryUri, nchars=1e6) 72 | # Find and replace the table name placeholder with our table name. 73 | querySql <- sub("@THE_TABLE", theTable, querySql, fixed=TRUE) 74 | # Display the updated SQL. 75 | cat(querySql) 76 | # Dispatch the query to BigQuery for execution. 77 | query_exec(querySql, project, use_legacy_sql = FALSE) 78 | } 79 | ``` 80 | 81 | ## Running a Query in R 82 | 83 | Now we're ready to execute our query, bringing the results down to our R session for further examination: 84 | 85 | ```r 86 | result <- DisplayAndDispatchQuery("../sql/sample-variant-counts-for-brca1.sql") 87 | ``` 88 | 89 | ``` 90 | # Sample variant counts within BRCA1. 91 | WITH brca1_calls AS ( 92 | SELECT 93 | reference_name, 94 | start, 95 | `end`, 96 | reference_bases, 97 | ARRAY_TO_STRING(v.alternate_bases, ',') AS alts, 98 | call.call_set_name, 99 | call.genotype[SAFE_ORDINAL(1)] AS first_allele, 100 | call.genotype[SAFE_ORDINAL(2)] AS second_allele 101 | FROM 102 | `genomics-public-data.platinum_genomes.variants` v, v.call call 103 | WHERE 104 | reference_name IN ('chr17', '17') 105 | AND start BETWEEN 41196311 AND 41277499 # per GRCh37 106 | ) 107 | 108 | SELECT 109 | call_set_name, 110 | COUNT(call_set_name) AS variant_count 111 | FROM brca1_calls 112 | WHERE 113 | first_allele > 0 OR second_allele > 0 114 | GROUP BY 115 | call_set_name 116 | ORDER BY 117 | call_set_name 118 | ``` 119 | 120 | ``` 121 | 10.7 gigabytes processed 122 | ``` 123 | 124 | Let us examine our query result: 125 | 126 | ```r 127 | head(result) 128 | ``` 129 | 130 | ``` 131 | call_set_name variant_count 132 | 1 NA12877 27 133 | 2 NA12878 198 134 | 3 NA12889 198 135 | 4 NA12890 33 136 | 5 NA12891 37 137 | 6 NA12892 209 138 | ``` 139 | 140 | ```r 141 | summary(result) 142 | ``` 143 | 144 | ``` 145 | call_set_name variant_count 146 | Length:6 Min. : 27.0 147 | Class :character 1st Qu.: 34.0 148 | Mode :character Median :117.5 149 | Mean :117.0 150 | 3rd Qu.:198.0 151 | Max. :209.0 152 | ``` 153 | 154 | ```r 155 | str(result) 156 | ``` 157 | 158 | ``` 159 | 'data.frame': 6 obs. of 2 variables: 160 | $ call_set_name: chr "NA12877" "NA12878" "NA12889" "NA12890" ... 161 | $ variant_count: int 27 198 198 33 37 209 162 | ``` 163 | We can see that what we get back from bigrquery is an R dataframe holding our query results. 164 | 165 | ## Data Visualization of Query Results 166 | 167 | Now that our results are in a dataframe, we can easily apply data visualization to our results: 168 | 169 | ```r 170 | ggplot(result, aes(x=call_set_name, y=variant_count)) + 171 | geom_bar(stat="identity") + coord_flip() + 172 | ggtitle("Count of Variants Per Sample") 173 | ``` 174 | 175 | plot of chunk viz 176 | 177 | Its clear to see that number of variants within BRCA1 for each sample corresponds roughly to two levels. 178 | 179 | We can then examine the variant level data more closely: 180 | 181 | ```r 182 | result <- DisplayAndDispatchQuery("../sql/variant-level-data-for-brca1.sql") 183 | ``` 184 | 185 | ``` 186 | # Retrieve variant-level information for BRCA1 variants. 187 | SELECT 188 | reference_name, 189 | start, 190 | `end`, 191 | reference_bases, 192 | ARRAY_TO_STRING(v.alternate_bases, ',') AS alts, 193 | quality, 194 | ARRAY_TO_STRING(v.filter, ',') AS filter, 195 | ARRAY_TO_STRING(v.names, ',') AS names, 196 | ARRAY_LENGTH(v.call) AS num_samples 197 | FROM 198 | `genomics-public-data.platinum_genomes.variants` v 199 | WHERE 200 | reference_name IN ('17', 'chr17') 201 | AND start BETWEEN 41196311 AND 41277499 # per GRCh37 202 | # Skip non-variant segments. 203 | AND EXISTS (SELECT alt FROM UNNEST(v.alternate_bases) alt WHERE alt NOT IN ("", "<*>")) 204 | ORDER BY 205 | start, 206 | alts 207 | ``` 208 | 209 | ``` 210 | 9.9 gigabytes processed 211 | ``` 212 | Number of rows returned by this query: 281. 213 | 214 | Displaying the first few rows of the dataframe of results: 215 | 216 | |reference_name | start| end|reference_bases |alts | quality|filter |names | num_samples| 217 | |:--------------|--------:|--------:|:---------------|:----|-------:|:-------------------------------------------------|:-----|-----------:| 218 | |chr17 | 41196407| 41196408|G |A | 733.47|PASS | | 3| 219 | |chr17 | 41196820| 41196823|CTT |C,CT | 287.18|PASS | | 1| 220 | |chr17 | 41197273| 41197274|C |A | 1011.08|PASS | | 3| 221 | |chr17 | 41197957| 41197958|G |T | 178.48|TruthSensitivityTranche99.90to100.00 | | 4| 222 | |chr17 | 41198182| 41198183|A |C | 98.02|TruthSensitivityTranche99.00to99.90 | | 1| 223 | |chr17 | 41198186| 41198187|A |C | 7.68|TruthSensitivityTranche99.90to100.00,LowGQX,LowQD | | 4| 224 | 225 | 226 | And also work with the sample level data: 227 | 228 | ```r 229 | result <- DisplayAndDispatchQuery("../sql/sample-level-data-for-brca1.sql") 230 | ``` 231 | 232 | ``` 233 | # Retrieve sample-level information for BRCA1 variants. 234 | SELECT 235 | reference_name, 236 | start, 237 | `end`, 238 | reference_bases, 239 | ARRAY_TO_STRING(v.alternate_bases, ',') AS alts, 240 | quality, 241 | ARRAY_TO_STRING(v.filter, ',') AS filters, 242 | ARRAY_TO_STRING(v.names, ',') AS names, 243 | call.call_set_name, 244 | (SELECT STRING_AGG(CAST(gt AS STRING)) from UNNEST(call.genotype) gt) AS genotype 245 | FROM 246 | `genomics-public-data.platinum_genomes.variants` v, v.call call 247 | WHERE 248 | reference_name IN ('17', 'chr17') 249 | AND start BETWEEN 41196311 AND 41277499 # per GRCh37 250 | # Skip non-variant segments. 251 | AND EXISTS (SELECT alt FROM UNNEST(v.alternate_bases) alt WHERE alt NOT IN ("", "<*>")) 252 | ORDER BY 253 | start, 254 | alts, 255 | call_set_name 256 | ``` 257 | 258 | ``` 259 | 17.0 gigabytes processed 260 | ``` 261 | Number of rows returned by this query: 706. 262 | 263 | 264 | Displaying the first few rows of the dataframe of results: 265 | 266 | |reference_name | start| end|reference_bases |alts | quality|filters |names |call_set_name |genotype | 267 | |:--------------|--------:|--------:|:---------------|:----|-------:|:-------|:-----|:-------------|:--------| 268 | |chr17 | 41196407| 41196408|G |A | 733.47|PASS | |NA12878 |0,1 | 269 | |chr17 | 41196407| 41196408|G |A | 733.47|PASS | |NA12889 |0,1 | 270 | |chr17 | 41196407| 41196408|G |A | 733.47|PASS | |NA12892 |0,1 | 271 | |chr17 | 41196820| 41196823|CTT |C,CT | 287.18|PASS | |NA12889 |1,2 | 272 | |chr17 | 41197273| 41197274|C |A | 1011.08|PASS | |NA12878 |0,1 | 273 | |chr17 | 41197273| 41197274|C |A | 1011.08|PASS | |NA12889 |0,1 | 274 | 275 | ## Provenance 276 | 277 | Lastly, let us capture version information about R and loaded packages for the sake of provenance. 278 | 279 | ```r 280 | sessionInfo() 281 | ``` 282 | 283 | ``` 284 | R version 3.4.0 (2017-04-21) 285 | Platform: x86_64-apple-darwin15.6.0 (64-bit) 286 | Running under: macOS Sierra 10.12.5 287 | 288 | Matrix products: default 289 | BLAS: /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib 290 | LAPACK: /Library/Frameworks/R.framework/Versions/3.4/Resources/lib/libRlapack.dylib 291 | 292 | locale: 293 | [1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8 294 | 295 | attached base packages: 296 | [1] stats graphics grDevices utils datasets methods base 297 | 298 | other attached packages: 299 | [1] ggplot2_2.2.1 bigrquery_0.3.0.9000 knitr_1.16 300 | 301 | loaded via a namespace (and not attached): 302 | [1] Rcpp_0.12.11 magrittr_1.5 progress_1.1.2 303 | [4] munsell_0.4.3 colorspace_1.3-2 R6_2.2.1 304 | [7] rlang_0.1.1 highr_0.6 stringr_1.2.0 305 | [10] httr_1.2.1 plyr_1.8.4 tools_3.4.0 306 | [13] grid_3.4.0 gtable_0.2.0 DBI_0.6-1 307 | [16] htmltools_0.3.6 rprojroot_1.2 digest_0.6.12 308 | [19] openssl_0.9.6 lazyeval_0.2.0 assertthat_0.2.0 309 | [22] tibble_1.3.3 curl_2.6 evaluate_0.10 310 | [25] rmarkdown_1.5 labeling_0.3 stringi_1.1.5 311 | [28] compiler_3.4.0 backports_1.1.0 scales_0.4.1 312 | [31] prettyunits_1.0.2 jsonlite_1.5 313 | ``` 314 | --------------------------------------------------------------------------------