Data processing with R tidyverse - Ginolhac and al. 2017.
69 | 70 | It aims to provide readibility and understanding rather than performance (althought it sometimes provides both). 71 | 72 | The Tidyverse is modular as you can load each packages separatly. We'll use mostly the {dplyr} package in this workshop. 73 | 74 | data.table in the other end was created to be more efficient and can easily [compete with Python Pandas for example](https://h2oai.github.io/db-benchmark/). 75 | data.table can handle very large datasets. Its syntax is close to Python Pandas but less readible by non programmers. 76 | 77 |  78 | 79 | It is not closed worlds. You can mix all of these in your script like [Suzan Baert](https://suzan.rbind.io/) explained at [SatRday Paris 2019](https://youtu.be/vSU3o3c0-7U?t=1923). 80 | 81 | You can even mix R with Python if you need to with the Rpackage [reticulate](https://rstudio.github.io/reticulate/) or [rpy2](https://rpy2.readthedocs.io). 82 | 83 | This document is based on geodata handling with {sf} and {dplyr} (from the Tidyverse toolset). 84 | 85 | This can be followed on Rstudio or Jupyter R notebook. -------------------------------------------------------------------------------- /docs/libs/gitbook-2.6.7/js/plugin-fontsettings.js: -------------------------------------------------------------------------------- 1 | gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { 2 | var fontState; 3 | 4 | var THEMES = { 5 | "white": 0, 6 | "sepia": 1, 7 | "night": 2 8 | }; 9 | 10 | var FAMILY = { 11 | "serif": 0, 12 | "sans": 1 13 | }; 14 | 15 | // Save current font settings 16 | function saveFontSettings() { 17 | gitbook.storage.set("fontState", fontState); 18 | update(); 19 | } 20 | 21 | // Increase font size 22 | function enlargeFontSize(e) { 23 | e.preventDefault(); 24 | if (fontState.size >= 4) return; 25 | 26 | fontState.size++; 27 | saveFontSettings(); 28 | }; 29 | 30 | // Decrease font size 31 | function reduceFontSize(e) { 32 | e.preventDefault(); 33 | if (fontState.size <= 0) return; 34 | 35 | fontState.size--; 36 | saveFontSettings(); 37 | }; 38 | 39 | // Change font family 40 | function changeFontFamily(index, e) { 41 | e.preventDefault(); 42 | 43 | fontState.family = index; 44 | saveFontSettings(); 45 | }; 46 | 47 | // Change type of color 48 | function changeColorTheme(index, e) { 49 | e.preventDefault(); 50 | 51 | var $book = $(".book"); 52 | 53 | if (fontState.theme !== 0) 54 | $book.removeClass("color-theme-"+fontState.theme); 55 | 56 | fontState.theme = index; 57 | if (fontState.theme !== 0) 58 | $book.addClass("color-theme-"+fontState.theme); 59 | 60 | saveFontSettings(); 61 | }; 62 | 63 | function update() { 64 | var $book = gitbook.state.$book; 65 | 66 | $(".font-settings .font-family-list li").removeClass("active"); 67 | $(".font-settings .font-family-list li:nth-child("+(fontState.family+1)+")").addClass("active"); 68 | 69 | $book[0].className = $book[0].className.replace(/\bfont-\S+/g, ''); 70 | $book.addClass("font-size-"+fontState.size); 71 | $book.addClass("font-family-"+fontState.family); 72 | 73 | if(fontState.theme !== 0) { 74 | $book[0].className = $book[0].className.replace(/\bcolor-theme-\S+/g, ''); 75 | $book.addClass("color-theme-"+fontState.theme); 76 | } 77 | }; 78 | 79 | function init(config) { 80 | var $bookBody, $book; 81 | 82 | //Find DOM elements. 83 | $book = gitbook.state.$book; 84 | $bookBody = $book.find(".book-body"); 85 | 86 | // Instantiate font state object 87 | fontState = gitbook.storage.get("fontState", { 88 | size: config.size || 2, 89 | family: FAMILY[config.family || "sans"], 90 | theme: THEMES[config.theme || "white"] 91 | }); 92 | 93 | update(); 94 | }; 95 | 96 | 97 | gitbook.events.bind("start", function(e, config) { 98 | var opts = config.fontsettings; 99 | 100 | // Create buttons in toolbar 101 | gitbook.toolbar.createButton({ 102 | icon: 'fa fa-font', 103 | label: 'Font Settings', 104 | className: 'font-settings', 105 | dropdown: [ 106 | [ 107 | { 108 | text: 'A', 109 | className: 'font-reduce', 110 | onClick: reduceFontSize 111 | }, 112 | { 113 | text: 'A', 114 | className: 'font-enlarge', 115 | onClick: enlargeFontSize 116 | } 117 | ], 118 | [ 119 | { 120 | text: 'Serif', 121 | onClick: _.partial(changeFontFamily, 0) 122 | }, 123 | { 124 | text: 'Sans', 125 | onClick: _.partial(changeFontFamily, 1) 126 | } 127 | ], 128 | [ 129 | { 130 | text: 'White', 131 | onClick: _.partial(changeColorTheme, 0) 132 | }, 133 | { 134 | text: 'Sepia', 135 | onClick: _.partial(changeColorTheme, 1) 136 | }, 137 | { 138 | text: 'Night', 139 | onClick: _.partial(changeColorTheme, 2) 140 | } 141 | ] 142 | ] 143 | }); 144 | 145 | 146 | // Init current settings 147 | init(opts); 148 | }); 149 | }); 150 | 151 | 152 | -------------------------------------------------------------------------------- /packages.bib: -------------------------------------------------------------------------------- 1 | @Manual{R-base, 2 | title = {R: A Language and Environment for Statistical Computing}, 3 | author = {{R Core Team}}, 4 | organization = {R Foundation for Statistical Computing}, 5 | address = {Vienna, Austria}, 6 | year = {2018}, 7 | url = {https://www.R-project.org/}, 8 | } 9 | @Manual{R-bookdown, 10 | title = {bookdown: Authoring Books and Technical Documents with R Markdown}, 11 | author = {Yihui Xie}, 12 | year = {2018}, 13 | note = {R package version 0.9}, 14 | url = {https://CRAN.R-project.org/package=bookdown}, 15 | } 16 | @Manual{R-dplyr, 17 | title = {dplyr: A Grammar of Data Manipulation}, 18 | author = {Hadley Wickham and Romain François and Lionel Henry and Kirill Müller}, 19 | year = {2019}, 20 | note = {R package version 0.8.1}, 21 | url = {https://CRAN.R-project.org/package=dplyr}, 22 | } 23 | @Manual{R-forcats, 24 | title = {forcats: Tools for Working with Categorical Variables (Factors)}, 25 | author = {Hadley Wickham}, 26 | year = {2019}, 27 | note = {R package version 0.4.0}, 28 | url = {https://CRAN.R-project.org/package=forcats}, 29 | } 30 | @Manual{R-ggplot2, 31 | title = {ggplot2: Create Elegant Data Visualisations Using the Grammar of Graphics}, 32 | author = {Hadley Wickham and Winston Chang and Lionel Henry and Thomas Lin Pedersen and Kohske Takahashi and Claus Wilke and Kara Woo}, 33 | year = {2018}, 34 | note = {R package version 3.1.0}, 35 | url = {https://CRAN.R-project.org/package=ggplot2}, 36 | } 37 | @Manual{R-ggspatial, 38 | title = {ggspatial: Spatial Data Framework for ggplot2}, 39 | author = {Dewey Dunnington}, 40 | year = {2018}, 41 | note = {R package version 1.0.3}, 42 | url = {https://CRAN.R-project.org/package=ggspatial}, 43 | } 44 | @Manual{R-here, 45 | title = {here: A Simpler Way to Find Your Files}, 46 | author = {Kirill Müller}, 47 | year = {2017}, 48 | note = {R package version 0.1}, 49 | url = {https://CRAN.R-project.org/package=here}, 50 | } 51 | @Manual{R-knitr, 52 | title = {knitr: A General-Purpose Package for Dynamic Report Generation in R}, 53 | author = {Yihui Xie}, 54 | year = {2018}, 55 | note = {R package version 1.21}, 56 | url = {https://CRAN.R-project.org/package=knitr}, 57 | } 58 | @Manual{R-purrr, 59 | title = {purrr: Functional Programming Tools}, 60 | author = {Lionel Henry and Hadley Wickham}, 61 | year = {2019}, 62 | note = {R package version 0.3.2}, 63 | url = {https://CRAN.R-project.org/package=purrr}, 64 | } 65 | @Manual{R-readr, 66 | title = {readr: Read Rectangular Text Data}, 67 | author = {Hadley Wickham and Jim Hester and Romain Francois}, 68 | year = {2018}, 69 | note = {R package version 1.3.1}, 70 | url = {https://CRAN.R-project.org/package=readr}, 71 | } 72 | @Manual{R-rmarkdown, 73 | title = {rmarkdown: Dynamic Documents for R}, 74 | author = {JJ Allaire and Yihui Xie and Jonathan McPherson and Javier Luraschi and Kevin Ushey and Aron Atkins and Hadley Wickham and Joe Cheng and Winston Chang and Richard Iannone}, 75 | year = {2018}, 76 | note = {R package version 1.11}, 77 | url = {https://CRAN.R-project.org/package=rmarkdown}, 78 | } 79 | @Manual{R-sf, 80 | title = {sf: Simple Features for R}, 81 | author = {Edzer Pebesma}, 82 | year = {2019}, 83 | note = {R package version 0.7-4}, 84 | url = {https://CRAN.R-project.org/package=sf}, 85 | } 86 | @Manual{R-spData, 87 | title = {spData: Datasets for Spatial Analysis}, 88 | author = {Roger Bivand and Jakub Nowosad and Robin Lovelace}, 89 | year = {2019}, 90 | note = {R package version 0.3.0}, 91 | url = {https://CRAN.R-project.org/package=spData}, 92 | } 93 | @Manual{R-stringr, 94 | title = {stringr: Simple, Consistent Wrappers for Common String Operations}, 95 | author = {Hadley Wickham}, 96 | year = {2019}, 97 | note = {R package version 1.4.0}, 98 | url = {https://CRAN.R-project.org/package=stringr}, 99 | } 100 | @Manual{R-tibble, 101 | title = {tibble: Simple Data Frames}, 102 | author = {Kirill Müller and Hadley Wickham}, 103 | year = {2019}, 104 | note = {R package version 2.1.3}, 105 | url = {https://CRAN.R-project.org/package=tibble}, 106 | } 107 | @Manual{R-tidyr, 108 | title = {tidyr: Easily Tidy Data with 'spread()' and 'gather()' Functions}, 109 | author = {Hadley Wickham and Lionel Henry}, 110 | year = {2018}, 111 | note = {R package version 0.8.2}, 112 | url = {https://CRAN.R-project.org/package=tidyr}, 113 | } 114 | @Manual{R-tidyverse, 115 | title = {tidyverse: Easily Install and Load the 'Tidyverse'}, 116 | author = {Hadley Wickham}, 117 | year = {2017}, 118 | note = {R package version 1.2.1}, 119 | url = {https://CRAN.R-project.org/package=tidyverse}, 120 | } 121 | @Manual{R-tmap, 122 | title = {tmap: Thematic Maps}, 123 | author = {Martijn Tennekes}, 124 | year = {2019}, 125 | note = {R package version 2.2}, 126 | url = {https://CRAN.R-project.org/package=tmap}, 127 | } 128 | -------------------------------------------------------------------------------- /02-R_basics.Rmd: -------------------------------------------------------------------------------- 1 | # R basics 2 | 3 | How to manipulate data in R ? How to install and load a package ? Let's see.. 4 | 5 | ## Getting some help 6 | 7 | ## Where am I ? 8 | 9 | To get the current directory, use `getwd()` : 10 | 11 | ```{r getwd} 12 | getwd() 13 | ``` 14 | 15 | If you need to change the directory, there is `setwd()` 16 | 17 | ```{r setwd, eval=FALSE} 18 | setwd("path/to/my/directory") 19 | ``` 20 | 21 | 22 | If you use Rstudio, I can only recommand to work with a [project workflow](https://www.tidyverse.org/articles/2017/12/workflow-vs-script/) to avoid path issues on another computer. 23 | 24 | ## Make calculations 25 | 26 | ```{r calculations} 27 | 1+1 28 | 29 | 3 * 4 30 | 31 | 7/3 32 | 33 | 7%%3 # rest of the division 34 | 35 | ``` 36 | ## Arthmetic functions 37 | R provides a lot of arithmetic functions by default : 38 | 39 | ```{r arithmetic_functions} 40 | sqrt(4.0) 41 | abs(-625) 42 | log10(12900) 43 | ``` 44 | 45 | ## Assign values to a variable 46 | 47 | ```{r fruits} 48 | fruits <- c("apples", "pears", "lemons") 49 | fruits 50 | ``` 51 | 52 | 53 | ```{r quantities} 54 | quantities <- c(3, 2, 1) 55 | print(quantities) 56 | ``` 57 | 58 | ```{block2, type='rmdcaution'} 59 | Indices in R start at 1 ! 60 | ``` 61 | 62 | ```{r show_indices} 63 | print(fruits[1]) 64 | print(fruits[0]) # returns nothing 65 | ``` 66 | 67 | ## For loop and print 68 | ### Simple for loop 69 | ```{r print} 70 | for (fruit in fruits) { 71 | print(fruit) 72 | } 73 | ``` 74 | 75 | ### For loop with indices 76 | 77 | ```{r loop_indices} 78 | for (x in seq(length(fruits))) { 79 | print(paste0("I have ", quantities[x]," ", fruits[x],".")) 80 | } 81 | ``` 82 | 83 | ```{block2, type='rmdwarning'} 84 | For loops in R are possible but not memory efficient. 85 | So if you need to walk through a large amount of data, please consider using functions instead. 86 | ``` 87 | ## Data types 88 | 89 | ### Vectors 90 | `fruits` and `quantities` are character and numeric vectors. 91 | 92 | ```{r, vectors} 93 | class(fruits) 94 | class(quantities) 95 | ``` 96 | 97 | Vectors are the most basic R data object. There is six types of atomic vectors: logical, integer, double, complex, character and raw. You can't mix types in vectors. 98 | 99 | ### Dataframes 100 | 101 | Another frequently encountered data type is the **dataframe**. It is a collection data organized by rows and columns. Columns that can be of different types. Rows don't have to unique but having [tidy data](https://cran.r-project.org/web/packages/tidyr/vignettes/tidy-data.html) is known as a good pratice : 102 | 103 | 104 | > 1. Each variable forms a column. 105 | 1. Each observation forms a row. 106 | 1. Each type of observational unit forms a table. 107 | 108 | Good thing is, in GIS, we tend to have tidy data, right ? 109 | 110 | How to create a data frame from our vectors ? 111 | 112 | #### With `cbind.data.frame()` 113 | 114 | 115 | ```{r build_dataframe} 116 | df1 <- cbind.data.frame(fruits, quantities) # column binding 117 | print(df1) 118 | class(df1) 119 | ``` 120 | 121 | 122 | ```{r build_dataframe2} 123 | df2 <- as_data_frame(fruits) # column binding 124 | colnames(df2) <- "fruits" # change column name 125 | print(df2) 126 | class(df2) 127 | ``` 128 | ```{block2, type='rmdnote'} 129 | Tibbles (`tbl` // `tbl_df`) are dataframes on steroids from the tidyverse. 130 | ``` 131 | 132 | ### Add columns to a dataframe 133 | ```{r, add_column_to_df} 134 | 135 | df3 <- cbind(df2, # entry dataframe 136 | quantities, # column with quantities 137 | price = c(4,7,9) # new colum with price 138 | ) 139 | df3 140 | ``` 141 | 142 | 143 | 144 | ### Other datatypes 145 | 146 | * Matrices 147 | * Lists (`list()`) : collection of objects of different kind 148 | 149 | ```{block2, type='rmdcaution'} 150 | List in R are not like lists in Python. 151 | ``` 152 | 153 | ## Filtering / Subsetting 154 | 155 | In R, you can subset your data by value or variable. There is several way to do it, here is some of them. 156 | 157 | 158 | ### Select variables 159 | 160 | ```{r, get_colnames} 161 | names(df3) 162 | ``` 163 | 164 | ```{r select_indices} 165 | df3[, 2:3] 166 | df3[, c("fruits","price")] 167 | df3 %>% # pipe symbol 168 | select(fruits, quantities) # select from dplyr 169 | ``` 170 | 171 | ### Filter values 172 | 173 | ```{r filtering_data} 174 | df3[df3["price"] > 5,] # don't forget the column comma 175 | 176 | df3 %>% 177 | filter(quantities >= 2) 178 | ``` 179 | ### Mixing selection and filtering 180 | 181 | ```{r mix_select_filter} 182 | 183 | df3[df3["price"] > 5, 1] # select the prices > 5 184 | 185 | df3 %>% 186 | filter(price > 5) %>% # filter first 187 | select(fruits) # select second 188 | ``` 189 | 190 | 191 | ## Joins 192 | 193 | Let's create a new dataframe to join 194 | 195 | ```{r df4} 196 | df4 <- cbind.data.frame(fruits = fruits, buyer = c("Sophie", "Marc", "Nathan")) 197 | df4 198 | ``` 199 | 200 | ### Merge 201 | 202 | ```{r merger_left} 203 | merged_df <- merge(x = df3, y = df4, by = "fruits", all = TRUE) # OUTER JOIN 204 | merged_df 205 | ``` 206 | 207 | 208 | See that [answer on StackOverflow](https://stackoverflow.com/questions/1299871/how-to-join-merge-data-frames-inner-outer-left-right) for more details on left, right, inner and outer joins with `merge()`. 209 | 210 | ### Dplyr 211 | 212 | ```{r dplyr_join} 213 | merged_df <- df3 %>% 214 | full_join(df4) ## or full_join(df4, by = "fruits") 215 | 216 | merged_df 217 | ``` 218 | 219 | See the [documentation of {dplyr}](https://dplyr.tidyverse.org/reference/join.html) for more information on joins. 220 | 221 | ## Going further 222 | 223 | If you want to go further in the learning of the R language and the Tidyverse tools, there is a lot of resources online. You might want to start by those : 224 | 225 | * Base R : [R manuals](https://colinfay.me/r-manuals/) 226 | * Tidyverse : [R for Data Science (free ebook)](https://r4ds.had.co.nz) 227 | 228 | 229 | -------------------------------------------------------------------------------- /docs/libs/gitbook-2.6.7/js/plugin-search.js: -------------------------------------------------------------------------------- 1 | gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { 2 | var index = null; 3 | var $searchInput, $searchLabel, $searchForm; 4 | var $highlighted, hi = 0, hiOpts = { className: 'search-highlight' }; 5 | var collapse = false; 6 | 7 | // Use a specific index 8 | function loadIndex(data) { 9 | // [Yihui] In bookdown, I use a character matrix to store the chapter 10 | // content, and the index is dynamically built on the client side. 11 | // Gitbook prebuilds the index data instead: https://github.com/GitbookIO/plugin-search 12 | // We can certainly do that via R packages V8 and jsonlite, but let's 13 | // see how slow it really is before improving it. On the other hand, 14 | // lunr cannot handle non-English text very well, e.g. the default 15 | // tokenizer cannot deal with Chinese text, so we may want to replace 16 | // lunr with a dumb simple text matching approach. 17 | index = lunr(function () { 18 | this.ref('url'); 19 | this.field('title', { boost: 10 }); 20 | this.field('body'); 21 | }); 22 | data.map(function(item) { 23 | index.add({ 24 | url: item[0], 25 | title: item[1], 26 | body: item[2] 27 | }); 28 | }); 29 | } 30 | 31 | // Fetch the search index 32 | function fetchIndex() { 33 | return $.getJSON(gitbook.state.basePath+"/search_index.json") 34 | .then(loadIndex); // [Yihui] we need to use this object later 35 | } 36 | 37 | // Search for a term and return results 38 | function search(q) { 39 | if (!index) return; 40 | 41 | var results = _.chain(index.search(q)) 42 | .map(function(result) { 43 | var parts = result.ref.split("#"); 44 | return { 45 | path: parts[0], 46 | hash: parts[1] 47 | }; 48 | }) 49 | .value(); 50 | 51 | // [Yihui] Highlight the search keyword on current page 52 | hi = 0; 53 | $highlighted = results.length === 0 ? undefined : $('.page-inner') 54 | .unhighlight(hiOpts).highlight(q, hiOpts).find('span.search-highlight'); 55 | scrollToHighlighted(); 56 | toggleTOC(results.length > 0); 57 | 58 | return results; 59 | } 60 | 61 | // [Yihui] Scroll the chapter body to the i-th highlighted string 62 | function scrollToHighlighted() { 63 | if (!$highlighted) return; 64 | var n = $highlighted.length; 65 | if (n === 0) return; 66 | var $p = $highlighted.eq(hi), p = $p[0], rect = p.getBoundingClientRect(); 67 | if (rect.top < 0 || rect.bottom > $(window).height()) { 68 | ($(window).width() >= 1240 ? $('.body-inner') : $('.book-body')) 69 | .scrollTop(p.offsetTop - 100); 70 | } 71 | $highlighted.css('background-color', ''); 72 | // an orange background color on the current item and removed later 73 | $p.css('background-color', 'orange'); 74 | setTimeout(function() { 75 | $p.css('background-color', ''); 76 | }, 2000); 77 | } 78 | 79 | // [Yihui] Expand/collapse TOC 80 | function toggleTOC(show) { 81 | if (!collapse) return; 82 | var toc_sub = $('ul.summary').children('li[data-level]').children('ul'); 83 | if (show) return toc_sub.show(); 84 | var href = window.location.pathname; 85 | href = href.substr(href.lastIndexOf('/') + 1); 86 | if (href === '') href = 'index.html'; 87 | var li = $('a[href^="' + href + location.hash + '"]').parent('li.chapter').first(); 88 | toc_sub.hide().parent().has(li).children('ul').show(); 89 | li.children('ul').show(); 90 | } 91 | 92 | // Create search form 93 | function createForm(value) { 94 | if ($searchForm) $searchForm.remove(); 95 | if ($searchLabel) $searchLabel.remove(); 96 | if ($searchInput) $searchInput.remove(); 97 | 98 | $searchForm = $('