├── GBPUSD_2014_01.csv ├── GBPUSD_2014_02.csv ├── README.md ├── personsData.json └── personsDataSchema.json /README.md: -------------------------------------------------------------------------------- 1 | ## Optimizing Google BigQuery 2 | This file contains text you can copy and paste for the examples in Cloud Academy's _Optimizing Google BigQuery_ course. 3 | 4 | ### Reducing the Amount of Data Processed 5 | #### Stock exchange data files 6 | GBPUSD_2014_01.csv and GBPUSD_2014_02.csv are no longer available from Google, but you can download smaller versions of them from this repository. 7 | 8 | The schema for the above tables is: 9 | venue:STRING,currencies:STRING,time:TIMESTAMP,bid:FLOAT,ask:FLOAT 10 | 11 | #### BETWEEN operator 12 | ```sql 13 | SELECT time, bid 14 | FROM examples.gbpusd_201401 15 | WHERE time 16 | BETWEEN TIMESTAMP("2014-01-01 00:00:00") 17 | AND TIMESTAMP("2014-01-01 00:30:00") 18 | ORDER BY time ASC 19 | ``` 20 | 21 | #### Wildcard in table reference 22 | ``` 23 | SELECT MIN(time) AS mintime, MAX(time) AS maxtime 24 | FROM `examples.gbpusd_20140*` 25 | ``` 26 | 27 | ### Partitioned Tables 28 | #### Create partitioned table 29 | ``` 30 | bq query --use_legacy_sql=false --replace --destination_table 'examples.gbpusd_201401p' --time_partitioning_field time "SELECT * from examples.gbpusd_201401" 31 | ``` 32 | 33 | ### Denormalized Data Structures 34 | #### MusicBrainz data files 35 | __***Note: The files below are no longer available. We are working on uploading smaller versions of them to this repository.__ 36 | 37 | | Table | Data | Schema | 38 | | ------------------ | ----------------------------- | ------------------------------------------------------ | 39 | | artist | gs://solutions-public-assets/bqetl/artist.json | https://storage.googleapis.com/solutions-public-assets/bqetl/artist_schema.json | 40 | | artist_credit_name | gs://solutions-public-assets/bqetl/artist_credit_name.json | https://storage.googleapis.com/solutions-public-assets/bqetl/artist_credit_name_schema.json| 41 | | recording | gs://solutions-public-assets/bqetl/recording.json | https://storage.googleapis.com/solutions-public-assets/bqetl/recording_schema.json | 42 | 43 | #### Create denormalized table 44 | ``` 45 | SELECT 46 | artist.id, artist.gid AS artist_gid, artist.name AS artist_name, artist.area, 47 | recording.name AS recording_name, recording.length, recording.gid, recording.video 48 | FROM 49 | `examples.artist` AS artist 50 | INNER JOIN 51 | `examples.artist_credit_name` AS artist_credit_name 52 | ON 53 | artist.id = artist_credit_name.artist 54 | INNER JOIN 55 | `examples.recording` AS recording 56 | ON 57 | artist_credit_name.artist_credit = recording.artist_credit 58 | ``` 59 | 60 | #### Query denormalized table 61 | ``` 62 | SELECT artist_name, recording_name 63 | FROM `cloud-academy-content-team.examples.recording_by_artist` 64 | WHERE artist_name LIKE '%Elvis%' 65 | ``` 66 | 67 | ### Nested Repeated Fields 68 | #### Example data 69 | | Table | Data | Schema | 70 | | ------------------ | ----------------------------- | ------------------------------------------------------ | 71 | | persons_data | https://raw.githubusercontent.com/cloudacademy/optimizing-bigquery/master/personsData.json | https://raw.githubusercontent.com/cloudacademy/optimizing-bigquery/master/personsDataSchema.json | 72 | 73 | #### Query a nested field 74 | ``` 75 | SELECT fullName, phoneNumber.number 76 | FROM `cloud-academy-content-team.examples.persons_data` 77 | ``` 78 | 79 | #### Unnest a field 80 | ``` 81 | SELECT fullName, place 82 | FROM `cloud-academy-content-team.examples.persons_data`, 83 | UNNEST(citiesLived) 84 | WHERE place = "Austin" 85 | ``` 86 | 87 | #### Query a view 88 | ``` 89 | SELECT fullName, place 90 | FROM `cloud-academy-content-team.examples.cities_by_person` 91 | WHERE place = 'Stockholm' 92 | ``` 93 | -------------------------------------------------------------------------------- /personsData.json: -------------------------------------------------------------------------------- 1 | { "fullName": "John Doe", "age": 22, "gender": "Male", "phoneNumber": { "areaCode": "206", "number": "1234567"}, "children": [{ "name": "Jane", "gender": "Female", "age": "6"}, {"name": "John", "gender": "Male", "age": "15"}], "citiesLived": [{ "place": "Seattle", "yearsLived": ["1995"]}, {"place": "Stockholm", "yearsLived": ["2005"]}]} 2 | { "fullName": "Mike Jones", "age": 35, "gender": "Male", "phoneNumber": { "areaCode": "622", "number": "1567845"}, "children": [{ "name": "Earl", "gender": "Male", "age": "10"}, {"name": "Sam", "gender": "Male", "age": "6"}, { "name": "Kit", "gender": "Male", "age": "8"}], "citiesLived": [{"place": "Los Angeles", "yearsLived": ["1989", "1993", "1998", "2002"]}, {"place": "Washington DC", "yearsLived": ["1990", "1993", "1998", "2008"]}, {"place": "Portland", "yearsLived": ["1993", "1998", "2003", "2005"]}, {"place": "Austin", "yearsLived": ["1973", "1998", "2001", "2005"]}]} 3 | { "fullName": "Anna Karenina", "age": 45, "gender": "Female", "phoneNumber": { "areaCode": "425", "number": "1984783"}, "citiesLived": [{"place": "Stockholm", "yearsLived": ["1992", "1998", "2000", "2010"]}, {"place": "Russia", "yearsLived": ["1998", "2001", "2005"]}, {"place": "Austin", "yearsLived": ["1995", "1999"]}]} -------------------------------------------------------------------------------- /personsDataSchema.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "name": "kind", 3 | "mode": "nullable", 4 | "type": "string" 5 | }, 6 | { 7 | "name": "fullName", 8 | "type": "string", 9 | "mode": "required" 10 | }, 11 | { 12 | "name": "age", 13 | "type": "integer", 14 | "mode": "nullable" 15 | }, 16 | { 17 | "name": "gender", 18 | "type": "string", 19 | "mode": "nullable" 20 | }, 21 | { "name": "phoneNumber", 22 | "type": "record", 23 | "mode": "nullable", 24 | "fields": [ 25 | { 26 | "name": "areaCode", 27 | "type": "integer", 28 | "mode": "nullable" 29 | }, 30 | { 31 | "name": "number", 32 | "type": "integer", 33 | "mode": "nullable" 34 | } 35 | ] 36 | }, 37 | { 38 | "name": "children", 39 | "type": "record", 40 | "mode": "repeated", 41 | "fields": [ 42 | { 43 | "name": "name", 44 | "type": "string", 45 | "mode": "nullable" 46 | }, 47 | { 48 | "name": "gender", 49 | "type": "string", 50 | "mode": "nullable" 51 | }, 52 | { 53 | "name": "age", 54 | "type": "integer", 55 | "mode": "nullable" 56 | } 57 | ] 58 | }, 59 | { 60 | "name": "citiesLived", 61 | "type": "record", 62 | "mode": "repeated", 63 | "fields": [ 64 | { 65 | "name": "place", 66 | "type": "string", 67 | "mode": "nullable" 68 | }, 69 | { 70 | "name": "yearsLived", 71 | "type": "integer", 72 | "mode": "repeated" 73 | } 74 | ] 75 | } 76 | ] 77 | 78 | 79 | --------------------------------------------------------------------------------