├── .gitignore ├── scrrenshot-01.png ├── images ├── 2752723036.jpg ├── 3936430428.jpg └── 3960968780.jpg ├── sreality.Rproj ├── R └── sreality-funcs.R ├── README.Rmd └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | -------------------------------------------------------------------------------- /scrrenshot-01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MarekProkop/sreality/HEAD/scrrenshot-01.png -------------------------------------------------------------------------------- /images/2752723036.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MarekProkop/sreality/HEAD/images/2752723036.jpg -------------------------------------------------------------------------------- /images/3936430428.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MarekProkop/sreality/HEAD/images/3936430428.jpg -------------------------------------------------------------------------------- /images/3960968780.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MarekProkop/sreality/HEAD/images/3960968780.jpg -------------------------------------------------------------------------------- /sreality.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: knitr 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | -------------------------------------------------------------------------------- /R/sreality-funcs.R: -------------------------------------------------------------------------------- 1 | add_property <- function(data_path, img_dir = NULL, url) { 2 | if (file.exists(data_path)) { 3 | properties <- readr::read_rds(data_path) 4 | } else { 5 | properties <- create_archive() 6 | warning(paste0("File ", data_path, "doesn't exist. Creating a new one.")) 7 | } 8 | id = id_from_url(url) 9 | stopifnot("Id already exists." = !(id %in% properties$id)) 10 | new_properties <- id |> 11 | purrr::map_dfr(fetch_prop_info, img_dir = img_dir) |> 12 | dplyr::mutate(status = "init", url = url) 13 | properties |> 14 | dplyr::bind_rows(new_properties) |> 15 | readr::write_rds(data_path) 16 | new_properties 17 | } 18 | 19 | id_from_url <- function(url) { 20 | stringr::str_extract(url, "\\d+$") 21 | } 22 | 23 | update_properties <- function(rds_path) { 24 | archived_data <- readr::read_rds(rds_path) 25 | fresh_data <- archived_data |> 26 | dplyr::pull(id) |> 27 | unique() |> 28 | purrr::map_dfr(fetch_prop_info) 29 | archived_data |> 30 | dplyr::bind_rows(fresh_data) |> 31 | readr::write_rds(rds_path) 32 | } 33 | 34 | list_properies <- function(rds_path) { 35 | readr::read_rds(rds_path) |> 36 | dplyr::group_by(id) |> 37 | tidyr::fill(location, name, price, description, url, .direction = "down") |> 38 | dplyr::summarise( 39 | id = dplyr::first(id), 40 | status = dplyr::last(status), 41 | checked_0 = min(checked), 42 | checked_last = max(checked), 43 | dur = checked_last - checked_0, 44 | location = dplyr::last(location), 45 | name = dplyr::last(name), 46 | price_0 = dplyr::first(price), 47 | price_last = dplyr::last(price), 48 | description = dplyr::last(description), 49 | url = dplyr::first(url) 50 | ) 51 | } 52 | 53 | 54 | # Private functions ------------------------------------------------------- 55 | 56 | 57 | create_archive <- function(data_path) { 58 | tibble::tibble( 59 | id = character(), 60 | status = character(), 61 | checked = as.POSIXct(numeric()), 62 | location = character(), 63 | name = character(), 64 | price = numeric(), 65 | description = character(), 66 | url = character() 67 | ) 68 | } 69 | 70 | fetch_prop_info <- function(prop_id, img_dir = NULL) { 71 | base_url <- "https://www.sreality.cz/api/cs/v2/estates/" 72 | resp <- httr::GET(paste0(base_url, prop_id)) 73 | if (httr::http_error(resp)) { 74 | if (resp$status_code == 410) { 75 | return(list( 76 | id = prop_id, 77 | status = "gone", 78 | checked = Sys.time() 79 | )) 80 | } else { 81 | stop(httr::http_status(resp)$message) 82 | } 83 | } else { 84 | json <- jsonlite::fromJSON(httr::content(resp, as = "text", encoding = "utf-8")) 85 | if (!is.null(img_dir)) { 86 | json[["_embedded"]][["images"]][["_links"]][["gallery"]][["href"]][1] |> 87 | download.file(destfile = paste0(img_dir, prop_id, ".jpg"), mode = "wb") 88 | } 89 | list( 90 | id = prop_id, 91 | status = "live", 92 | checked = Sys.time(), 93 | location = json$locality$value, 94 | name = stringr::str_squish(json$name$value), 95 | price = json$price_czk$value_raw, 96 | description = json$meta_description 97 | ) 98 | } 99 | } 100 | 101 | -------------------------------------------------------------------------------- /README.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | output: github_document 3 | --- 4 | 5 | 6 | 7 | ```{r setup, include = FALSE} 8 | knitr::opts_chunk$set( 9 | collapse = TRUE, 10 | comment = "#>" 11 | ) 12 | 13 | file.remove("data/my_properties.rds") 14 | ``` 15 | 16 | # sreality 17 | 18 | 19 | 20 | 21 | With the functions from this repository you can create your personal database of property offers from the Sreality website, update their status and analyse their history. You need some knowledge of R to do this. 22 | 23 | ## How to use 24 | 25 | 1. Clone this repo, or create a new project in RStudio and copy the R folder. 26 | 2. Create subfolders `data` and `images`. 27 | 28 | ### Load packages and functions 29 | 30 | ```{r message=FALSE, warning=FALSE} 31 | library(tidyverse) 32 | library(httr) 33 | library(jsonlite) 34 | 35 | source("R/sreality-funcs.R") 36 | ``` 37 | 38 | ### Add property 39 | 40 | Adds the property with the given URL to the `data/my_properties.rds` file. If you specify the `img_dir` parameter, a small preview image is downloaded and saved in this folder under the name _id_property.jpg_ 41 | 42 | If a property with the given id already exists in the file, an error will occur. 43 | 44 | ```{r} 45 | add_property( 46 | data_path = "data/my_properties.rds", 47 | img_dir = "images/", 48 | url = "https://www.sreality.cz/detail/prodej/dum/rodinny/melnik-melnik-namesti-miru/3960968780" 49 | ) 50 | ``` 51 | 52 | Adds multiple properties to the `data/my_properties.rds` file. If any property id already exists in the file, an error will occur. 53 | 54 | ```{r} 55 | add_property( 56 | data_path = "data/my_properties.rds", 57 | img_dir = "images/", 58 | url = c( 59 | "https://www.sreality.cz/detail/prodej/dum/rodinny/vernerice-rychnov-/2752723036", 60 | "https://www.sreality.cz/detail/prodej/dum/rodinny/tabor-tabor-provaznicka/3936430428" 61 | ) 62 | ) 63 | ``` 64 | 65 | ### Update properties 66 | 67 | Fetches all properties in the `data/my_properties.rds` file from Sreality and appends their current data to the end of the file. 68 | 69 | ```{r} 70 | update_properties(rds_path = "data/my_properties.rds") 71 | ``` 72 | 73 | ### List archived data 74 | 75 | #### Full content of the archive 76 | 77 | ```{r} 78 | read_rds("data/my_properties.rds") 79 | ``` 80 | 81 | #### Current state of properties 82 | 83 | Displays the current state of all properties in the `data/my_properties.rds` file. 84 | 85 | ```{r} 86 | list_properies("data/my_properties.rds") 87 | ``` 88 | 89 | You can filter the previous dataframe by status (only valid: `status %in% c("init", "live")` or only removed: `status == "gone"`), by price change (`price_last != price_0`), etc. 90 | 91 | #### List of properties using the {gt} package 92 | 93 | ```{r} 94 | list_properies("data/my_properties.rds") |> 95 | mutate( 96 | name = paste0("[", name, "](", url, ")"), 97 | name = map(name, gt::md) 98 | ) |> 99 | select(name, location, price_last) |> 100 | gt::gt() |> 101 | gt::cols_label(price_last = "price") |> 102 | gt::fmt_number(price_last, decimals = 0, suffixing = "K") |> 103 | gt::as_raw_html() 104 | ``` 105 | 106 | You can also add images to the listing. 107 | 108 | ```{r eval=FALSE} 109 | list_properies("data/my_properties.rds") |> 110 | mutate( 111 | title = paste0(location, "\n\n", replace_na(name, "")), 112 | title = map(title, gt::md) 113 | ) |> 114 | select(id, title, price_last) |> 115 | gt::gt() |> 116 | gt::cols_label(price_last = "price") |> 117 | gt::fmt_number(price_last, decimals = 0, suffixing = "K") |> 118 | gt::text_transform( 119 | locations = gt::cells_body(columns = id), 120 | fn = function(x) { 121 | map_chr(x, function(x) { 122 | img_path <- here::here("images", paste0(x, ".jpg")) 123 | if (file.exists(img_path)) { 124 | gt::local_image(filename = img_path, height = 166) 125 | } else { 126 | x 127 | } 128 | }) 129 | } 130 | ) 131 | ``` 132 | 133 | ![listing preview](scrrenshot-01.png) 134 | 135 | ```{r cleanup, include=FALSE} 136 | file.remove("data/my_properties.rds") 137 | ``` 138 | 139 | 140 | ## To-do 141 | 142 | Maybe I'll add those features some day: 143 | 144 | - Downloading preview images of the property. 145 | - Shiny app to make it more user-friendly. 146 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # sreality 5 | 6 | 7 | 8 | 9 | 10 | With the functions from this repository you can create your personal 11 | database of property offers from the Sreality website, update their 12 | status and analyse their history. You need some knowledge of R to do 13 | this. 14 | 15 | ## How to use 16 | 17 | 1. Clone this repo, or create a new project in RStudio and copy the R 18 | folder. 19 | 2. Create subfolders `data` and `images`. 20 | 21 | ### Load packages and functions 22 | 23 | ``` r 24 | library(tidyverse) 25 | library(httr) 26 | library(jsonlite) 27 | 28 | source("R/sreality-funcs.R") 29 | ``` 30 | 31 | ### Add property 32 | 33 | Adds the property with the given URL to the `data/my_properties.rds` 34 | file. If you specify the `img_dir` parameter, a small preview image is 35 | downloaded and saved in this folder under the name *id\_property.jpg* 36 | 37 | If a property with the given id already exists in the file, an error 38 | will occur. 39 | 40 | ``` r 41 | add_property( 42 | data_path = "data/my_properties.rds", 43 | img_dir = "images/", 44 | url = "https://www.sreality.cz/detail/prodej/dum/rodinny/melnik-melnik-namesti-miru/3960968780" 45 | ) 46 | #> Warning in add_property(data_path = "data/my_properties.rds", img_dir = 47 | #> "images/", : File data/my_properties.rdsdoesn't exist. Creating a new one. 48 | #> # A tibble: 1 × 8 49 | #> id status checked location name price description url 50 | #> 51 | #> 1 3960968780 init 2022-07-07 09:43:11 náměstí … Prod… 1.30e7 Rodinný dů… http… 52 | ``` 53 | 54 | Adds multiple properties to the `data/my_properties.rds` file. If any 55 | property id already exists in the file, an error will occur. 56 | 57 | ``` r 58 | add_property( 59 | data_path = "data/my_properties.rds", 60 | img_dir = "images/", 61 | url = c( 62 | "https://www.sreality.cz/detail/prodej/dum/rodinny/vernerice-rychnov-/2752723036", 63 | "https://www.sreality.cz/detail/prodej/dum/rodinny/tabor-tabor-provaznicka/3936430428" 64 | ) 65 | ) 66 | #> # A tibble: 2 × 8 67 | #> id status checked location name price description url 68 | #> 69 | #> 1 2752723036 init 2022-07-07 09:43:11 Verneřic… Prod… 5.79e6 Rodinný dů… http… 70 | #> 2 3936430428 init 2022-07-07 09:43:12 Provazni… Prod… 9.7 e6 Rodinný dů… http… 71 | ``` 72 | 73 | ### Update properties 74 | 75 | Fetches all properties in the `data/my_properties.rds` file from 76 | Sreality and appends their current data to the end of the file. 77 | 78 | ``` r 79 | update_properties(rds_path = "data/my_properties.rds") 80 | ``` 81 | 82 | ### List archived data 83 | 84 | #### Full content of the archive 85 | 86 | ``` r 87 | read_rds("data/my_properties.rds") 88 | #> # A tibble: 6 × 8 89 | #> id status checked location name price description url 90 | #> 91 | #> 1 3960968780 init 2022-07-07 09:43:11 náměstí … Prod… 1.30e7 Rodinný dů… http… 92 | #> 2 2752723036 init 2022-07-07 09:43:11 Verneřic… Prod… 5.79e6 Rodinný dů… http… 93 | #> 3 3936430428 init 2022-07-07 09:43:12 Provazni… Prod… 9.7 e6 Rodinný dů… http… 94 | #> 4 3960968780 live 2022-07-07 09:43:12 náměstí … Prod… 1.30e7 Rodinný dů… 95 | #> 5 2752723036 live 2022-07-07 09:43:12 Verneřic… Prod… 5.79e6 Rodinný dů… 96 | #> 6 3936430428 live 2022-07-07 09:43:13 Provazni… Prod… 9.7 e6 Rodinný dů… 97 | ``` 98 | 99 | #### Current state of properties 100 | 101 | Displays the current state of all properties in the 102 | `data/my_properties.rds` file. 103 | 104 | ``` r 105 | list_properies("data/my_properties.rds") 106 | #> # A tibble: 3 × 11 107 | #> id status checked_0 checked_last dur location name 108 | #> 109 | #> 1 2752723036 live 2022-07-07 09:43:11 2022-07-07 09:43:12 1.18… Verneři… Prod… 110 | #> 2 3936430428 live 2022-07-07 09:43:12 2022-07-07 09:43:13 1.07… Provazn… Prod… 111 | #> 3 3960968780 live 2022-07-07 09:43:11 2022-07-07 09:43:12 1.37… náměstí… Prod… 112 | #> # … with 4 more variables: price_0 , price_last , description , 113 | #> # url 114 | ``` 115 | 116 | You can filter the previous dataframe by status (only valid: `status 117 | %in% c("init", "live")` or only removed: `status == "gone"`), by price 118 | change (`price_last != price_0`), etc. 119 | 120 | #### List of properties using the {gt} package 121 | 122 | ``` r 123 | list_properies("data/my_properties.rds") |> 124 | mutate( 125 | name = paste0("[", name, "](", url, ")"), 126 | name = map(name, gt::md) 127 | ) |> 128 | select(name, location, price_last) |> 129 | gt::gt() |> 130 | gt::cols_label(price_last = "price") |> 131 | gt::fmt_number(price_last, decimals = 0, suffixing = "K") |> 132 | gt::as_raw_html() 133 | ``` 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 |
namelocationprice
Prodej rodinného domu 208 m², pozemek 2 069 m²Verneřice - Rychnov, okres Děčín5,790K
Prodej rodinného domu 231 m², pozemek 107 m²Provaznická, Tábor9,700K
Prodej rodinného domu 150 m², pozemek 250 m²náměstí Míru, Mělník12,990K
158 | 159 | You can also add images to the listing. 160 | 161 | ``` r 162 | list_properies("data/my_properties.rds") |> 163 | mutate( 164 | title = paste0(location, "\n\n", replace_na(name, "")), 165 | title = map(title, gt::md) 166 | ) |> 167 | select(id, title, price_last) |> 168 | gt::gt() |> 169 | gt::cols_label(price_last = "price") |> 170 | gt::fmt_number(price_last, decimals = 0, suffixing = "K") |> 171 | gt::text_transform( 172 | locations = gt::cells_body(columns = id), 173 | fn = function(x) { 174 | map_chr(x, function(x) { 175 | img_path <- here::here("images", paste0(x, ".jpg")) 176 | if (file.exists(img_path)) { 177 | gt::local_image(filename = img_path, height = 166) 178 | } else { 179 | x 180 | } 181 | }) 182 | } 183 | ) 184 | ``` 185 | 186 | ![listing preview](scrrenshot-01.png) 187 | 188 | ## To-do 189 | 190 | Maybe I’ll add those features some day: 191 | 192 | - Downloading preview images of the property. 193 | - Shiny app to make it more user-friendly. 194 | --------------------------------------------------------------------------------