The Pine fire started Thursday morning about 11 miles north of Ojai. As of Sunday morning, the fire had consumed 1,590 acres and was threatening 50 structures, said...
27 | ]]>Officials say Ahran Haugley, 41, drove off on April 28 when officers approached the Honda Accord in which he was...
56 | ]]>Williams, the defending women's champion and six-time winner, overwhelmed Annika Beck, 6-3, 6-0, in just...
73 | ]]>The two Mercedes drivers touched as Hamilton sought to overtake and Formula One championship leader Rosberg ended up losing his front wing, which...
83 | ]]>Sagan, who pulled on the coveted shirt for the first time, used his power on the 1.9-kilometer Cote de la Glacerie...
93 | ]]>Fire officials say it happened shortly before 11 a.m., inside the park at 68th Street and Fifth Avenue. The man suffered serious injuries, possibly requiring...
103 | ]]>The attack, claimed by Islamic State, was the deadliest in months in...
113 | ]]>Heading into his start against the Rockies on Saturday, Kazmir had a 9.00 ERA in the first inning of his 16 starts. Opponents were hitting a healthy .342 against him in the first. More than...
133 | ]]>Resident Martha Hall told the San Bernardino Sun that she saw the flames rushing up the hill toward her home. She ran into her house, grabbed her pet cockatiel and fled. As she was leaving, she...
153 | ]]>Trump himself came to Colorado for the Western Conservative...
163 | ]]>On the surface, the community of Porter Ranch is returning to normal four months after the largest methane leak in American history was capped in...
173 | ]]>Name: Lola app
What it does: Connects travelers with travel agents who can find the best options for you based on your preferences, including favorite airlines, hotel...
183 | ]]>Upon leaving for London, I took a taxi to the train...
211 | ]]>Once behind the wheel...
221 | ]]>Facebook founder and CEO Mark Zuckerberg said at a developer conference this year that "I hear fearful voices calling...
231 | ]]>Workshop
REI experts will share tips on exploring Yosemite National Park.
When, where: 7 p.m. Thursday at the REI store in Arcadia, 214 N. Santa Anita Ave.
Admission, info: Free. (626) 447-1062
PATAGONIA
Presentation
Explorer and mountain guide Tad McCrea will share his experiences and...
249 | ]]>stuff
") 74 | "stuff" 75 | """ 76 | 77 | @spec without_html(String.t()) :: String.t() 78 | 79 | def without_html(text) do 80 | text 81 | |> Floki.parse() 82 | |> Floki.text() 83 | end 84 | 85 | @doc """ 86 | A text paragraph shall not include any whitespace except single spaces 87 | between words. 88 | 89 | ## Example 90 | iex> Scrape.IR.Text.normalize_whitespace("\\r\\thello world\\r ") 91 | "hello world" 92 | """ 93 | 94 | @spec normalize_whitespace(String.t()) :: String.t() 95 | 96 | def normalize_whitespace(text) do 97 | text 98 | |> String.replace(~r/\s+/, " ") 99 | |> String.replace(~r/\s+/, " ") 100 | |> String.trim() 101 | end 102 | 103 | @doc """ 104 | Removes all junk from a given text, like javascript, html or mixed whitespace. 105 | 106 | ## Example 107 | iex> Scrape.IR.Text.clean("\\t hello, \\rworld!") 108 | "hello, world!" 109 | """ 110 | def clean(text) do 111 | text 112 | |> without_js() 113 | |> without_html() 114 | |> normalize_whitespace() 115 | end 116 | 117 | @doc """ 118 | Dissect a text into word tokens. 119 | 120 | The resulting list is a list of downcased words with all non-word-characters 121 | stripped. 122 | 123 | ## Examples 124 | iex> Scrape.IR.Text.tokenize("Hello, world!") 125 | ["hello", "world"] 126 | """ 127 | 128 | @spec tokenize(String.t()) :: [String.t()] 129 | 130 | def tokenize(text) do 131 | text 132 | |> String.replace(~r/[^\w\s]/u, " ") 133 | |> normalize_whitespace() 134 | |> String.downcase() 135 | |> String.split() 136 | end 137 | 138 | @doc """ 139 | Dissect a text into word tokens. 140 | 141 | The resulting list is a list of downcased words with all non-word-characters 142 | stripped, but common phrase delimiters still included. 143 | 144 | ## Examples 145 | iex> Scrape.IR.Text.tokenize_preserve_delimiters("Hello, world!") 146 | ["hello", ",", "world", "!"] 147 | """ 148 | 149 | @spec tokenize_preserve_delimiters(String.t()) :: [String.t()] 150 | 151 | def tokenize_preserve_delimiters(text) do 152 | text 153 | |> String.replace(~r/([,\.\!\?])/u, " \\1 ") 154 | |> String.replace(~r/[^\w\s,\.\!\?]/u, " ") 155 | |> normalize_whitespace() 156 | |> String.downcase() 157 | |> String.split() 158 | end 159 | 160 | @doc """ 161 | Dissect a text into word tokens similar to `tokenize/1` but strips words 162 | that carry no semantic value. 163 | 164 | ## Examples 165 | iex> Scrape.IR.Text.semantic_tokenize("A beautiful day!", :en) 166 | ["beautiful", "day"] 167 | """ 168 | 169 | @spec semantic_tokenize(String.t(), :de | :en) :: [String.t()] 170 | 171 | def semantic_tokenize(text, language \\ :en) do 172 | text 173 | |> tokenize() 174 | |> Enum.filter(fn word -> Word.is_meaningful?(word, language) end) 175 | end 176 | 177 | @doc """ 178 | Similar to `semantic_tokenize/2`, but also determines the n (default: 20) 179 | most relevant **stemmed** tokens from the list. 180 | """ 181 | 182 | def semantic_keywords(text, n \\ 20, language \\ :en) do 183 | text 184 | |> semantic_tokenize(language) 185 | |> Enum.map(&Word.stem(&1, language)) 186 | |> Enum.reduce(%{}, &aggregate_word_scores/2) 187 | |> Map.to_list() 188 | |> Enum.sort_by(fn {_word, score} -> score end, &>=/2) 189 | |> Enum.take(n) 190 | |> Enum.map(&elem(&1, 0)) 191 | end 192 | 193 | defp aggregate_word_scores(word, acc) do 194 | existing = Map.get(acc, word, 0) 195 | Map.put(acc, word, existing + 1) 196 | end 197 | end 198 | -------------------------------------------------------------------------------- /lib/scrape/ir/text/rake.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.IR.Text.RAKE do 2 | alias Scrape.IR.Text 3 | alias Scrape.Tools.Word 4 | 5 | def sample_text() do 6 | """ 7 | Compatibility of systems of linear constraints over the set of natural numbers 8 | 9 | Criteria of compatibility of a system of linear Diophantine equations, strict inequations, 10 | and nonstrict inequations are considered. Upper bounds for components of a minimal set of 11 | solutions and algorithms of construction of minimal generating sets of solutions for all 12 | types of systems are given. These criteria and the corresponding algorithms for constructing 13 | a minimal supporting set of solutions can be used in solving all the considered types of 14 | systems and systems of mixed types. 15 | """ 16 | end 17 | 18 | def execute(text, language \\ :en) do 19 | text 20 | |> Text.tokenize_preserve_delimiters() 21 | |> calculate_candidates(language) 22 | |> calculate_keyword_scores() 23 | end 24 | 25 | defp calculate_candidates(tokens, language) do 26 | calculate_candidates(tokens, language, [], []) 27 | end 28 | 29 | defp calculate_candidates([], _language, candidates, current_candidate) do 30 | candidates 31 | |> Kernel.++([current_candidate]) 32 | |> List.flatten() 33 | |> Enum.filter(&(String.length(String.trim(&1)) > 0)) 34 | end 35 | 36 | defp calculate_candidates([token | tokens], language, candidates, current_candidate) do 37 | if Word.is_stopword?(token, language) || token in [",", ".", "?", "!"] do 38 | calculate_candidates( 39 | tokens, 40 | language, 41 | candidates ++ [current_candidate |> Enum.join(" ")], 42 | [] 43 | ) 44 | else 45 | calculate_candidates(tokens, language, candidates, current_candidate ++ [token]) 46 | end 47 | end 48 | 49 | defp calculate_keyword_scores(candidates) do 50 | words = candidates |> Enum.map(&String.split(&1, " ")) |> List.flatten() |> Enum.uniq() 51 | len = length(words) 52 | 53 | word_index = 54 | 0..len |> Stream.zip(words) |> Enum.map(fn {k, v} -> {v, k} end) |> Enum.into(%{}) 55 | 56 | table = :ets.new(:co_occurence_matrix, [:set]) 57 | 58 | for candidate <- candidates do 59 | chunks = String.split(candidate, " ") 60 | 61 | Enum.each(chunks, fn chunk -> 62 | i = word_index[chunk] 63 | value = matrix_value(table, i, i) 64 | :ets.insert(table, {{i, i}, value + 1}) 65 | end) 66 | 67 | if length(chunks) > 1 do 68 | Enum.each(permutations(chunks), fn words -> 69 | i1 = word_index[Enum.at(words, 0)] 70 | i2 = word_index[Enum.at(words, 1)] 71 | value = matrix_value(table, i1, i2) 72 | :ets.insert(table, {{i1, i2}, value + 1}) 73 | end) 74 | end 75 | end 76 | 77 | word_scores = 78 | words 79 | |> Enum.map(fn word -> {word, matrix_row(table, word_index[word], len)} end) 80 | |> Enum.into(%{}) 81 | 82 | candidates 83 | |> Enum.uniq() 84 | |> Enum.map(fn candidate -> 85 | chunks = candidate |> String.split(" ") 86 | score = chunks |> Enum.map(&word_scores[&1]) |> Enum.sum() 87 | {candidate, score} 88 | end) 89 | |> Enum.sort_by(fn {_candidate, score} -> score end, &>=/2) 90 | |> Enum.take(length(words) |> Integer.floor_div(3)) 91 | |> Enum.map(fn {candidate, _score} -> candidate end) 92 | end 93 | 94 | defp matrix_value(table, i1, i2) do 95 | case :ets.lookup(table, {i1, i2}) do 96 | [] -> 0 97 | [{_, value}] -> value 98 | end 99 | end 100 | 101 | defp matrix_row(table, index, max) do 102 | 0..max 103 | |> Enum.map(fn i -> matrix_value(table, i, index) end) 104 | |> Enum.sum() 105 | end 106 | 107 | defp permutations(list), do: for(x <- list, y <- list, x != y, do: [x, y]) 108 | end 109 | -------------------------------------------------------------------------------- /lib/scrape/ir/text/tfidf.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.IR.Text.TFIDF do 2 | @moduledoc false 3 | 4 | defstruct [:text, :language, :corpus, :idf] 5 | 6 | def generate_database(text, language) do 7 | %__MODULE__{text: text, language: language} 8 | |> create_corpus() 9 | |> tokenize_sentences() 10 | |> calculate_idf_scores() 11 | end 12 | 13 | def query(%__MODULE__{corpus: corpus} = data, words) do 14 | idf = 15 | words 16 | |> Enum.map(fn word -> {word, calculate_inverse_document_frequency(data, word)} end) 17 | |> Enum.into(%{}) 18 | 19 | find_id = fn excludes -> find_best_sentence_id(corpus, idf, words, excludes) end 20 | s1_id = find_id.([]) 21 | s1_sentence = corpus[s1_id] 22 | 23 | s2_id = find_id.(s1_sentence.words) 24 | s2_sentence = corpus[s2_id] 25 | 26 | s3_id = find_id.(List.flatten([s1_sentence.words, s2_sentence.words])) 27 | s3_sentence = corpus[s3_id] 28 | 29 | [s1_sentence, s2_sentence, s3_sentence] 30 | |> Enum.map(fn %{sentence: sentence} -> sentence end) 31 | |> Enum.map(fn sentence -> sentence <> "." end) 32 | |> Enum.join(" ") 33 | end 34 | 35 | defp find_best_sentence_id(corpus, idf, words, blacklist) do 36 | for {id, %{tf: tf}} <- corpus do 37 | score = 38 | tf 39 | |> Enum.filter(fn {word, _} -> word in words end) 40 | |> Enum.filter(fn {word, _} -> word not in blacklist end) 41 | |> Enum.map(fn {word, value} -> value * idf[word] end) 42 | |> Enum.sum() 43 | 44 | {id, score} 45 | end 46 | |> Enum.sort_by(fn {_id, score} -> score end, &>=/2) 47 | |> List.first() 48 | |> elem(0) 49 | end 50 | 51 | defp create_corpus(%__MODULE__{text: text} = data) do 52 | corpus = 53 | text 54 | |> String.replace(~r/(\s\S+[a-zäöüß]+)([A-ZÄÖÜ]\S+\s)/u, "\\1. \\2") 55 | |> String.split(~r/[\?!\.\s]\s/) 56 | |> Enum.map(&String.trim/1) 57 | |> Enum.map(&String.replace(&1, ~r/\.+$/, "")) 58 | |> Enum.reject(fn sentence -> String.length(sentence) < 3 end) 59 | |> Enum.uniq() 60 | |> Enum.with_index() 61 | |> Enum.map(fn {sentence, i} -> {i, %{sentence: sentence}} end) 62 | |> Enum.into(%{}) 63 | 64 | %{data | corpus: corpus} 65 | end 66 | 67 | defp tokenize_sentences(%__MODULE__{corpus: corpus, language: language} = data) do 68 | updated_corpus = 69 | for {id, %{sentence: sentence} = document} <- corpus do 70 | words = tokenize(sentence, language) 71 | 72 | updated_document = 73 | document 74 | |> Map.put(:words, words) 75 | |> Map.put(:tf, calculate_term_frequency(words)) 76 | 77 | {id, updated_document} 78 | end 79 | |> Enum.into(%{}) 80 | 81 | %{data | corpus: updated_corpus} 82 | end 83 | 84 | defp calculate_idf_scores(%__MODULE__{corpus: corpus} = data) do 85 | idf = 86 | corpus 87 | |> Enum.map(fn {_id, %{words: words}} -> words end) 88 | |> List.flatten() 89 | |> Enum.uniq() 90 | |> Enum.map(fn word -> {word, calculate_inverse_document_frequency(data, word)} end) 91 | |> Enum.into(%{}) 92 | 93 | %{data | idf: idf} 94 | end 95 | 96 | defp calculate_term_frequency(list) do 97 | len = length(list) 98 | 99 | list 100 | |> Enum.group_by(& &1) 101 | |> Enum.map(fn {word, occurences} -> {word, length(occurences) / len} end) 102 | |> Enum.into(%{}) 103 | end 104 | 105 | defp calculate_inverse_document_frequency(%__MODULE__{corpus: corpus}, word) do 106 | num_docs = corpus |> Map.keys() |> length 107 | 108 | num_hits = 109 | corpus |> Map.values() |> Enum.filter(fn %{words: words} -> word in words end) |> length 110 | 111 | if num_hits == 0, do: 0, else: :math.log(num_docs / num_hits) 112 | end 113 | 114 | defp tokenize(str, language) do 115 | str 116 | |> String.replace(~r/[^\w\s]/u, "") 117 | |> Scrape.IR.Text.semantic_tokenize(language) 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /lib/scrape/options.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.Options do 2 | @moduledoc false 3 | 4 | @defaults num_stems: 30 5 | 6 | def merge(opts \\ []) do 7 | Keyword.merge(@defaults, opts) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/scrape/source/disk.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.Source.Disk do 2 | @moduledoc """ 3 | Abstraction over the native `File` functions. Currently without additional logic. 4 | """ 5 | 6 | @doc """ 7 | Same as `File.read/1`. 8 | """ 9 | def get(path) do 10 | File.read(path) 11 | end 12 | 13 | @doc """ 14 | Same as `File.read!/1`. 15 | """ 16 | def get!(path) do 17 | File.read!(path) 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/scrape/source/http.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.Source.HTTP do 2 | alias Scrape.Source.HTTP.Charset 3 | alias Scrape.Source.HTTP.Get 4 | alias Scrape.Source.HTTP.Transcode 5 | 6 | @doc """ 7 | Perform a HTTP GET request against the given url. 8 | 9 | This function is optimized for *text*-based data, not binary like images. 10 | It will try to normalize the response into valid utf-8 and transcode if needed. 11 | 12 | Everything that is not a status code 200 with valid encoding will result in 13 | some error object. 14 | 15 | ## Examples: 16 | iex> HTTP.get("http://example.com") 17 | {:ok, }"some response"} 18 | """ 19 | 20 | @spec get(String.t()) :: {:ok, String.t()} | {:error, any()} | {:http_error, any()} 21 | 22 | def get(url) do 23 | url |> Get.execute() |> evaluate() 24 | end 25 | 26 | @doc """ 27 | Same as `get/1`, but will raise if the result is not `:ok`. 28 | """ 29 | 30 | @spec get!(String.t()) :: String.t() 31 | 32 | def get!(url) do 33 | {:ok, data} = get(url) 34 | data 35 | end 36 | 37 | defp evaluate({:error, _} = response), do: response 38 | 39 | defp evaluate({:ok, %{status_code: 200, headers: headers, body: body}}) do 40 | case Charset.from_headers(headers) do 41 | nil -> {:ok, body} 42 | charset -> {:ok, Transcode.execute(charset, body)} 43 | end 44 | end 45 | 46 | defp evaluate({:ok, %{body: body}}), do: {:http_error, body} 47 | 48 | defp evaluate(response), do: response 49 | end 50 | -------------------------------------------------------------------------------- /lib/scrape/source/http/charset.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.Source.HTTP.Charset do 2 | @moduledoc false 3 | 4 | def from_headers(headers) do 5 | header = 6 | headers 7 | |> Enum.filter(fn {k, _} -> k == "Content-Type" end) 8 | |> first 9 | 10 | if header do 11 | {_name, content} = header 12 | 13 | ~r/charset=(ISO-8859-[1-9])/i 14 | |> Regex.run(content, capture: :all_but_first) 15 | |> first 16 | else 17 | nil 18 | end 19 | end 20 | 21 | defp first([h | _]), do: h 22 | defp first(_), do: nil 23 | end 24 | -------------------------------------------------------------------------------- /lib/scrape/source/http/get.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.Source.HTTP.Get do 2 | @moduledoc false 3 | 4 | @opts [ 5 | follow_redirect: true, 6 | timeout: 33_000, 7 | recv_timeout: 30_000, 8 | ssl: [{:versions, [:"tlsv1.2"]}] 9 | ] 10 | 11 | @headers [ 12 | "user-agent": 13 | "Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36", 14 | accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" 15 | ] 16 | 17 | def execute(url, http_headers \\ @headers, http_opts \\ @opts) do 18 | HTTPoison.get(url, http_headers, http_opts) 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/scrape/source/http/transcode.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.Source.HTTP.Transcode do 2 | @moduledoc false 3 | 4 | def execute(charset, text) do 5 | encoding = charset_to_encoding(charset) 6 | {_status, result} = Codepagex.to_string(text, encoding) 7 | result 8 | end 9 | 10 | defp charset_to_encoding(charset) do 11 | charset 12 | |> String.replace("-", "_") 13 | |> String.downcase() 14 | |> to_charlist 15 | |> List.to_atom() 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/scrape/tools/dom.ex: -------------------------------------------------------------------------------- 1 | defmodule Scrape.Tools.DOM do 2 | @moduledoc """ 3 | Utility module for selecting/extracting data from a "DOM" (HTML/XML tree-like 4 | structure). Can find text values and attribute values, inspired by jQuery and 5 | implemented with Floki. 6 | """ 7 | 8 | @typedoc """ 9 | DOM tree representation, same as Floki's html_tree. 10 | 11 | Can be created via `from_string/1`. 12 | """ 13 | 14 | @type dom :: String.t() | tuple() | [any()] 15 | 16 | @doc """ 17 | Create a DOM from a given (HTML/XML) string. 18 | 19 | ## Examples 20 | iex> DOM.from_string("") 21 | [] 22 | 23 | iex> DOM.from_string("") 24 | {"html", [], []} 25 | """ 26 | 27 | @spec from_string(String.t()) :: dom 28 | 29 | def from_string(string) do 30 | Floki.parse(string) 31 | end 32 | 33 | @doc """ 34 | Builds a (HTML/XML) string from a DOM structure. 35 | 36 | ## Examples 37 | iex> DOM.to_string([]) 38 | "" 39 | 40 | iex> DOM.to_string({"html", [], []}) 41 | "" 42 | """ 43 | 44 | @spec to_string(dom) :: String.t() 45 | 46 | def to_string(dom) do 47 | case dom do 48 | dom when is_tuple(dom) or is_list(dom) -> Floki.raw_html(dom) 49 | _ -> "" 50 | end 51 | end 52 | 53 | @doc """ 54 | Get the text value of a DOM node (including nested nodes). 55 | 56 | If many nodes match the selector, the first one is used. 57 | 58 | ## Examples 59 | iex> "a
b
" |> DOM.from_string() |> DOM.texts("p") 89 | ["a", "b"] 90 | """ 91 | 92 | @spec texts(dom, String.t()) :: [String.t()] 93 | 94 | def texts(dom, selector) do 95 | dom 96 | |> Floki.find(selector) 97 | |> Enum.map(&get_text/1) 98 | |> Enum.map(&unwrap_string/1) 99 | |> Enum.reject(&is_nil/1) 100 | |> List.wrap() 101 | end 102 | 103 | @doc """ 104 | Similar to `text/2` but but returns a chosen attribute value instead of the 105 | node's text value (or nil). 106 | 107 | ## Examples 108 | iex> "" |> DOM.from_string |> DOM.attr("meta", "unknown") 109 | nil 110 | 111 | iex> "" |> DOM.from_string |> DOM.attr("meta", "content") 112 | "b" 113 | 114 | iex> "" |> DOM.from_string |> DOM.attr("meta[name=a]", "content") 115 | "b" 116 | """ 117 | 118 | @spec attr(dom, String.t(), String.t()) :: nil | String.t() 119 | 120 | def attr(dom, selector, name) do 121 | dom 122 | |> Floki.find(selector) 123 | |> List.first() 124 | |> get_attr(name) 125 | |> unwrap_string() 126 | end 127 | 128 | @doc """ 129 | Similar to `attr/3` but returns a list of all matching results. 130 | 131 | ## Examples 132 | iex> "b
" |> DOM.from_string() |> DOM.attrs("div", "class") 133 | [] 134 | 135 | iex> "b
" |> DOM.from_string() |> DOM.attrs("p", "id") 136 | [] 137 | 138 | iex> "b
" |> DOM.from_string() |> DOM.attrs("p", "class") 139 | ["a", "c"] 140 | """ 141 | 142 | @spec attrs(dom, String.t(), String.t()) :: [String.t()] 143 | 144 | def attrs(dom, selector, name) do 145 | dom 146 | |> Floki.find(selector) 147 | |> Enum.map(&get_attr(&1, name)) 148 | |> Enum.map(&unwrap_string/1) 149 | |> Enum.reject(&is_nil/1) 150 | end 151 | 152 | @doc """ 153 | Cascading query helper, applies either `text/2` or `attr/3` until something 154 | returns a non-nil result or all queries are tried. 155 | 156 | ## Examples 157 | iex> DOM.first([], []) 158 | nil 159 | 160 | iex> DOM.first([], [{"b"}, {"i"}, {"div", "class"}]) 161 | nil 162 | 163 | iex> "hello world
") 48 | assert DOM.text(dom, "div") == nil 49 | end 50 | 51 | test "returns string if something is found" do 52 | dom = DOM.from_string("hello world
") 53 | assert DOM.text(dom, "p") == "hello world" 54 | end 55 | 56 | test "returns first string if many matches are found" do 57 | dom = DOM.from_string("hello
world
") 58 | assert DOM.text(dom, "p") == "hello" 59 | end 60 | end 61 | 62 | describe "DOM#texts/2" do 63 | test "returns empty list if nothing is found" do 64 | dom = DOM.from_string("hello world
") 65 | assert DOM.texts(dom, "div") == [] 66 | end 67 | 68 | test "returns string list if one match is found" do 69 | dom = DOM.from_string("hello world
") 70 | assert DOM.texts(dom, "p") == ["hello world"] 71 | end 72 | 73 | test "returns string list if many matches are found" do 74 | dom = DOM.from_string("hello
world
") 75 | assert DOM.texts(dom, "p") == ["hello", "world"] 76 | end 77 | end 78 | 79 | describe "DOM#attr/3" do 80 | test "returns nil if nothing is found" do 81 | dom = DOM.from_string("") 82 | assert DOM.attr(dom, "unknown", "unknown") == nil 83 | assert DOM.attr(dom, "meta", "unknown") == nil 84 | assert DOM.attr(dom, "meta[name='unknown']", "unknown") == nil 85 | assert DOM.attr(dom, "unknown", "content") == nil 86 | end 87 | 88 | test "returns string if something is found" do 89 | dom = DOM.from_string("") 90 | assert DOM.attr(dom, "meta[name='a']", "content") == "b" 91 | end 92 | 93 | test "returns first string if many matches are found" do 94 | dom = DOM.from_string("") 95 | assert DOM.attr(dom, "meta[name='a']", "content") == "b" 96 | end 97 | end 98 | 99 | describe "DOM#attrs/3" do 100 | test "returns empty list if nothing is found" do 101 | dom = DOM.from_string("") 102 | assert DOM.attrs(dom, "unknown", "unknown") == [] 103 | end 104 | 105 | test "returns string list if one match is found" do 106 | dom = DOM.from_string("") 107 | assert DOM.attrs(dom, "meta[name=a]", "content") == ["b"] 108 | end 109 | 110 | test "returns string list if many matches are found" do 111 | dom = DOM.from_string("") 112 | assert DOM.attrs(dom, "meta[name=a]", "content") == ["b", "c"] 113 | end 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /test/tools/tree_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Scrape.Tools.TreeTest do 2 | use ExUnit.Case 3 | 4 | alias Scrape.Tools.Tree 5 | 6 | doctest Tree 7 | 8 | describe "Tree#from_string/1" do 9 | test "works with nil" do 10 | assert Tree.from_xml_string(nil) == %{} 11 | end 12 | 13 | test "works with empty string" do 14 | assert Tree.from_xml_string("") == %{} 15 | end 16 | 17 | test "works with xml string" do 18 | assert Tree.from_xml_string("