├── .gitignore ├── .rspec ├── Gemfile ├── crypto-currency.jpg ├── spec ├── scraper_spec.rb ├── main_spec.rb └── spec_helper.rb ├── .github └── workflows │ └── linters.yml ├── bin └── main.rb ├── Gemfile.lock ├── lib └── scraper.rb ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gem 'httparty' 3 | gem 'nokogiri' 4 | gem 'rspec' 5 | -------------------------------------------------------------------------------- /crypto-currency.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gango-anan/crypto-scraper/HEAD/crypto-currency.jpg -------------------------------------------------------------------------------- /spec/scraper_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../lib/scraper' 2 | 3 | describe Scraper do 4 | describe '#scrape' do 5 | let(:new_scraper) { Scraper.new('https://coinmarketcap.com/') } 6 | it 'returns an array of the top 10 most traded cryptos' do 7 | expect(new_scraper.scrape.length).to eql(10) 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /spec/main_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../bin/main' 2 | 3 | describe '#generate_csv' do 4 | let(:test_array) do 5 | [{ 6 | name: 'Gango', 7 | price: 'Ugx 9,999', 8 | marketcap: 'Ugx 2B', 9 | volume: '9,999 UTC', 10 | circulatingsupply: '18B UTC' 11 | }] 12 | end 13 | it 'should return nil if a csv file is successfully generated' do 14 | expect(generate_csv(test_array)).to eql(nil) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: Linters 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | rubocop: 7 | name: Rubocop 8 | runs-on: ubuntu-18.04 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: actions/setup-ruby@v1 12 | with: 13 | ruby-version: 2.6.x 14 | - name: Setup Rubocop 15 | run: | 16 | gem install --no-document rubocop:'~>0.81.0' # https://docs.rubocop.org/en/stable/installation/ 17 | [ -f .rubocop.yml ] || wget https://raw.githubusercontent.com/microverseinc/linters-config/master/ruby/.rubocop.yml 18 | - name: Rubocop Report 19 | run: rubocop --color 20 | -------------------------------------------------------------------------------- /bin/main.rb: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env ruby 2 | 3 | require 'csv' 4 | require_relative '../lib/scraper' 5 | 6 | def generate_csv(crypto_details) 7 | filepath = 'scrape.csv' 8 | csv_options = { headers: :first_row, col_sep: ',' } 9 | CSV.open(filepath, 'wb', csv_options) do |csv| 10 | csv << ['Name', 'Price', 'Market Cap', 'Volume', 'Circulating Supply'] 11 | crypto_details.each do |crypto| 12 | csv << crypto.values 13 | end 14 | end 15 | nil 16 | end 17 | 18 | scraped_page = Scraper.new('https://coinmarketcap.com/') 19 | cryptos = scraped_page.scrape 20 | crypto_details = [] 21 | 10.times do |index| 22 | crypto_details << cryptos[index] 23 | end 24 | generate_csv(crypto_details) 25 | puts '' 26 | puts ' ========Thanks for using Crypto Scraper.========' 27 | puts '' 28 | puts "Your scrape csv file has been created in the Application's directory." 29 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | diff-lcs (1.4.4) 5 | httparty (0.18.1) 6 | mime-types (~> 3.0) 7 | multi_xml (>= 0.5.2) 8 | mime-types (3.3.1) 9 | mime-types-data (~> 3.2015) 10 | mime-types-data (3.2020.1104) 11 | mini_portile2 (2.4.0) 12 | multi_xml (0.6.0) 13 | nokogiri (1.10.10-x64-mingw32) 14 | mini_portile2 (~> 2.4.0) 15 | rspec (3.10.0) 16 | rspec-core (~> 3.10.0) 17 | rspec-expectations (~> 3.10.0) 18 | rspec-mocks (~> 3.10.0) 19 | rspec-core (3.10.0) 20 | rspec-support (~> 3.10.0) 21 | rspec-expectations (3.10.0) 22 | diff-lcs (>= 1.2.0, < 2.0) 23 | rspec-support (~> 3.10.0) 24 | rspec-mocks (3.10.0) 25 | diff-lcs (>= 1.2.0, < 2.0) 26 | rspec-support (~> 3.10.0) 27 | rspec-support (3.10.0) 28 | 29 | PLATFORMS 30 | x64-mingw32 31 | 32 | DEPENDENCIES 33 | httparty 34 | nokogiri 35 | rspec 36 | 37 | BUNDLED WITH 38 | 2.2.2 39 | -------------------------------------------------------------------------------- /lib/scraper.rb: -------------------------------------------------------------------------------- 1 | require 'nokogiri' 2 | require 'httparty' 3 | 4 | # This is a class designed to scrape CoinMarketCap 5 | class Scraper 6 | attr_reader :url 7 | 8 | def initialize(new_url) 9 | @url = new_url 10 | end 11 | 12 | def scrape 13 | row_page = HTTParty.get(@url) 14 | filtered_page = Nokogiri::HTML(row_page.body) 15 | cryptos = [] 16 | crypto_details = filtered_page.css('tbody tr:not([class])') 17 | populate_crypto(crypto_details, cryptos) 18 | cryptos 19 | end 20 | 21 | private 22 | 23 | def populate_crypto(crypto_details, cryptos) 24 | crypto_details.each do |crypto_detail| 25 | crypto = { 26 | name: crypto_detail.css('div > p.sc-1eb5slv-0.iJjGCS').text, 27 | price: crypto_detail.css('div.price___3rj7O>a.cmc-link').text, 28 | marketcap: crypto_detail.css('td>p.sc-1eb5slv-0.kDEzev').text, 29 | volume: crypto_detail.css('p.sc-1eb5slv-0.etpvrL').text, 30 | circulatingsupply: crypto_detail.css('div>p.sc-1eb5slv-0.hNpJqV').text 31 | } 32 | cryptos << crypto 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Galiwango Ananiya 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](https://img.shields.io/badge/Microverse-blueviolet) 2 | 3 | # Crypto Scrape 4 | 5 | > Crypto Scrape is a project that involves designing a web scrapping tool that runs from the terminal, to help scrape https://coinmarketcap.com/ (a real-time cryptocurrency market cap rankings platform) 6 | for data on the top 10 most traded crypto currencies. I choose to create design crypto scrape because am a cryptocurrency ethusiast and do trade in cryptocurrency 7 | trading during my free time soo a tool that helps give me updates on how a given cryptocurrency is performing is handy to me. 8 | 9 | ## Additional description about the project and its features. 10 | - This tool scrapes coinmarket cap and returns the upto date details of a crypto name, the price, the volume, market cap and the circulating supply. 11 | - The tool is customized to give details of the top 10 highly traded crypto currencies. 12 | - After scrapping, the tool generates a CSV file with the required data 13 | 14 | ## Built With 15 | 16 | - Major languages: Ruby 17 | - Frameworks: Nokogiri and HTTParty 18 | - Technologies used: MS Excel for viewing the CSV file 19 | 20 | ## Getting Started 21 | Install all the required dependecies by following these simple instructions: 22 | - To run Crypto Scrape, make sure you have a internet connection 23 | - Install the most recent version of ruby by typing the following command in your terminal: 24 | gem install ruby 25 | - Install the required libraries by typing the following command in your terminal: 26 | bundle install 27 | 28 | ## Setup and run the application 29 | To get a local copy up and running follow these simple example steps. 30 | - Clone the repository by typing the following command in your terminal 31 | git clone https://github.com/gango-anan/crypto-scraper.git 32 | - Change directory by typing the following command in your terminal 33 | cd crypto-scraper 34 | - Run the program by typing the following command in your terminal 35 | bin/main.rb 36 | - Open scrape.csv file crypto-scraper in using any spreadsheet application such as MS Excel for windows platform and OpenOffice Spreadsheet for linux platforms. 37 | 38 | ## Run Test 39 | - Open your terminal and run the following test command to test the application 40 | rspec --format documentation 41 | 42 | 43 | ## Author 44 | 45 | Galiwango Ananiya 46 | 47 | - GitHub: [@gango-anan](https://github.com/gango-anan) 48 | - Twitter: [@gango_anan](https://twitter.com/gango_anan) 49 | - LinkedIn: [@galiwango-ananiya](https://www.linkedin.com/in/galiwango-ananiya-0800821b4/) 50 | 51 | ## 🤝 Contributing 52 | 53 | Contributions, issues, and feature requests are welcome! 54 | 55 | Feel free to check the [issues page](https://github.com/gango-anan/crypto-scraper/issues). 56 | 57 | ## Show your support 58 | 59 | Give a ⭐️ if you like this project! 60 | 61 | ## Acknowledgments 62 | 63 | - I would like to thank my standup team for the support and positive code reviews during the standup team meetings, thumbs up to the Odin Project for the resourceful 64 | materials they offer. I would also like to thank the Nokogiri and HTTParty teams for the job well done to make such great libraries, without them this project would ghave beena night mare to complete. 65 | 66 | 67 | ## 📝 License 68 | 69 | This project is [MIT](https://github.com/gango-anan/crypto-scraper/blob/develop/LICENSE) licensed. -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # This file was generated by the `rspec --init` command. Conventionally, all 2 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 3 | # The generated `.rspec` file contains `--require spec_helper` which will cause 4 | # this file to always be loaded, without a need to explicitly require it in any 5 | # files. 6 | # 7 | # Given that it is always loaded, you are encouraged to keep this file as 8 | # light-weight as possible. Requiring heavyweight dependencies from this file 9 | # will add to the boot time of your test suite on EVERY test run, even for an 10 | # individual file that may not need all of that loaded. Instead, consider making 11 | # a separate helper file that requires the additional dependencies and performs 12 | # the additional setup, and require it from the spec files that actually need 13 | # it. 14 | # 15 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 16 | RSpec.configure do |config| 17 | # rspec-expectations config goes here. You can use an alternate 18 | # assertion/expectation library such as wrong or the stdlib/minitest 19 | # assertions if you prefer. 20 | config.expect_with :rspec do |expectations| 21 | # This option will default to `true` in RSpec 4. It makes the `description` 22 | # and `failure_message` of custom matchers include text for helper methods 23 | # defined using `chain`, e.g.: 24 | # be_bigger_than(2).and_smaller_than(4).description 25 | # # => "be bigger than 2 and smaller than 4" 26 | # ...rather than: 27 | # # => "be bigger than 2" 28 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 29 | end 30 | 31 | # rspec-mocks config goes here. You can use an alternate test double 32 | # library (such as bogus or mocha) by changing the `mock_with` option here. 33 | config.mock_with :rspec do |mocks| 34 | # Prevents you from mocking or stubbing a method that does not exist on 35 | # a real object. This is generally recommended, and will default to 36 | # `true` in RSpec 4. 37 | mocks.verify_partial_doubles = true 38 | end 39 | 40 | # This option will default to `:apply_to_host_groups` in RSpec 4 (and will 41 | # have no way to turn it off -- the option exists only for backwards 42 | # compatibility in RSpec 3). It causes shared context metadata to be 43 | # inherited by the metadata hash of host groups and examples, rather than 44 | # triggering implicit auto-inclusion in groups with matching metadata. 45 | config.shared_context_metadata_behavior = :apply_to_host_groups 46 | 47 | # The settings below are suggested to provide a good initial experience 48 | # with RSpec, but feel free to customize to your heart's content. 49 | # # This allows you to limit a spec run to individual examples or groups 50 | # # you care about by tagging them with `:focus` metadata. When nothing 51 | # # is tagged with `:focus`, all examples get run. RSpec also provides 52 | # # aliases for `it`, `describe`, and `context` that include `:focus` 53 | # # metadata: `fit`, `fdescribe` and `fcontext`, respectively. 54 | # config.filter_run_when_matching :focus 55 | # 56 | # # Allows RSpec to persist some state between runs in order to support 57 | # # the `--only-failures` and `--next-failure` CLI options. We recommend 58 | # # you configure your source control system to ignore this file. 59 | # config.example_status_persistence_file_path = "spec/examples.txt" 60 | # 61 | # # Limits the available syntax to the non-monkey patched syntax that is 62 | # # recommended. For more details, see: 63 | # # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ 64 | # # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 65 | # # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode 66 | # config.disable_monkey_patching! 67 | # 68 | # # This setting enables warnings. It's recommended, but in some cases may 69 | # # be too noisy due to issues in dependencies. 70 | # config.warnings = true 71 | # 72 | # # Many RSpec users commonly either run the entire suite or an individual 73 | # # file, and it's useful to allow more verbose output when running an 74 | # # individual spec file. 75 | # if config.files_to_run.one? 76 | # # Use the documentation formatter for detailed output, 77 | # # unless a formatter has already been configured 78 | # # (e.g. via a command-line flag). 79 | # config.default_formatter = "doc" 80 | # end 81 | # 82 | # # Print the 10 slowest examples and example groups at the 83 | # # end of the spec run, to help surface which specs are running 84 | # # particularly slow. 85 | # config.profile_examples = 10 86 | # 87 | # # Run specs in random order to surface order dependencies. If you find an 88 | # # order dependency and want to debug it, you can fix the order by providing 89 | # # the seed, which is printed after each run. 90 | # # --seed 1234 91 | # config.order = :random 92 | # 93 | # # Seed global randomization in this process using the `--seed` CLI option. 94 | # # Setting this allows you to use `--seed` to deterministically reproduce 95 | # # test failures related to randomization by passing the same `--seed` value 96 | # # as the one that triggered the failure. 97 | # Kernel.srand config.seed 98 | end 99 | --------------------------------------------------------------------------------