Sorry but I don't know how to flip from: <%= params[:url] %>
7 |
8 |
14 |
--------------------------------------------------------------------------------
/INSTALL.md:
--------------------------------------------------------------------------------
1 | # Installing dbpedialite on your machine
2 |
3 | Clone the repo from github:
4 |
5 | git clone git://github.com/njh/dbpedialite.git
6 | cd dbpedialite
7 |
8 | Install the required gems using [bundler](http://gembundler.com/):
9 |
10 | bundle install
11 |
12 | Start the application:
13 |
14 | bundle exec rackup -p 4567
15 |
16 | Or use shotgun, which causes the app to reload after every request:
17 |
18 | bundle exec shotgun -p 4567
19 |
20 | Visit the application in your browser at:
21 | [http://localhost:4567/](http://localhost:4567/)
22 |
23 | You can then run the tests using:
24 |
25 | bundle exec rake spec
26 |
--------------------------------------------------------------------------------
/spec/spec_helper.rb:
--------------------------------------------------------------------------------
1 | $:.unshift(File.join(File.dirname(__FILE__),'..','lib'))
2 | $:.unshift(File.join(File.dirname(__FILE__),'..'))
3 |
4 | require 'rubygems'
5 | require 'bundler'
6 |
7 | Bundler.require(:default, :test)
8 |
9 | unless RUBY_VERSION =~ /^1\.8/
10 | SimpleCov.start
11 | end
12 |
13 | RSpec.configure do |config|
14 | config.before(:each) do
15 | FakeWeb.clean_registry
16 | end
17 | end
18 |
19 | FakeWeb.allow_net_connect = false
20 |
21 | def fixture(filename)
22 | File.join(File.dirname(__FILE__), 'fixtures', filename)
23 | end
24 |
25 | def fixture_data(filename)
26 | File.read(fixture(filename)).force_encoding('UTF-8')
27 | end
28 |
--------------------------------------------------------------------------------
/views/gems.erb:
--------------------------------------------------------------------------------
1 | <% @title = 'Ruby Gems' %>
2 |
3 | Alternative formats:
9 | <% DbpediaLite::FORMATS.each_with_index do |format,i| %> 10 | <%= link_to(format.name, object.doc_path(format.file_extension.first)) %> 11 | <%= '|' unless i == DbpediaLite::FORMATS.count - 1 %> 12 | <% end %> 13 |
4 |
7 |
8 | Ruby Gems
5 |dbpedialite.org is powered by Heroku and the following ruby gems:
6 |
9 |
10 |
23 |
24 |
--------------------------------------------------------------------------------
/spec/fixtures/categories-934787.json:
--------------------------------------------------------------------------------
1 | {"query":{"pages":{"27476290":{"pageid":27476290,"ns":14,"title":"Category:Articles with OS grid coordinates","touched":"2012-05-07T12:56:46Z","lastrevid":364853048,"counter":"","length":30,"displaytitle":"Category:Articles with OS grid coordinates"},"8528555":{"pageid":8528555,"ns":14,"title":"Category:Churches in Fife","touched":"2012-03-25T11:01:47Z","lastrevid":432954881,"counter":"","length":189,"displaytitle":"Category:Churches in Fife"},"34751738":{"pageid":34751738,"ns":14,"title":"Category:Parishes in Fife","touched":"2012-05-06T21:23:01Z","lastrevid":477101254,"counter":"","length":51,"displaytitle":"Category:Parishes in Fife"},"4309010":{"pageid":4309010,"ns":14,"title":"Category:Villages in Fife","touched":"2012-05-07T12:15:28Z","lastrevid":405486096,"counter":"","length":295,"displaytitle":"Category:Villages in Fife"}}}}
--------------------------------------------------------------------------------
/public/css/cssreset-min.css:
--------------------------------------------------------------------------------
1 | /*
2 | YUI 3.5.1 (build 22)
3 | Copyright 2012 Yahoo! Inc. All rights reserved.
4 | Licensed under the BSD License.
5 | http://yuilibrary.com/license/
6 | */
7 | html{color:#000;background:#FFF}body,div,dl,dt,dd,ul,ol,li,h1,h2,h3,h4,h5,h6,pre,code,form,fieldset,legend,input,textarea,p,blockquote,th,td{margin:0;padding:0}table{border-collapse:collapse;border-spacing:0}fieldset,img{border:0}address,caption,cite,code,dfn,em,strong,th,var{font-style:normal;font-weight:normal}ol,ul{list-style:none}caption,th{text-align:left}h1,h2,h3,h4,h5,h6{font-size:100%;font-weight:normal}q:before,q:after{content:''}abbr,acronym{border:0;font-variant:normal}sup{vertical-align:text-top}sub{vertical-align:text-bottom}input,textarea,select{font-family:inherit;font-size:inherit;font-weight:inherit}input,textarea,select{*font-size:100%}legend{color:#000}#yui3-css-stamp.cssreset{display:none}
--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source "https://rubygems.org"
2 | ruby File.read('.ruby-version').chomp
3 |
4 | gem 'thin'
5 |
6 | gem 'sinatra'
7 | gem 'sinatra-contrib', :require => 'sinatra/content_for'
8 | gem 'emk-sinatra-url-for', :require => 'sinatra/url_for'
9 |
10 | gem 'json_pure', :require => 'json'
11 | gem 'nokogiri'
12 | gem 'rdiscount'
13 | gem 'doodle'
14 |
15 | gem 'rdf', "~>1.0.8"
16 | gem 'rdf-json', "~>1.0.0", :require => 'rdf/json'
17 | gem 'rdf-turtle', "~>1.0.9", :require => 'rdf/turtle'
18 | gem 'rdf-trix', "~>1.0.0", :require => 'rdf/trix'
19 | gem 'rdf-rdfxml', "~>1.0.2"
20 | gem 'json-ld', "~>1.0.7"
21 |
22 | group :development do
23 | gem 'rake'
24 | gem 'shotgun'
25 | end
26 |
27 | group :test do
28 | gem 'rspec', '>=2.7.0'
29 | gem 'fakeweb'
30 | gem 'simplecov'
31 | gem 'rack-test', :require => 'rack/test'
32 | gem 'rdf-rdfa', :require => 'rdf/rdfa'
33 | gem 'equivalent-xml'
34 | end
35 |
--------------------------------------------------------------------------------
/spec/fixtures/wbgetentities-bse.json:
--------------------------------------------------------------------------------
1 | {"entities":{"Q154666":{"pageid":155902,"ns":0,"title":"Q154666","lastrevid":751038337,"modified":"2018-09-23T05:54:57Z","type":"item","id":"Q154666","labels":{"en":{"language":"en","value":"bovine sponginess encephalitic"}},"descriptions":{"en":{"language":"en","value":"bovine counterpart to variant Creutzfeldt-Jakob disease"}},"aliases":{"en":[{"language":"en","value":"Bovine spongiform encephalopathy"},{"language":"en","value":"BSE"},{"language":"en","value":"variant Creutzfeldt-Jakob disease"},{"language":"en","value":"Mad Cow Diseases"},{"language":"en","value":"Mad Cow Disease"},{"language":"en","value":"BSE (Bovine Spongiform Encephalopathy)"},{"language":"en","value":"Encephalitis, Bovine Spongiform"},{"language":"en","value":"BSEs (Bovine Spongiform Encephalopathy)"},{"language":"en","value":"Bovine Spongiform Encephalitis"},{"language":"en","value":"Spongiform Encephalopathy, Bovine"}]}}},"success":1}
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | | Name | 12 |Version | 13 |Summary | 14 |
|---|---|---|
| <%= link_to(spec.name, spec.homepage) %> | 18 |<%= link_to(spec.version, "http://rubygems.org/gems/#{spec.name}/versions/#{spec.version}") %> | 19 |<%= spec.summary %> | 20 |
2 | dbpedia lite is no-more.
3 | What it was attempting to do is now handled much better by Wikidata.
4 |
5 | Pages now redirect to the equivalent Wikidata pages.
6 |
7 |
8 |
9 | [dbpedia lite](http://www.dbpedialite.org) used to take some of the structured data in [Wikipedia](http://wikipedia.org/) and presents it as [Linked Data](http://linkeddata.org/). It contained a small subset of the data that [dbpedia](http://dbpedia.org/) contains; it did not attempt to extract data from the Wikipedia infoboxes. Data is fetched live from the [Wikipedia API](http://en.wikipedia.org/w/api.php).
10 |
11 | Unlike dbpedia is it uses stable Wikipedia pageIds in its URIs to attempt to mitigate the problems of article titles changing over time. If the title of a Wikipedia page changes, the dbpedia lite URI will stay the same. This makes it safer to store dbpedia lite identifiers in your own database.
12 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) Nicholas J Humfrey
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 |
--------------------------------------------------------------------------------
/lib/wikidata_api.rb:
--------------------------------------------------------------------------------
1 | require 'mediawiki_api'
2 |
3 | class WikidataApi < MediaWikiApi
4 |
5 | def self.api_uri
6 | URI.parse('https://www.wikidata.org/w/api.php')
7 | end
8 |
9 | def self.get_sitelink(id, site='enwiki')
10 | data = self.get('wbgetentities', {
11 | :ids => id,
12 | :sites => site,
13 | :props => 'sitelinks',
14 | :languages => 'en'
15 | })
16 |
17 | key = id.upcase
18 | if data['entities'].nil?
19 | raise MediaWikiApi::Exception.new('Empty response')
20 | elsif data['entities'][key].nil?
21 | raise MediaWikiApi::NotFound.new('Wikidata identifier does not exist')
22 | elsif data['entities'][key]['sitelinks'][site].nil?
23 | raise MediaWikiApi::NotFound.new('Sitelink does not exist for Wikidata identifier')
24 | else
25 | return data['entities'][key]['sitelinks'][site]
26 | end
27 | end
28 |
29 | def self.find_by_title(title, site='enwiki')
30 | data = self.get('wbgetentities', {
31 | :titles => title,
32 | :sites => site,
33 | :props => 'info|aliases|labels|descriptions',
34 | :languages => 'en'
35 | })
36 |
37 | if data['entities'].nil?
38 | raise MediaWikiApi::Exception.new('Empty response')
39 | elsif data['entities'].empty? or data['entities'].keys.first == "-1"
40 | raise MediaWikiApi::NotFound.new('Failed to lookup title in Wikidata')
41 | end
42 |
43 | data['entities'].values.first
44 | end
45 |
46 | end
47 |
--------------------------------------------------------------------------------
/lib/base_model.rb:
--------------------------------------------------------------------------------
1 | require 'wikipedia_api'
2 | require 'uri'
3 |
4 | class BaseModel < Doodle
5 | has :pageid
6 | has :title, :default => nil
7 | has :displaytitle, :default => nil
8 |
9 | class << self
10 | has :identifier_path, :default => 'base'
11 |
12 | def load(pageid)
13 | object = self.new(pageid)
14 | object.load ? object : nil
15 | end
16 | end
17 |
18 | def initialize(pageid, args={})
19 | if pageid.kind_of?(Hash)
20 | args.merge!(pageid)
21 | else
22 | args.merge!(:pageid => pageid)
23 | end
24 | update(args)
25 | super(args)
26 | end
27 |
28 | # FIXME: can doodle be told to ignore unknown attributes?
29 | def update(args={})
30 | args.each_pair do |key,value|
31 | unless self.respond_to?(key)
32 | args.delete(key)
33 | end
34 | end
35 | doodle.update(args)
36 | end
37 |
38 | def uri
39 | @uri ||= RDF::URI.parse("http://www.dbpedialite.org/#{self.class.identifier_path}/#{pageid}#id")
40 | end
41 |
42 | def doc_uri=(uri)
43 | @doc_uri = RDF::URI.parse(uri.to_s)
44 | end
45 |
46 | def doc_uri
47 | @doc_uri || RDF::URI.parse("http://www.dbpedialite.org/#{self.class.identifier_path}/#{pageid}")
48 | end
49 |
50 | def doc_path(format=nil)
51 | "/#{self.class.identifier_path}/#{pageid}#{format ? '.' + format.to_s : ''}"
52 | end
53 |
54 | def wikipedia_uri
55 | unless title.nil?
56 | @wikipedia_uri ||= RDF::URI.parse("http://en.wikipedia.org/wiki/#{WikipediaApi.escape_title(title)}")
57 | end
58 | end
59 |
60 | def dbpedia_uri
61 | unless title.nil?
62 | @dbpedia_uri ||= RDF::URI.parse("http://dbpedia.org/resource/#{WikipediaApi.title_to_dbpedia_key(title)}")
63 | end
64 | end
65 | end
66 |
--------------------------------------------------------------------------------
/lib/category.rb:
--------------------------------------------------------------------------------
1 | require 'base_model'
2 | require 'wikipedia_api'
3 | require 'thing'
4 |
5 | class Category < BaseModel
6 | identifier_path "categories"
7 |
8 | has :things, :collect => Thing
9 | has :subcategories, :collect => Category
10 |
11 | def load
12 | data = WikipediaApi.page_info(:pageids => pageid)
13 |
14 | # Is it actually a category?
15 | unless data['ns'] == 14
16 | raise MediaWikiApi::NotFound.new("Page #{pageid} is not a category")
17 | end
18 |
19 | # Update object properties with the data that was loaded
20 | update(data)
21 |
22 | data = WikipediaApi.category_members(pageid)
23 | data.each do |member|
24 | case member['ns']
25 | when 0
26 | self.things << Thing.new(member)
27 | when 14
28 | self.subcategories << Category.new(member)
29 | end
30 | end
31 |
32 | true
33 | end
34 |
35 | def label
36 | @label ||= displaytitle.sub(/^Category:/,'')
37 | end
38 |
39 | def to_rdf
40 | RDF::Graph.new(doc_uri) do |graph|
41 | # Triples about the Document
42 | graph << [doc_uri, RDF.type, RDF::FOAF.Document]
43 | graph << [doc_uri, RDF::DC.title, "dbpedia lite category - #{label}"]
44 | graph << [doc_uri, RDF::FOAF.primaryTopic, self.uri]
45 |
46 | # Triples about the Concept
47 | graph << [self.uri, RDF.type, RDF::OWL.Class]
48 | graph << [self.uri, RDF::RDFS.label, label]
49 | graph << [self.uri, RDF::FOAF.isPrimaryTopicOf, wikipedia_uri]
50 | graph << [self.uri, RDF::OWL.sameAs, dbpedia_uri]
51 |
52 | things.each do |thing|
53 | graph << [thing.uri, RDF.type, self.uri]
54 | graph << [thing.uri, RDF::RDFS.label, thing.label]
55 | end
56 |
57 | subcategories.each do |subcat|
58 | graph << [subcat.uri, RDF::RDFS.subClassOf, self.uri]
59 | graph << [subcat.uri, RDF::RDFS.label, subcat.label]
60 | end
61 | end
62 | end
63 | end
64 |
--------------------------------------------------------------------------------
/spec/wikidata_api_spec.rb:
--------------------------------------------------------------------------------
1 | # encoding: utf-8
2 | require 'spec_helper'
3 | require 'wikidata_api'
4 |
5 | describe WikidataApi do
6 |
7 | context "getting Wikipedia page title from a Wikidata identifier" do
8 | before :each do
9 | FakeWeb.register_uri(
10 | :get, 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&ids=Q9531&languages=en&props=sitelinks&sites=enwiki',
11 | :body => fixture_data('wbgetentities-Q9531.json'),
12 | :content_type => 'application/json'
13 | )
14 | @data = WikidataApi.get_sitelink('Q9531')
15 | end
16 |
17 | it "should return a Hash" do
18 | @data.should be_a(Hash)
19 | end
20 |
21 | it "should have the correct site identifier" do
22 | @data['site'].should == 'enwiki'
23 | end
24 |
25 | it "should have the correct page title" do
26 | @data['title'].should == 'BBC'
27 | end
28 | end
29 |
30 | context "getting an entity from Wikidata from an Wikipedia page title" do
31 | before :each do
32 | FakeWeb.register_uri(
33 | :get, 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&languages=en&props=info%7Caliases%7Clabels%7Cdescriptions&sites=enwiki&titles=Ceres,%20Fife',
34 | :body => fixture_data('wbgetentities-ceres.json'),
35 | :content_type => 'application/json'
36 | )
37 | @data = WikidataApi.find_by_title('Ceres, Fife')
38 | end
39 |
40 | it "should return a Hash" do
41 | @data.should be_a(Hash)
42 | end
43 |
44 | it "should have the correct title/identifier" do
45 | @data['title'].should == 'Q33980'
46 | end
47 |
48 | it "should be of type 'item" do
49 | @data['type'].should == 'item'
50 | end
51 |
52 | it "should have an English label" do
53 | @data['labels']['en']['value'].should == 'Ceres'
54 | end
55 |
56 | it "should have an English description" do
57 | @data['descriptions']['en']['value'].should == 'village in Fife, Scotland'
58 | end
59 | end
60 |
61 | end
62 |
--------------------------------------------------------------------------------
/views/index.erb:
--------------------------------------------------------------------------------
1 | <% content_for :head do %>
2 |
3 |
4 |
5 |
6 | <% end %>
7 |
8 |
9 |
11 |
12 | dbpedia lite
10 |
13 | <%= @readme.to_html %>
14 |
15 |
46 |
--------------------------------------------------------------------------------
/spec/mediawiki_api_spec.rb:
--------------------------------------------------------------------------------
1 | # encoding: utf-8
2 | require 'spec_helper'
3 | require 'mediawiki_api'
4 |
5 | describe MediaWikiApi do
6 |
7 | context "escaping a page title" do
8 | it "should convert 'AC/DC' to 'AC/DC'" do
9 | MediaWikiApi.escape_title('AC/DC').should == 'AC/DC'
10 | end
11 |
12 | it "should convert 'Category:Villages in Fife' to 'Category:Villages_in_Fife'" do
13 | MediaWikiApi.escape_title('Category:Villages in Fife').should == 'Category:Villages_in_Fife'
14 | end
15 |
16 | it "should convert 'Who Censored Roger Rabbit?' to 'Who_Censored_Roger_Rabbit%3F'" do
17 | MediaWikiApi.escape_title('Who Censored Roger Rabbit?').should == 'Who_Censored_Roger_Rabbit%3F'
18 | end
19 |
20 | it "should convert '100% (song)' to '100%25_(song)'" do
21 | MediaWikiApi.escape_title('100% (song)').should == '100%25_(song)'
22 | end
23 |
24 | it "should convert 'C#' to 'C%23'" do
25 | MediaWikiApi.escape_title('C#').should == 'C%23'
26 | end
27 |
28 | it "should convert '2 + 2 = 5' to 'C%23'" do
29 | MediaWikiApi.escape_title('2 + 2 = 5').should == '2_%2B_2_%3D_5'
30 | end
31 |
32 | it "should convert 'Nat \"King\" Cole' to 'Nat_%22King%22_Cole'" do
33 | MediaWikiApi.escape_title('Nat "King" Cole').should == 'Nat_%22King%22_Cole'
34 | end
35 |
36 | it "should not convert '—We Also Walk Dogs' to '—We_Also_Walk_Dogs'" do
37 | MediaWikiApi.escape_title('—We Also Walk Dogs').force_encoding('utf-8').should == '—We_Also_Walk_Dogs'
38 | end
39 | end
40 |
41 | context "escaping a query parameter" do
42 | it "should convert 'Florence + the Machine' to 'Florence%20%2B%20the%20Machine'" do
43 | MediaWikiApi.escape_query('Florence + the Machine').should == 'Florence%20%2B%20the%20Machine'
44 | end
45 |
46 | it "should convert 'C#' to 'C%23'" do
47 | MediaWikiApi.escape_query('C#').should == 'C%23'
48 | end
49 |
50 | it "should convert 'Café' to 'Café'" do
51 | MediaWikiApi.escape_query('Café').should == 'Caf%C3%A9'
52 | end
53 | end
54 |
55 | end
56 |
--------------------------------------------------------------------------------
/lib/mediawiki_api.rb:
--------------------------------------------------------------------------------
1 | require 'net/http'
2 | require 'uri'
3 |
4 | class MediaWikiApi
5 |
6 | USER_AGENT = 'DbpediaLite/1'
7 | HTTP_TIMEOUT = 5
8 | NBSP = Nokogiri::HTML(" ").text
9 | UNSAFE_REGEXP = Regexp.new('[^-_\.!~*\'()a-zA-Z0-9;/:@&=$,]', false, 'N').freeze
10 |
11 | class Exception < Exception
12 | end
13 |
14 | class NotFound < MediaWikiApi::Exception
15 | end
16 |
17 | def self.escape_query(str)
18 | URI::escape(str, UNSAFE_REGEXP)
19 | end
20 |
21 | def self.escape_title(title)
22 | URI::escape(title.gsub(' ','_'), ' ?#%"+=').force_encoding('UTF-8')
23 | end
24 |
25 | def self.get(action, args={})
26 | items = []
27 | args.merge!(:action => action, :format => 'json')
28 |
29 | keys = args.keys.sort {|a,b| a.to_s <=> b.to_s}
30 | keys.each do |key|
31 | items << escape_query(key.to_s)+'='+escape_query(args[key].to_s)
32 | end
33 |
34 | uri = self.api_uri
35 | uri.query = items.join('&')
36 |
37 | http = Net::HTTP.new(uri.host, uri.port)
38 | if uri.scheme == 'https'
39 | http.use_ssl = true
40 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE ## FIXME: bad
41 | end
42 | http.read_timeout = HTTP_TIMEOUT
43 | http.open_timeout = HTTP_TIMEOUT
44 | res = http.get(uri.request_uri, {'User-Agent' => USER_AGENT})
45 |
46 | # Throw exception if unsuccessful
47 | res.value
48 |
49 | # Parse the response if it is JSON
50 | if res.content_type == 'application/json'
51 | data = JSON.parse(res.body)
52 | else
53 | raise MediaWikiApi::Exception.new(
54 | "Response from MediaWiki API was not of type application/json."
55 | )
56 | end
57 |
58 | # Check for errors in the response
59 | if data.nil?
60 | raise MediaWikiApi::Exception.new('Empty response')
61 | elsif data.has_key?('error')
62 | if data['error']['code'] == 'nosuchpageid'
63 | raise MediaWikiApi::NotFound.new(
64 | data['error']['info']
65 | )
66 | else
67 | raise MediaWikiApi::Exception.new(
68 | data['error']['info']
69 | )
70 | end
71 | end
72 |
73 | return data
74 | end
75 |
76 | end
77 |
--------------------------------------------------------------------------------
/views/layout.erb:
--------------------------------------------------------------------------------
1 |
2 |
4 | >
5 |
6 | Search
16 | 20 | 21 |Bookmarklet
22 |23 | Drag this dbpedialite flipr 24 | link to your bookmarks bar in your browser. Now, when you're on Wikipedia page click on the bookmarklet to switch to the coresponding dbpedia lite page. 25 | When you're on a dbpedia lite page, click on the bookmarklet to take you back to the respective Wikipedia page. 26 |
27 | 28 |Example Things
29 |-
30 |
- U2 31 |
- U2 (band) 32 |
- Florence + the Machine 33 |
- Stoat 34 |
- Ermine 35 |
- Ceres, Fife 36 |
- Berlin 37 |
- British Broadcasting Corporation 38 |
Example Categories
41 |-
42 |
- Towns in Fife 43 |
- James Bond films 44 |
27 |
52 |
53 |
54 |
55 |
--------------------------------------------------------------------------------
/scripts/daily_ping.rb:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env ruby
2 | #
3 | # This script fetches the Wikipedia english front page
4 | # scans it for links to articles, looks each of them up,
5 | # and submits the dbpedialite.org URI for them to Sindice.
6 | #
7 |
8 | $:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
9 |
10 | require 'rubygems'
11 | require 'net/http'
12 | require 'nokogiri'
13 | require 'json'
14 | require 'wikipedia_api'
15 |
16 | USER_AGENT = 'Dbpedialite/1'
17 | HTTP_TIMEOUT = 10
18 |
19 | def get_links(url)
20 | url = URI.parse(url) unless url.is_a?(URI)
21 |
22 | res = Net::HTTP.start(url.host, url.port) do |http|
23 | http.get(url.request_uri)
24 | end
25 |
26 | # Check it was a 200
27 | if res.code != '200'
28 | raise "Failed to GET #{url} #{res.message}"
29 | end
30 |
31 | links = []
32 | doc = Nokogiri::HTML(res.body)
33 | doc.xpath("//body//a").each do |a|
34 | if a.has_attribute?('href') and a['href'].match(%r[/wiki/(.+)$])
35 | links << URI.unescape($1)
36 | end
37 | end
38 | links.sort.uniq
39 | end
40 |
41 | def ping_sindice(data)
42 | uri = URI.parse("http://api.sindice.com/v2/ping")
43 | begin
44 | req = Net::HTTP::Post.new(uri.request_uri)
45 | req['User-Agent'] = USER_AGENT
46 | req['Content-Type'] = 'text/plain'
47 | req['Content-Length'] = data.length
48 | req['Accept'] = 'text/plain'
49 | req.body = data
50 | res = Net::HTTP.start(uri.host, uri.port) do |http|
51 | http.read_timeout = HTTP_TIMEOUT
52 | http.open_timeout = HTTP_TIMEOUT
53 | http.request(req)
54 | end
55 | raise res.body unless res.code == '200'
56 | puts "Sindice Response: #{res.body}"
57 | rescue Exception => e
58 | $stderr.puts "Failed to ping sindice: #{e}"
59 | end
60 | end
61 |
62 |
63 |
64 | rdf_urls = []
65 | links = get_links("https://en.wikipedia.org/wiki/Main_Page")
66 | links.each do |link|
67 | puts "Looking up: #{link}"
68 | begin
69 | info = WikipediaApi.page_info(:titles => link)
70 | puts " ns=#{info['ns']} pageid=#{info['pageid']}"
71 | #lastmod = Time.parse(info['touched'])
72 | if info['ns'] == 0
73 | rdf_url = "http://www.dbpedialite.org/things/#{info['pageid']}.rdf"
74 | rdf_urls << rdf_url
75 | end
76 | rescue MediaWikiApi::NotFound
77 | end
78 | end
79 |
80 | ping_sindice(
81 | rdf_urls.join("\n")
82 | )
83 |
--------------------------------------------------------------------------------
/spec/fixtures/parse-2008435.json:
--------------------------------------------------------------------------------
1 | {"parse":{"title":"IMAC","text":{"*":"
28 | 
31 |
32 | 
37 |
38 |
45 |
31 |
32 |
37 |
46 | Source code: <%= link_to('http://github.com/njh/dbpedialite') %>
47 |
48 |
49 | Code last updated: <%= link_to(APP_LAST_UPDATED, 'http://github.com/njh/dbpedialite/tree/' + GIT_LAST_COMMIT) %>
50 |
51 | iMac<\/a><\/b> is a line of Apple Macintosh computers.<\/p>\n IMAC<\/b> or Imac<\/b> may also refer to:<\/p>\n\n
![\"Disambiguation](\"\/\/upload.wikimedia.org\/wikipedia\/en\/thumb\/5\/5f\/Disambig_gray.svg\/30px-Disambig_gray.svg.png\")
<\/a><\/td>\n![\"flag\"](\"//upload.wikimedia.org/wikipedia/commons/thumb/1/10/Flag_of_Scotland.svg/32px-Flag_of_Scotland.svg.png\")
![\"\"](\"//upload.wikimedia.org/wikipedia/en/thumb/4/4a/Commons-logo.svg/30px-Commons-logo.svg.png\"){kind=logo}
![\"About](\"\/\/upload.wikimedia.org\/wikipedia\/commons\/thumb\/8\/8a\/Loudspeaker.svg\/11px-Loudspeaker.svg.png\")
<\/a> ![\"\"](\"\/\/upload.wikimedia.org\/wikipedia\/commons\/thumb\/8\/89\/Auchtertool_Kirk.jpg\/220px-Auchtertool_Kirk.jpg\")
<\/a>\n![\"\"](\"\/\/bits.wikimedia.org\/skins-1.18\/common\/images\/magnify-clip.png\")
<\/a><\/div>\nThe present kirk in Auchtertool replaced an earlier building said to date to 1178<\/div>\n<\/div>\n<\/div>\n
