├── test ├── package.lisp ├── cl-feedparser-tests.asd ├── suite.lisp ├── data │ ├── xxe-exploit.rss │ ├── lol.xml │ ├── foolnews_rss091.xml │ ├── 2013-06-28HypocriteReader.rss │ ├── push-pub.xml │ ├── guid-mask │ │ └── mythics.xml │ ├── distrijob.xml │ ├── yayitsrob.rss │ ├── what-if.atom │ ├── hydro74.rss │ ├── statwing-poison.html │ └── ideachampions.rdf └── tests.lisp ├── all.lisp ├── cl-feedparser.asd ├── time.lisp ├── feed-sanitizer.lisp ├── README.md ├── xml-namespaces.lisp ├── handlers.lisp └── parser.lisp /test/package.lisp: -------------------------------------------------------------------------------- 1 | (defpackage :cl-feedparser/test 2 | (:use :cl :alexandria :serapeum :cl-feedparser :fiveam) 3 | (:import-from :html5-parser :parse-html5) 4 | (:shadowing-import-from :serapeum :string+) 5 | (:export :run-tests)) 6 | -------------------------------------------------------------------------------- /test/cl-feedparser-tests.asd: -------------------------------------------------------------------------------- 1 | (defsystem "cl-feedparser-tests" 2 | :description "Test suite for cl-feedparser." 3 | :author "Paul M. Rodriguez " 4 | :license "MIT" 5 | :depends-on ("cl-feedparser" "fiveam" "local-time" "fxml/html5") 6 | :serial t 7 | :components ((:file "package") 8 | (:file "suite") 9 | (:file "tests"))) 10 | -------------------------------------------------------------------------------- /all.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package :cl-feedparser/all 2 | (:nicknames :cl-feedparser :feedparser) 3 | (:use :cl-feedparser/parser :cl-feedparser/handlers) 4 | (:export 5 | :parse-feed :feed-ref 6 | :*keys* :feedparser-key :gethash* 7 | :repair :return-feed 8 | :feed-sanitizer 9 | :unsanitized-string :unsanitized-string-string :string+ 10 | :sanitize-title :sanitize-content :sanitize-text 11 | :feed-string 12 | :*base* 13 | :parse-time 14 | :masked?)) 15 | -------------------------------------------------------------------------------- /test/suite.lisp: -------------------------------------------------------------------------------- 1 | (in-package :cl-feedparser/test) 2 | 3 | (def-suite cl-feedparser) 4 | 5 | (defun run-tests () 6 | (run! 'cl-feedparser)) 7 | 8 | (defun debug-test (test &key (error t) (failure t)) 9 | "Run TEST, breaking on error or failure." 10 | (let ((5am:*on-error* (and error :debug)) 11 | (5am:*on-failure* (and failure :debug))) 12 | (run! test))) 13 | 14 | (defparameter *test-data-dir* 15 | (asdf:system-relative-pathname :cl-feedparser "test/data/")) 16 | 17 | (defun find-test-file (name) 18 | (uiop:merge-pathnames* name *test-data-dir*)) 19 | 20 | (defun load-test-file (name &key external-format) 21 | (read-file-into-string (find-test-file name) :external-format external-format)) 22 | -------------------------------------------------------------------------------- /test/data/xxe-exploit.rss: -------------------------------------------------------------------------------- 1 | 2 | 3 | ]> 4 | 5 | 6 | The Blog 7 | http://example.com/ 8 | A blog about things 9 | Mon, 03 Feb 2014 00:00:00 -0000 10 | 11 | &xxe; 12 | http://example.com 13 | a post 14 | author@example.com 15 | Mon, 03 Feb 2014 00:00:00 -0000 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /test/data/lol.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | ]> 15 | &lol9; 16 | -------------------------------------------------------------------------------- /cl-feedparser.asd: -------------------------------------------------------------------------------- 1 | ;;;; cl-feedparser.asd 2 | 3 | (defsystem "cl-feedparser" 4 | :serial t 5 | :description "Common Lisp universal feed parser" 6 | :author "Paul M. Rodriguez " 7 | :license "LLGPL" 8 | :version "1.2.0" 9 | :class :package-inferred-system 10 | :defsystem-depends-on (:asdf-package-system) 11 | :depends-on ("cl-feedparser/all") 12 | :in-order-to ((test-op (test-op "cl-feedparser/test")))) 13 | 14 | (defsystem "cl-feedparser/test" 15 | :serial t 16 | :description "Test suite for cl-feedparser." 17 | :author "Paul M. Rodriguez " 18 | :license "MIT" 19 | :perform (test-op (o c) (symbol-call :cl-feedparser/test :run-tests)) 20 | :depends-on ("cl-feedparser" "fiveam" "local-time" "fxml/html5") 21 | :pathname "test/" 22 | :components ((:file "package") 23 | (:file "suite") 24 | (:file "tests"))) 25 | 26 | (asdf:register-system-packages :cxml '(:cxml :klacks :cxml-dom :sax)) 27 | (asdf:register-system-packages :fxml '(:fxml :fxml-dom :fxml.klacks :fxml.sax)) 28 | (asdf:register-system-packages :cxml-stp '(:stp :cxml-stp)) 29 | (asdf:register-system-packages :fxml/stp '(:fxml.stp)) 30 | (asdf:register-system-packages :fxml/html5 '(:fxml.html5)) 31 | (asdf:register-system-packages :fxml/sanitize '(:fxml.sanitize)) 32 | (asdf:register-system-packages :cl-html5-parser '(:html5-parser)) 33 | (asdf:register-system-packages :net-telent-date '(:net.telent.date)) 34 | 35 | -------------------------------------------------------------------------------- /test/data/foolnews_rss091.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | The Motley Fool 7 | http://www.fool.com 8 | To Educate, Amuse, and Enrich 9 | en-us 10 | 11 | The Motley Fool 12 | http://www.fool.com/art/partners/bbac/88x31F.gif 13 | http://www.fool.com 14 | 88 15 | 31 16 | To Educate, Amuse, and Enrich 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /time.lisp: -------------------------------------------------------------------------------- 1 | (defpackage :cl-feedparser/time 2 | (:use :cl :alexandria :serapeum :local-time) 3 | (:import-from :cl-ppcre :regex-replace) 4 | (:import-from :net.telent.date) 5 | (:export :parse-time :time=)) 6 | 7 | (in-package :cl-feedparser/time) 8 | 9 | (defun parse-time (string) 10 | "Parse STRING as a date. 11 | As a second value, return the parser used: either :net.telent.date 12 | or :local-time." 13 | (let ((string (regex-replace "UT$" string "GMT"))) 14 | (if-let (date (net.telent.date:parse-time string)) 15 | (values date :net.telent.date) 16 | (ignoring local-time::invalid-timestring ;XXX 17 | (handler-case 18 | (when-let (date 19 | (timestamp-to-universal 20 | (parse-timestring string))) 21 | (values date :local-time))))))) 22 | 23 | (assert (= 3645907200 (parse-time "Wed, 15 Jul 2015 00:00:00 UT"))) 24 | 25 | (defmethod time= ((t1 integer) (t2 integer)) 26 | (= t1 t2)) 27 | 28 | (defmethod time= ((t1 timestamp) (t2 timestamp)) 29 | (timestamp= t1 t2)) 30 | 31 | (defmethod time= ((t1 timestamp) (t2 integer)) 32 | (time= t2 t1)) 33 | 34 | (defmethod time= ((t1 integer) (t2 timestamp)) 35 | (mvlet* ((ss1 mm1 hh1 day1 month1 year1 (decode-universal-time t1)) 36 | (nsec2 ss2 mm2 hh2 day2 month2 year2 (decode-timestamp t2))) 37 | (declare (ignore nsec2)) 38 | (and (= ss1 ss2) 39 | (= mm1 mm2) 40 | (= hh1 hh2) 41 | (= day1 day2) 42 | (= month1 month2) 43 | (= year1 year2)))) 44 | 45 | (defmethod time= ((t1 string) t2) (time= (parse-time t1) t2)) 46 | (defmethod time= (t1 (t2 string)) (time= t1 (parse-time t2))) 47 | 48 | ;; (assert 49 | ;; (let* ((time (get-universal-time)) 50 | ;; (timestamp (local-time:universal-to-timestamp time))) 51 | ;; (and (time= time timestamp) 52 | ;; (not (time= (1+ time) timestamp))))) 53 | -------------------------------------------------------------------------------- /feed-sanitizer.lisp: -------------------------------------------------------------------------------- 1 | (defpackage :cl-feedparser/feed-sanitizer 2 | (:use :cl :alexandria :serapeum) 3 | (:import-from :fxml.sanitize :define-sanitize-mode) 4 | (:export :feed-sanitizer)) 5 | 6 | (in-package :cl-feedparser/feed-sanitizer) 7 | 8 | (define-sanitize-mode feed-sanitizer 9 | :elements ("a" "abbr" "acronym" "address" "area" "aside" "audio" 10 | "b" "big" "bdo" "blockquote" "br" 11 | "caption" "center" "cite" "code" "col" "colgroup" 12 | "dd" "del" "details" "dfn" "dir" "div" "dl" "dt" 13 | "em" 14 | "figcaption" "figure" 15 | "h1" "h2" "h3" "h4" "h5" "h6" "hgroup" 16 | "i" "img" "ins" 17 | "kbd" 18 | "li" 19 | "m" "map" "mark" "noscript" 20 | "ol" 21 | "p" "pre" 22 | "q" 23 | "rp" "rt" "ruby" 24 | "s" "samp" "section" "small" "span" "strike" "strong" "sub" "sup" 25 | "table" "tbody" "td" "tfoot" "th" "thead" "time" "tr" 26 | "u" "ul" "var" 27 | "wbr") 28 | 29 | :remove-elements ("script" "style") 30 | 31 | :allow-data-attributes t 32 | 33 | :attributes ((:all . ("dir" "lang" "title" "class")) 34 | ("a" . ("href")) 35 | ("blockquote" . ("cite")) 36 | ("col" . ("span" "width")) 37 | ("colgroup" . ("span" "width")) 38 | ("del" . ("cite" "datetime")) 39 | ("img" . ("align" "alt" "height" "src" "width")) 40 | ("ins" . ("cite" "datetime")) 41 | ("ol" . ("start" "reversed" "type")) 42 | ("p" . ("align")) ;XXX style? 43 | ("q" . ("cite")) 44 | ("table" . ("summary" "width")) 45 | ("td" . ("abbr" "axis" "colspan" "rowspan" "width")) 46 | ("th" . ("abbr" "axis" "colspan" "rowspan" "scope" "width")) 47 | ("time" . ("datetime" "pubdate")) 48 | ("ul" . ("type"))) 49 | 50 | :protocols (("a" . (("href" . (:ftp :http :https :mailto :relative)))) 51 | ("blockquote" . (("cite" . (:http :https :relative)))) 52 | ("del" . (("cite" . (:http :https :relative)))) 53 | ("img" . (("src" . (:http :https :relative)))) 54 | ("ins" . (("cite" . (:http :https :relative)))) 55 | ("q" . (("cite" . (:http :https :relative)))))) 56 | 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | CL-FEEDPARSER is a Common Lisp port of a subset of Python’s 2 | [feedparser][]. It does not do fetching; it only does parsing. 3 | 4 | The API of cl-feedparser is very close to the API of feedparser. As in 5 | Feedparser, the parsed feed is returned as a set of nested 6 | dictionaries (hash tables). Unsurprisingly, keys that are strings in 7 | Python are keywords in cl-feedparser, with dashes instead of 8 | underscores. 9 | 10 | feed['author'] -> (feedparser:feed-ref feed :author) 11 | feed['author_detail'] -> (feedparser:feed-ref feed :author-detail) 12 | 13 | (What is `feed-ref`? It is just like `gethash`, but if the provided 14 | key is constant, then it does a compile-time check that the key 15 | provided is valid. You can always just use `gethash`, of course.) 16 | 17 | There are a few important differences: 18 | 19 | - Parsed times are returned as universal times, rather than Python 20 | time tuples. 21 | - The sanitizer strips all inline CSS. 22 | - Text is always sanitized, regardless of declared media type. (Python 23 | feedparser does not sanitize `text/plain`.) 24 | - `bozo-exception` is a list of exceptions. 25 | 26 | In regard to sanitizing, cl-feedparser offers a guarantee Python’s 27 | feedparser does not: values that are returned as strings are *always* 28 | sanitized. It is possible to turn sanitizing off, but with sanitizing 29 | off, instances of `unsanitized-string` are returned in place of 30 | strings. 31 | 32 | A warning: cl-feedparser is not nearly as sophisticated as Python 33 | feedparser about handling timestamps. On the other hand, timestamps 34 | are one area in which real-world feeds have actually improved since 35 | feedparser was written. 36 | 37 | CL-FEEDPARSER is a classic case of a [strangler application][]. 38 | Originally it was just a wrapper around the Python library; then 39 | parsing well-formed feeds was moved into Lisp, with a fallback to 40 | Python; finally, Lisp took over. 41 | 42 | # EXAMPLE 43 | 44 | ``` lisp 45 | (ql:quickload '(:cl-feedparser :drakma)) 46 | 47 | (defparameter *feed* (feedparser:parse-feed (drakma:http-request "http://planet.lisp.org/rss20.xml"))) 48 | 49 | (feedparser:feed-ref *feed* :title) 50 | => "Planet Lisp" 51 | ``` 52 | 53 | # FEEDBURNER 54 | 55 | There is an extra key, `:proxy`, which is set to `"feedburner"` when 56 | the feed is from Feedburner (or uses elements in the Feedburner 57 | namespace, as Feedblitz does). 58 | 59 | It is also the case that the Feedburner link is always overridden with 60 | the feedburner:origLink. This is in the interest of future-proofing: 61 | Feedburner may not be around forever, so for archival purposes you want the real links 62 | instead of the Feedburner redirects. 63 | 64 | # SKIPPING ENTRIES 65 | 66 | In the wild, you will sometimes come across enormous feeds that have 67 | simply never been truncated: they contain every post ever posted. And 68 | even for ordinary feeds, you may only be interested in the latest 69 | entry or two out of 10 or 20. 70 | 71 | CL-FEEDPARSER lets you skip entries with two options: `:max-entries` 72 | and `:guid-mask`. 73 | 74 | If you know how many entries you want, pass `:max-entries`. 75 | 76 | (length (feed-ref (parse-feed giant-feed :max-entries 10) :entries)) 77 | => 10 78 | 79 | If you know which entries you *don’t* want (because you already have them), pass `guid-mask`. 80 | 81 | Each entry of the mask can be either an id or an (id . timestamp) 82 | pair. Any entry whose guid matches the supplied id is ignored. If the 83 | timestamp is supplied, then the matching entry will not be ignored if 84 | it has a timestamp that is newer than the timestamp on record. (That 85 | is, it will still be ignored if it does not have a timestamp.) 86 | 87 | It’s important to understand how the limit and the mask interact. The 88 | limit takes effect first, and only then are the entries filtered with 89 | the mask. This might seem strange, but think about a feed with 90 | thousands of entries. If the mask were applied before the limit, you 91 | would get another set of older entries every time you tried to parse 92 | the feed. 93 | 94 | # SANITIZING 95 | 96 | By default, everything that might possibly be HTML is sanitized. This 97 | includes cases where the media type is ambiguous; as such, information 98 | might be lost. If that information is important, you may want to ask 99 | `cl-feedparser` not to sanitize feeds. 100 | 101 | There are actually two knobs: one turns off sanitizing for entry 102 | contents, and the other turns it off for everything else (titles and 103 | other metadata). 104 | 105 | ;; Don't sanitize entry contents. 106 | (parse-feed feed :sanitize-content nil) 107 | ;; Don’t sanitize titles or other metadata. 108 | (parse-feed feed :sanitize-title nil) 109 | 110 | The idea here is that if you are going to be re-parsing the entry 111 | contents anyway, you may want to skip the round-trip from string to 112 | DOM and back to string again, and do the sanitizing more cheaply as 113 | part of your own processing. 114 | 115 | Turning off sanitizing has consequences. Any time `cl-feedparser` 116 | returns a string, you may rely on it being sanitized. So, if you turn 117 | off sanitizing, what you get are not strings, but instances of 118 | `unsanitized-string`; to get the underlying string, you must call 119 | `unsanitized-string-string` on them. (There is a type available, 120 | `feed-string`, defined as `(or string unsanitized string)`, if you 121 | want to do exhaustiveness checking.) 122 | 123 | # HANDLING ERRORS 124 | 125 | Since cl-feedparser builds on [FXML][] and [Plump][], it can can 126 | understand and recover from many common XML errors. Sometimes, 127 | however, you may not want automatic recovery. You might, for example, 128 | want to validate a feed. In this case, it is enough to pass `:safe 129 | nil` to `parse-feed`: 130 | 131 | (parse-feed buggy-feed :safe nil) 132 | => 133 | 134 | Otherwise, errors during parsing are stored under the `:bozo-exception` key of the feed. 135 | 136 | [feedparser]: https://pythonhosted.org/feedparser/ 137 | [TBRSS]: https://tbrss.com 138 | [strangler application]: http://martinfowler.com/bliki/StranglerApplication.html 139 | [FXML]: https://github.com/TBRSS/FXML 140 | [Plump]: https://shinmera.github.io/plump 141 | -------------------------------------------------------------------------------- /test/data/2013-06-28HypocriteReader.rss: -------------------------------------------------------------------------------- 1 | 2 | Hypocrite Readerhttp://hypocritereader.com/An online monthly, of essays conceptual and timely, based in Brooklyn, New York.en-usFri, 28 Jun 2013 15:32:36 -0000The Student’s Blushhttp://hypocritereader.com/29/the-students-blushnot the doctor famous for his oath - wisdom is a calculus of pleasures - an agonistic logic of its own - Thrasymachus getting angry - blind spots that he can exploit - the other Platonic reflex, laughterhttp://hypocritereader.com/29/the-students-blushFacebookhttp://hypocritereader.com/29/facebookI don’t use OKCupid - public and private, virtual semiocapital - the message of the piece is unambiguous - not dark enough to be Yves Klein’s blue - Coca-Cola, Jacques Derrida, NYC - a novel called <i>Facebook</i>http://hypocritereader.com/29/facebookConfessionhttp://hypocritereader.com/29/under-the-oak-treedo you want to make out - a cask and a casket - two lies underpin a confession - a rare event unfolds - and more of them belowhttp://hypocritereader.com/29/under-the-oak-treeThe Five Senseshttp://hypocritereader.com/29/the-five-sensesthis is no fragrant ode for weekends - the likes of artichoke - dreams of my opposite - some sort of wieldy élan - between epsilon and epaulet - O!http://hypocritereader.com/29/the-five-sensesTalking to Dirthttp://hypocritereader.com/29/talking-to-dirthalf-Sharpie’d on the breast - the infant-bug resemblance - a bongo -themed rave going on in my chest cavity - I wanted so desperately to create a perfect sphere - secret nervous snail self - the true core of its being: dirthttp://hypocritereader.com/29/talking-to-dirt http://hypocritereader.com/29/29cover<center><img src="http://hypocritereader.com/media/images/29/calvin_cover.jpg" style="margin: auto -100px; width:800px;"><br>Illustration by Ellis Calvin</center>http://hypocritereader.com/29/29coverThe Gods Show Uphttp://hypocritereader.com/29/gods-show-uptwo poles of the erotic - home for Dionysos - gods are <i>complete</i> and <i>simple</i> - hesitant hen-pecked - counsel of the chorus 3 | http://hypocritereader.com/29/gods-show-upOn Severalnesshttp://hypocritereader.com/29/on-severalnessthey do not exist in the limen - but which thing - I am compassionate and over-sensitive - the self I was for her - a kind of Holi - embodiment of my face - a sin against authenticityhttp://hypocritereader.com/29/on-severalnessTender Violationshttp://hypocritereader.com/28/tender-violationsor, if the porno has a storyline - the tickler and the tickled - the « money shot » - a comeuppance to her impotent husband - <i>Straw Dogs</i> - « cheers from feminists » - a fragment of consent - hand-to-hand combat - magic circle - How can one be deflated without being full of hot air? - skipping to the punchlinehttp://hypocritereader.com/28/tender-violationsAfterthought to “A Robo-Poet Does Not Scare Us”http://hypocritereader.com/28/afterthought-to-robo-poethttp://hypocritereader.com/28/afterthought-to-robo-poet http://hypocritereader.com/28/28cover<center><img src="http://hypocritereader.com/media/images/28/ellis28cover.jpg" style="margin: auto -100px; width:800px;"><br>“Funny Story” by Ellis Calvin</center>http://hypocritereader.com/28/28coverOn the Jerkhttp://hypocritereader.com/28/on-the-jerkmoments of intense humiliation - what color the walls were - my hand shot up - <i>what</i> a little <i>turd</i>! - the iron law of egos - when the event burst through the door - certain vicissitudes or derivations - Louis CK and Jerking Off - half the studio audience would be openly sobbing - Baudelaire, That Jerk - a prose poem from <i>Paris Spleen</i> entitled « Beat the Poor »http://hypocritereader.com/28/on-the-jerkI find America before Paul Simon canhttp://hypocritereader.com/28/i-find-america-before-paul-simon-candowning weak cups of coffee at the Waffle House, - listening to the waitresses sigh about daughtershttp://hypocritereader.com/28/i-find-america-before-paul-simon-canAt the Front Line of the Punch Line: Or, Simone Weil goes to Warhttp://hypocritereader.com/28/the-front-linean anarchist militia - Simone Weil - uncompromising politics and inventive theology - Buenaventura Durruti - playing the clown - expectation transformed into nothing - the Dad-Humor Phenomenon - smash the lightbulb with a hammer - a pitchfork in the airhttp://hypocritereader.com/28/the-front-lineCremainshttp://hypocritereader.com/28/cremainsmy grandmother’s urn tucked under his arm - unbreakable, waterproof, light-tight and ugly - 1 hour per 100 lbs. of body - Do The Honors - The first myth about « spreading ashes » - dance around rusted anchors - U.S. law forbids the cremation of - cracked irreparablyhttp://hypocritereader.com/28/cremains 4 | -------------------------------------------------------------------------------- /test/data/push-pub.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Publisher example 4 | 5 | 6 | 2015-02-20T03:26:09Z 7 | http://push-pub.appspot.com/feed 8 | 9 | nobody 10 | 11 | 12 | 13 | 14 | Ho 15 | http://push-pub.appspot.com/feed/5738275486564352 16 | 2015-02-20T03:26:09Z 17 | Ho 18 | 19 | 20 | 21 | 22 | Fo 23 | http://push-pub.appspot.com/feed/6205728989642752 24 | 2015-02-20T03:25:32Z 25 | Fum 26 | 27 | 28 | 29 | 30 | Fee 31 | http://push-pub.appspot.com/feed/4791524302782464 32 | 2015-02-20T03:24:28Z 33 | Fi 34 | 35 | 36 | 37 | 38 | Flee 39 | http://push-pub.appspot.com/feed/6220228396580864 40 | 2015-02-20T02:39:24Z 41 | Floo 42 | 43 | 44 | 45 | 46 | Fibble 47 | http://push-pub.appspot.com/feed/5920826226376704 48 | 2015-02-20T02:38:47Z 49 | Whee 50 | 51 | 52 | 53 | 54 | Wibble 55 | http://push-pub.appspot.com/feed/5718922095493120 56 | 2015-02-20T02:37:50Z 57 | Thunk 58 | 59 | 60 | 61 | 62 | Quux 63 | http://push-pub.appspot.com/feed/6299857224466432 64 | 2015-02-20T02:36:53Z 65 | Wibble 66 | 67 | 68 | 69 | 70 | Bar 71 | http://push-pub.appspot.com/feed/4794926319534080 72 | 2015-02-20T02:36:14Z 73 | Quux 74 | 75 | 76 | 77 | 78 | z10iuq8Wegee6OdFYj5wyS4T3VCnqP4t 79 | http://push-pub.appspot.com/feed/5094328489738240 80 | 2015-02-20T02:35:46Z 81 | z10iuq8Wegee6OdFYj5wyS4T3VCnqP4t 82 | 83 | 84 | 85 | 86 | Foo 87 | http://push-pub.appspot.com/feed/5647435049205760 88 | 2015-02-20T02:34:51Z 89 | Bar 90 | 91 | 92 | 93 | 94 | Messenger 95 | http://push-pub.appspot.com/feed/5079829082800128 96 | 2015-02-20T02:15:31Z 97 | Message 98 | 99 | 100 | 101 | 102 | Messenger 103 | http://push-pub.appspot.com/feed/6198899186335744 104 | 2015-02-20T02:14:51Z 105 | Message 106 | 107 | 108 | 109 | 110 | PGwoNvlcon8f3MXj38o6eRG9Tk5NiSTR 111 | http://push-pub.appspot.com/feed/5686812383117312 112 | 2015-02-19T22:51:11Z 113 | PGwoNvlcon8f3MXj38o6eRG9Tk5NiSTR 114 | 115 | 116 | 117 | 118 | OHLYxQxbAUVvOEplDWLJPwpZuTwwW7FA 119 | http://push-pub.appspot.com/feed/5740240702537728 120 | 2015-02-19T22:50:50Z 121 | OHLYxQxbAUVvOEplDWLJPwpZuTwwW7FA 122 | 123 | 124 | 125 | 126 | lxDBzOvpsTYCY6fnejDF7kIW2jp3FsVt 127 | http://push-pub.appspot.com/feed/5187211452481536 128 | 2015-02-19T22:50:08Z 129 | lxDBzOvpsTYCY6fnejDF7kIW2jp3FsVt 130 | 131 | 132 | 133 | 134 | iYLrzA04DmtugIVpyilld6Ah5pmjXVLL 135 | http://push-pub.appspot.com/feed/5644309118320640 136 | 2015-02-19T22:49:31Z 137 | iYLrzA04DmtugIVpyilld6Ah5pmjXVLL 138 | 139 | 140 | 141 | 142 | x2LS3WUNxCT8Ubrs0AJebJNmEnjurh43 143 | http://push-pub.appspot.com/feed/5657278443159552 144 | 2015-02-19T22:48:52Z 145 | x2LS3WUNxCT8Ubrs0AJebJNmEnjurh43 146 | 147 | 148 | 149 | 150 | O time thy pyramids oCf7JCVnV9lUZ4EwrVPjrVc7GQLdRTNI 151 | http://push-pub.appspot.com/feed/5700983627710464 152 | 2015-02-19T22:47:54Z 153 | O time thy pyramids oCf7JCVnV9lUZ4EwrVPjrVc7GQLdRTNI 154 | 155 | 156 | 157 | 158 | Message 159 | http://push-pub.appspot.com/feed/5155492582129664 160 | 2015-02-19T22:26:58Z 161 | O time thy pyramids 162 | 163 | 164 | 165 | 166 | test 167 | http://push-pub.appspot.com/feed/5725107787923456 168 | 2015-02-19T11:44:55Z 169 | test 170 | 171 | 172 | 173 | -------------------------------------------------------------------------------- /test/data/guid-mask/mythics.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 11 | <![CDATA[Mythics Blog]]> 12 | http://www.mythics.com/blog 13 | en 14 | Copyright 2015 15 | 2015-02-06T16:48:16-05:00 16 | 17 | 18 | 19 | <![CDATA[Oracle Database 11gR2 Support Has Expired! But there is a Waiver? Confused? See Our Guide Inside]]> 20 | http://www.mythics.com/about/blog/mythics-oracle-database-11gr2-de-support-guide 21 | http://www.mythics.com/about/blog/mythics-oracle-database-11gr2-de-support-guide 22 | Did you know that Premier Support for the Oracle Database 11gR2 has expired on 1/31/2015?   Did you also know that Oracle has now issued a…]]> 23 | 24 | 2015-02-03T12:50+00:00 25 | 26 | 27 | 28 | <![CDATA[Oracle Exadata X5-2:  More Than Just A Hardware Upgrade]]> 29 | http://www.mythics.com/about/blog/oracle-exadata-x5-2-more-than-just-a-hardware-upgrade 30 | http://www.mythics.com/about/blog/oracle-exadata-x5-2-more-than-just-a-hardware-upgrade 31 | Last week, Oracle announced the X5 upgrades for several of their Engineered Systems, all triggered by the release of the Oracle Server X5-2 and X5-2L…]]> 32 | 33 | 2015-01-30T19:20+00:00 34 | 35 | 36 | 37 | <![CDATA[GoldenGate 12c Setup with Oracle 12c Pluggable Multi-Tenant DB and OEM GoldenGate Plugins]]> 38 | http://www.mythics.com/about/blog/goldengate-12c-setup-with-oracle-12c-pluggable-multi-tenant-db-and-oem-gold 39 | http://www.mythics.com/about/blog/goldengate-12c-setup-with-oracle-12c-pluggable-multi-tenant-db-and-oem-gold 40 | I recently built out a new installation of GoldenGate using the latest versions and features, namely the Integrated Capture and Replication capabilities for…]]> 41 | 42 | 2015-01-28T16:02+00:00 43 | 44 | 45 | 46 | <![CDATA[ODA X5-2: Just Plain More Everything!]]> 47 | http://www.mythics.com/about/blog/oda-x52-just-plain-more-everything 48 | http://www.mythics.com/about/blog/oda-x52-just-plain-more-everything 49 | Every time a major manufacturer announces a new system, we hear the same common messages from marketing.  More CPU, more RAM, faster than…]]> 50 | 51 | 2015-01-26T13:54+00:00 52 | 53 | 54 | 55 | <![CDATA[Practical Oracle WebCenter Content UI]]> 56 | http://www.mythics.com/about/blog/practical-oracle-webcenter-content-ui 57 | http://www.mythics.com/about/blog/practical-oracle-webcenter-content-ui 58 | This article focuses on practical aspects of Oracle's new WebCenter Content User Interface, frequently referred as Content UI (aka ADF UI and Web…]]> 59 | 60 | 2015-01-16T18:33+00:00 61 | 62 | 63 | 64 | <![CDATA[Debbie’s Outreach and Volunteer Efforts (DOVE) by Team Mythics]]> 65 | http://www.mythics.com/about/blog/debbies-outreach-and-vollunteer-efforts-dove-by-team-mythics 66 | http://www.mythics.com/about/blog/debbies-outreach-and-vollunteer-efforts-dove-by-team-mythics 67 | On behalf of the Mythics family and all of our employees we are so proud to continue the spirit of giving by our longtime employee…]]> 68 | 69 | 2014-12-22T14:33+00:00 70 | 71 | 72 | 73 | <![CDATA[It’s Time to Upgrade to Oracle Database 12c, Here is Why, and Here is How]]> 74 | http://www.mythics.com/about/blog/its-time-to-upgrade-to-oracle-database-12c-here-is-why-and-here-is-how 75 | http://www.mythics.com/about/blog/its-time-to-upgrade-to-oracle-database-12c-here-is-why-and-here-is-how 76 | Well, it's that time again, when the whole Oracle database community will be dealing with the questions around upgrading to Database 12c from 11g (and some…]]> 77 | 78 | 2014-10-16T18:16+00:00 79 | 80 | 81 | 82 | <![CDATA[Enterprise Storage Reinvented]]> 83 | http://www.mythics.com/about/blog/enterprise-storage-reinvented 84 | http://www.mythics.com/about/blog/enterprise-storage-reinvented 85 | At Oracle OpenWorld 2014, Oracle introduced the world to the latest member of the Axiom family, the FS1 array. This is a hybrid flash array,…]]> 86 | 87 | 2014-10-04T14:59+00:00 88 | 89 | 90 | 91 | <![CDATA[An Introduction to Your Server’s Next CPU:  the SPARC M7]]> 92 | http://www.mythics.com/about/blog/an-introduction-to-your-servers-next-cpu-the-sparc-m7 93 | http://www.mythics.com/about/blog/an-introduction-to-your-servers-next-cpu-the-sparc-m7 94 | Some changes in technology are evolutionary, some are disruptive but the rarest of technologies are revolutionary in nature.

Examples of evolutionary technology are…]]> 95 | 96 | 2014-10-03T17:27+00:00 97 | 98 | 99 | 100 | <![CDATA[Answers to Common Questions on Java Versions/Editions]]> 101 | http://www.mythics.com/about/blog/answers-to-common-questions-on-java-versions-editions 102 | http://www.mythics.com/about/blog/answers-to-common-questions-on-java-versions-editions 103 | In the past several months, our team has received many inquires about Java Virtual Machine (JVM) and Java Development Kit (JDK) versions and…]]> 104 | 105 | 2014-10-01T14:31+00:00 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /test/data/distrijob.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Distrijob.fr : Infos du secteur distribution 5 | http://www.distrijob.fr 6 | Sélection d'informations 7 | fr-FR 8 | Copyright AlphaDISTRI 2015 9 | mar., 24 févr. 2015 14:01:01 GMT 10 | 9171960Distrijob.fr16042http://www.distrijob.frhttp://www.distrijob.fr/img/logo_distrijob_rss.gif 11 | Dia en hausse de près de 10% en 2014 12 | http://www.lsa-conso.fr/dia-en-hausse-de-pres-de-10-en-2014,202369 13 | http://www.lsa-conso.fr/dia-en-hausse-de-pres-de-10-en-2014,202369 14 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.lsa-conso.fr/dia-en-hausse-de-pres-de-10-en-2014,202369">LSA</A></LI></UL> 15 | mar., 24 févr. 2015 00:00:00 GMT 16 | 17 | 18 | Courir (Go Sport) reprend 18 magasins Bata 19 | http://www.franchise-magazine.com/actualite/breves/courir-reprend-18-magasins-bata-8930.html 20 | http://www.franchise-magazine.com/actualite/breves/courir-reprend-18-magasins-bata-8930.html 21 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.franchise-magazine.com/actualite/breves/courir-reprend-18-magasins-bata-8930.html">Franchise Magazine</A></LI></UL> 22 | mar., 24 févr. 2015 00:00:00 GMT 23 | 24 | 25 | Les nouvelles règles d'urbanisme commercial entrent en vigueur 26 | http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Les-nouvelles-regles-d-urbanisme-commercial-entrent-en-vigueur-45665 27 | http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Les-nouvelles-regles-d-urbanisme-commercial-entrent-en-vigueur-45665 28 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Les-nouvelles-regles-d-urbanisme-commercial-entrent-en-vigueur-45665">Linéaires</A></LI></UL> 29 | mar., 24 févr. 2015 00:00:00 GMT 30 | 31 | 32 | Le magasin Marionnaud des Champs-Elysées peut rester ouvert après 21 heures 33 | http://www.lsa-conso.fr/le-magasin-marionnaud-des-champs-elysees-peut-rester-ouvert-apres-21-heures,202372 34 | http://www.lsa-conso.fr/le-magasin-marionnaud-des-champs-elysees-peut-rester-ouvert-apres-21-heures,202372 35 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.lsa-conso.fr/le-magasin-marionnaud-des-champs-elysees-peut-rester-ouvert-apres-21-heures,202372">LSA</A></LI></UL> 36 | mar., 24 févr. 2015 00:00:00 GMT 37 | 38 | 39 | Travail le dimanche : Les magasins Carrefour peuvent ouvrir dans le Bas-Rhin 40 | http://www.20minutes.fr/strasbourg/1547551-20150223-travail-dimanche-magasins-carrefour-peuvent-ouvrir-bas-rhin 41 | http://www.20minutes.fr/strasbourg/1547551-20150223-travail-dimanche-magasins-carrefour-peuvent-ouvrir-bas-rhin 42 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.20minutes.fr/strasbourg/1547551-20150223-travail-dimanche-magasins-carrefour-peuvent-ouvrir-bas-rhin">20 minutes</A></LI></UL> 43 | mar., 24 févr. 2015 00:00:00 GMT 44 | 45 | 46 | Carrefour allonge les dates de consommation de ses MDD 47 | http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Carrefour-allonge-les-dates-de-consommation-de-ses-MDD-45663 48 | http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Carrefour-allonge-les-dates-de-consommation-de-ses-MDD-45663 49 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Carrefour-allonge-les-dates-de-consommation-de-ses-MDD-45663">Linéaires</A></LI></UL> 50 | lun., 23 févr. 2015 00:00:00 GMT 51 | 52 | 53 | Pourquoi la baisse du panier moyen est une bonne nouvelle pour lculent-encore-en-grande-distribution-l-inflation-au-plus-bas-depuis-60-ans,202021 54 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.lsa-conso.fr/les-prix-reculent-encore-en-grande-distribution-l-inflation-au-plus-bas-depuis-60-ans,202021">LSA</A></LI></UL> 55 | ven., 20 févr. 2015 00:00:00 GMT 56 | 57 | 58 | Le bio, une arme de séduction massive ? 59 | http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Le-bio-une-arme-de-seduction-massive-45648 60 | http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Le-bio-une-arme-de-seduction-massive-45648 61 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.lineaires.com/LA-DISTRIBUTION/Les-actus/Le-bio-une-arme-de-seduction-massive-45648">Linéaires</A></LI></UL> 62 | ven., 20 févr. 2015 00:00:00 GMT 63 | 64 | 65 | Lidl sigaro</A></LI></UL> 66 | mer., 18 févr. 2015 00:00:00 GMT 67 | 68 | 69 | Entre attentats et Saint-Valentin, le bilan contrasté des soldes 70 | http://www.lesechos.fr/industrie-services/conso-distribution/0204163750989-entre-attentats-et-saint-valentin-le-bilan-contraste-des-soldes-1094093.php 71 | http://www.lesechos.fr/industrie-services/conso-distribution/0204163750989-entre-attentats-et-saint-valentin-le-bilan-contraste-des-soldes-1094093.php 72 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.lesechos.fr/industrie-services/conso-distribution/0204163750989-entre-attentats-et-saint-valentin-le-bilan-contraste-des-soldes-1094093.php">Les Echos</A></LI></UL> 73 | mer., 18 févr. 2015 00:00:00 GMT 74 | 75 | 76 | Bata France : 96 des 136 magasins sauvés 77 | http://www.lsa-conso.fr/bata-france-96-des-136-magasins-sauves,201574 78 | http://www.lsa-conso.fr/bata-france-96-des-136-magasins-sauves,201574 79 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.lsa-conso.fr/bata-france-96-des-136-magasins-sauves,201574">LSA</A></LI></UL> 80 | mer., 18 févr. 2015 00:00:00 GMT 81 | 82 | 83 | Tesco se choisit un nouveau président 84 | http://www.capital.fr/bourse/actualites/tesco-se-choisit-un-nouveau-president-1013770 85 | http://www.capital.fr/bourse/actualites/tesco-se-choisit-un-nouveau-president-1013770 86 | <UL><LI><FONT color=maroon>Source : </FONT><A target="_blank" href="http://www.capital.fr/bourse/actualites/tesco-se-choisit-un-nouveau-president-1013770">Capital</A></LI></UL> 87 | mer., 18 févr. 2015 00:00:00 GMT 88 | 89 | 90 | Le délit d -------------------------------------------------------------------------------- /xml-namespaces.lisp: -------------------------------------------------------------------------------- 1 | (defpackage :cl-feedparser/xml-namespaces 2 | (:use :cl :alexandria :serapeum) 3 | (:shadowing-import-from :fset :map) 4 | (:import-from :fset :with :lookup) 5 | (:export :find-ns :prefix-uri :namespace?)) 6 | 7 | (in-package :cl-feedparser/xml-namespaces) 8 | 9 | (def namespaces 10 | (dict "http://backend.userland.com/rss" nil 11 | "http://blogs.law.harvard.edu/tech/rss" nil 12 | "http://purl.org/rss/1.0/" nil 13 | "http://my.netscape.com/rdf/simple/0.9/" nil 14 | "http://example.com/newformat#" nil 15 | "http://example.com/necho" nil 16 | "http://purl.org/echo/" nil 17 | "uri/of/echo/namespace#" nil 18 | "http://purl.org/pie/" nil 19 | "http://purl.org/atom/ns#" :atom03 20 | "http://www.w3.org/2005/Atom" :atom ;atom10 21 | "http://purl.org/rss/1.0/modules/rss091#" nil 22 | "http://www.bloglines.com/about/specs/fac-1.0" :access 23 | "http://webns.net/mvcb/" :admin 24 | "http://purl.org/rss/1.0/modules/aggregation/" :ag 25 | "http://purl.org/rss/1.0/modules/annotate/" :annotate 26 | "http://www.w3.org/2007/app" :app 27 | "http://media.tangent.org/rss/1.0/" :audio 28 | "http://backend.userland.com/blogChannelModule" :blog-channel 29 | "http://web.resource.org/cc/" :cc 30 | "http://www.microsoft.com/schemas/rss/core/2005" :cf 31 | "http://backend.userland.com/creativeCommonsRssModule" :creative-commons 32 | "http://purl.org/rss/1.0/modules/company" :co 33 | "http://purl.org/rss/1.0/modules/content/" :content 34 | "http://conversationsnetwork.org/rssNamespace-1.0/" :conversations-network 35 | "http://my.theinfo.org/changed/1.0/rss/" :cp 36 | "http://purl.org/dc/elements/1.1/" :dc 37 | "http://purl.org/dc/terms/" :dcterms 38 | "http://purl.org/rss/1.0/modules/email/" :email 39 | "http://purl.org/rss/1.0/modules/event/" :ev 40 | "http://rssnamespace.org/feedburner/ext/1.0" :feedburner 41 | "http://purl.org/syndication/history/1.0" :fh 42 | "http://freshmeat.net/rss/fm/" :fm 43 | "http://xmlns.com/foaf/0.1" :foaf 44 | "http://xmlns.com/foaf/0.1/" :foaf 45 | "http://www.w3.org/2003/01/geo/wgs84_pos#" :geo 46 | "http://www.georss.org/georss" :georss 47 | "http://geourl.org/rss/module/" :geourl 48 | "http://base.google.com/ns/1.0" :g 49 | "http://www.opengis.net/gml" :gml 50 | "http://postneo.com/icbm/" :icbm 51 | "http://purl.org/rss/1.0/modules/image/" :image 52 | "urn:atom-extension:indexing" :indexing 53 | "http://www.itunes.com/DTDs/PodCast-1.0.dtd" :itunes 54 | "http://example.com/DTDs/PodCast-1.0.dtd" :itunes 55 | "http://earth.google.com/kml/2.0" :kml20 56 | "http://earth.google.com/kml/2.1" :kml21 57 | "http://www.opengis.net/kml/2.2" :kml22 58 | "http://purl.org/rss/1.0/modules/link/" :l 59 | "http://www.w3.org/1998/Math/MathML" :mathml 60 | "http://search.yahoo.com/mrss" :media 61 | ;; Version 1.1.2 of the Media RSS spec added the trailing slash on 62 | ;; the namespace 63 | "http://search.yahoo.com/mrss/" :media 64 | "http://openid.net/xmlns/1.0" :openid 65 | "http://a9.com/-/spec/opensearchrss/1.0/" :opensearch10 66 | "http://a9.com/-/spec/opensearch/1.1/" :opensearch 67 | "http://www.opml.org/spec2" :opml 68 | "http://madskills.com/public/xml/rss/module/pingback/" :pingback 69 | "http://prismstandard.org/namespaces/1.2/basic/" :prism 70 | "http://www.w3.org/1999/02/22-rdf-syntax-ns#" :rdf 71 | "http://www.w3.org/2000/01/rdf-schema#" :rdfs 72 | "http://purl.org/rss/1.0/modules/reference/" :ref 73 | "http://purl.org/rss/1.0/modules/richequiv/" :reqv 74 | "http://purl.org/rss/1.0/modules/search/" :search 75 | "http://purl.org/rss/1.0/modules/slash/" :slash 76 | "http://schemas.xmlsoap.org/soap/envelope/" :soap 77 | "http://purl.org/rss/1.0/modules/servicestatus/" :ss 78 | "http://hacks.benhammersley.com/rss/streaming/" :str 79 | "http://purl.org/rss/1.0/modules/subscription/" :sub 80 | "http://www.w3.org/2000/svg" :svg 81 | "http://feedsync.org/2007/feedsync" :sx 82 | "http://purl.org/rss/1.0/modules/syndication/" :sy 83 | "http://schemas.pocketsoap.com/rss/myDescModule/" :szf 84 | "http://purl.org/rss/1.0/modules/taxonomy/" :taxo 85 | "http://purl.org/rss/1.0/modules/threading/" :thr 86 | "http://purl.org/syndication/thread/1.0" :thr 87 | "http://purl.org/rss/1.0/modules/textinput/" :ti 88 | "http://madskills.com/public/xml/rss/module/trackback/" :trackback 89 | "http://wellformedweb.org/commentAPI/" :wfw 90 | "http://purl.org/rss/1.0/modules/wiki/" :wiki 91 | "http://www.w3.org/1999/xhtml" :xhtml 92 | "http://www.w3.org/1999/xlink" :xlink 93 | "http://www.w3.org/XML/1998/namespace" :xml 94 | "xri://$xrd*($v*2.0)" :xrd 95 | "xri://$xrds" :xrds 96 | "http://www.youtube.com/xml/schemas/2015" :yt 97 | "http://podlove.org/simple-chapters" :psc) 98 | "Table from namespace to prefix. 99 | Collected from the source of feedparser.py and the list at the W3C 100 | feed validator.") 101 | 102 | (defun nstring-camel-case (string) 103 | "Destructively convert STRING to camelCase and return it." 104 | (prog1 string 105 | (loop for i from 0 below (length string) 106 | for c2 = (aref string i) 107 | for c1 = #\Space then c2 108 | do (unless (eql c2 #\-) 109 | (if (eql c1 #\-) 110 | (setf (aref string i) (char-upcase c2)) 111 | (setf (aref string i) (char-downcase c2))))))) 112 | 113 | (defun string-camel-case (string) 114 | "Return a copy of STRING in camelCase." 115 | (nstring-camel-case (copy-seq (string string)))) 116 | 117 | (def namespace-map 118 | (let ((map (map))) 119 | (maphash (lambda (k v) 120 | (when k 121 | (setf map (with map (string-camel-case v) k)))) 122 | namespaces) 123 | map) 124 | "Map from prefix to namespace.") 125 | 126 | (def namespace-prefixes 127 | (adjoin nil (remove-duplicates (hash-table-values namespaces)))) 128 | 129 | (defun namespace? (x) 130 | (member x namespace-prefixes)) 131 | 132 | (defun find-ns (uri) 133 | (gethash uri namespaces)) 134 | 135 | (defun prefix-uri (prefix) 136 | (lookup namespace-map prefix)) 137 | -------------------------------------------------------------------------------- /test/tests.lisp: -------------------------------------------------------------------------------- 1 | (in-package :cl-feedparser/test) 2 | 3 | (in-suite cl-feedparser) 4 | 5 | (defun entry-content (entry) 6 | (let ((content 7 | (~> (href entry :content) first (href :value)))) 8 | (is (stringp content)) 9 | content)) 10 | 11 | (local* 12 | (defsubst control-char? (char) 13 | (let ((code (char-code char))) 14 | (or (<= 0 code #x001f) 15 | (<= #x007f code #x009f)))) 16 | 17 | (defun print-bozo (bozo) 18 | "Print BOZO as with `princ-to-string', removing any control chars." 19 | (remove-if #'control-char? (princ-to-string bozo)))) 20 | 21 | (defun parse-date-safe (date) 22 | (and (stringp date) 23 | (ignoring local-time::invalid-timestring 24 | (local-time:parse-timestring date)))) 25 | 26 | (defun extract-pubdate (hash) 27 | (let* ((given 28 | (or (gethash :pubdate hash) 29 | (feed-ref hash :published-parsed) 30 | (feed-ref hash :updated-parsed))) 31 | (final 32 | (if (and given (< given (get-universal-time))) 33 | given 34 | (get-universal-time)))) 35 | (values final (and given t)))) 36 | 37 | (def bad-mtime-feeds 38 | (let* ((dir (find-test-file "guid-mask/")) 39 | (files (uiop:directory-files dir))) 40 | (keep "xml" files 41 | :key #'pathname-type 42 | :test #'equal))) 43 | 44 | (test bad-mtimes 45 | "Make sure trying to parse invalid modified dates in entries won't 46 | error." 47 | (dolist (bmf bad-mtime-feeds) 48 | (let* ((file (load-test-file bmf)) 49 | (feed (parse-feed file)) 50 | (entries (href feed :entries)) 51 | (mtimes (mapcar (op (href _ :updated)) entries))) 52 | (finishes (mapc #'parse-date-safe mtimes))))) 53 | 54 | (test kinlay/ftp 55 | "Can we handle ftp links?" 56 | (is (member "A Study in Gold" 57 | (~> (load-test-file "kinlay.xml") 58 | parse-feed 59 | (href :entries)) 60 | :key (op (href _ :title)) 61 | :test #'equal))) 62 | 63 | (test iframes 64 | (let ((parse (~> (load-test-file "slashdot-iframes.rss") 65 | parse-feed))) 66 | (cl:loop for entry in (href parse :entries) 67 | for content = (href entry :summary) 68 | do (is (not (string*= "iframe" content)))))) 69 | 70 | (test xenomachina 71 | "Torture test for HTML sanitizers." 72 | (let ((testbed (load-test-file "testbed.xml"))) 73 | (let ((hash (parse-feed testbed))) 74 | (is (hash-table-p hash)) 75 | (is (= 1 (length (href hash :entries))))))) 76 | 77 | (test no-articles 78 | "Check that article elements are sanitized." 79 | (let ((html (~> (parse-feed (load-test-file "what-if.atom")) 80 | (href :entries) 81 | first 82 | (href :summary)))) 83 | (is (not (string*= " hash (href :entries) first entry-content))) 128 | (is (stringp contents)) 129 | (is (string*= " feed (href :entries) first entry-content))))) 174 | 175 | (test strip-style-element 176 | "Are the contents of style elements stripped?" 177 | (let ((feed (parse-feed (load-test-file "teche.xml")))) 178 | (dolist (entry (href feed :entries)) 179 | (let ((content (entry-content entry))) 180 | (is (not (string*= "font-face" content))))))) 181 | 182 | (test no-content-if-no-content 183 | (let ((files '("notes.unwieldy.rss" "yayitsrob.rss"))) 184 | (dolist (file files) 185 | (let* ((text (load-test-file file)) 186 | (feed (parse-feed text))) 187 | (dolist (entry (href feed :entries)) 188 | (is-true (href entry :summary)) 189 | (is-false (href entry :content))))))) 190 | 191 | (test junk-before-declaration 192 | "Can we work around junk before the declaration?" 193 | (is (hash-table-p (parse-feed (load-test-file "binnsblog.rss")))) 194 | (is (hash-table-p (parse-feed (load-test-file "hydro74.rss"))))) 195 | 196 | (test hub-is-none 197 | "What happens if hub is None (Python)?" 198 | (let ((feed (~> "samuel-clay.rss" 199 | load-test-file 200 | parse-feed))) 201 | (is-false (gethash :hub feed)))) 202 | 203 | (test xhtml-content 204 | "Can we handle inline XHTML content?" 205 | (is (= 10 (length 206 | (href (parse-feed (load-test-file "hypercritical.xml") 207 | :max-entries 10) 208 | :entries))))) 209 | 210 | (test statwing 211 | (is (= 10 (length (href (parse-feed (load-test-file "statwing.xml") 212 | :max-entries 10) 213 | :entries)))) 214 | (finishes (html5-parser:parse-html5 (load-test-file "statwing-poison.html")))) 215 | 216 | (test mailto-href 217 | ;; The rub is "http://mailto:alex@iphonelife.com" 218 | (let ((feed (load-test-file "iphonelife"))) 219 | (finishes (parse-feed feed)))) 220 | 221 | (test html5sec-rss 222 | (let ((feed (load-test-file "html5sec.rss"))) 223 | ;; Looks like HTML. 224 | (let* ((parse (parse-feed feed))) 225 | (is (equal (href parse :link) "")) 226 | (dolist (entry (href parse :entries)) 227 | (is (equal (href entry :link) "")) 228 | (macrolet ((test-key (key) 229 | `(is-false (find #\< (href entry ,key)) 230 | "~a is: ~a" ,key (href entry ,key)))) 231 | (test-key :author) 232 | (test-key :published) 233 | (test-key :description)))))) 234 | 235 | (test feedburner-namespaces 236 | "Regression for bug in Klacks." 237 | (is (not (emptyp (href (parse-feed (load-test-file "3q-atom.xml")) :entries))))) 238 | 239 | (test locklin-etx 240 | (let ((parse (parse-feed (load-test-file "locklin-etx.xml")))) 241 | (is (null (find #\ETX (print-bozo (href parse :bozo-exception))))))) 242 | 243 | (test xxe-exploit 244 | ;; http://blog.detectify.com/post/82370846588/how-we-got-read-access-on-googles-production 245 | ;; http://mikeknoop.com/lxml-xxe-exploi/t 246 | (finishes (parse-feed (load-test-file "xxe-exploit.rss")))) 247 | 248 | (test billion-laughs 249 | (signals fxml:entities-forbidden 250 | (fxml:parse (load-test-file "lol.xml") nil))) 251 | 252 | (test undefined-entity 253 | "The expansion has to be supplied as a string, not a character." 254 | (let* ((file "arrdem.xml") 255 | (string (load-test-file file)) 256 | (feed (parse-feed string)) 257 | (entries (href feed :entries))) 258 | (is (= (length entries) 20)))) 259 | 260 | (test content-sans-markup 261 | (let* ((file (load-test-file "push-pub.xml")) 262 | (feed (parse-feed file))) 263 | (let ((entries (href feed :entries))) 264 | (is (every (op (href _ :content)) entries))))) 265 | 266 | (test distrijob 267 | (finishes 268 | (parse-feed 269 | (load-test-file "distrijob.xml")))) 270 | 271 | (test rss-no-cdata 272 | (let* ((file (load-test-file "mfarmer-no-cdata.rss")) 273 | (feed (parse-feed file))) 274 | (is (notany #'emptyp 275 | (mapcar (op (href _ :summary)) 276 | (href feed :entries)))))) 277 | 278 | (test guid-overrides-link 279 | "Test that the guid doesn't override the link." 280 | (let* ((feed (parse-feed (load-test-file "hpr_ogg_rss.php")))) 281 | (dolist (entry (href feed :entries)) 282 | (is (string*= "eps.php?id=" (href entry :link)))))) 283 | 284 | (test rss-enclosure 285 | (let ((feed (parse-feed (load-test-file "substack.rss")))) 286 | (is (= 4 (hash-table-size (href feed :links)))))) 287 | -------------------------------------------------------------------------------- /handlers.lisp: -------------------------------------------------------------------------------- 1 | (defpackage :cl-feedparser/handlers 2 | (:use :cl :alexandria :serapeum :anaphora 3 | :cl-feedparser/parser) 4 | (:import-from :cl-feedparser/parser :lispify) 5 | (:shadowing-import-from :cl-feedparser/parser :string+) 6 | (:import-from :fxml.klacks 7 | :map-attributes 8 | :get-attribute)) 9 | 10 | (in-package :cl-feedparser/handlers) 11 | 12 | (defun add-content-key (table) 13 | (prog1 table 14 | (let ((content (get-text))) 15 | (unless (emptyp content) 16 | (setf (@ table :content) content))))) 17 | 18 | (defun defaulted-key (table key value) 19 | (ensure-gethash key table value) 20 | table) 21 | 22 | (defun resolve-attr (table attr) 23 | (callf #'resolve-uri (@ table attr)) 24 | table) 25 | 26 | (defmacro attrs-table (&rest attrs) 27 | (with-unique-names (ht source value) 28 | `(lret ((,source *source*) 29 | (,ht (make-hash-table :size ,(length attrs)))) 30 | ,@(loop for attr in attrs 31 | for keyword = (make-keyword (lispify attr)) 32 | collect `(when-let (,value (get-attribute ,source ,attr)) 33 | (setf (@ ,ht ,keyword) ,value)))))) 34 | 35 | ;;; Atom 1.0. 36 | 37 | (defhandler :atom :title 38 | (when-let (text (sanitize-title (get-text))) 39 | (let ((title (trim-whitespace text))) 40 | ;; Cf. Grantland. 41 | (ensure2 (gethash* :title (or *entry* *feed*)) 42 | title)))) 43 | 44 | (defhandler :atom :subtitle 45 | (when-let (text (sanitize-title (get-text))) 46 | (setf (gethash* :subtitle *feed*) text))) 47 | 48 | (defhandler :atom :rights 49 | (when-let (text (sanitize-title (get-text))) 50 | (setf (gethash* :rights *feed*) text))) 51 | 52 | (defhandler :atom :link 53 | (let* ((source *source*) 54 | (rel (get-attribute source "rel")) 55 | (type (get-attribute source "type")) 56 | (href (get-attribute source "href")) 57 | (title (get-attribute source "title")) 58 | (link (make-hash-table))) 59 | 60 | ;; E.g. Quora. 61 | (when (or (equal href "None") (equal href "/None")) 62 | (return-from handle-tag)) 63 | 64 | (when href 65 | (setf href (resolve-uri href))) 66 | 67 | ;; TODO Invalid for Atom 0.3? 68 | (when (and href (or (not rel) (equal rel "alternate"))) 69 | (setf (gethash* :link (or *entry* *feed*)) 70 | href)) 71 | 72 | (setf (gethash* :rel link) rel 73 | (gethash* :type link) type 74 | (gethash* :href link) href 75 | (gethash* :title link) title) 76 | 77 | (when (and *entry* (equal rel "enclosures")) 78 | (push link (gethash* :enclosures *entry*))) 79 | 80 | (push link (gethash* :links (or *entry* *feed*))))) 81 | 82 | (defhandler :atom :name 83 | (let ((name (sanitize-text (get-text)))) 84 | (setf (gethash* :author (or *entry* *feed*)) name 85 | (gethash* :name *author*) name))) 86 | 87 | (defhandler :atom :email 88 | (setf (gethash* :email *author*) (get-text/sanitized))) 89 | 90 | (defhandler :atom :uri 91 | (setf (gethash* :href *author*) 92 | (resolve-uri (get-text)))) 93 | 94 | (defhandler :atom :feed 95 | (block nil 96 | (map-attributes 97 | (lambda (ns lname qname value dtdp) 98 | (declare (ignore ns lname dtdp)) 99 | (when (equal qname "xml:lang") 100 | (setf (gethash* :language *feed*) 101 | (sanitize-text value)) 102 | (return))) 103 | *source*))) 104 | 105 | (defhandler :atom :icon 106 | (setf (gethash* :icon *feed*) 107 | (resolve-uri (get-text)))) 108 | 109 | (defhandler :atom :summary 110 | (if-let (entry *entry*) 111 | (when-let (content (get-content)) 112 | (setf (gethash* :summary entry) (gethash* :value content) 113 | (gethash* :summary-detail entry) content)) 114 | 115 | (setf (gethash* :subtitle *feed*) 116 | (sanitize-title (get-text))))) 117 | 118 | (defhandler :atom :published 119 | (let ((target (or *entry* *feed*))) 120 | (setf (values (gethash* :published target) 121 | (gethash* :published-parsed target)) 122 | (get-timestring)))) 123 | 124 | (defhandler :atom :updated 125 | (let ((target (or *entry* *feed*))) 126 | (setf (values (gethash* :updated target) 127 | (gethash* :updated-parsed target)) 128 | (get-timestring))) 129 | 130 | (and *entry* (check-guid-mask))) 131 | 132 | (defhandler :atom :id 133 | (let ((id (get-text))) 134 | (when-let (entry *entry*) 135 | (check-guid-mask entry)) 136 | (setf (gethash* :id (or *entry* *feed*)) id))) 137 | 138 | (defhandler :atom :content 139 | (push (get-content) (gethash* :content *entry*))) 140 | 141 | (defhandler :atom :entry 142 | (entry-context)) 143 | 144 | (defhandler :atom :contributor 145 | (let ((*author* (dict))) 146 | (parser-loop *source* :recursive t) 147 | (push *author* (gethash* :contributors *entry*)))) 148 | 149 | ;;; Atom 0.3. 150 | 151 | (defmethod handle-tag ((ns (eql :atom03)) tag) 152 | "Fall back to Atom 1.0 parsing." 153 | (handle-tag :atom tag)) 154 | 155 | (defhandler :atom03 :tagline 156 | (handle-tag :atom :subtitle)) 157 | 158 | (defhandler :atom :info 159 | (when-let (text (sanitize-title (get-text))) 160 | (setf (gethash* :info *feed*) text))) 161 | 162 | (defhandler :atom03 :copyright 163 | (handle-tag :atom :rights)) 164 | 165 | (defhandler :atom03 :modified 166 | (handle-tag :atom :updated)) 167 | 168 | (defhandler :atom03 :created 169 | (awhen (get-timestring) 170 | (setf (gethash* :created *entry*) t))) 171 | 172 | ;;; E.g. 3QD. 173 | (defhandler :atom03 :issued 174 | (handle-tag :atom :published)) 175 | 176 | ;;; Dublin Core. 177 | 178 | (defmethod handle-tag ((ns (eql :dcterms)) lname) 179 | (handle-tag :dc lname)) 180 | 181 | (defhandler :dc :title 182 | (handle-tag :atom :title)) 183 | 184 | (defhandler :dc :rights 185 | (handle-tag :atom :rights)) 186 | 187 | (defhandler :dc :creator 188 | (handle-tag nil :author)) 189 | 190 | (defhandler :dc :date 191 | (handle-tag :atom :updated)) 192 | 193 | (defhandler :dcterms :modified 194 | (handle-tag :atom :updated)) 195 | 196 | (defhandler :dcterms :issued 197 | (handle-tag :atom :published)) 198 | 199 | (defhandler :dcterms :contributor 200 | (push (dict :name (get-text)) 201 | (gethash* :contributors *entry*))) 202 | 203 | (defhandler :dc :created 204 | (handle-tag :atom03 :created)) 205 | 206 | (defhandler :dc :language 207 | (handle-tag nil :language)) 208 | 209 | ;;; Feedburner. 210 | 211 | (defmethod handle-tag ((ns (eql :feedburner)) lname) 212 | "The feed is from Feedburner." 213 | (declare (ignore lname)) 214 | (ensure2 (gethash* :proxy *feed*) "feedburner")) 215 | 216 | (defhandler :feedburner :browser-friendly 217 | (handle-tag :atom :info)) 218 | 219 | (defhandler :feedburner :orig-link 220 | ;; Eg. 3QD. 221 | (when *entry* 222 | (setf (gethash* :link *entry*) 223 | (resolve-uri (get-text))))) 224 | 225 | ;;; Itunes. 226 | 227 | (defhandler :itunes :subtitle 228 | (handle-tag :atom :subtitle)) 229 | 230 | (defhandler :itunes :author 231 | (handle-tag nil :author)) 232 | 233 | ;;; RDF. 234 | 235 | (defhandler :rdf :title 236 | (handle-tag :atom :title)) 237 | 238 | (defhandler :rdf :description 239 | (handle-tag :dc :description)) 240 | 241 | (defhandler :rdf :item 242 | (when-let (id (get-attribute *source* "about")) 243 | (entry-context :id id))) 244 | 245 | ;;; Media RSS. 246 | 247 | (defhandler :media :category 248 | (setf (gethash* :media-category *entry*) 249 | (~> (attrs-table "scheme" "label") 250 | (defaulted-key :schema "http://search.yahoo.com/mrss/category_schema") 251 | add-content-key))) 252 | 253 | (defhandler :media :hash 254 | (setf (gethash* :media-hash *entry*) 255 | (~> (attrs-table "algo") 256 | (add-content-key)))) 257 | 258 | (defhandler :media :description 259 | (setf (gethash* :media-description *entry*) 260 | (get-text/sanitized))) 261 | 262 | (defhandler :media :content 263 | (push 264 | (~> (attrs-table "url" 265 | "fileSize" 266 | "type" 267 | "medium" 268 | "isDefault" 269 | "expression" 270 | "bitrate" 271 | "framerate" 272 | "samplingrate" 273 | "channels" 274 | "duration" 275 | "height" 276 | "width" 277 | "lang") 278 | add-content-key 279 | (resolve-attr :url)) 280 | (gethash* :media-content *entry*))) 281 | 282 | (defhandler :media :credit 283 | (push 284 | (~> (attrs-table "role" "schema") 285 | add-content-key 286 | (defaulted-key :schema "urn:ebu")) 287 | (gethash* :media-credit *entry*))) 288 | 289 | (defhandler :media :keywords 290 | (setf (gethash* :media-keywords *entry*) 291 | (~>> (get-text) 292 | (split-sequence #\,) 293 | (mapcar #'trim-whitespace) 294 | (remove-if #'emptyp)))) 295 | 296 | (defhandler :media :player 297 | (setf (gethash* :media-player *entry*) 298 | (~> (attrs-table "url" "height" "width") 299 | (resolve-attr :url)))) 300 | 301 | (defhandler :media :copyright 302 | (setf (gethash* :media-copyright *entry*) 303 | (~> (attrs-table "url") 304 | (resolve-attr :url) 305 | add-content-key))) 306 | 307 | (defhandler :media :rating 308 | (setf (gethash* :media-rating *entry*) 309 | (~> (attrs-table "schema") 310 | (defaulted-key :schema "urn:simple") 311 | add-content-key))) 312 | 313 | (defhandler :media :restriction 314 | (setf (gethash* :media-restriction *entry*) 315 | (~> (attrs-table "relationship" "type") 316 | add-content-key))) 317 | 318 | (defhandler :media :statistics 319 | (setf (gethash* :media-statistics *entry*) 320 | (attrs-table "views" "favorites"))) 321 | 322 | (defhandler :media :star-rating 323 | (setf (gethash* :media-star-rating *entry*) 324 | (attrs-table "count" "max" "average" "min"))) 325 | 326 | (defhandler :media :tags 327 | (setf (gethash* :media-tags *entry*) 328 | (~>> (get-text) 329 | (split-sequence #\,) 330 | (mapcar #'trim-whitespace) 331 | (remove-if #'emptyp)))) 332 | 333 | (defhandler :media :thumbnail 334 | (push 335 | (~> (attrs-table "url" "height" "width" "time") 336 | (resolve-attr :url)) 337 | (gethash* :media-thumbnail *entry*))) 338 | 339 | (defhandler :media :title 340 | (handle-tag :atom :title)) 341 | 342 | ;;; RSS. 343 | 344 | (defhandler nil :title 345 | (handle-tag :atom :title)) 346 | 347 | (defhandler nil :description 348 | (if *entry* 349 | (handle-tag :atom :summary) 350 | (handle-tag :atom :subtitle))) 351 | 352 | (defhandler nil :copyright 353 | (handle-tag :atom :rights)) 354 | 355 | (defhandler nil :link 356 | (when-let (string (get-text)) 357 | (setf (gethash* :link (or *entry* *feed*)) 358 | (resolve-uri string)))) 359 | 360 | (defhandler nil :author 361 | (let* ((author (get-text/sanitized)) 362 | (email? (find #\@ author)) 363 | creator) 364 | 365 | (if email? 366 | (let ((space (position #\Space author))) 367 | (setf (gethash* :email *author*) (subseq author 0 space)) 368 | (when space 369 | (ensure creator (strip-parens (subseq author space))))) 370 | (ensure creator author)) 371 | 372 | (let ((name (strip-parens creator))) 373 | (setf (gethash* :name *author*) name 374 | (gethash* :author (or *entry* *feed*)) name)))) 375 | 376 | (defhandler nil :language 377 | (setf (gethash* :language *feed*) (get-text))) 378 | 379 | (defhandler nil :pub-date 380 | (handle-tag :atom :published)) 381 | 382 | (defhandler nil :last-build-date 383 | (unless *entry* 384 | (handle-tag :atom :published))) 385 | 386 | (defhandler nil :guid 387 | ;; todo rdf:about 388 | (when-let (entry *entry*) 389 | (let ((permalinkp 390 | (equal "true" (get-attribute *source* "isPermaLink"))) 391 | (id (get-text))) 392 | (when id 393 | (check-guid-mask entry) 394 | (setf (href entry :id) id) 395 | (symbol-macrolet ((link (gethash* :link entry))) 396 | (when (or permalinkp 397 | ;; Use GUID as a fallback link. 398 | (and (urlish? id) 399 | (null link))) 400 | (setf link (resolve-uri id)))))))) 401 | 402 | (defhandler nil :item 403 | (handle-tag :atom :entry)) 404 | 405 | (defhandler nil :comments 406 | (setf (gethash* :comments *entry*) (get-text))) 407 | 408 | (defhandler nil :ttl 409 | (when-let (string (get-text)) 410 | (setf (gethash* :ttl *feed*) string))) 411 | 412 | (defhandler :content :encoded 413 | (handle-tag :atom :content)) 414 | 415 | (defhandler :xhtml :body 416 | (handle-tag :atom :content)) 417 | 418 | (defhandler nil :body 419 | (handle-tag :atom :content)) 420 | 421 | (defhandler nil :fullitem 422 | (handle-tag :atom :content)) 423 | 424 | (defhandler nil :enclosure 425 | (let ((source *source*)) 426 | (push (dict :rel "enclosure" 427 | :type (get-attribute source "type") 428 | :length (get-attribute source "length") 429 | :href (get-attribute source "url")) 430 | (gethash* :enclosures (or *entry* *feed*))))) 431 | -------------------------------------------------------------------------------- /test/data/yayitsrob.rss: -------------------------------------------------------------------------------- 1 | 2 | I’m an associate editor at The Atlantic. On Twitter, I’m @yayitsrob. 3 | 4 | 5 | 6 | The hectograph — or jellygraph — was a near-print process used in the late 19th and early 20th centuries. It could make cheap copies of original documents, usually for business or professional purposes, by transferring the image of the original to a gelatin bed with colored dye. Copies could then be made from the bed as long as the dye lasted.Says Wikipedia: “At least eight different colors of hectographic ink were available at one time, but purple was the most popular because of its density and contrast.” 7 | 8 | 9 | 10 | I use my Tumblr like a combination of a hectograph, a commonplace book, and a wunderkammer.JellygraphTumblr (3.0; @yayitsrob)http://yayitsrob.tumblr.com/Peter Buffett, the son of Warren Buffett, writes of the “Charitable-Industrial Complex” in the...<p>Peter Buffett, the son of Warren Buffett, writes of the “Charitable-Industrial Complex” in the Times. I found it resonant and revealing, and <a href="http://www.nytimes.com/2013/07/27/opinion/the-charitable-industrial-complex.html">you should read it.</a></p> 11 | 12 | <p>It’s resonant because I think Buffett correlates, as I’ve seen few other authors successfully do, the global rise in inequality with a rise in non-profit activity and power. “Inside any important philanthropy meeting,” he <a href="http://www.nytimes.com/2013/07/27/opinion/the-charitable-industrial-complex.html">writes</a>, “you witness heads of state meeting with investment managers and corporate leaders. All are searching for answers with their right hand to problems that others in the room have created with their left.”</p> 13 | 14 | <p>It reminded me of a phenomenon at Northwestern University, which I graduated from in June.</p> 15 | 16 | <p>At Northwestern, there was a special kind of non-profit group, and groups of that kind were flourishing. They prized concepts like <em>engagement</em>, <em>design</em> and <em>community</em>. Almost all of them focused on the <em>global.</em> They were social justice-oriented&#8212;if anything, that was the moniker for this kind of group&#8212;but they were neither activist nor leftist. Almost all their members would ultimately declare, with Buffett: “I’m really not calling for an end to capitalism; I’m calling for humanism.”</p> 17 | 18 | <p>They had a politics, but it was separate&#8212;and meant to be separate&#8212;from American party politics. Their politics sought to end global poverty, mostly with or within global capitalism; they sought sustainable, rather bland anti-climate change policies; and they sought <em>designed solutions,</em> small alterations to systems, probably conceived by collaborating amateurs, which tweaked an aspect of someone’s everyday routine so that some other goal might be accomplished. These are the loose politics of the global progressive elite.</p> 19 | 20 | <p>These groups weren&#8217;t ineffectual, and there were jewels among them. <a href="http://globemed.org">Globemed</a>, for instance, had a concise mission and took its structure from the community-centered approach of Partners in Health.</p> 21 | 22 | <p>Yet groups of this kind had certain tendencies. They were (and this is not a technical, nor technically correct, term) almost <em>franchises</em>: they had a structure easy to duplicate, the intention to expand to other colleges and universities, and a financial system to buy into. They were meant to foster (a favorite word) thoughtful leaders, to propel non-profit entrepreneurs into professional charities. They were elite preparation organizations, in other words, so there was a premium on founding, on opening a new branch of something, on being notable in a way that fits inside a Twitter bio. The young, attractive entrepreneur of technology mythology roams beyond Silicon Valley: with charitable accomplishments swapped out for technological ones, he or she also stalks non-profit conferences, lectures to other world-making devotees, advocates design and development in places beyond social media.</p> 23 | 24 | <p>If you went to college recently, you probably recognize this cohort. I’m friends with many people who worked in the “change-maker” space, and I respect them and their work. (I, meanwhile, spent college on Twitter, and <a href="http://apps.northbynorthwestern.com/commencement/2013/">writing bad poems about newspaper history</a>.) But these groups are unmistakably pre-professional, preparing for an industry (Buffett says nonprofits employ 9.4 million people in the US) and they sit snug beside all the other factories of the elite: the technological industrial complex, the consulting industrial complex, the financial.</p> 25 | 26 | <hr><p>And it&#8217;s here I found Buffett’s op-ed revealing, because he cannot process, cannot provide a path, for how to break down these factories. He laments a “crisis of imagination,” and calls for a new operating system, “new code.” I am not sure whether this is out of humility&#8212;he does not have an answer&#8212;but it is odd to watch his op-ed become a TED talk.</p> 27 | 28 | <p>And reading his closing, I could hear leftist friends responding. Buffett explicitly rejects anti-capitalism, but then he tumbles into platitudes, and it’s easy to see all this and shout: <em>Look where his privilege cannot take him!</em></p> 29 | 30 | <p>I am not so certain. A crisis of imagination strikes me as a crisis of vocabulary, and I wonder if an unspoken first principle of these groups&#8212;to prize the often apolitical&#8212;is the mistake. I am not sure that I want “technology” and “philanthropy,” the industries which, thanks to their novelty, feel “exciting,” to become plainly intelligible to boring old politics. I’m not sure I have a choice, though: <a href="http://www.washingtonpost.com/community-relations/the-washington-post-to-launch-new-tech-policy-blog-called-the-switch/2013/07/19/2bf2d0e2-f0ae-11e2-a1f9-ea873b7e0424_story.html">The gargantuans of tech are gearing up for politicization</a>. I wonder if these new, “world-changing” non-profits will gear up soon, too.</p>http://yayitsrob.tumblr.com/post/56618708680http://yayitsrob.tumblr.com/post/56618708680Sat, 27 Jul 2013 12:59:00 -0500politicswritingnon-profitinequalityeducationcollegesexpigeon: 31 | 32 | Notable whites in America.  33 | (via) 34 | 35 | The culmination...<img src="http://31.media.tumblr.com/e20dc1aac9a72b5d512a453d58752b25/tumblr_mqk8qkiALv1qzp87ao1_500.png"/><br/><br/><p><a href="http://sexpigeon.tumblr.com/post/56542149379/notable-whites-in-america-via" class="tumblr_blog">sexpigeon</a>:</p> 36 | 37 | <blockquote><p>Notable whites in America. </p> 38 | <p>(<a href="https://twitter.com/yayitsrob/status/360831989575729154">via</a>)</p></blockquote> 39 | 40 | <p>The culmination of all my Internet work, right there in that little “via.”</p>http://yayitsrob.tumblr.com/post/56547854213http://yayitsrob.tumblr.com/post/56547854213Fri, 26 Jul 2013 16:41:11 -0500“Maybe our grandparents also suffered from alien laser attacks,...<img src="http://31.media.tumblr.com/af650ba46aa7dc6ad91de100d5d3181f/tumblr_mqjzhdpyT21re0ihfo1_500.gif"/><br/><br/><p>“Maybe our grandparents <em>also</em> suffered from alien laser attacks, but just couldn’t <em>identify</em> it as such.” (via <a href="http://wondermark.com/952/">Wondermark » Archive » #952; In which Everybody is Lasered</a>)</p>http://yayitsrob.tumblr.com/post/56527889977http://yayitsrob.tumblr.com/post/56527889977Fri, 26 Jul 2013 12:05:36 -0500historynostalgiaTHIS IS THE BESTPowerless not to reblog Maurice Sendak, in a robe, in a forest, ONE HAND ON HIS DOG NAMED AFTER...<p>Powerless not to reblog <em>Maurice Sendak,</em> <strong>in a robe</strong>, <em><strong>in a forest</strong></em>, <em><strong>ONE HAND ON HIS DOG NAMED AFTER HERMAN MELVILLE</strong></em>.</p> 41 | 42 | <p>Powerless.</p>http://yayitsrob.tumblr.com/post/56272424810http://yayitsrob.tumblr.com/post/56272424810Tue, 23 Jul 2013 16:42:00 -0500explore-blog: 43 | 44 | 45 | Try not to smile: The late Maurice Sendak, with...<img src="http://31.media.tumblr.com/76f540a7ea1b81880d092db2e17f003d/tumblr_mopm5eda1r1rqpa8po1_500.jpg"/><br/><br/><p><a class="tumblr_blog" href="http://exp.lore.com/post/53463500696/try-not-to-smile-the-late-maurice-sendak-with">explore-blog</a>:</p> 46 | 47 | <blockquote> 48 | <p>Try not to smile: The late <a href="http://exp.lore.com/tagged/maurice-sendak">Maurice Sendak</a>, with his beloved dog Herman (named after Herman Melville, Sendak’s great hero), photographed by <a href="http://www.cookstudio.com/">Mariana Cook</a>.</p> 49 | 50 | </blockquote>http://yayitsrob.tumblr.com/post/56272210846http://yayitsrob.tumblr.com/post/56272210846Tue, 23 Jul 2013 16:40:00 -0500Tuesday noon.<img src="http://31.media.tumblr.com/f5b7510d8b3327bf9d45286882440b95/tumblr_mqef3iAMI61re0ihfo1_500.jpg"/><br/><br/><p>Tuesday noon.</p>http://yayitsrob.tumblr.com/post/56250183338http://yayitsrob.tumblr.com/post/56250183338Tue, 23 Jul 2013 11:57:18 -0500Photo<img src="http://31.media.tumblr.com/005b81325567eac633fb8a2a92a4de59/tumblr_mqeajiZHR81re0ihfo1_500.jpg"/><br/><br/>http://yayitsrob.tumblr.com/post/56243801300http://yayitsrob.tumblr.com/post/56243801300Tue, 23 Jul 2013 10:18:51 -0500Photo<img src="http://24.media.tumblr.com/8aebc0f048aca973e9ff18849095a62b/tumblr_mqdawmEJWv1qattsgo1_500.png"/><br/><br/>http://yayitsrob.tumblr.com/post/56206944394http://yayitsrob.tumblr.com/post/56206944394Mon, 22 Jul 2013 22:30:35 -0500chicagogeek: 51 | 52 | 53 | In honor of Bertrand Goldberg’s 100th birthday...<img src="http://31.media.tumblr.com/d1a59080ed7fa9bb93dfa344a1da8454/tumblr_mq3q1ygb5s1qcaxqeo1_500.jpg"/><br/><br/><p><a class="tumblr_blog" href="http://chicagogeek.tumblr.com/post/55725477302/in-honor-of-bertrand-goldbergs-100th-birthday">chicagogeek</a>:</p> 54 | 55 | <blockquote> 56 | <p>In honor of Bertrand Goldberg’s 100th birthday today, here is a picture of Marina City under construction.</p> 57 | <p>(Photographer: Richard Nickel)</p> 58 | </blockquote>http://yayitsrob.tumblr.com/post/55742234214http://yayitsrob.tumblr.com/post/55742234214Wed, 17 Jul 2013 21:06:43 -0500Asbury Park, NJ. Photos by Matthew Gallaway.<img src="http://24.media.tumblr.com/c72075c9a00d2308fa76b524ab781095/tumblr_mq1y92RwMN1qzse3ho1_500.jpg"/><br/> <br/><img src="http://31.media.tumblr.com/cf6c8f06967e25dc21cd4a6a51e4ea69/tumblr_mq1y92RwMN1qzse3ho3_500.jpg"/><br/> <br/><img src="http://24.media.tumblr.com/77967ee02117ce675f9b8613f44604a4/tumblr_mq1y92RwMN1qzse3ho2_500.jpg"/><br/> <br/><img src="http://24.media.tumblr.com/fb9f553a60e80d82eb94fbf5091818f9/tumblr_mq1y92RwMN1qzse3ho4_500.jpg"/><br/> <br/><img src="http://31.media.tumblr.com/86ad1ac82e7d3b7563313de01b3ae896/tumblr_mq1y92RwMN1qzse3ho7_500.jpg"/><br/> <br/><img src="http://24.media.tumblr.com/90aeb77fc9145de2ad6ecbb94d748ab2/tumblr_mq1y92RwMN1qzse3ho6_500.jpg"/><br/> <br/><img src="http://24.media.tumblr.com/965491c6afecea723c1aa863018ae11a/tumblr_mq1y92RwMN1qzse3ho8_500.jpg"/><br/> <br/><p>Asbury Park, NJ. Photos by Matthew Gallaway.</p>http://yayitsrob.tumblr.com/post/55689500459http://yayitsrob.tumblr.com/post/55689500459Wed, 17 Jul 2013 08:28:00 -0500liartownusa: 59 | 60 | Ryan Gosling New Movies 1<img src="http://31.media.tumblr.com/ba50dafd13771720ffdc6fe4882cd8fe/tumblr_mmof58p4lP1s71q1zo1_500.png"/><br/><br/><p><a href="http://liartownusa.tumblr.com/post/50236858207/ryan-gosling-new-movies-1" class="tumblr_blog">liartownusa</a>:</p> 61 | 62 | <blockquote><p><em><strong>Ryan Gosling New Movies 1</strong></em></p></blockquote>http://yayitsrob.tumblr.com/post/55688821899http://yayitsrob.tumblr.com/post/55688821899Wed, 17 Jul 2013 08:13:16 -0500laaaaaaandsat: 63 | 64 | Path 155, Row 23 2013-07-15 at...<img src="http://31.media.tumblr.com/ef84198b43dc6dc89d548394c8bf9dbd/tumblr_mpz6ck0wwC1swt5zko1_500.jpg"/><br/><br/><p><a href="http://laaaaaaandsat.tumblr.com/post/55505185949/path-155-row-23-2013-07-15-at" class="tumblr_blog">laaaaaaandsat</a>:</p> 65 | 66 | <blockquote><p><a href="http://bit.ly/18XRYOs">Path 155, Row 23 2013-07-15 at 2013-07-15 06:09:25.461878 GMT</a></p></blockquote>http://yayitsrob.tumblr.com/post/55508287616http://yayitsrob.tumblr.com/post/55508287616Mon, 15 Jul 2013 07:44:58 -0500‘Where Did You Sleep Last Night’ by Leadbelly is my new jam.<img src="http://24.media.tumblr.com/6adcb0a4356e3cb7aa491a3f6ae6a6a9/tumblr_mpyhtrDqoI1re0ihfo1_400.jpg"/><br/><br/><p><b><a href="http://www.thisismyjam.com/yayitsrob/_69dz613?utm_source=tumblr&utm_medium=sharing&utm_campaign=user">‘Where Did You Sleep Last Night’ by Leadbelly</a></b> is my new jam.</p>http://yayitsrob.tumblr.com/post/55475555574http://yayitsrob.tumblr.com/post/55475555574Sun, 14 Jul 2013 21:34:38 -0500thisismyjamLitany and wish<p><a href="http://vruba.tumblr.com/post/55396370871/litany-and-wish" class="tumblr_blog">vruba</a>:</p> 67 | 68 | <blockquote><p>America, we are a great nation.</p> 69 | 70 | <p>Greatness is not goodness.</p> 71 | 72 | <p>Greatness is not the lack of gruesome failure.</p> 73 | 74 | <p>Greatness is not liberty and justice for all.</p> 75 | 76 | <p>Greatness is less useful. It’s the ability to find solutions within problems. It’s the ability to, after it’s too late, after the irreparable harm has been done, change more than seemed possible.</p> 77 | 78 | <p>Greatness overlooks. Greatness is hypocritical. Greatness betrays.</p> 79 | 80 | <p>Greatness is not enough.</p> 81 | 82 | <p>America, what I want is kindness. I want us to keep each other’s subjectivities close. America, we live together. We make each other’s neighborhoods, and laws, and food, and lives. America, I do not own what I can defend, or what I can attack from. I only own what I can care for. And America, I own you.</p> 83 | 84 | <p>America, you are the guilty one who goes free. And you are the innocent ones who go to prison, the innocent ones who die, the innocent ones who mourn. You are the ones asking what they can tell their children tomorrow at breakfast, and you are the children. America, you are the ones who are held back, deferred, told “no” every way but aloud, whose dignity is put up to vote, whose bodies are not their own, whose rights come with asterisks, who are invited to the fancy dinners but have no suit, who are beaten for admitting they were beaten, whose safety is voided on technicalities, who cannot say what they feel, who are blamed for what is visited upon them, who are told they are not themselves, whose self-respect is called anger, who are made to pretend that unfair things are fair, who are belittled and ignored and killed.</p> 85 | 86 | <p>I have trouble, on a night like tonight, caring much about any one idea. No telling detail seems all that telling. I find myself wishing, knowing it’s too much and too little, that we can each find some way to push and pull and scold and coax it all, the thing we are, the whole America, toward kindness.</p></blockquote>http://yayitsrob.tumblr.com/post/55428021802http://yayitsrob.tumblr.com/post/55428021802Sun, 14 Jul 2013 10:34:42 -0500Refugee autumn leaves; the smaller one has an unusual blister...<img src="http://31.media.tumblr.com/542c58bf3f82c880049496060e2660ee/tumblr_mpvpdi2k8y1re0ihfo1_500.jpg"/><br/><br/><p>Refugee autumn leaves; the smaller one has an unusual blister pattern. #nofilter (at Camp No-Be-Bo-Sco)</p>http://yayitsrob.tumblr.com/post/55341974261http://yayitsrob.tumblr.com/post/55341974261Sat, 13 Jul 2013 09:24:54 -0500nofilterA lightning strike at 1,100 frames per second. Worth it to watch...<iframe width="400" height="225" src="http://www.youtube.com/embed/fiMHDsWbiOk?wmode=transparent&autohide=1&egm=0&hd=1&iv_load_policy=3&modestbranding=1&rel=0&showinfo=0&showsearch=0" frameborder="0" allowfullscreen></iframe><br/><br/><p>A lightning strike at 1,100 frames per second. Worth it to watch the physicist—whom National Geographic identifies as a “lightning physicist”— watch the film of lightning for the first time. Joyful.</p>http://yayitsrob.tumblr.com/post/55049930867http://yayitsrob.tumblr.com/post/55049930867Tue, 09 Jul 2013 21:12:25 -0500scienceweatherlightningEchinopsis. (“Apricot Glow”) #nofilter (at Chicago Botanic...<img src="http://24.media.tumblr.com/281becd3e69a06cb048b6924cc19a594/tumblr_mox2l2aFjF1re0ihfo1_500.jpg"/><br/><br/><p>Echinopsis. (“Apricot Glow”) #nofilter (at Chicago Botanic Garden)</p>http://yayitsrob.tumblr.com/post/53787410321http://yayitsrob.tumblr.com/post/53787410321Mon, 24 Jun 2013 16:34:14 -0500nofilterWeird Cactus #nofilter (at Chicago Botanic Garden)<img src="http://24.media.tumblr.com/4d4cd594e037bc0110ea9c9e1ff9f266/tumblr_mox285OdtA1re0ihfo1_500.jpg"/><br/><br/><p>Weird Cactus #nofilter (at Chicago Botanic Garden)</p>http://yayitsrob.tumblr.com/post/53786813216http://yayitsrob.tumblr.com/post/53786813216Mon, 24 Jun 2013 16:26:29 -0500nofilterIn disbelief this document exists. (at Pick Staiger Concert...<img src="http://31.media.tumblr.com/757dd4458096c195c461a41c857c5c96/tumblr_mosvopaJkg1re0ihfo1_500.jpg"/><br/><br/><p>In disbelief this document exists. (at Pick Staiger Concert Hall)</p>http://yayitsrob.tumblr.com/post/53598320145http://yayitsrob.tumblr.com/post/53598320145Sat, 22 Jun 2013 10:14:49 -0500ayjay: 87 | 88 | ZXX, a typeface unreadable by computers. (Via...<img src="http://31.media.tumblr.com/70e2503d189ae56b310e86483bd9eed7/tumblr_mopwmnB4mK1qz4v5ho1_500.jpg"/><br/><br/><p><a href="http://ayjay.tumblr.com/post/53478275349/zxx-a-typeface-unreadable-by-computers-via" class="tumblr_blog">ayjay</a>:</p> 89 | 90 | <blockquote><p><a href="http://blogs.walkerart.org/design/2013/06/20/sang-mun-defiant-typeface-nsa-privacy/">ZXX</a>, a typeface unreadable by computers. (Via @notjessewalker on Twitter.)</p></blockquote>http://yayitsrob.tumblr.com/post/53478410399http://yayitsrob.tumblr.com/post/53478410399Thu, 20 Jun 2013 19:44:27 -0500 91 | -------------------------------------------------------------------------------- /test/data/what-if.atom: -------------------------------------------------------------------------------- 1 | 2 | 3 | What If? 4 | 5 | 6 | http://what-if.xkcd.com/feed.atom 7 | 8 | xkcd 9 | whatif@xkcd.com 10 | 11 | 2013-06-25T00:00:00Z 12 | 13 | Free Fall 14 | 15 | http://what-if.xkcd.com/51/ 16 | 2013-06-25T00:00:00Z 17 | 2013-06-25T00:00:00Z 18 |

19 |

Free Fall

 20 |

What place on Earth would allow you to freefall the longest by jumping off it? What about using a squirrel suit?

21 |

—Dhash Shrivathsa

22 |

The largest purely vertical drop on Earth is the face of Canada's Mount Thor, which is shaped like this:

23 | 24 |

To make things a little less gruesome, let's put a pit at the bottom of the cliff filled with something fluffy, like cotton candy, to safely break your fall. (Would this work? Hmm ...)

25 | 26 |

A human falling with arms and legs outstretched has a terminal velocity in the neighborhood of 55 meters per second. It takes a few hundred meters to get up to speed, so it would take you a little over 26 seconds to fall the full distance.

27 |

How long is that?

28 |

It's long enough to finish the first level of the original Super Mario.[1]

29 |

Sprint's ring cycle—the time the phone rings before going to voicemail—is 23 seconds.[2] (For those keeping score, Wagner's is 2,350 times longer.)

30 |

This means that if someone called your phone, and it started ringing the moment you jumped, it would go to voicemail three seconds before you reached the bottom.

31 | 32 |

On the other hand, if you jumped off Ireland's 210-meter Cliffs of Moher, you would only be able to fall for about eight seconds—nine, if you jumped upward.

33 |

That's not very long, but according to River Tam, it would be enough time to drain all the blood from your body given adequate vacuuming systems. (So much for making things less gruesome.)

34 |

But you don't have to drop vertically.

35 |

Even without any special equipment, a skilled skydiver—once they get up to full speed—can glide at almost a 45-degree angle.[3] By gliding away from the base of the cliff, you could conceivably extend your fall substantially.

36 | 37 |

It's hard to say exactly how far; it all depends on your clothes. As a comment on a BASE jumping records wiki puts it,

38 |
39 |

The record for longest [fall time] without a wingsuit is hard to find since the line between jeans and wingsuits has blurred since the introduction of more advanced tracking apparel.

40 |
41 |

Which brings us to wingsuits.

42 |

Wingsuits are what you get when you take the average of parachute pants and parachutes.

43 |

One wingsuit operator posted tracking data from a series of jumps.[4] It shows that in a glide, a wingsuit can lose altitude as slowly as 18 meters per second.

44 |

Even ignoring horizontal travel, that would stretch out our fall to over a minute. That's long enough for a chess game. It's also long enough to sing the first verse of—appropriately enough—REM's It's the End of the World as We Know It followed by—less appropriately—the rap breakdown from the end of the Spice Girls' Wannabe.

45 | 46 |

When we include horizontal glides, the times get even longer.

47 |

There are a lot of mountains that could probably support very long wingsuit flights. For example, Nanga Parbat, a mountain in Pakistan, has a drop of more than three kilometers at a fairly steep angle.[5] (Surprisingly, a wingsuit still works fine at those altitudes,[6] though the jumper needs oxygen and glides a little faster than normal.)

48 |

So far, the record for longest wingsuit BASE jump is held by Dean Potter, who jumped from the Eiger—a mountain in Switzerland—and flew for three minutes and twenty seconds.[7]

49 |

Joey Chestnut and Takeru Kobayashi are the world's top competitive eaters.

50 |

If we can find a way for them to operate wingsuits while eating at full speed, and they jumped from the Eiger, they could—in theory—finish as many as 45 hot dogs between them before reaching the ground ...

51 | 52 |

... making them the joint holders of what just might be the strangest world record of all time.

53 | 54 | 55 | ]]> 56 | 57 | 58 | Extreme Boating 59 | 60 | http://what-if.xkcd.com/50/ 61 | 2013-06-18T00:00:00Z 62 | 2013-06-18T00:00:00Z 63 | 64 |

Extreme Boating

 65 | Question:What would it be like to navigate a rowboat through a lake of mercury? 66 | What about bromine? Liquid gallium? Liquid tungsten? Liquid nitrogen? 67 | Liquid helium? 68 | By:–Nicholas Aron 69 | 70 | Let's take these one at a time. 71 | 72 | Bromine and mercury are the only known pure elements that are liquid at 73 | room temperature. 74 | 75 | Rowing a boat on a sea of mercury just might be possible. 76 | 77 | **Mercury** is so dense that [steel ball bearings float on the 78 | surface](http://www.youtube.com/watch?v=EGv_YVQHu7U). Your boat would be 79 | so buoyant that you'd barely make a dent in the mercury, and you'd have 80 | to lean your weight into the paddle to get the end of it below the 81 | surface. 82 | 83 | Image:boat_mercury.png:'Michael, row the boat ashore.' 'I'm TRYING!' 84 | 85 | In the end, it certainly wouldn't be easy, and you wouldn't be able to 86 | move *fast*. But you could probably row a little bit. 87 | 88 | You should probably avoid splash fights. 89 | 90 | **Bromine** is about as dense as water, so a standard rowboat could in 91 | theory float on it. 92 | 93 | However, Bromine is awful. For one thing, it smells terrible; the name 94 | "bromine" comes from the ancient Greek "brōmos", meaning "stench". If 95 | that weren't enough, it [violently 96 | reacts](http://www.youtube.com/watch?v=uCwHzTsx5yY) with a lot of 97 | materials. Hopefully, you're not in an aluminium rowboat. 98 | 99 | Imageboat_bromine_aluminium.png:The mercury one was going to be the least deadly, wasn't it. 100 | 101 | If that's not incentive enough to avoid it, the [Materials Safety Data 102 | Sheet on bromine](http://avogadro.chem.iastate.edu/MSDS/Br2.htm) 103 | includes the following phrases: 104 | 105 | - "severe burns and ulceration" 106 | - "perforation of the digestive tract" 107 | - "permanent corneal opacification" 108 | - "vertigo, anxiety, depression, muscle incoordination, and emotional 109 | instability" 110 | - "diarrhea, possibly with blood" 111 | 112 | You should not get in a splash fight on a bromine lake. 113 | 114 | **Liquid gallium** is weird stuff. Gallium melts just above room 115 | temperature, like butter, so you can't hold it in your hand for too 116 | long. 117 | 118 | It's fairly dense, though not anywhere near as dense as mercury, and 119 | would be easier to row a boat on. 120 | 121 | However, once again, you'd better hope the boat isn't made of aluminium, 122 | because aluminium (like many metals) absorbs gallium like a sponge 123 | absorbs water. The gallium spreads throughout the aluminium, 124 | dramatically changing its chemical properties. The modified aluminium is 125 | so weak it can be [pulled apart like wet 126 | paper](http://www.youtube.com/watch?v=FaMWxLCGY0U). This is something 127 | gallium has in common with mercury—both will [destroy 128 | aluminium](http://www.youtube.com/watch?v=Z7Ilxsu-JlY). 129 | 130 | Like my grandma used to say, don't sail an aluminium boat on a gallium 131 | lake. (My grandma was a little strange.) 132 | 133 | **Liquid tungsten** is really hard to work with. 134 | 135 | Tungsten has the highest melting point of any element. This means 136 | there's a lot we don't know about its properties. The reason for 137 | this—and this may sound a little stupid—is that it's hard to study, 138 | because we can't find a container to hold it in. For almost any 139 | container, the material in the container will melt before the tungsten 140 | does. There are a few compounds, like tantalum hafnium carbide, with 141 | slightly higher melting points, but no one has been able to make a 142 | liquid tungsten container with them. 143 | 144 | To give you an idea of how hot liquid tungsten is, I could tell you the 145 | exact temperature that it melts at (3422°C). But a better point might be 146 | this: 147 | 148 | *Liquid tungsten is so hot, if you dropped it into a lava flow, the lava 149 | would freeze the tungsten.* 150 | 151 | Needless to say, if you set a boat on a sea of liquid tungsten, both you 152 | and the boat would rapidly combust and be incinerated. 153 | 154 | **Liquid nitrogen** is very cold. 155 | 156 | Liquid helium is colder, but they're both closer to absolute zero than 157 | to the coldest temperatures in Antarctica, so to someone floating on 158 | them in a boat, the temperature difference is not that significant. 159 | 160 | A [Dartmouth engineering page on liquid nitrogen 161 | safety](http://engineering.dartmouth.edu/microeng/ln2.html) includes the 162 | following phrases: 163 | 164 | - "violent reactions with organic materials" 165 | - "it will explode" 166 | - "displace oxygen in the room" 167 | - "severe clothing fire" 168 | - "suffocation without warning" 169 | 170 | Liquid nitrogen has a density similar to that of water, so a rowboat 171 | would float on it, but if you were in it, you wouldn't survive for long. 172 | 173 | If the air above the nitrogen was room temperature when you started, it 174 | would cool rapidly, and you and the boat would be smothered in a thick 175 | fog as the water condensed out of the air. (This is the same effect that 176 | causes steam when you pour out liquid nitrogen.) The condensation would 177 | freeze, quickly covering your boat in a layer of frost. 178 | 179 | The warm air would cause the nitrogen on the surface to evaporate. This 180 | would displace the oxygen over the lake, causing you to asphyxiate. 181 | 182 | If the air (or the nitrogen) were both cold enough to avoid evaporation, 183 | you would instead develop hypothermia and die of exposure. 184 | 185 | **Liquid helium** would be worse. 186 | 187 | For one thing, it's only about one-eighth as dense as water, so your 188 | boat would have to be eight times larger to support a given weight. 189 | 190 | Imageboat_large.png:Frankly, what they needed was a smaller shark. 191 | 192 | But helium has a trick. When cooled below about two degrees kelvin, it 193 | becomes a superfluid, which has the odd property that it crawls up and 194 | over the walls of containers by capillary forces. 195 | 196 | It crawls along at about 20 centimeters per second, so it would take the 197 | liquid helium less than 30 seconds to start collecting in the bottom of 198 | your boat. 199 | 200 | This would, as in the liquid nitrogen scenario, cause rapid death from 201 | hypothermia. 202 | 203 | If it's any consolation, as you lay dying, you would be able to observe 204 | an odd phenomenon. 205 | 206 | Superfluid helium films, like the one rapidly covering you, carry the 207 | same types of ordinary sound waves that most materials do. But they also 208 | exhibit an additional type of wave, a slow-moving ripple that propogates 209 | along thin films of helium. It's only observed in superfluids, and has 210 | the mysterious and poetic name "[third 211 | sound](http://www.physics.berkeley.edu/research/packard/current_research/schechter's%20web/page2.html)." 212 | 213 | Your eardrums may no longer function, and wouldn't be able to detect 214 | this type of vibration anyway, but as you froze to death in the floor of 215 | a giant boat, your ears would be filled—literally—with a sound no human 216 | can ever hear: The third sound. 217 | 218 | And that, at least, is pretty cool. 219 | 220 | Imageboat_cool.png:Worth it. 221 | 222 | 223 | ]]> 224 | 225 | 226 | Extreme Boating 227 | 228 | http://what-if.xkcd.com/50/ 229 | 2013-06-18T00:00:00Z 230 | 2013-06-18T00:00:00Z 231 | 232 |

Extreme Boating

 233 |

What would it be like to navigate a rowboat through a lake of mercury? What about bromine? Liquid gallium? Liquid tungsten? Liquid nitrogen? Liquid helium?

234 |

-Nicholas Aron

235 |

Let's take these one at a time.

236 |

Bromine and mercury are the only known pure elements that are liquid at room temperature.

237 |

Rowing a boat on a sea of mercury just might be possible.

238 |

Mercury is so dense that steel ball bearings float on the surface. Your boat would be so buoyant that you'd barely make a dent in the mercury, and you'd have to lean your weight into the paddle to get the end of it below the surface.

239 | 240 |

In the end, it certainly wouldn't be easy, and you wouldn't be able to move fast. But you could probably row a little bit.

241 |

You should probably avoid splash fights.

242 |

Bromine is about as dense as water, so a standard rowboat could in theory float on it.

243 |

However, Bromine is awful. For one thing, it smells terrible; the name "bromine" comes from the ancient Greek "brōmos", meaning "stench". If that weren't enough, it violently reacts with a lot of materials. Hopefully, you're not in an aluminium rowboat.

244 | 245 |

If that's not incentive enough to avoid it, the Materials Safety Data Sheet on bromine includes the following phrases:

246 |
    247 |
  • "severe burns and ulceration"
    • 248 |
  • "perforation of the digestive tract"
    • 249 |
  • "permanent corneal opacification"
    • 250 |
  • "vertigo, anxiety, depression, muscle incoordination, and emotional instability"
    • 251 |
  • "diarrhea, possibly with blood"
    • 252 |
253 |

You should not get in a splash fight on a bromine lake.

254 |

Liquid gallium is weird stuff. Gallium melts just above room temperature, like butter, so you can't hold it in your hand for too long.

255 |

It's fairly dense, though not anywhere near as dense as mercury, and would be easier to row a boat on.

256 |

However, once again, you'd better hope the boat isn't made of aluminium, because aluminium (like many metals) absorbs gallium like a sponge absorbs water. The gallium spreads throughout the 257 | aluminium, dramatically changing its chemical properties. The modified aluminium is so weak it can be pulled apart like wet paper. This is something gallium has in common with mercury—both will destroy aluminium.

258 |

Like my grandma used to say, don't sail an aluminium boat on a gallium lake. (My grandma was a little strange.)

259 |

Liquid tungsten is really hard to work with.

260 |

Tungsten has the highest melting point of any element. This means there's a lot we don't know about its properties. The reason for this—and this may sound a little stupid—is that it's hard to study, because we can't find a container to hold it in. For almost any container, the material in the container will melt before the tungsten does. There are a few compounds, like tantalum hafnium carbide, with slightly higher melting points, but no one has been able to make a liquid tungsten container with them.

261 |

To give you an idea of how hot liquid tungsten is, I could tell you the exact temperature that it melts at (3422°C). But a better point might be this:

262 |

Liquid tungsten is so hot, if you dropped it into a lava flow, the lava would freeze the tungsten.

263 |

Needless to say, if you set a boat on a sea of liquid tungsten, both you and the boat would rapidly combust and be incinerated.

264 |

Liquid nitrogen is very cold.

265 |

Liquid helium is colder, but they're both closer to absolute zero than to the coldest temperatures in Antarctica, so to someone floating on them in a boat, the temperature difference is not that significant.

266 |

A Dartmouth engineering page on liquid nitrogen safety includes the following phrases:

267 |
    268 |
  • "violent reactions with organic materials"
    • 269 |
  • "it will explode"
    • 270 |
  • "displace oxygen in the room"
    • 271 |
  • "severe clothing fire"
    • 272 |
  • "suffocation without warning"
    • 273 |
274 |

Liquid nitrogen has a density similar to that of water, so a rowboat would float on it, but if you were in it, you wouldn't survive for long.

275 |

If the air above the nitrogen was room temperature when you started, it would cool rapidly, and you and the boat would be smothered in a thick fog as the water condensed out of the air. (This is the same effect that causes steam when you pour out liquid nitrogen.) The condensation would freeze, quickly covering your boat in a layer of frost.

276 |

The warm air would cause the nitrogen on the surface to evaporate. This would displace the oxygen over the lake, causing you to asphyxiate.

277 |

If the air (or the nitrogen) were cold enough to avoid immediate evaporation, you would instead develop hypothermia and die of exposure.

278 |

Liquid helium would be worse.

279 |

For one thing, it's only about one-eighth as dense as water, so your boat would have to be eight times larger to support a given weight.

280 | 281 |

But helium has a trick. When cooled to within two degrees of absolute zero, it becomes a superfluid, which has the odd property that it crawls up and over the walls of containers by capillary forces.

282 |

It advances up the walls at about 20 centimeters per second, so it would take the liquid helium less than 30 seconds to start collecting in the bottom of your boat.

283 |

This would, as in the liquid nitrogen scenario, cause rapid death from hypothermia.

284 |

If it's any consolation, as you lay dying, you would be able to observe an odd phenomenon.

285 |

Superfluid helium films, like the one rapidly covering you, carry the same types of ordinary sound waves that most materials do. But they also exhibit an additional type of wave, a slow-moving ripple that propagates gates along thin films of helium. It's only observed in superfluids, and has the mysterious and poetic name "third sound."

286 |

Your eardrums may no longer function, and wouldn't be able to detect this type of vibration anyway, but as you froze to death in the floor of a giant boat, your ears would be filled—literally—with a sound no human can ever hear: The third sound.

287 |

And that, at least, is pretty cool.

288 | 289 | 290 | 291 | ]]> 292 | 293 | 294 | 295 | -------------------------------------------------------------------------------- /test/data/hydro74.rss: -------------------------------------------------------------------------------- 1 |
2 | Deprecated: Assigning the return value of new by reference is deprecated in /nfs/c02/h14/mnt/22237/domains/hydro74.com/html/H74/wordpress/wp-includes/cache.php on line 36
3 |
4 | Deprecated: Assigning the return value of new by reference is deprecated in /nfs/c02/h14/mnt/22237/domains/hydro74.com/html/H74/wordpress/wp-includes/query.php on line 21
5 |
6 | Deprecated: Assigning the return value of new by reference is deprecated in /nfs/c02/h14/mnt/22237/domains/hydro74.com/html/H74/wordpress/wp-includes/theme.php on line 540
7 |
8 | Warning: Cannot modify header information - headers already sent by (output started at /nfs/c02/h14/mnt/22237/domains/hydro74.com/html/H74/wordpress/wp-includes/cache.php:36) in /nfs/c02/h14/mnt/22237/domains/hydro74.com/html/H74/wordpress/wp-includes/feed-rss2.php on line 2
9 | 10 | 11 | 16 | 17 | 18 | Hydro74 19 | http://hydro74.com/H74 20 | HydroSeventyFour™ | The rebirth of Digital Tyranny 21 | Thu, 16 Jul 2009 17:05:29 +0000 22 | http://wordpress.org/?v=2.3.3 23 | en 24 | 25 | Hydro74 | New Products & Fonts: 26 | http://hydro74.com/H74/2009/07/16/hydro74-new-products-fonts-2/ 27 | http://hydro74.com/H74/2009/07/16/hydro74-new-products-fonts-2/#comments 28 | Thu, 16 Jul 2009 17:05:29 +0000 29 | admin 30 | 31 | 32 | 33 | http://hydro74.com/H74/2009/07/16/hydro74-new-products-fonts-2/ 34 | 36 | hydro_hipsterprintjpg.jpeghydro_nrsmnprintjpg.jpeghydro_samuraiprintjpg.jpeghydro_pyrmdschmjpg.jpeg Hydro74 & Merchline joined forces to bring you the official Hydro74 Store. All orders will be shipped out that day or next day as well as all fonts & vector will be a instant download. Check out the store here: http://www.legacyofdefeat.com

37 | ]]> 38 | http://hydro74.com/H74/2009/07/16/hydro74-new-products-fonts-2/feed/ 39 | 40 | 41 | Hydro74 | New Products & Fonts: 42 | http://hydro74.com/H74/2009/07/16/hydro74-new-products-fonts/ 43 | http://hydro74.com/H74/2009/07/16/hydro74-new-products-fonts/#comments 44 | Thu, 16 Jul 2009 17:04:11 +0000 45 | admin 46 | 47 | 48 | 49 | http://hydro74.com/H74/2009/07/16/hydro74-new-products-fonts/ 50 | 53 | hydro_hipsterprintjpg.jpeghydro_nrsmnprintjpg.jpeghydro_samuraiprintjpg.jpeghydro_pyrmdschmjpg.jpeg

54 |

Hydro74 & Merchline joined forces to bring you the official Hydro74 Store. All orders will be shipped out that day or next day as well as all fonts & vector will be a instant download. Check out the store here: http://www.legacyofdefeat.com

55 | ]]> 56 | http://hydro74.com/H74/2009/07/16/hydro74-new-products-fonts/feed/ 57 | 58 | 59 | New Event Posters 60 | http://hydro74.com/H74/2009/06/29/new-event-posters/ 61 | http://hydro74.com/H74/2009/06/29/new-event-posters/#comments 62 | Mon, 29 Jun 2009 13:46:34 +0000 63 | admin 64 | 65 | 66 | 67 | http://hydro74.com/H74/2009/06/29/new-event-posters/ 68 | 70 | alwaysnever_final.jpgalwaysnever_july2_v3.jpgredwhite_bootay_1.jpg

71 |

One project I always love to do and look forward to is doing these Scion event Posters for Revolt Marketing. Maybe to the viewer, it’s not that exciting, but to me, it’s a way to do whatever I want and have limited creative feedback to develop something I think is fitting for a event. Simply put, I love it. I’ve been really busy on many other things as well and also trying to get the new site design sorted in every small bit of free-time I’ve been able to grab. So just a heads up, something rad is coming soon:)

72 | ]]> 73 | http://hydro74.com/H74/2009/06/29/new-event-posters/feed/ 74 | 75 | 76 | Peaches & Margaret Cho 77 | http://hydro74.com/H74/2009/06/29/peaches-margaret-cho/ 78 | http://hydro74.com/H74/2009/06/29/peaches-margaret-cho/#comments 79 | Mon, 29 Jun 2009 13:41:16 +0000 80 | admin 81 | 82 | 83 | 84 | 85 | 86 | http://hydro74.com/H74/2009/06/29/peaches-margaret-cho/ 87 | 89 | peaches_cho.jpg

90 |

Recently at a Scion event, Revolt Marketing was giving away Peaches Posters that I designed for the event. Little did I know, Margaret Cho was in attendance and Revolt was able to get all paparazzi on her ass for a quick second. I can’t stand her VH1 show, but I utterly love her humor and stand up shows she does. So this… is awesome!

91 | ]]> 92 | http://hydro74.com/H74/2009/06/29/peaches-margaret-cho/feed/ 93 | 94 | 95 | NUE HYDRO74 FONTS 96 | http://hydro74.com/H74/2009/06/08/nue-hydro74-fonts-2/ 97 | http://hydro74.com/H74/2009/06/08/nue-hydro74-fonts-2/#comments 98 | Mon, 08 Jun 2009 22:04:13 +0000 99 | admin 100 | 101 | 102 | 103 | http://hydro74.com/H74/2009/06/08/nue-hydro74-fonts-2/ 104 | 107 | calypso.jpgburialblack.jpg

108 |

Two new fonts from Hydro74 are now up. These faces are custom display typefaces that are easy to use. Order them from the Hydro Stock Section

109 | ]]> 110 | http://hydro74.com/H74/2009/06/08/nue-hydro74-fonts-2/feed/ 111 | 112 | 113 | NUE HYDRO74 FONTS 114 | http://hydro74.com/H74/2009/06/08/nue-hydro74-fonts/ 115 | http://hydro74.com/H74/2009/06/08/nue-hydro74-fonts/#comments 116 | Mon, 08 Jun 2009 22:03:59 +0000 117 | admin 118 | 119 | 120 | 121 | http://hydro74.com/H74/2009/06/08/nue-hydro74-fonts/ 122 | 125 | calypso.jpgburialblack.jpg

126 |

Two new fonts from Hydro74 are now up. These faces are custom display typefaces that are easy to use. Order them from the Hydro Stock Section

127 | ]]> 128 | http://hydro74.com/H74/2009/06/08/nue-hydro74-fonts/feed/ 129 | 130 | 131 | Nike - All Star Game Art 132 | http://hydro74.com/H74/2009/06/02/nike-all-star-game-art/ 133 | http://hydro74.com/H74/2009/06/02/nike-all-star-game-art/#comments 134 | Wed, 03 Jun 2009 06:42:53 +0000 135 | admin 136 | 137 | 138 | 139 | http://hydro74.com/H74/2009/06/02/nike-all-star-game-art/ 140 | 142 | nike_allstar.jpg

143 |

This past NBA season, I had a great opportunity to work on some fun pieces for Nike for their Apparel side for the NBA All Star Game. It’s not everyday you get a chance to work on pieces for LeBron, Kobe, & Nash, as well as have them review it for approval. It was a fun project and finally got a thumbs up to be able to show it!

144 | ]]> 145 | http://hydro74.com/H74/2009/06/02/nike-all-star-game-art/feed/ 146 | 147 | 148 | Back in Black 2 | The Late Post & Thank You 149 | http://hydro74.com/H74/2009/06/02/back-in-black-2-the-late-post-thank-you/ 150 | http://hydro74.com/H74/2009/06/02/back-in-black-2-the-late-post-thank-you/#comments 151 | Wed, 03 Jun 2009 06:40:21 +0000 152 | admin 153 | 154 | 155 | 156 | http://hydro74.com/H74/2009/06/02/back-in-black-2-the-late-post-thank-you/ 157 | 159 | 3544923346_04c599bca3_o.jpg3544111847_e2f2367194_o.jpg3544920358_daa633935e_o.jpg

160 |

Few weeks back, Back in Black 2 hit Ybor Florida with great success. Over 950 people attended the event that night with another 350 hitting the after party. It personally took me a week to recover from this event, then found myself neck deep in client work. All in all, everything was fantastic. Pale Horse did so much on the back end with prepping, setting up and keeping things in order that a special thanks goes to them. Revolt Marketing came in and developed a great party with Dave Nada, Monk and some residence DJ’s. One highlight was having ISO50 do his set at the closing of the show. Of course, by that time, I was fueled by too many free drinks, A.D.D and keeping up with everyone else. However from what I did catch, pretty amazing.

161 |

If you didn’t get a chance to check it out already, hit up Back in Black for a recap. Shirts are no longer available, however I know Back in Black 3 is already getting some whispers. Thank you to all who attended, ordered and just supported both Chris from Pale Horse & I as we just put together a show we felt very passionate about.

162 | ]]> 163 | http://hydro74.com/H74/2009/06/02/back-in-black-2-the-late-post-thank-you/feed/ 164 | 165 | 166 | New Posters 167 | http://hydro74.com/H74/2009/06/02/new-posters/ 168 | http://hydro74.com/H74/2009/06/02/new-posters/#comments 169 | Wed, 03 Jun 2009 06:23:36 +0000 170 | admin 171 | 172 | 173 | 174 | http://hydro74.com/H74/2009/06/02/new-posters/ 175 | 177 | peaches.jpgsteveaoki.jpgchimaira.jpgdavenada.jpgfuckyesss_june4th.jpg

178 |

One of my guilty pleasures lately has been doing event posters for Scion & Myspace via Revolt Marketing and Vahalla Studios. It’s not every day you can do as you wish on piece and develop something that will be given away at various events as well as having your name attached to it. Over this past month I’ve had the opportunity to develop posters for Steve Aoki, Peaches, Chimaira, Le Castle Vania, Dave Nada, Cut Copy, The Presets, & a few others. I am pretty stoked to see these go from digital ideas to silk screened pieces.

179 | ]]> 180 | http://hydro74.com/H74/2009/06/02/new-posters/feed/ 181 | 182 | 183 | FREE FONT FROM HYDRO74 184 | http://hydro74.com/H74/2009/05/08/free-font-from-hydro74-2/ 185 | http://hydro74.com/H74/2009/05/08/free-font-from-hydro74-2/#comments 186 | Fri, 08 May 2009 17:28:37 +0000 187 | admin 188 | 189 | 190 | 191 | http://hydro74.com/H74/2009/05/08/free-font-from-hydro74-2/ 192 | 196 | freebie.jpg

197 |

Just developed this font as a give-away for Back in Black 2. This font is for the taking and will only be available for the month of May.

198 |

Grab it over at Back in Black

199 | ]]> 200 | http://hydro74.com/H74/2009/05/08/free-font-from-hydro74-2/feed/ 201 | 202 | 203 | 204 | -------------------------------------------------------------------------------- /test/data/statwing-poison.html: -------------------------------------------------------------------------------- 1 | 2 | Statwing is an easy-to-use data analysis tool. Click through the visualizations below to see analyses in Statwing (and to play around with the rest of the dataset). 3 |

4 |

5 | It’s third-and-2 and you desperately need a first down. What do you do, run or pass? 6 |

7 |

8 | A few months ago, Advanced NFL Stats released an 9 | incredible dataset containing information on every play of the 2012 NFL regular season. We structured that dataset and 10 | 11 | uploaded it into Statwing for analysis. 12 | Now you can test your coaching instincts against the data. 13 |

14 | 15 |
16 | 17 |
18 |

Early in the game, the score is tied. You have fourth-and-goal at the 2-yard line. What should you do?

19 |
20 | 21 | 22 | 23 |
24 |

Correct

25 |

Wrong. You should go for it.

26 |

Wrong. You should go for it.

27 |
28 |

29 | About 1.5% of field goal attempts are unsuccessful at that range, so you’re virtually assured three points if you kick. 30 | When teams go for it on fourth-and-goal from the 2, they get a touchdown 44% of the time. 31 | So on average teams get 3.1 points when they go for it—roughly the same amount they’d expect if they kicked a field goal. 32 |

33 |
34 | Touchdowns on 4th and Goal 35 |

Click the image to explore this analysis and play with a dataset consisting of the last ten years of plays from third-and-goal and fourth-and-goal from within the 10-yard line.

36 |

37 | But that’s not all. If you’re stopped on fourth-and-goal, the opponent starts with terrible field position. You’ll even get a safety about 5% of the time. 38 | By comparison, you can expect the opponent to start from the 23-yard line after a kickoff following a made field goal, 39 | and they will even have a 0.5% chance of returning the kickoff for a touchdown 40 |

41 |
42 | Yards to Endzone Immediately After Kickoff 43 |

Click the image to see the full NFL 2012 regular season dataset in Statwing. It contains all the analyses cited above.

44 |

45 | Going for it and kicking a field goal both yield about 3 points on average, and the field position is much better if you go for it. 46 | Despite this, coaches usually kick a field goal on fourth-and-goal. 47 | In fact, during the last ten seasons coaches only went for the touchdown twice in this situation. 48 |

49 |

50 | You’ve gotten out of answers correct so far. 51 |

52 |
53 |
54 | 55 |
56 |

It’s third-and-1. Which type of run is most likely to result in a first down?

57 |
58 | 59 | 60 |
61 |

Correct

62 |

Wrong. Side story: your author’s mom always yelled at the Chiefs not to go up the middle on third-and-short. 63 | It saddens your author to find out that she was mostly likely leading the Chiefs astray.

64 |
65 |

66 | Going up the gut just barely beats running around the end. 67 |

68 |
69 | Run Direction vs. First Down 70 |

Click the image to see an even more detailed breakdown of running plays (e.g., off-center versus off-guard).

71 |

72 | You’ve gotten out of answers correct so far. 73 |

74 |
75 |
76 | 77 |
78 |

On third-and-2, are you more likely to pick up a first down by running the ball or passing it?

79 |
80 | 81 | 82 | 83 |
84 |

Wrong

85 |

Wrong

86 |

Correct

87 |
88 |

89 | We used Statwing to look at every third-and-2 in the 2012 NFL regular season dataset. 90 | Teams picked up first downs a bit more than half of the time, regardless of whether they ran or passed. 91 |

92 |
93 | Run vs. Pass for First 94 |

This difference is not statistically significant. 95 |
96 | Click the image to explore this analysis and the rest of the dataset in Statwing.

97 |

98 |

99 | In case you’re curious, here are the odds of picking up a first down on third-and-x, split by running versus passing: 100 |
Likelihood of Getting 1st Down, by Play Type

Runs are statistically significantly more effective than passes on third-and-1 and similarly effective on third-and-2, third-and-3, and third-and-4. 101 | In 2012 passes were not statistically significantly more successful for third-and-5 through third-and-10.

102 | As an aside, coaches tend to pass on third with more than a yard to go. 103 |
Proportion of 3rd Down Plays That Are Runs

Coaches very rarely run on third down with three or more yards to go.

104 |

105 | You’ve gotten out of questions correct so far. 106 |

107 |
108 |
109 | 110 | 111 |
112 |

You need a two-point conversion. What kind of play should you call?

113 |
114 | 115 | 116 | 117 |
118 |

Correct

119 |

Wrong

120 |

Wrong

121 |
122 |

123 | During the last ten years running has succeeded 61% of the time, versus 45% for passing. 124 |

125 |

126 | This seems odd because we just found out that running wasn’t better than passing on third-and-2. 127 | But the two-point conversion situation is different from a typical third-and-2. 128 | For example, the defense isn’t spread out, so it’s hard for receivers to find gaps in the coverage. 129 |

130 |
131 | Two-point Conversions - Running vs Passing 132 |

Click the image to see statistical data, explore this analysis, and play with the rest of the 2003–2012 two-point conversion dataset in Statwing.

133 |

134 | This suggests that coaches should run more often than they currently do. 135 |

136 |
137 | Coaches Tend to Pass for Two-point Conversions 138 |

Click the image to see the confidence intervals and play with the rest of the 2003–2012 two-point conversion dataset in Statwing.

139 |

140 | You’ve gotten out of answers correct so far. 141 |

142 |
143 |
144 | 145 | 146 | 147 |
148 |

How depressing was it to be a Chiefs fan last year?

149 |
150 | 151 | 152 | 153 |
154 |

Indeed

155 |

Wrong-ish. This is a judgment call, but the author claims the expertise of having 156 | specialized in this field for some time, and thus feels confident to assert that one does not get used to it. 157 | Though please feel free to credit yourself as correct if you are a Chiefs fan or root for another team that 158 | 159 | hasn’t won a playoff game since George Bush was President (the other George Bush).

160 |

No, not really. Last year’s Chiefs made the Royals look competitive.

161 |
162 |

163 | Here’s how depressing it was. On an average defensive play in 2012, the Chiefs were losing by a touchdown. And that calculation includes the first quarter. 164 |

165 |
166 | Chiefs Score Differential Before Defensive Play 167 |

This data allows you to precisely measure how seldom a Chiefs fan thinks, “Wow, I can’t believe we’re winning this game!” 168 |
169 | Click the image to see more and explore the 2012 regular season dataset in Statwing.

170 |
171 |
172 |
173 | 174 |
175 |
176 |

Thanks for playing

177 |

178 | You got out of answers correct. 179 | Not ideal. But, hey, such is life. 180 | Not ideal. But, hey, such is life. 181 | Not ideal. But, hey, such is life. 182 | That’ll do just fine. 183 | That’ll do just fine. 184 | Change your name to Harbaugh. 185 |

186 | Tweet your quiz results 187 |
188 |
189 | 190 |
191 |
192 |

Discussion on Hacker News

193 |

A special thanks to David Laughlin, who wrote most of the copy for this post. David is available for freelance work at 194 | davidclaughlin@gmail.com.

195 | 196 | 197 | 198 |
199 |

Notes

200 | 201 |

202 | 203 | The original data from 204 | Advanced NFL Stats is mostly free-text play descriptions, 205 | which we interpreted into structured data using Excel. 206 | The original data does have a few errors here and there, not all of which could be cleaned up. 207 | Some plays are missing, and a very small number of plays have some inaccurate data. 208 | But overall the dataset is comprehensive and accurate. 209 | For example, the dataset only omits four of the roughly 1,000 field goals attempted in 2012. 210 |

211 |

212 | You can download the structured data in Excel to see the specifics of how we calculated various fields from the original data: 213 |

    214 |
  • 2012 regular season data (39MB). 215 | This data is excellent, though not perfect.
    • 216 |
  • All data from 2003–2012 except the 2012 playoffs (101MB). 217 | Our formulas are only tailored to the 2012 season, so there are a few difficulties when applying our formulas to the multi-year dataset. 218 | For example, the older dataset doesn’t always distinguish quarterback scrambles from run plays. 219 | Still, the data for the other seasons seems similar in most respects.
    • 220 |
  • Two-point conversions (200kB). 221 | A subset of the 2003–2012 dataset. It contains every two-point conversion during that time frame.
    • 222 |
  • Third- or fourth-and-goal (4MB). 223 | A subset of the 2003–2012 dataset. It contains every third- or fourth-and-goal from inside the 10-yard line during that time frame.
    • 224 |
225 |

226 | 227 |

228 | 229 | We make the following assumptions throughout: 230 |

    231 |
  • You are coaching the “average” team. Individual teams would vary, but the data we use is the average of all teams’ behaviors in whatever situation we are analyzing.
    • 232 |
  • There are more than five minutes left in the half or game. Last minute plays have slightly different outcomes, so we exclude them.
    • 233 |
  • Yardage from penalties committed during a play is included in the outcome.
    • 234 |
  • The hypothetical coach does not always call the same type of play in the same situation. 235 | That is, the coach randomizes play calling enough to be unpredictable while still favoring the more advantageous plays. 236 | The very awesome Brian at Advanced NFL Stats does a great job of 237 | describing randomization and game theory. 238 |
239 |

240 | 241 |

242 | 243 | Click through the image (or this link) 244 | to see the analysis that yielded the 1.5% number. 245 |

246 | 247 |

248 | 249 | In the last ten years, there have only been twenty-five instances where teams went for it on fourth-and-goal at the 2-yard line. It turns out, though, that going for the end 250 | zone on third-and-short is pretty similarly successful to going for it on fourth-and-short, so we used both types of plays in this analysis. For example, the 44% figure 251 | was calculated using both third- and fourth-and-2. Here, as with the rest of this fourth-and-2 analysis, we were inspired by a 252 | 2002 paper by David Romer. Romer is a notable Berkeley economist, and 253 | his wife chaired the White House Council of Economic Advisors in 2009 and 2010. 254 |

255 | 256 |

257 | 258 | This was barely statistically significant in the 2012 dataset, 259 | so we ran the same analysis in the 2003–2012 dataset. There we found a very clearly statistically significant relationship. 260 | The magnitude of the difference between the two running directions was only 6.5% in the ten-year dataset, 261 | as opposed to 11% (with wide confidence intervals) in the 2012 dataset. 262 |

263 | 264 |

265 | 266 | If objectives other than just picking up a first down are considered, there is actually some evidence that running is better. Brian at Advanced NFL Stats 267 | does a great job of diving into that question, 268 | though you might want to learn about the concept of 269 | expected points before reading Brian’s analysis. 270 |

271 | 272 |

273 | 274 | Brian at Advanced NFL Stats looked at this same question 275 | with data from 2000 to 2007. He found that running was more likely to have resulted in a first down on third-and-2. 276 | He brought more data to bear, but our confidence intervals were pretty narrow. This suggests that in 2012 running and passing were 277 | more similar in their success on third-and-2 than during the years Brian looked at. 278 |

279 | 280 |
281 | 282 | 283 | 284 | 285 | 286 |

The post NFL Play-by-Play Data—Analyzed, Visualized, and Quizzified appeared first on Statwing Blog.

375 | -------------------------------------------------------------------------------- /parser.lisp: -------------------------------------------------------------------------------- 1 | (in-package :cl-user) 2 | (defpackage :cl-feedparser/parser 3 | (:use 4 | :cl :alexandria :serapeum :anaphora 5 | :cl-feedparser/xml-namespaces 6 | :cl-feedparser/feed-sanitizer 7 | :cl-feedparser/time) 8 | (:import-from :local-time :timestamp) 9 | (:import-from :cl-ppcre :regex-replace-all) 10 | (:shadowing-import-from :cl-ppcre :scan) 11 | (:import-from :quri) 12 | (:import-from :fxml) 13 | (:import-from :fxml.sanitize) 14 | (:import-from :fxml.html5) 15 | (:import-from :fxml.stp) 16 | (:import-from :plump) 17 | (:import-from :html5-parser :parse-html5) 18 | (:import-from :uiop :file-exists-p) 19 | (:shadow :string+) 20 | (:export 21 | :parse-feed 22 | :*keys* :feedparser-key :feed-ref :gethash* 23 | :repair :return-feed 24 | :feed-sanitizer 25 | :unsanitized-string :unsanitized-string-string :string+ 26 | :sanitize-title :sanitize-content :sanitize-text 27 | :feed-string 28 | :*base* 29 | :parse-time 30 | :masked? 31 | 32 | :gethash* :ensure-gethash* 33 | :defhandler :handle-tag 34 | :urlish? 35 | :get-content :get-text :get-text/sanitized 36 | :sanitize-title 37 | :*entry* 38 | :*feed* 39 | :*source* 40 | :*author* 41 | :resolve-uri 42 | :get-timestring 43 | :check-guid-mask 44 | :parser-loop 45 | :entry-context 46 | :strip-parens)) 47 | 48 | (in-package #:cl-feedparser/parser) 49 | 50 | ;;; "cl-feedparser" goes here. Hacks and glory await! 51 | 52 | (defparameter *version* 53 | "0.3") 54 | 55 | (defvar *base* nil 56 | "Default value for `current-xml-base'.") 57 | 58 | (defvar *text-length-limit* nil 59 | "Limit for text length, if one is in effect.") 60 | 61 | (defun version-string () 62 | (fmt "cl-feedparser ~a" *version*)) 63 | 64 | (deftype universal-time () 65 | 'integer) 66 | 67 | (deftype time-designator () 68 | '(or universal-time timestamp)) 69 | 70 | (deftype entry-mtime () 71 | '(or time-designator null)) 72 | 73 | (deftype uri () 74 | '(or string quri:uri)) 75 | 76 | (deftype id () 77 | '(or null uri)) 78 | 79 | (deftype mask-time () 80 | '(or time-designator null string)) 81 | 82 | (deftype mask () 83 | '(or string (cons string mask-time))) 84 | 85 | (deftype max-entries () 86 | '(or null wholenum)) 87 | 88 | (deftype uri-protocol () 89 | '(or null string)) 90 | 91 | (deftype klacks-event () 92 | '(member :start-document 93 | :characters 94 | :processing-instruction 95 | :comment 96 | :dtd 97 | :start-element 98 | :end-element 99 | :end-document 100 | nil)) 101 | 102 | 103 | 104 | ;;; This is a hack to statically check that the wrong key can't be set 105 | ;;; due to a typo. 106 | 107 | (eval-and-compile 108 | (defparameter *keys* 109 | '(:entries :bozo :title :info :rights :subtitle 110 | :proxy :link :rel :type :href :title 111 | :author :name :email :uri :email 112 | :name :author :language :icon 113 | :summary :value :summary-detail :subtitle 114 | :published :published-parsed 115 | :updated :updated-parsed 116 | :ttl :id :content :proxy 117 | :base :value :type 118 | :author-detail :links 119 | :enclosures :contributors 120 | :created :comments 121 | ;; mrss 122 | :media-category 123 | :media-community 124 | :media-content 125 | :media-copyright 126 | :media-credit 127 | :media-description 128 | :media-group 129 | :media-hash 130 | :media-keywords 131 | :media-license 132 | :media-player 133 | :media-rating 134 | :media-restriction 135 | :media-star-rating 136 | :media-statistics 137 | :media-tags 138 | :media-title 139 | :media-thumbnail))) 140 | 141 | (deftype feedparser-key () 142 | `(member ,@*keys*)) 143 | 144 | (defmacro gethash* (key hash &optional default) 145 | "Like `gethash', but check (statically) that KEY is a member of 146 | `*keys*'." 147 | (check-type key feedparser-key) 148 | `(gethash ,key ,hash ,@(unsplice default))) 149 | 150 | (defmacro ensure-gethash* (key hash &optional default) 151 | "Like `ensure-gethash', but check (statically) that KEY is a member 152 | of `*keys*'." 153 | (check-type key feedparser-key) 154 | `(ensure-gethash ,key ,hash ,@(unsplice default))) 155 | 156 | (defsubst feed-ref (feed key &optional default) 157 | (gethash (assure feedparser-key key) feed default)) 158 | 159 | (define-compiler-macro feed-ref (&whole call feed key &optional default &environment env) 160 | (cond ((constantp key) 161 | `(gethash ,(assure feedparser-key key) ,feed ,default)) 162 | ((constantp key env) 163 | `(gethash (load-time-value (assure feedparser-key ,key)) 164 | ,feed 165 | ,default)) 166 | (t call))) 167 | 168 | (defsubst (setf feed-ref) (value feed key) 169 | (setf (gethash (assure feedparser-key key) feed) value)) 170 | 171 | (define-compiler-macro (setf feed-ref) (&whole call value feed key &environment env) 172 | (cond ((constantp key) 173 | `(setf (gethash ,(assure feedparser-key key) 174 | ,feed) 175 | ,value)) 176 | ((constantp key env) 177 | `(setf (gethash (load-time-value (assure feedparser-key ,key)) 178 | ,feed) 179 | ,value)) 180 | (t call))) 181 | 182 | 183 | 184 | (deftype sanitizer () 185 | '(or null fxml.sanitize:mode)) 186 | 187 | (deftype sanitizer-designator () 188 | '(or sanitizer (eql t))) 189 | 190 | (defconstructor unsanitized-string 191 | "A string that has yet to be sanitized." 192 | (string string)) 193 | 194 | (deftype feed-string () 195 | '(or string unsanitized-string)) 196 | 197 | (def empty-uri (quri:uri "")) 198 | 199 | (defun empty-uri? (uri) 200 | (etypecase-of uri uri 201 | (string (emptyp uri)) 202 | (quri:uri (quri:uri= uri empty-uri)))) 203 | 204 | (def http :http) 205 | (def https :https) 206 | (def relative nil) 207 | 208 | (defun protocol-allowed? (protocol) 209 | (etypecase-of uri-protocol protocol 210 | (null t) 211 | (string 212 | (string-case protocol 213 | (("http" "https") t))))) 214 | 215 | (defun string+ (&rest strings) 216 | "Concatenate STRINGS, ensuring that if any are unsanitized, the 217 | result is an unsanitized string." 218 | (let* ((unsanitized nil) 219 | (s (with-output-to-string (s) 220 | (dolist (string strings) 221 | (etypecase-of feed-string string 222 | (string (write-string string s)) 223 | (unsanitized-string 224 | (setf unsanitized t) 225 | (write-string (unsanitized-string-string string) s))))))) 226 | (if unsanitized 227 | (unsanitized-string s) 228 | s))) 229 | 230 | (defun make-html-sink (&key base sanitizer) 231 | (~> (fxml.html5:make-html5-sink) 232 | (make-absolute-uri-handler :base base) 233 | (fxml.sanitize:wrap-sanitize sanitizer))) 234 | 235 | (defun has-markup? (string) 236 | (scan "[<>&]" string)) 237 | 238 | (defun sanitize-aux (x sanitizer) 239 | (cond ((typep x 'fxml.stp:node) 240 | (let ((sink (make-html-sink :sanitizer sanitizer))) 241 | (if (typep x 'fxml.stp:document) 242 | (fxml.stp:serialize x sink) 243 | (progn 244 | (fxml.stp:serialize x sink) 245 | (fxml.sax:end-document sink))))) 246 | ((not (stringp x)) x) 247 | ((emptyp x) x) 248 | ((not (has-markup? x)) x) 249 | (t (etypecase-of sanitizer sanitizer 250 | (null (unsanitized-string x)) 251 | (fxml.sanitize:mode 252 | (fxml.html5:serialize-dom 253 | (parse-html5 x) 254 | (make-html-sink :sanitizer sanitizer))))))) 255 | 256 | (defparameter *content-sanitizer* feed-sanitizer) 257 | 258 | (defparameter *title-sanitizer* fxml.sanitize:restricted) 259 | 260 | (defun sanitize-content (x &optional (sanitizer t)) 261 | (etypecase-of sanitizer-designator sanitizer 262 | ((eql t) (sanitize-aux x *content-sanitizer*)) 263 | (sanitizer (sanitize-aux x sanitizer)))) 264 | 265 | (defun sanitize-title (x &optional (sanitizer t)) 266 | (etypecase-of sanitizer-designator sanitizer 267 | ((eql t) (sanitize-aux x *title-sanitizer*)) 268 | (sanitizer (sanitize-aux x sanitizer)))) 269 | 270 | (defun sanitize-text (x) 271 | (cond ((not (stringp x)) x) 272 | ((emptyp x) x) 273 | ((not (has-markup? x)) x) 274 | ;; If there are no entities, just strip all tags. 275 | ((not (find #\& x)) 276 | ;; Tags that never close... 277 | (regex-replace-all "<[^>]*>?" x " ")) 278 | ;; If there are entities, we have to resort to a real parser. 279 | (t (sanitize-aux x fxml.sanitize:default)))) 280 | 281 | ;; Unclosed tags. 282 | (assert (equal " " (sanitize-text "