├── .gitmodules ├── icons ├── cats.png ├── Ammonite.png ├── doobie.png ├── spray.io.png ├── scala-sbt.org.png ├── learning-scalaz.png ├── playframework.com.png ├── slick.typesafe.com.png ├── docs.scala-lang.org.png └── learn-you-a-haskell.png ├── .gitignore ├── sync.sh ├── Readme.md └── src └── main └── scala ├── main.scala ├── EntryType.scala ├── dash.scala └── docsets.scala /.gitmodules: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /icons/cats.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/cats.png -------------------------------------------------------------------------------- /icons/Ammonite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/Ammonite.png -------------------------------------------------------------------------------- /icons/doobie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/doobie.png -------------------------------------------------------------------------------- /icons/spray.io.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/spray.io.png -------------------------------------------------------------------------------- /icons/scala-sbt.org.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/scala-sbt.org.png -------------------------------------------------------------------------------- /icons/learning-scalaz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/learning-scalaz.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | docsets/ 3 | downloads/ 4 | project.sublime-project 5 | project.sublime-workspace 6 | -------------------------------------------------------------------------------- /icons/playframework.com.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/playframework.com.png -------------------------------------------------------------------------------- /icons/slick.typesafe.com.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/slick.typesafe.com.png -------------------------------------------------------------------------------- /icons/docs.scala-lang.org.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/docs.scala-lang.org.png -------------------------------------------------------------------------------- /icons/learn-you-a-haskell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvogt/scala-dash/HEAD/icons/learn-you-a-haskell.png -------------------------------------------------------------------------------- /sync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | rsync -avz --delete --exclude '.DS_Store' --exclude '.htaccess' --progress docsets/ ~/domainfactory/webseiten/cvogt.org/releases/docsets/ -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | Docset feeds: https://github.com/cvogt/cvogt.github.io/tree/master/releases/docsets (use Raw view of xml files to add them) 2 | 3 | Script to generate Dash docsets for the manuals (not API docs) of many Scala libraries. 4 | 5 | Why? Offline support, fast, unified search. API was already available via maven, manuals weren't. 6 | 7 | Includes 8 | [Scala](http://docs.scala-lang.org/), 9 | [Slick](http://slick.typesafe.com/docs/), 10 | [Spray](http://spray.io/documentation/), 11 | [Playframework](https://www.playframework.com/documentation/), 12 | [Sbt](http://www.scala-sbt.org/documentation.html), 13 | [Ammonite](https://lihaoyi.github.io/Ammonite/), 14 | [cats](https://non.github.io/cats/), 15 | [leaning scalaz](http://eed3si9n.com/learning-scalaz/), 16 | [doobie](http://tpolecat.github.io/doobie-0.2.2/00-index.html), 17 | [fastparse](http://lihaoyi.github.io/fastparse/), 18 | [Hand-on Scala.js](http://lihaoyi.github.io/hands-on-scala-js/), 19 | [PPrint](http://lihaoyi.github.io/upickle-pprint/pprint/), 20 | [ScalaTags](http://lihaoyi.github.io/scalatags/), 21 | [Scalatex](http://lihaoyi.github.io/Scalatex/), 22 | [uPickle](http://lihaoyi.github.io/upickle-pprint/upickle/) 23 | 24 | To run this yourself, you need a recent version of wget that supports the --exclude argument. You also need Dashing installed (https://github.com/technosophos/dashing) 25 | 26 | How to contribute? 27 | - improve existing docsets: Tweak css selectors, add ignores, improve text via regexes 28 | - add docsets for more libraries 29 | -------------------------------------------------------------------------------- /src/main/scala/main.scala: -------------------------------------------------------------------------------- 1 | package org.cvogt.doc 2 | import ammonite.ops._ 3 | import org.cvogt.dash._ 4 | 5 | object OfflineDocsBuilder{ 6 | val docsets = Docsets.docsets 7 | def main(args: Array[String]): Unit = { 8 | args match{ 9 | case Array("all") => 10 | docsets.foreach{ d => 11 | d.download 12 | d.build 13 | d.feed 14 | } 15 | case Array("clean", "all") => 16 | rm! cwd/'downloads 17 | rm! cwd/'docsets 18 | case Array("download", "clean") => rm! cwd/'downloads 19 | case Array("build", "clean") => rm! cwd/'docsets 20 | case Array("download", "all") => docsets.foreach(_.download) 21 | case Array("build", "all") => docsets.foreach(_.build) 22 | case Array("download", docset) => docsets.find(_.docsetName == docset).map(_.download).getOrElse(usage) 23 | case Array("download", docset, version) => docsets.collect{ 24 | case d: VersionedDocset if d.docsetName == docset => d 25 | }.headOption.flatMap(_.docsets.find(_.version == Some(version))).map(_.download).getOrElse(usage) 26 | case Array("build", docset) => docsets.find(_.docsetName == docset).map(_.build).getOrElse(usage) 27 | case Array("build", docset, version) => docsets.collect{ 28 | case d: VersionedDocset if d.docsetName == docset => d 29 | }.headOption.flatMap(_.docsets.find(_.version == Some(version))).map(_.build).getOrElse(usage) 30 | case _ => usage 31 | } 32 | } 33 | 34 | def usage = println(s""" 35 | USAGE: 36 | sbt run []] 37 | 38 | supported s 39 | 40 | download - downloads the given docset 41 | build - creates docset 42 | all - download and build all docsets 43 | 44 | supported s and versions 45 | 46 | all - all docsets 47 | ${docsets.map{ 48 | d => d.docsetName ++ ( d match { 49 | case v: VersionedDocset => v.versions.mkString(" ["," | ","]") 50 | case _ => "" 51 | }) 52 | }.mkString("\n ")} 53 | """) 54 | } 55 | -------------------------------------------------------------------------------- /src/main/scala/EntryType.scala: -------------------------------------------------------------------------------- 1 | package org.cvogt.dash 2 | import play.api.libs.json._ 3 | import org.cvogt.play.json._ 4 | import implicits.optionWithNull 5 | import SingletonEncoder.simpleName 6 | import implicits.formatSingleton 7 | 8 | sealed trait SelectorTarget 9 | 10 | case class Replacement( 11 | `type` : EntryType, 12 | regexp: String = "", 13 | replacement: String = "" 14 | ) extends SelectorTarget 15 | 16 | object Replacement{ 17 | implicit val jsonFormat: Format[Replacement] = Jsonx.formatCaseClass[Replacement] 18 | } 19 | 20 | sealed trait EntryType extends SelectorTarget 21 | case object Annotation extends EntryType 22 | case object Attribute extends EntryType 23 | case object Binding extends EntryType 24 | case object Builtin extends EntryType 25 | case object Callback extends EntryType 26 | case object Category extends EntryType 27 | case object Class extends EntryType 28 | case object Command extends EntryType 29 | case object Component extends EntryType 30 | case object Constant extends EntryType 31 | case object Constructor extends EntryType 32 | case object Define extends EntryType 33 | case object Delegate extends EntryType 34 | case object Diagram extends EntryType 35 | case object Directive extends EntryType 36 | case object Element extends EntryType 37 | case object Entry extends EntryType 38 | case object Enum extends EntryType 39 | case object Environment extends EntryType 40 | case object Error extends EntryType 41 | case object Event extends EntryType 42 | case object Exception extends EntryType 43 | case object Extension extends EntryType 44 | case object Field extends EntryType 45 | case object File extends EntryType 46 | case object Filter extends EntryType 47 | case object Framework extends EntryType 48 | case object Function extends EntryType 49 | case object Global extends EntryType 50 | case object Guide extends EntryType 51 | case object Hook extends EntryType 52 | case object Instance extends EntryType 53 | case object Instruction extends EntryType 54 | case object Interface extends EntryType 55 | case object Keyword extends EntryType 56 | case object Library extends EntryType 57 | case object Literal extends EntryType 58 | case object Macro extends EntryType 59 | case object Method extends EntryType 60 | case object Mixin extends EntryType 61 | case object Modifier extends EntryType 62 | case object Module extends EntryType 63 | case object Namespace extends EntryType 64 | case object Notation extends EntryType 65 | case object Object extends EntryType 66 | case object Operator extends EntryType 67 | case object Option extends EntryType 68 | case object Package extends EntryType 69 | case object Parameter extends EntryType 70 | case object Plugin extends EntryType 71 | case object Procedure extends EntryType 72 | case object Property extends EntryType 73 | case object Protocol extends EntryType 74 | case object Provider extends EntryType 75 | case object Provisioner extends EntryType 76 | case object Query extends EntryType 77 | case object Record extends EntryType 78 | case object Resource extends EntryType 79 | case object Sample extends EntryType 80 | case object Section extends EntryType 81 | case object Service extends EntryType 82 | case object Setting extends EntryType 83 | case object Shortcut extends EntryType 84 | case object Statement extends EntryType 85 | case object Struct extends EntryType 86 | case object Style extends EntryType 87 | case object Subroutine extends EntryType 88 | case object Tag extends EntryType 89 | case object Test extends EntryType 90 | case object Trait extends EntryType 91 | case object Type extends EntryType 92 | case object Union extends EntryType 93 | case object Value extends EntryType 94 | case object Variable extends EntryType 95 | case object Word extends EntryType 96 | 97 | object EntryType{ 98 | implicit val jsonFormat: Format[EntryType] = Jsonx.formatSealed[EntryType] 99 | } 100 | object SelectorTarget{ 101 | implicit val jsonFormat: Format[SelectorTarget] = Jsonx.formatSealed[SelectorTarget] 102 | } 103 | -------------------------------------------------------------------------------- /src/main/scala/dash.scala: -------------------------------------------------------------------------------- 1 | package org.cvogt.dash 2 | 3 | import ammonite.ops._ 4 | import java.net.URL 5 | import play.api.libs.json._ 6 | import org.cvogt.play.json._ 7 | import org.cvogt.play.json.implicits.optionWithNull 8 | import scala.collection.immutable.ListMap 9 | 10 | trait AbstractDocset{ 11 | def docsetName: String 12 | def feed: Unit 13 | def build: Unit 14 | def download: Unit 15 | } 16 | 17 | final class VersionedDocset( 18 | val docsets: Vector[Docset] 19 | ) extends AbstractDocset{ 20 | self => 21 | val docsetName = { 22 | assert(docsets.sliding(2).forall{ 23 | case Seq(_) => true 24 | case Seq(a,b) => a.docsetName == b.docsetName 25 | }) 26 | docsets.head.docsetName 27 | } 28 | val versions = { 29 | assert(docsets.forall(_.version.isDefined)) 30 | docsets.flatMap(_.version) 31 | } 32 | 33 | def download = docsets.map(_.download) 34 | def build = { 35 | // reverse, so first version remains as most recent 36 | docsets.map(_.build) 37 | val latest = cwd/'docsets/(docsetName + ".tgz") 38 | rm(latest) 39 | mkdir(docsets.head.tarFile / RelPath.up) 40 | cp(docsets.head.tarFile, latest) 41 | feed 42 | } 43 | def feed = { 44 | val feedXml = 45 | 46 | {versions.head}/{java.time.LocalDateTime.now.toString} 47 | http://cvogt.org/releases/docsets/{docsetName}.tgz 48 | { 49 | versions.map( v => 50 | {v} 51 | ) 52 | } 53 | 54 | 55 | val feedFile = cwd/'docsets/(docsetName++".xml") 56 | mkdir(feedFile/RelPath.up) 57 | write.over(feedFile, feedXml.toString) 58 | } 59 | } 60 | 61 | case class DashingConfig( 62 | name: String, 63 | `package`: String, 64 | index: String, 65 | selectors: Map[String,SelectorTarget], 66 | ignore: Vector[String], 67 | icon32x32: String, 68 | allowJs: Boolean 69 | ){ 70 | require(selectors.isInstanceOf[ListMap[_,_]]) 71 | } 72 | 73 | class Docset( 74 | val docsetName: String, 75 | wgetExtraArgs: Vector[Shellable] = Vector(), 76 | val version: Option[String] = None 77 | ) extends AbstractDocset{ 78 | def url = s"http://$docsetName/$urlPath" 79 | def urls = Vector(url) 80 | val wgetArgs = Seq[Shellable]( 81 | "--mirror", 82 | "--html-extension", 83 | "--convert-links", 84 | "--page-requisites", 85 | "--no-parent", 86 | url 87 | ) ++ wgetExtraArgs 88 | def urlPath = "" 89 | def now = java.time.ZonedDateTime.now.withZoneSameInstant(java.time.ZoneId.of("UTC")) 90 | 91 | // dashing config 92 | def name = docsetName 93 | def `package` = "scala" 94 | def index: RelPath = docsRootFolder / "index.html" 95 | def icon32x32 = docsetName ++ ".png" 96 | def selectors: ListMap[String,SelectorTarget] = 97 | ListMap( 98 | (1 to 6).map("h"+_).flatMap( 99 | h => Vector( 100 | h+":matches(^[^a-zA-Z]+$)" -> Command, 101 | h+":matches(^([a-z]|[A-Z])$)" -> Shortcut, 102 | h+":matches(^([a-z][a-z]+|[A-Z][A-Z]+)$)" -> Command, 103 | h+":not(:matches(^([a-z]+|[A-Z]+|[^a-zA-Z]+)$))" -> Section 104 | ) 105 | ):_*) ++ ListMap( 106 | "dt a" -> Command, 107 | "title" -> Guide 108 | ) 109 | def ignore: Vector[String] = Vector("ABOUT") 110 | def allowJs: Boolean = true 111 | 112 | final val versionedDocsetName = docsetName++version.map("-"++_).getOrElse("") 113 | final val downloadFolder = cwd/'downloads/versionedDocsetName 114 | final def buildFolder = downloadFolder 115 | 116 | final def docsRootFolder = { 117 | val _url = new URL(url) 118 | val fileOrDir = _url.getHost / { 119 | _url.getPath match { 120 | case "" | "/" => RelPath.empty 121 | case path => 122 | assert(path(0) == '/') 123 | RelPath(path.drop(1)) 124 | } 125 | } 126 | if( exists(downloadFolder / fileOrDir) ){ 127 | val info = stat! (downloadFolder / fileOrDir) 128 | if( info.isDir ){ 129 | fileOrDir 130 | } else if(info.isFile) { 131 | fileOrDir / RelPath.up 132 | } else { 133 | ??? 134 | } 135 | } else { 136 | fileOrDir / RelPath.up 137 | } 138 | } 139 | 140 | 141 | final val tarFile = cwd / 'docsets / version.map( 142 | 'versions / docsetName / _ 143 | ).getOrElse( 144 | RelPath.empty 145 | ) / (docsetName + ".tgz") 146 | 147 | final def build = { 148 | val dashingConfig = buildFolder / "dashing.json" 149 | val icons = cwd/'icons 150 | val logo = icons/icon32x32 151 | if( exists(logo) ){ 152 | rm( buildFolder/icon32x32 ) 153 | cp( logo, buildFolder/icon32x32 ) 154 | } else { 155 | println("[warning] no icon found for "+docsetName) 156 | } 157 | write.over( dashingConfig, json ) 158 | rm(buildFolder / (`package` ++ ".docset")) 159 | %.dashing("build")(buildFolder) 160 | archive 161 | feed 162 | } 163 | 164 | final def archive = { 165 | println(tarFile) 166 | mkdir( tarFile/RelPath.up ) 167 | rm(tarFile) 168 | %.tar( 169 | "--exclude='.DS_Store'", 170 | "-cvzf", 171 | tarFile, 172 | `package` ++ ".docset" 173 | )(buildFolder) 174 | } 175 | 176 | final def feed = { 177 | val feedXml = 178 | 179 | {version.getOrElse(now.format(java.time.format.DateTimeFormatter.ISO_LOCAL_DATE))}/{now.toString} 180 | http://cvogt.org/releases/docsets/{docsetName}.tgz 181 | 182 | 183 | val feedFile = cwd/'docsets/(docsetName++".xml") 184 | mkdir(feedFile/RelPath.up) 185 | write.over(feedFile, feedXml.toString) 186 | } 187 | 188 | final def download: Unit = { 189 | rm( downloadFolder ) 190 | mkdir( downloadFolder ) 191 | mirror 192 | } 193 | 194 | def mirror: Unit = { 195 | /*val args = Vector[Shellable]( "--mirror", "-n", url ) ++ httrackArguments 196 | %(downloadFolder).applyDynamic("httrack")(args: _*)*/ 197 | 198 | %.applyDynamic("wget")(wgetArgs: _*)(downloadFolder) 199 | } 200 | 201 | private def json = { 202 | import play.api.libs.json.{`package` => _,_} 203 | val config = DashingConfig( 204 | name, `package`, index.toString, selectors, ignore, icon32x32, allowJs 205 | ) 206 | Json.prettyPrint( Json.toJson( config ) ) 207 | } 208 | } 209 | 210 | object DashingConfig{ 211 | implicit val jsonFormat: Format[DashingConfig] = Json.format[DashingConfig] 212 | } 213 | -------------------------------------------------------------------------------- /src/main/scala/docsets.scala: -------------------------------------------------------------------------------- 1 | package org.cvogt.doc 2 | import org.cvogt.dash._ 3 | import ammonite.ops.{Command => _,_} 4 | import scala.collection.immutable.ListMap 5 | 6 | object Docsets{ 7 | val docsets = Vector( 8 | "Ammonite" -> "http://lihaoyi.github.io/Ammonite/", 9 | "Fastparse" -> "http://lihaoyi.github.io/fastparse/", 10 | "ScalaTags" -> "http://lihaoyi.github.io/scalatags/", 11 | "Scalatex" -> "http://lihaoyi.github.io/Scalatex/", 12 | "PPrint" -> "http://lihaoyi.github.io/upickle-pprint/pprint/", 13 | "uPickle" -> "http://lihaoyi.github.io/upickle-pprint/upickle/", 14 | "Hands-on-Scala.js" -> "http://lihaoyi.github.io/hands-on-scala-js/" 15 | ).map{ 16 | case (name, _url) => new ScalatexDocset( name ){ override def url = _url } 17 | } ++ Vector( 18 | new Docset( 19 | "docs.scala-lang.org", 20 | Vector("-X/de/","-X/es/","-X/fr/","-X/ja/","-X/ko/","-X/pt-br/","-X/zh-cn/") 21 | ){ 22 | override def ignore = Vector( 23 | "404", 24 | "API", 25 | "Current", 26 | "Nightly", 27 | "Learn", 28 | "Guides & Overviews", 29 | "Tutorials", 30 | "Scala Style Guide", 31 | "Quickref", 32 | "Glossary", 33 | "Cheatsheets", 34 | "Contribute", 35 | "Source Code", 36 | "Contributors Guide", 37 | "Suggestions", 38 | "Other Resources", 39 | "Scala Improvement Process" 40 | ) 41 | override def selectors = super.selectors.map{ 42 | case (s,e:EntryType) => s -> Replacement(e, " \\- Scala Documentation$", "") 43 | case _ => ??? 44 | } 45 | }, 46 | new Docset( "cats", Vector("http://plastic-idolatry.com/erik/cats2.png", "-X/cats/api/") ){ 47 | override def url = "http://typelevel.org/cats/" 48 | override def ignore = Vector( 49 | "Cats", "Page not found · GitHub Pages", "404", "useful links" 50 | ) 51 | override def selectors = super.selectors.map{ 52 | case ("title",e:EntryType) => "title" -> Replacement(e, "^Cats Documentation - ", "") 53 | case other => other 54 | } 55 | }, 56 | new VersionedDocset( 57 | Vector("0.2.2").map( v => 58 | new Docset( "doobie", Vector(), Some(v) ){ 59 | override def url = s"http://tpolecat.github.io/doobie-$v/00-index.html" 60 | override def index = docsRootFolder / "00-index.html" 61 | override def ignore = Vector( "Setting Up" ) 62 | override def selectors = super.selectors.filterNot(v => (v._1 startsWith "h1") || (v._1 startsWith "title") ) 63 | } 64 | ) 65 | ), 66 | new VersionedDocset( 67 | Vector("3.0.3","3.1.0-RC1","2.1.0").map( v => 68 | new Docset( "slick.typesafe.com", Vector(s"-X/doc/$v/api/",s"-X/doc/$v/testkit-api/",s"-X/doc/$v/codegen-api/",s"-X/doc/$v/direct-api/",s"-X/doc/$v/extensions-api/",s"-X/doc/$v/hikaricp-api/"), Some(v) ){ 69 | override def urlPath = s"doc/$v/" 70 | override def ignore = Vector( "[1]","[2]","[3]","[4]","[5]","[6]", "Slick¶", "Search", "SQL¶", "Table Of Contents", "Index" ) 71 | override def selectors = super.selectors.map{ 72 | case (s,e:EntryType) if s startsWith "h" => s -> Replacement(e,"¶$","") 73 | case ("title",e:EntryType) => "title" -> Replacement(e," — Slick ([^ ]+) documentation$","") 74 | case other => other 75 | } 76 | } 77 | ) 78 | ), 79 | new VersionedDocset( 80 | Vector("1.2.3").map( v => 81 | new Docset( "spray.io", Vector(), Some(v) ){ 82 | override def urlPath = s"documentation/$v/" 83 | override def ignore = Vector( "Dependencies", "Description", "Example", "Signature" ) 84 | override def selectors = super.selectors.map{ 85 | case ("title",e:EntryType) => "title" -> Replacement(e, "^(.* » )+", "") 86 | case other => other 87 | } 88 | } 89 | ) 90 | ), 91 | new VersionedDocset( 92 | Vector("0.13").map( v => 93 | new Docset( "scala-sbt.org", Vector( 94 | "-I/0.13/", 95 | "-X/0.13/api/", 96 | "-X/0.13/tutorial/es/", 97 | "-X/0.13/tutorial/ja/", 98 | "-X/0.13/tutorial/zh-cn/", 99 | "-RCombined+Pages.html" 100 | ), Some(v) ){ 101 | override def url = "http://www.scala-sbt.org/documentation.html" 102 | override def ignore = Vector( 103 | "Community Support", "Commercial Support", "Contents" 104 | ) 105 | override def index = docsRootFolder / "0.13" / 'docs / "index.html" 106 | override def selectors = super.selectors.map{ 107 | case ("title",e:EntryType) => 108 | "title:not(:matches(\\.(scala|java)$))" -> 109 | Replacement(e, "^(Getting Started with sbt — |sbt Reference Manual — )", "") 110 | case other => other 111 | } 112 | /* 113 | override def wgetArgs = Vector( 114 | "--html-extension", 115 | "--convert-links", 116 | "--page-requisites", 117 | "--no-parent", 118 | "http://www.scala-sbt.org/0.13/docs/Combined+Pages.html", 119 | "http://www.scala-sbt.org/0.13/tutorial/Combined+Pages.html" 120 | ) 121 | */ 122 | } 123 | ) 124 | ), 125 | new VersionedDocset( 126 | Vector("2.4.x", "3.0.x", "2.3.x").map( v => 127 | new Docset( "playframework.com", Vector(s"-X/documentation/$v/api/"), Some(v) ){ 128 | override def url = s"https://www.playframework.com/documentation/$v/" 129 | override def ignore = Vector( 130 | "Documentation", "Books", "Home", "Play releases", "Browse versions", "Browse APIs", "Scala Java Language", 131 | "Community support", "Professional support" 132 | ) 133 | override def selectors = super.selectors.filterNot(_._1 == "title").map{ 134 | case (s,e:EntryType) if s startsWith "h" => ("article "+s) -> Replacement(if(s startsWith "h1") Guide else e, "^§", "") 135 | case other => other 136 | } 137 | } 138 | ) 139 | ), 140 | new Docset( "learning-scalaz", Vector("-X/learning-scalaz/ja/","-X/learning-scalaz/7.0/") ){ 141 | override def url = "http://eed3si9n.com/learning-scalaz/Combined+Pages.html" 142 | override def name = "learning Scalaz" 143 | override def index = docsRootFolder / "Combined+Pages.html" 144 | override def ignore = Vector("Page not found | eed3si9n") 145 | override def selectors = super.selectors.filterNot(_._1 == "title").filterNot(_._1 startsWith "h1") 146 | override val wgetArgs = { 147 | Vector[Shellable]( 148 | "--convert-links", 149 | "--page-requisites", 150 | "-e", "robots=off", 151 | url 152 | ) 153 | } 154 | }, 155 | new Docset( "learn-you-a-haskell", Vector() ){ 156 | override def url = "http://learnyouahaskell.com/chapters" 157 | override def index = docsRootFolder / "chapters.html" 158 | override def name = "Learn You a Haskell for Great Good!" 159 | override def ignore = Vector() 160 | override def selectors = ListMap( 161 | "h1" -> Guide, 162 | "h2" -> Section, 163 | "h3" -> Section 164 | ) 165 | override def `package` = "haskell" 166 | }, 167 | new Docset( "wish-I-knew-haskell", Vector() ){ 168 | override def url = "http://dev.stephendiehl.com/hask/" 169 | override def name = "What I Wish I Knew When Learning Haskell" 170 | override def ignore = Vector() 171 | override def selectors = ListMap( 172 | "h1" -> Guide, 173 | "h2" -> Section, 174 | "h3" -> Section 175 | ) 176 | override def `package` = "haskell" 177 | } 178 | ) 179 | } 180 | 181 | class ScalatexDocset(name: String) extends Docset(name){ 182 | override def selectors = super.selectors.filterNot(_._1 == "title") 183 | private val digits = (0 to 9).toVector 184 | override def ignore = Vector("404") ++ (for{ 185 | i <- digits 186 | j <- digits 187 | k <- digits 188 | } yield s"$i.$j.$k") 189 | } 190 | --------------------------------------------------------------------------------