├── .gitignore ├── .jenkins-scripts ├── generate.sh └── publish.sh ├── Jenkinsfile ├── README.md ├── adoptopenjdk.groovy ├── allure.groovy ├── ant.groovy ├── buckminster.groovy ├── chromedriver.groovy ├── cmake.groovy ├── codeql.groovy ├── consul.groovy ├── dependencycheck.groovy ├── dotnetSdk.groovy ├── flyway.groovy ├── golang.groovy ├── gradle.groovy ├── grails.groovy ├── grapeConfig.xml ├── groovy.groovy ├── jdk.groovy ├── jdk.hack.json ├── leiningen.groovy ├── lib ├── DataWriter.groovy ├── init.groovy ├── runner.groovy └── signer.groovy ├── maven.groovy ├── mongodb.groovy ├── nodejs.groovy ├── packer.groovy ├── play.groovy ├── pom.xml ├── recipe.groovy ├── sbt.groovy ├── sbuild.groovy ├── scala.groovy ├── scriptler.groovy ├── sonarqubescanner.groovy ├── sonarqubescannermsbuild.groovy └── terraform.groovy /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.ipr 3 | *.iws 4 | target 5 | /.idea 6 | *~ 7 | *.sw* 8 | -------------------------------------------------------------------------------- /.jenkins-scripts/generate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # .jenkins-scripts/generate.sh: generate Jenkins tool installer metadatas for all Groovy files 3 | 4 | set -eux -o pipefail 5 | 6 | # Provide signing keys in the directory $SECRET if you want to sign metadatas. Not mandatory. 7 | if test -d "${SECRET:-/notexisting}" 8 | then 9 | export JENKINS_SIGNER="-key \"$SECRET/update-center.key\" -certificate \"$SECRET/update-center.cert\" -root-certificate \"$SECRET/jenkins-update-center-root-ca.crt\"" 10 | fi 11 | 12 | for f in *.groovy 13 | do 14 | echo "= Crawler '$f':" 15 | groovy -Dgrape.config=./grapeConfig.xml ./lib/runner.groovy "$f" \ 16 | || true # Do not fail immediatly 17 | done 18 | -------------------------------------------------------------------------------- /.jenkins-scripts/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # .jenkins-scripts/publish.sh: execute publication of crawler generated metadatas to the different Update Center sync targets 3 | # NOTE: NEVER delete any remote files 4 | # TODO: find a way to reuse 'SYNC_UC_TASKS' from https://github.com/jenkins-infra/update-center2/blob/master/site/publish.sh#L9 to avoid repetition and automate delivery. 5 | 6 | set -eux -o pipefail 7 | 8 | # Prepare Artifacts for publication 9 | mkdir -p updates 10 | cp target/*.json target/*.html updates 11 | 12 | # Rsync sync tasks 13 | rsync_publish_tasks=("rsync-archives.jenkins.io" "rsync-updates.jenkins.io-data-content" "rsync-updates.jenkins.io-data-redirections-unsecured" "rsync-updates.jenkins.io-data-redirections-secured") 14 | 15 | for rsync_publish_task in "${rsync_publish_tasks[@]}" 16 | do 17 | envToLoad="${UPDATE_CENTER_FILESHARES_ENV_FILES}/.env-${rsync_publish_task}" 18 | 19 | test -f "${envToLoad}" 20 | 21 | # shellcheck source=/dev/null 22 | source "${envToLoad}" 23 | 24 | # Required variables that should now be set from the .env file 25 | : "${RSYNC_HOST?}" "${RSYNC_USER?}" "${RSYNC_GROUP?}" "${RSYNC_REMOTE_DIR?}" "${RSYNC_IDENTITY_NAME?}" 26 | 27 | ## TODO: retrieve SSH known_hosts file from a credential to avoid StrictHostKeyChecking=no (context: ephemeral VMs) 28 | time rsync --recursive --links --perms --times -D \ 29 | --chown="${RSYNC_USER}":"${RSYNC_GROUP}" \ 30 | --checksum --verbose --compress \ 31 | --rsh="ssh -o StrictHostKeyChecking=no -i ${UPDATE_CENTER_FILESHARES_ENV_FILES}/${RSYNC_IDENTITY_NAME}" `# rsync identity file is stored with .env files` \ 32 | --exclude=.svn `# TODO: still needed?` \ 33 | ./updates/ "${RSYNC_USER}"@"${RSYNC_HOST}":"${RSYNC_REMOTE_DIR}"/updates/ 34 | done 35 | 36 | # Cloudflare R2 (uses AWS S3 protocol) sync tasks 37 | export AWS_DEFAULT_REGION=auto 38 | 39 | # sanity check for build observability 40 | aws --version 41 | 42 | # Cloudflare R2 does not support more than 2 concurent requests - https://community.cloudflare.com/t/is-it-actually-possible-to-upload-to-r2-buckets-using-wrangler/388762/7 43 | aws configure set default.s3.max_concurrent_requests 2 44 | aws configure set default.s3.multipart_threshold "50MB" 45 | 46 | sync_s3_tasks=("s3sync-westeurope" "s3sync-eastamerica") 47 | 48 | for bucket in "${sync_s3_tasks[@]}" 49 | do 50 | test -f "${UPDATE_CENTER_FILESHARES_ENV_FILES}/.env-${bucket}" 51 | 52 | # Don't print any command to avoid exposing credentials 53 | set +x 54 | 55 | # Pipeline usually uses '/bin/sh' so no 'source' shell keyword available 56 | # shellcheck source=/dev/null 57 | source "${UPDATE_CENTER_FILESHARES_ENV_FILES}/.env-${bucket}" 58 | 59 | # Required variables that should now be set from the .env file 60 | : "${BUCKET_NAME?}" "${BUCKET_ENDPOINT_URL?}" "${AWS_ACCESS_KEY_ID?}" "${AWS_SECRET_ACCESS_KEY?}" "${AWS_DEFAULT_REGION?}" 61 | 62 | # It's now safe 63 | set -x 64 | 65 | ## Note: AWS CLI are configured through environment variables (from Jenkins credentials) - https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html 66 | # Do NOT delete files as part of the 'sync' command (aws s3 sync tends to randomly delete after uploading - https://github.com/jenkins-infra/helpdesk/issues/4403) 67 | aws s3 sync ./updates/ s3://"${BUCKET_NAME}"/updates/ \ 68 | --no-progress \ 69 | --no-follow-symlinks \ 70 | --endpoint-url "${BUCKET_ENDPOINT_URL}" \ 71 | --checksum-algorithm CRC32 72 | done 73 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env groovy 2 | 3 | 4 | List p = [buildDiscarder(logRotator(numToKeepStr: '5'))] 5 | 6 | /* When we're running inside our trusted infrastructure, we want to 7 | * re-generate the tools meta-data every four hours 8 | */ 9 | if (infra.isTrusted()) { 10 | p.add(pipelineTriggers([cron('H */4 * * *')])) 11 | p.add(disableConcurrentBuilds()) 12 | } 13 | 14 | properties(p) 15 | 16 | node('linux') { 17 | stage ('Prepare') { 18 | deleteDir() 19 | checkout scm 20 | } 21 | 22 | withEnv([ 23 | "PATH+GROOVY=${tool 'groovy'}/bin", 24 | "PATH+MVN=${tool 'mvn'}/bin", 25 | "JAVA_HOME=${tool 'jdk17'}", 26 | "PATH+JAVA=${tool 'jdk17'}/bin" 27 | ]) { 28 | stage('Build') { 29 | sh 'mvn -e clean install' 30 | } 31 | 32 | stage('Generate') { 33 | timestamps { 34 | if (infra.isTrusted()) { 35 | withCredentials([[$class: 'ZipFileBinding', credentialsId: 'update-center-signing', variable: 'SECRET']]) { 36 | sh 'bash ./.jenkins-scripts/generate.sh' 37 | } 38 | } 39 | else { 40 | sh 'bash ./.jenkins-scripts/generate.sh' 41 | } 42 | } 43 | } 44 | } 45 | 46 | stage('Archive') { 47 | dir ('target') { 48 | archiveArtifacts '**' 49 | } 50 | } 51 | 52 | if (infra.isTrusted()) { 53 | stage('Publish') { 54 | withCredentials([[$class: 'ZipFileBinding', credentialsId: 'update-center-publish-env', variable: 'UPDATE_CENTER_FILESHARES_ENV_FILES']]) { 55 | sh 'bash ./.jenkins-scripts/publish.sh' 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Infra Backend Crawlers 2 | 3 | These scripts generate machine readable tool installer metadata. 4 | You need Groovy 2.0 or later to run them. 5 | You can add new one by doing monkey-see monkey-do. Any one of these scripts 6 | are individually runnable, for example: 7 | 8 | ```shell 9 | groovy -Dgrape.config=./grapeConfig.xml ./lib/runner.groovy mongodb.groovy 10 | ``` 11 | 12 | ## Signing Files 13 | 14 | If you need to "only" sign the JSON files (after an update center certificate renewal for instance): 15 | 16 | ```shell 17 | mkdir -p ./target/ 18 | rsync -av "${source}/*.json" ./target/ 19 | export JENKINS_SIGNER="-key path/to/uc-cert.key -certificate path/to/uc-cert.cert -root-certificate path/to/uc-root-ca.crt" 20 | groovy -Dgrape.config=./grapeConfig.xml ./lib/runner.groovy ./lib/signer.groovy 21 | # You can find the signed files in ./target/*.html 22 | ``` 23 | -------------------------------------------------------------------------------- /adoptopenjdk.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | import org.htmlunit.Page 3 | import org.htmlunit.WebClient 4 | import org.htmlunit.WebResponse 5 | import net.sf.json.JSONArray 6 | import net.sf.json.JSONObject 7 | 8 | class ListAdoptOpenJDK { 9 | private final WebClient wc; 10 | 11 | private final String API_URL = "https://api.adoptium.net/v3"; 12 | 13 | ListAdoptOpenJDK() { 14 | wc = new WebClient(); 15 | wc.addRequestHeader("accept", "application/json") 16 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 17 | wc.getOptions().setPrintContentOnFailingStatusCode(false); 18 | } 19 | 20 | void main() throws Exception { 21 | lib.DataWriter.write("io.jenkins.plugins.adoptopenjdk.AdoptOpenJDKInstaller", build()); 22 | } 23 | 24 | private JSONObject build() throws IOException { 25 | Page p = wc.getPage(API_URL + "/info/available_releases") 26 | WebResponse response = p.getWebResponse(); 27 | 28 | if (response.statusCode != 200) { 29 | throw new Exception("Could not load available_releases") 30 | } 31 | 32 | JSONArray result = new JSONArray() 33 | JSONObject data = JSONObject.fromObject(response.getContentAsString()) 34 | 35 | result.addAll(getReleases(data.available_releases as JSONArray, "HotSpot")) 36 | 37 | return new JSONObject() 38 | .element("version", 2) 39 | .element("data", result); 40 | } 41 | 42 | private JSONArray getReleases(JSONArray available_releases, String jvm_impl) { 43 | JSONArray result = new JSONArray() 44 | available_releases.each { feature_version -> 45 | JSONObject r = getRelease(feature_version as String, jvm_impl) 46 | result.add(r) 47 | } 48 | return result 49 | } 50 | 51 | private JSONObject getRelease(String feature_version, String jvm_impl){ 52 | String openjdk_impl = jvm_impl.toLowerCase() 53 | JSONObject r = new JSONObject() 54 | r.put("name", "OpenJDK " + feature_version + " - " + jvm_impl) 55 | Map releasesMap = new LinkedHashMap<>(); 56 | int page = 0 57 | boolean keepGoing = true 58 | while (keepGoing) { 59 | Page a = wc.getPage(API_URL + "/assets/feature_releases/" + feature_version + "/ga?vendor=eclipse&project=jdk&image_type=jdk&sort_method=DEFAULT&sort_order=DESC&page_size=20&page=" + (page++) + "&jvm_impl=" + openjdk_impl) 60 | WebResponse ar = a.getWebResponse() 61 | if (ar.getStatusCode() == 200) { 62 | JSONArray assets = JSONArray.fromObject(ar.getContentAsString()) 63 | assets.forEach { JSONObject asset -> releasesMap.merge(asset.release_name as String, asset, { v1, v2 -> 64 | if (v1.version_data?.adopt_build_number > v2.version_data?.adopt_build_number){ 65 | v1.binaries.addAll(v2.binaries); 66 | return v1; 67 | } else { 68 | v2.binaries.addAll(v1.binaries); 69 | return v2; 70 | } 71 | })} 72 | } else { 73 | keepGoing = false 74 | } 75 | } 76 | 77 | def releases = releasesMap.values().collect { 78 | [ 79 | release_name: it.release_name, 80 | openjdk_impl: openjdk_impl, 81 | binaries : it.binaries.collect { 82 | [ 83 | architecture: it.architecture, 84 | os : it.os, 85 | openjdk_impl: it.jvm_impl, 86 | binary_link : it.package.link 87 | ] 88 | } 89 | ] 90 | } 91 | r.put("releases", releases) 92 | return r 93 | } 94 | } 95 | 96 | new ListAdoptOpenJDK().main(); 97 | -------------------------------------------------------------------------------- /allure.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Allure 3 | import org.htmlunit.WebClient 4 | import org.htmlunit.html.DomElement 5 | import org.htmlunit.html.HtmlPage 6 | import org.htmlunit.xml.XmlPage 7 | import net.sf.json.JSONObject 8 | 9 | def getList() { 10 | List versions = new ArrayList() 11 | versions.addAll(getCentralVersions()) 12 | versions.addAll(getSonatypeVersions()) 13 | return versions 14 | } 15 | def getCentralVersions() { 16 | String baseUrl = 'https://repo1.maven.org/maven2/io/qameta/allure/allure-commandline' 17 | URL metaUrl = new URL("$baseUrl/maven-metadata.xml") 18 | 19 | WebClient wc = new WebClient() 20 | XmlPage meta = wc.getPage(metaUrl) 21 | 22 | List versions = meta.getByXPath("//metadata/versioning/versions/version") 23 | .collect() { DomElement e -> e.getTextContent() } 24 | .findAll() { e -> !e.contains('BETA') } 25 | .reverse() 26 | 27 | return versions.collect() { version -> 28 | return ["id" : version, 29 | "name": version, 30 | "url" : String.format('%s/%s/allure-commandline-%s.zip', baseUrl, version, version) 31 | ] 32 | } 33 | } 34 | 35 | def getSonatypeVersions() { 36 | String baseUrl = 'https://oss.sonatype.org/content/repositories/releases/ru/yandex/qatools/allure/allure-commandline' 37 | URL metaUrl = new URL("$baseUrl/maven-metadata.xml") 38 | 39 | WebClient wc = new WebClient() 40 | XmlPage meta = wc.getPage(metaUrl) 41 | 42 | List versions = meta.getByXPath("//metadata/versioning/versions/version") 43 | .collect() { DomElement e -> e.getTextContent() } 44 | .findAll() { e -> !e.contains('RC') } 45 | .reverse() 46 | 47 | return versions.collect() { version -> 48 | return ["id" : version, 49 | "name": version, 50 | "url" : getSonatypeArtifactUrl(baseUrl, version) 51 | ] 52 | } 53 | } 54 | 55 | def getSonatypeArtifactUrl(String baseUrl, String version) { 56 | def artifactName = getSonatypeArtifactName(version); 57 | return String.format('%s/%s/%s', baseUrl, version, artifactName); 58 | } 59 | 60 | def getSonatypeArtifactName(String version) { 61 | switch (version) { 62 | case '1.4.17': return String.format('allure-commandline-%s.zip', version) 63 | default: return String.format('allure-commandline-%s-standalone.zip', version) 64 | } 65 | } 66 | 67 | def store(key, o) { 68 | JSONObject envelope = JSONObject.fromObject(["list": o]) 69 | lib.DataWriter.write(key, envelope) 70 | } 71 | 72 | store("ru.yandex.qatools.allure.jenkins.tools.AllureCommandlineInstaller", getList()) 73 | -------------------------------------------------------------------------------- /ant.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Ant 3 | import org.htmlunit.WebClient 4 | import org.htmlunit.html.HtmlAnchor 5 | import org.htmlunit.html.HtmlPage 6 | import java.util.regex.Pattern 7 | import net.sf.json.JSONObject 8 | import hudson.util.VersionNumber 9 | 10 | def listUp(url,pattern) { 11 | wc = new WebClient() 12 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 13 | wc.getOptions().setJavaScriptEnabled(false); 14 | wc.getOptions().setThrowExceptionOnScriptError(false); 15 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 16 | 17 | HtmlPage p = wc.getPage(url); 18 | pattern=Pattern.compile(pattern); 19 | 20 | return p.getAnchors().collect { HtmlAnchor a -> 21 | m = pattern.matcher(a.hrefAttribute) 22 | if(m.find()) { 23 | ver=m.group(1) 24 | url = p.getFullyQualifiedUrl(a.hrefAttribute); 25 | return ["id":ver, "name":ver, "url":url.toExternalForm()] 26 | } 27 | return null; 28 | }.findAll { it!=null }.sort { o1,o2 -> 29 | try { 30 | def v1 = new VersionNumber(o1.id) 31 | try { 32 | new VersionNumber(o2.id).compareTo(v1) 33 | } catch (IllegalArgumentException _2) { 34 | -1 35 | } 36 | } catch (IllegalArgumentException _1) { 37 | try { 38 | new VersionNumber(o2.id) 39 | 1 40 | } catch (IllegalArgumentException _2) { 41 | o2.id.compareTo(o1.id) 42 | } 43 | } 44 | } 45 | } 46 | 47 | def store(key,o) { 48 | JSONObject envelope = JSONObject.fromObject(["list": o]); 49 | lib.DataWriter.write(key,envelope); 50 | } 51 | 52 | store("hudson.tasks.Ant.AntInstaller", listUp("https://archive.apache.org/dist/ant/binaries/", "ant-([0-9.]+)-bin.zip\$")) -------------------------------------------------------------------------------- /buckminster.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Buckminster auto-installation 3 | import org.htmlunit.html.*; 4 | 5 | import net.sf.json.* 6 | import org.htmlunit.WebClient 7 | 8 | def url = "https://raw.github.com/jenkinsci/buckminster-plugin/master/updates/buckminster.json".toURL() 9 | def text = url.text 10 | 11 | //println text 12 | 13 | lib.DataWriter.write("hudson.plugins.buckminster.BuckminsterInstallation.BuckminsterInstaller",JSONObject.fromObject(text)); 14 | -------------------------------------------------------------------------------- /chromedriver.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for chromedriver auto-installation 3 | import net.sf.json.JSONObject 4 | import groovy.json.JsonSlurper 5 | 6 | def url = 'https://googlechromelabs.github.io/chrome-for-testing/latest-versions-per-milestone-with-downloads.json' 7 | def jsonString = new URL(url).text 8 | def jsonData = new JsonSlurper().parseText(jsonString) 9 | 10 | def jsonList = [] 11 | 12 | jsonData.milestones.each { milestone -> 13 | def milestoneData = milestone.value 14 | 15 | def chromeDriverData = milestoneData.downloads.chromedriver 16 | chromeDriverData.each { entry -> 17 | def id = "${entry.platform}_${milestoneData.version}".toString() 18 | def entry_url = entry.url 19 | jsonList << ["id": id, "url": entry_url] 20 | } 21 | } 22 | 23 | lib.DataWriter.write("org.jenkins-ci.plugins.chromedriver.ChromeDriver",JSONObject.fromObject([list:jsonList])) 24 | -------------------------------------------------------------------------------- /cmake.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for CMake auto-installation 3 | import org.htmlunit.html.* 4 | import net.sf.json.* 5 | import org.htmlunit.WebClient 6 | import org.htmlunit.BrowserVersion 7 | 8 | def baseUrl = 'https://cmake.org/files/' 9 | 10 | def wc = new WebClient( 11 | new BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED) 12 | .setApplicationName("JenkinsBackendCrawler") 13 | .setApplicationVersion("1.0") 14 | .build() 15 | ) 16 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 17 | wc.getOptions().setJavaScriptEnabled(false); 18 | wc.getOptions().setThrowExceptionOnScriptError(false); 19 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 20 | def releases = [:] 21 | 22 | // Gather a list of top dirs 23 | def dirs = [] 24 | wc.getPage(baseUrl).getByXPath("//td/a").reverse().each { HtmlAnchor e -> 25 | dirs << e.getHrefAttribute() 26 | } 27 | dirs.each { dir -> 28 | // We only want download dir of v 2.6 (for RHEL7 !) and above 29 | if (!(dir =~ /^v[2]\.[6-9].*/ || dir =~ /^v[3-9]\.\d+.*/)) { 30 | return 31 | } 32 | // gather archive files 33 | def files= [] 34 | // get version urls 35 | wc.getPage(baseUrl+ dir).getByXPath("//td/a").reverse().each { HtmlAnchor e -> 36 | files << e.getHrefAttribute() 37 | } 38 | 39 | // Build a map of Cmake versions -> platform archives 40 | files.each { file -> 41 | // We only want release archives; ignore source packages, installers and beta/RC versions 42 | if (file =~ /\.src\.tar\.bz2$/ || file =~ /(alpha|beta|rc)\d+\./) { 43 | return 44 | } 45 | if (! (file =~ /\.(zip|tar\.gz)$/)) { 46 | return 47 | } 48 | 49 | // Extract the version info from archive filename 50 | // cmake-2.6.4-Darwin-x86_64.tar.gz 51 | // cmake-2.6.4-win32-i386.zip 52 | def parts = (file =~ /^cmake-(\d+\.\d+\.\d+)-(HP-UX|[^-]+)-([^-]+)\.(?:tar\.gz|zip)$/) 53 | if (!parts) { 54 | return 55 | } 56 | 57 | // Gather the info for this archive 58 | def variant = [:] 59 | variant.url = baseUrl+ dir+ file; 60 | variant.os = parts[0][2] 61 | variant.arch = parts[0][3] 62 | 63 | // Add it to the list of variants for this version of CMake 64 | def version = parts[0][1] 65 | if (!releases[version]) { 66 | releases[version] = [] 67 | } 68 | releases[version] << variant 69 | } 70 | } 71 | 72 | // Build the JSON structure: a list of release objects, each with its platform-specific variants 73 | def json = [list: releases.collect { key, value -> 74 | [ "id": key, "name": "${key}".toString(), 75 | "variants": value] }] 76 | // Write the JSON update file 77 | lib.DataWriter.write("hudson.plugins.cmake.CmakeInstaller",JSONObject.fromObject(json)); 78 | -------------------------------------------------------------------------------- /codeql.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Gradle auto-installation 3 | import org.htmlunit.Page 4 | import org.htmlunit.WebClient 5 | import org.htmlunit.WebResponse 6 | 7 | import net.sf.json.* 8 | 9 | def wc = new WebClient(); 10 | wc.addRequestHeader("accept", "application/json") 11 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 12 | wc.getOptions().setPrintContentOnFailingStatusCode(false); 13 | 14 | def API_URL = "https://api.github.com"; 15 | Page p = wc.getPage(API_URL + "/repos/github/codeql-action/releases") 16 | 17 | WebResponse response = p.getWebResponse(); 18 | 19 | if (response.statusCode != 200) { 20 | throw new Exception("Could not load available_releases") 21 | } 22 | 23 | JSONArray result = new JSONArray() 24 | JSONArray data = JSONArray.fromObject(response.getContentAsString()) 25 | 26 | 27 | data.each { release -> 28 | JSONObject r = new JSONObject() 29 | def tagname = release.tag_name 30 | if(!(tagname =~ /codeql-bundle-(.*)/)) { 31 | return 32 | } 33 | def bundleDate = (tagname =~ /codeql-bundle-(.*)/)[0][1] 34 | 35 | def bundleversion = ( release.body =~ /.*v([0-9]+\.[0-9]+\..*)/)[0][1] 36 | 37 | r.put("id", bundleversion ) 38 | r.put("name", bundleversion) 39 | r.put("url", "https://github.com/github/codeql-action/releases/download/" + tagname + '/') 40 | result.add(r) 41 | } 42 | 43 | JSONObject resultObj = new JSONObject() 44 | .element("list", result); 45 | 46 | lib.DataWriter.write("io.jenkins.plugins.codeql.CodeQLInstaller", resultObj); 47 | -------------------------------------------------------------------------------- /consul.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Consul auto-installation 3 | import org.htmlunit.html.*; 4 | import net.sf.json.* 5 | import org.htmlunit.WebClient 6 | 7 | def baseUrl = 'https://releases.hashicorp.com' 8 | def json = [] 9 | def wc = new WebClient() 10 | HtmlPage p = wc.getPage("${baseUrl}/consul/") 11 | p.getByXPath("//a[@href]").grep { it.hrefAttribute =~ /\/consul\/.*/ }.each { 12 | wc.getPage("${baseUrl}${it.hrefAttribute}").getByXPath("//a[@href]").each { 13 | def m = (it.textContent =~ /consul_.*(\d+.\d+.\d+)_(.*)_(.*).zip/) 14 | if (m) { 15 | def verId = "${m[0][1]}-${m[0][2]}-${m[0][3]}".toString() 16 | json << ["id": verId, "name": "Consul ${m[0][1]} ${m[0][2]} (${m[0][3]})".toString(), "url": "${it.hrefAttribute}".toString()]; 17 | } 18 | } 19 | } 20 | 21 | lib.DataWriter.write("com.inneractive.jenkins.plugins.consul.ConsulInstaller",JSONObject.fromObject([list:json])); 22 | -------------------------------------------------------------------------------- /dependencycheck.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for dependencycheck-launch auto-installation 3 | 4 | import hudson.util.VersionNumber 5 | import net.sf.json.* 6 | 7 | def listFromGithub(int page) { 8 | def url = ("https://api.github.com/repos/dependency-check/dependencycheck/releases?per_page=50&page=" + page).toURL() 9 | def releases = JSONArray.fromObject(url.text) 10 | 11 | releases.collect { 12 | release -> 13 | def version = release["tag_name"].substring(1) 14 | ["id": version, 15 | "name": "dependency-check ${version}".toString(), 16 | "url": "https://github.com/dependency-check/DependencyCheck/releases/download/v${version}/dependency-check-${version}-release.zip".toString()] 17 | } 18 | } 19 | 20 | 21 | def listAll() { 22 | Map versions = new HashMap() 23 | for (int page = 0;; page++) { 24 | def releases = listFromGithub(page) 25 | releases.each {version -> versions.put(version["id"], version)} 26 | if (releases.isEmpty()) { 27 | break; // do {} until {} 28 | } 29 | } 30 | 31 | return versions.values() 32 | .findAll { it != null } 33 | .sort { o1,o2 -> 34 | try { 35 | def v1 = new VersionNumber(o1.id) 36 | try { 37 | new VersionNumber(o2.id).compareTo(v1) 38 | } catch (IllegalArgumentException _2) { 39 | -1 40 | } 41 | } catch (IllegalArgumentException _1) { 42 | try { 43 | new VersionNumber(o2.id) 44 | 1 45 | } catch (IllegalArgumentException _2) { 46 | o2.id.compareTo(o1.id) 47 | } 48 | } 49 | } 50 | } 51 | 52 | lib.DataWriter.write("org.jenkinsci.plugins.DependencyCheck.tools.DependencyCheckInstaller",JSONObject.fromObject([list:listAll()])); 53 | -------------------------------------------------------------------------------- /dotnetSdk.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates data files used by the dotnet-sdk-plugin, both for SDK installation and for auto-completion. 3 | 4 | import org.htmlunit.Page 5 | import org.htmlunit.SilentCssErrorHandler 6 | import org.htmlunit.WebClient 7 | import net.sf.json.JSONArray 8 | import net.sf.json.JSONException 9 | import net.sf.json.JSONObject 10 | import org.apache.commons.lang3.StringUtils 11 | 12 | //region Utilities 13 | 14 | static WebClient createWebClient() { 15 | final WebClient wc = new WebClient() 16 | wc.setCssErrorHandler(new SilentCssErrorHandler()) 17 | wc.options.javaScriptEnabled = false 18 | wc.options.throwExceptionOnFailingStatusCode = false 19 | wc.options.throwExceptionOnScriptError = false 20 | return wc 21 | } 22 | 23 | void createDownloadable(String className, JSONObject object) { 24 | lib.DataWriter.write('io.jenkins.plugins.dotnet.data.' + className, object) 25 | } 26 | 27 | static JSONObject fetchJson(String url) { 28 | //System.out.printf('fetching JSON from %s%n', url) 29 | final WebClient wc = createWebClient() 30 | final Page p = wc.getPage(url) 31 | return JSONObject.fromObject(p.webResponse.contentAsString) 32 | } 33 | 34 | 35 | //endregion 36 | 37 | Throwable failure = null 38 | 39 | private Throwable addFailure(Throwable failure, Throwable t) { 40 | if (failure == null) { 41 | failure = new IOException("Failed to create one or more output files.") 42 | } 43 | failure.addSuppressed(t) 44 | return failure 45 | } 46 | 47 | //region File 1: Framework Monikers 48 | 49 | private void createFrameworkMonikers() { 50 | // https://docs.microsoft.com/en-us/dotnet/standard/frameworks is referenced by the .NET CLI docs as the reference 51 | // but does not actually have a complete list. Nor does there seem to be another source. So for now, this just takes 52 | // a file from the plugin repository as-is. 53 | final JSONObject tfmList = fetchJson('https://raw.githubusercontent.com/jenkinsci/dotnet-sdk-plugin/master/downloadables/tfm-list.json') 54 | createDownloadable('Framework', tfmList) 55 | } 56 | 57 | try { 58 | createFrameworkMonikers() 59 | } 60 | catch (Throwable t) { 61 | failure = addFailure(failure, t) 62 | } 63 | 64 | //endregion 65 | 66 | //region File 2: RID Catalog 67 | 68 | private void createRidCatalog() { 69 | final JSONObject ridCatalog = fetchJson('https://raw.githubusercontent.com/dotnet/runtime/main/src/libraries/Microsoft.NETCore.Platforms/src/runtime.json') 70 | final String[] rids = ridCatalog.getJSONObject('runtimes').keySet().toArray() 71 | // TODO: Maybe sort this list so that names with fewer parts sort before those with more. 72 | // TODO: Specifically, sort 'tizen-4.0.0' and 'tizen-5.0.0' before 'tizen-4.0.0-x64' and 'tizen-5.0.0-x64' 73 | final JSONObject data = JSONObject.fromObject([ 'ridCatalog': rids ]) 74 | createDownloadable('Runtime', data) 75 | } 76 | 77 | try { 78 | createRidCatalog() 79 | } 80 | catch (Throwable t) { 81 | failure = addFailure(failure, t) 82 | } 83 | 84 | //endregion 85 | 86 | //region File 3: SDK Downloads 87 | 88 | private String getSdkInfo(JSONObject s) { 89 | final StringBuilder info = new StringBuilder() 90 | def value = s.get('vs-support') 91 | if (value instanceof String && !value.isEmpty()) { 92 | info.append(value) 93 | } 94 | else { 95 | value = s.get('vs-version') 96 | if (value instanceof String && !value.isEmpty()) { 97 | info.append('Visual Studio ').append(value) 98 | } 99 | } 100 | value = s.get('csharp-version') 101 | if (value instanceof String && !value.isEmpty()) { 102 | if (info.size() > 0) { 103 | info.append(', ') 104 | } 105 | info.append('C# ').append(value) 106 | } 107 | value = s.get('fsharp-version') 108 | if (value instanceof String && !value.isEmpty()) { 109 | if (info.size() > 0) { 110 | info.append(', ') 111 | } 112 | info.append('F# ').append(value) 113 | } 114 | value = s.get('vb-version') 115 | if (value instanceof String && !value.isEmpty()) { 116 | if (info.size() > 0) { 117 | info.append(', ') 118 | } 119 | info.append('VB ').append(value) 120 | } 121 | if (info.size() == 0) { 122 | return null 123 | } 124 | return info.toString() 125 | } 126 | 127 | private def getSdk(sdks, JSONObject s) { 128 | def name = s.get('version-display') 129 | if (name instanceof JSONObject && name.isNullObject()) { 130 | name = null 131 | } 132 | if (name == null) { // some SDKs have version-display as null; fall back on the raw version 133 | name = s.get('version') 134 | if (name instanceof JSONObject && name.isNullObject()) { 135 | name = null 136 | } 137 | } 138 | if (name == null || !(name instanceof String)) { 139 | throw new JSONException('SDK has neither a version-display nor a version property set.') 140 | } 141 | def sdk = [:] 142 | sdk['name'] = name 143 | if (sdks.containsKey(name)) { 144 | return sdk // Assumption: SDK of same name has same contents 145 | } 146 | def info = getSdkInfo(s) 147 | if (info != null) { 148 | sdk['info'] = info 149 | } 150 | def packages = [] 151 | def urls = [] 152 | def rids = [:] 153 | for (JSONObject p : s.getJSONArray('files')) { 154 | final String fileName = p.getString('name') 155 | if (fileName == null || (!fileName.endsWith('.zip') && !fileName.endsWith('.tar.gz'))) { 156 | continue 157 | } 158 | def pkg = [:] 159 | final String rid = p.getString('rid') 160 | if (rids.containsKey(rid)) { 161 | // Avoid listing two files for the same rid; prefer the .tar.gz because it's slightly smaller. 162 | if (fileName.endsWith('.tar.gz')) { 163 | packages.remove(rids[rid]) 164 | } 165 | else { 166 | continue 167 | } 168 | } 169 | pkg['rid'] = rid 170 | rids[rid] = pkg 171 | if (rid != null) { 172 | String osAndVersion = rid 173 | String arch 174 | if (rid.endsWith('-arm')) { 175 | arch = 'ARM32' 176 | osAndVersion = rid.substring(0, rid.length() - 4) 177 | } 178 | else if (rid.endsWith('-arm64')) { 179 | arch = 'ARM64' 180 | osAndVersion = rid.substring(0, rid.length() - 6) 181 | } 182 | else if (rid.endsWith('-x64')) { 183 | arch = 'x64' 184 | osAndVersion = rid.substring(0, rid.length() - 4) 185 | } 186 | else if (rid.endsWith('-x86')) { 187 | arch = 'x86' 188 | osAndVersion = rid.substring(0, rid.length() - 4) 189 | } 190 | else { 191 | arch = "???" 192 | } 193 | final String[] parts = osAndVersion.split(/[.]/) 194 | String os = parts[0] 195 | String version = parts.length > 1 ? parts[1] : null 196 | switch (os) { 197 | case 'centos': 198 | os = 'CentOS' 199 | break 200 | case 'debian': 201 | os = 'Debian' 202 | break 203 | case 'fedora': 204 | os = 'Fedora' 205 | break 206 | case 'linux': 207 | os = 'Linux' 208 | break 209 | case 'linux-musl': 210 | os = 'Alpine Linux' 211 | break 212 | case 'opensuse': 213 | os = 'OpenSUSE' 214 | break 215 | case 'osx': 216 | os = 'macOS' 217 | break 218 | case 'rhel': 219 | os = 'RHEL' 220 | break 221 | case 'ubuntu': 222 | os = 'Ubuntu' 223 | break 224 | case 'win': 225 | os = 'Windows' 226 | break 227 | default: 228 | //System.err.printf('NO MAPPING DEFINED FOR OS PART (%s) OF RID "%s"%n', os, rid) 229 | os = '???' 230 | } 231 | version = (version != null) ? (' ' + version) : '' 232 | arch = (arch != null) ? ' - ' + arch : '' 233 | pkg['platform'] = os + version + arch 234 | } 235 | // Not currently used or needed, and it makes the file much bigger 236 | // pkg['hash'] = p.getString('hash') 237 | // Later SDKs also include an 'akams' key, with a (shorter) aka.ms link. 238 | // However, those links seem to be broken, so stick to 'url'. 239 | final String url = p.getString('url') 240 | pkg['url'] = url 241 | urls << url 242 | packages << pkg 243 | } 244 | if (urls.size() > 0) { // compute the URL prefix for the package (reduces file size) 245 | String urlPrefix = StringUtils.getCommonPrefix(urls as String[]) 246 | if (!urlPrefix.endsWith('/')) { 247 | urlPrefix = urlPrefix.substring(0, urlPrefix.lastIndexOf('/') + 1) 248 | } 249 | if (urlPrefix.size() > 15) { // shorter is not worth emitting the property for 250 | for (def pkg : packages) { 251 | pkg['url'] = pkg['url'].substring(urlPrefix.size()) 252 | } 253 | sdk['urlPrefix'] = urlPrefix 254 | } 255 | } 256 | sdk['packages'] = packages 257 | sdks[name] = sdk 258 | return sdk 259 | } 260 | 261 | private void createSdkDownloads() { 262 | def versions = [] 263 | def sdks = [:] 264 | JSONObject releaseIndex = fetchJson('https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/releases-index.json') 265 | for (JSONObject v : releaseIndex.getJSONArray('releases-index')) { 266 | def version = [:] 267 | version['name'] = v.getString('product') + ' ' + v.getString('channel-version') 268 | version['status'] = v.getString('support-phase').toUpperCase() 269 | version['type'] = v.getString('release-type').toUpperCase() 270 | version['endOfSupport'] = v.get('eol-date') 271 | def releases = [] 272 | final JSONObject channel = fetchJson(v.getString('releases.json')) 273 | for (JSONObject r : channel.getJSONArray('releases')) { 274 | def release = [:] 275 | release['name'] = r.getString('release-version') 276 | release['released'] = r.getString('release-date') 277 | if (r.getBoolean('security')) { 278 | release['securityFixes'] = true 279 | } 280 | // Assumption based on Semantic Versioning 281 | if (r.getString('release-version').contains('-')) { 282 | release['preview'] = true 283 | } 284 | def releaseNotes = r.get('release-notes') 285 | if (releaseNotes instanceof String) { 286 | release['releaseNotes'] = releaseNotes 287 | } 288 | release['sdks'] = [] 289 | // Older releases have only 'sdk'. Some have sdks: null. But when sdks is set, it always includes sdk. 290 | def releaseSdks = r.get('sdks') 291 | if (releaseSdks instanceof JSONObject && releaseSdks.isNullObject()) { 292 | releaseSdks = null 293 | } 294 | if (releaseSdks == null) { 295 | def s = r.get('sdk') 296 | if (s instanceof JSONObject && s.isNullObject()) { 297 | s = null 298 | } 299 | if (s != null) { 300 | def sdk = getSdk(sdks, s) 301 | release['sdks'] << sdk['name'] 302 | } 303 | } 304 | else { 305 | for (JSONObject s : releaseSdks) { 306 | def sdk = getSdk(sdks, s) 307 | release['sdks'] << sdk['name'] 308 | } 309 | } 310 | releases << release 311 | } 312 | version['releases'] = releases 313 | versions << version 314 | } 315 | createDownloadable('Downloads', JSONObject.fromObject([ 'versions' : versions, 'sdks': sdks.values()])) 316 | } 317 | 318 | try { 319 | createSdkDownloads() 320 | } 321 | catch (Throwable t) { 322 | failure = addFailure(failure, t) 323 | } 324 | 325 | //endregion 326 | 327 | if (failure != null) { 328 | throw failure 329 | } 330 | -------------------------------------------------------------------------------- /flyway.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | import org.htmlunit.WebClient 3 | import org.htmlunit.html.HtmlAnchor 4 | import org.htmlunit.html.HtmlPage 5 | import hudson.util.VersionNumber 6 | import net.sf.json.JSONObject 7 | 8 | import java.util.regex.Pattern 9 | // Generates server-side metadata for Flyway command-line 10 | def webclient (){ 11 | def wc = new WebClient() 12 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 13 | wc.getOptions().setThrowExceptionOnScriptError(false); 14 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 15 | return wc 16 | } 17 | 18 | def listFromMavenRepo() { 19 | def json = [] 20 | def url = "https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/"; 21 | 22 | def wc=webclient(); 23 | HtmlPage p = wc.getPage(url); 24 | def pattern = Pattern.compile("^([0-9]+.*)/\$"); 25 | 26 | p.getAnchors().collect { HtmlAnchor a -> 27 | m = pattern.matcher(a.hrefAttribute) 28 | if (m.find()) { 29 | ver = m.group(1) 30 | json.addAll( os_versions(url + ver, ver, wc)) 31 | } 32 | } 33 | return json 34 | } 35 | 36 | def os_versions(url, version, wc) { 37 | json=[] 38 | HtmlPage page= wc.getPage(url) 39 | def pattern = Pattern.compile("flyway-commandline-" + ver + "(\\.tar\\.gz|-([a-z]+)-x64(\\.zip|\\.tar.gz))\$") 40 | 41 | page.getAnchors().collect { HtmlAnchor a -> 42 | m = pattern.matcher(a.hrefAttribute) 43 | if (m.find()) { 44 | installer_url=page.getFullyQualifiedUrl(m.group(0)) 45 | 46 | def parsed_platform = m.group(2) 47 | platform = parsed_platform ? " (" + parsed_platform + ")": " (without JRE)" 48 | id = version + ( parsed_platform ? "_" + parsed_platform : "nojre") 49 | 50 | json << ["id": id, "name": ver + platform, "url": installer_url.toExternalForm()] 51 | } 52 | } 53 | return json 54 | } 55 | 56 | def flyway_distributions = listFromMavenRepo() 57 | 58 | flyway_distributions.sort { o1, o2 -> 59 | try { 60 | def v1 = new VersionNumber(o1.id) 61 | try { 62 | new VersionNumber(o2.id).compareTo(v1) 63 | } catch (IllegalArgumentException _2) { 64 | -1 65 | } 66 | } catch (IllegalArgumentException _1) { 67 | try { 68 | new VersionNumber(o2.id) 69 | 1 70 | } catch (IllegalArgumentException _2) { 71 | o2.id.compareTo(o1.id) 72 | } 73 | } 74 | } 75 | 76 | lib.DataWriter.write("sp.sd.flywayrunner.installation.FlywayInstaller", JSONObject.fromObject([list: flyway_distributions])); 77 | 78 | 79 | -------------------------------------------------------------------------------- /golang.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | import org.htmlunit.html.HtmlAnchor 3 | import org.htmlunit.html.HtmlPage 4 | import org.htmlunit.WebClient 5 | 6 | import net.sf.json.* 7 | 8 | // Fetch the list of downloads from the Go website 9 | def downloadsUrl = "https://golang.org/dl/" 10 | 11 | // Gather a list of URLs 12 | def urls = [] 13 | 14 | // Disable JS, as we don't care about it 15 | WebClient webClient = new WebClient() 16 | webClient.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 17 | webClient.getOptions().setJavaScriptEnabled(false); 18 | webClient.getOptions().setThrowExceptionOnScriptError(false); 19 | webClient.getOptions().setThrowExceptionOnFailingStatusCode(false); 20 | 21 | // Fetch the page and gather the links 22 | HtmlPage page = webClient.getPage(downloadsUrl) 23 | page.getByXPath("//td/a").each { HtmlAnchor e -> 24 | urls << e.getHrefAttribute() 25 | } 26 | 27 | // Build a map of Go versions -> platform archives 28 | def releases = [:] 29 | urls.each { url -> 30 | // We only want release archives; ignore source packages and beta/RC versions 31 | if (url =~ /\.src\.tar\.gz$/ || url =~ /(beta|rc)\d+\./ || url =~ /bootstrap/) { 32 | return 33 | } 34 | 35 | // Extract the version info from archive filename (ignore .msi or .pkg installers), e.g.: 36 | // go1.2.1.darwin-amd64-osx10.8.tar.gz 37 | // go.go1.windows-386.zip 38 | // go1.10.linux-amd64.tar.gz 39 | def parts = (url =~ /go(?:\.go)?(\d(?:\.\d+)*)\.(?:([^-]+)-([^-]+)(?:-(.+))?)\.(?:tar\.gz|zip)/) 40 | if (!parts) { 41 | return 42 | } 43 | 44 | // Gather the info for this archive 45 | def variant = [:] 46 | variant.url = "https://golang.org" + url 47 | variant.os = parts[0][2] 48 | variant.arch = parts[0][3] 49 | if (parts[0][4] && parts[0][4].startsWith("osx")) { 50 | variant.osxversion = parts[0][4].substring("osx".length()) 51 | } 52 | 53 | // Add it to the list of variants for this version of Go 54 | def version = parts[0][1] 55 | if (!releases[version]) { 56 | releases[version] = [] 57 | } 58 | releases[version] << variant 59 | } 60 | 61 | // Build the JSON structure: a list of release objects, each with its platform-specific variants 62 | def json = [releases: releases.collect { key, value -> ["id": key, "name": "Go ${key}".toString(), "variants": value] }] 63 | 64 | // Write the JSON update file 65 | lib.DataWriter.write("org.jenkinsci.plugins.golang.GolangInstaller", JSONObject.fromObject(json)) 66 | -------------------------------------------------------------------------------- /gradle.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Gradle auto-installation 3 | import org.htmlunit.html.*; 4 | import org.htmlunit.WebClient 5 | 6 | import net.sf.json.* 7 | 8 | def wc = new WebClient() 9 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 10 | wc.getOptions().setThrowExceptionOnScriptError(false); 11 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 12 | 13 | def baseUrl = 'https://services.gradle.org' 14 | HtmlPage p = wc.getPage(baseUrl + '/distributions'); 15 | 16 | def json = []; 17 | 18 | p.getByXPath("//a[@href]").collect { HtmlAnchor e -> 19 | def url = baseUrl + e.getHrefAttribute() 20 | println url 21 | def m = (url =~ /gradle-(.*)-bin.zip$/) 22 | if (m) { 23 | json << ["id":m[0][1], "name": "Gradle ${m[0][1]}".toString(), "url":url]; 24 | } 25 | } 26 | 27 | lib.DataWriter.write("hudson.plugins.gradle.GradleInstaller",JSONObject.fromObject([list:json])); 28 | -------------------------------------------------------------------------------- /grails.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for MongoDB auto-installation 3 | import net.sf.json.* 4 | 5 | def json = [] 6 | def versions = new URL('https://api.sdkman.io/2/candidates/grails/universal/versions/all').text.split(',').reverse() 7 | versions.each { 8 | json << [id:it, name:"Grails $it".toString(), url:"https://github.com/grails/grails-core/releases/download/v${it}/grails-${it}.zip".toString()] 9 | } 10 | 11 | lib.DataWriter.write("com.g2one.hudson.grails.GrailsInstaller",JSONObject.fromObject([list:json])); 12 | -------------------------------------------------------------------------------- /grapeConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /groovy.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Groovy auto-installation 3 | import org.htmlunit.html.*; 4 | 5 | import net.sf.json.* 6 | import org.htmlunit.WebClient 7 | import org.htmlunit.BrowserVersion 8 | 9 | final String applicationName = "crawler"; 10 | final String applicationVersion = "1.0"; 11 | final String userAgent = "jenkins-project-crawler"; 12 | 13 | final BrowserVersion browser = 14 | new BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED) 15 | .setApplicationName(applicationName) 16 | .setApplicationVersion(applicationVersion) 17 | .setUserAgent(userAgent) 18 | .build(); 19 | 20 | // Set a custom user agent so the server won't redirect to https://groovy.jfrog.io/ui/native/dist-release-local/groovy-zips/ 21 | // which contains a script with the unsuppported JS function `fetch` 22 | def wc = new WebClient(browser); 23 | def baseUrl = 'https://groovy.jfrog.io/artifactory/dist-release-local/groovy-zips/' 24 | HtmlPage p = wc.getPage(baseUrl); 25 | 26 | def json = []; 27 | 28 | p.getByXPath("//a[@href]").reverse().collect { HtmlAnchor e -> 29 | def url = baseUrl + e.getHrefAttribute() 30 | println url 31 | def m = (url =~ /groovy-binary-(\d+.\d+.\d+).zip$/) 32 | if (m) { 33 | json << ["id":m[0][1], "name": "Groovy ${m[0][1]}".toString(), "url":url]; 34 | } 35 | } 36 | 37 | lib.DataWriter.write("hudson.plugins.groovy.GroovyInstaller",JSONObject.fromObject([list:json.sort { it.id }.reverse()])); 38 | -------------------------------------------------------------------------------- /jdk.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Oracle JDK 3 | import org.htmlunit.WebClient 4 | import org.htmlunit.BrowserVersion 5 | import net.sf.json.JSONObject 6 | import org.kohsuke.args4j.CmdLineException 7 | import java.security.GeneralSecurityException 8 | import net.sf.json.JSONArray 9 | import java.util.regex.Pattern 10 | import java.util.regex.Matcher 11 | import org.htmlunit.html.HtmlPage 12 | 13 | public class ListJDK { 14 | private final WebClient wc; 15 | 16 | public ListJDK() { 17 | wc = new WebClient(BrowserVersion.BEST_SUPPORTED); // INTERNET_EXPLORER); 18 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 19 | wc.getOptions().setThrowExceptionOnScriptError(false); 20 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 21 | } 22 | 23 | public void main() throws Exception { 24 | lib.DataWriter.write("hudson.tools.JDKInstaller",build()); 25 | } 26 | 27 | private JSONObject build() throws IOException, CmdLineException, GeneralSecurityException { 28 | // HACK HACK HACK INFRA-2345 29 | // Temporary bandaid to give up scraping and use a snapshot 30 | return JSONObject.fromObject(new File("jdk.hack.json").text) 31 | 32 | return new JSONObject() 33 | .element("version", 2) 34 | .element("data", new JSONArray() 35 | // .element(family("JDK 10", 36 | // parse("http://www.oracle.com/technetwork/java/javase/downloads/jdk10-downloads-4416644.html"))) 37 | .element(family("JDK 9", combine( 38 | parse("http://www.oracle.com/technetwork/java/javase/downloads/java-archive-javase9-3934878.html")))) 39 | .element(family("JDK 8", combine( 40 | parse("http://www.oracle.com/technetwork/java/javase/downloads/java-archive-javase8-2177648.html"), 41 | parse("http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html")))) 42 | .element(family("JDK 7", parse("http://www.oracle.com/technetwork/java/javase/downloads/java-archive-downloads-javase7-521261.html"))) 43 | .element(family("JDK 6", parse("http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-javase6-419409.html"))) 44 | .element(family("JDK 5", parse("http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-javase5-419410.html"))) 45 | .element(family("JDK 1.4", parse("http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-javase14-419411.html")))); 46 | } 47 | 48 | private static final Pattern NUMBER = Pattern.compile("\\d+"); 49 | 50 | /** 51 | * Builds a data structure for a major release. 52 | */ 53 | private JSONObject family(String name, JSONObject data) { 54 | JSONObject o = new JSONObject(); 55 | o.put("name",name); 56 | JSONArray releases = new JSONArray(); 57 | List releaseNames = new ArrayList((Set) data.keySet()); 58 | Collections.sort(releaseNames,new Comparator() { 59 | public int compare(String o1, String o2) { 60 | return -versionNormalize(o1).compareTo(versionNormalize(o2)); // descending order 61 | } 62 | 63 | /** 64 | * We want to compare numbers among string tokens as numbers. To do it simply, we force the width of any string token to be 4 and then compare. 65 | */ 66 | private String versionNormalize(String s) { 67 | Matcher m = NUMBER.matcher(s); 68 | StringBuffer sb = new StringBuffer(); 69 | while (m.find()) { 70 | m.appendReplacement(sb, String.format("%4s",m.group())); 71 | } 72 | m.appendTail(sb); 73 | return sb.toString(); 74 | } 75 | }); 76 | for (String n : releaseNames) { 77 | if (n.contains("demo")) continue; // we don't care about demo & sample bundles 78 | if (n.contains("jdk") || n.contains("j2sdk")) 79 | releases.add(release(n,data.getJSONObject(n))); 80 | } 81 | o.put("releases",releases); 82 | return o; 83 | } 84 | 85 | private JSONObject release(String name, JSONObject src) { 86 | JSONObject input = src.getJSONObject("files"); 87 | JSONArray files = new JSONArray(); 88 | for (String n : (Set)input.keySet()) { 89 | if (n.contains("rpm")) continue; // we don't use RPM bundle 90 | if (n.contains("tar.Z")) continue; // we don't use RPM bundle 91 | if (n.contains("-iftw.exe")) continue; // online installers 92 | files.add(file(n, input.getJSONObject(n))); 93 | } 94 | return src.element("name",massageName(name)).element("files", files); 95 | } 96 | 97 | /** 98 | * JDK release names are inconsistent, so make it look uniform 99 | */ 100 | private String massageName(String name) { 101 | name = name.trim(); 102 | name = name.replace("(TM)", ""); 103 | name = name.replace(" Update ","u"); 104 | return name; 105 | } 106 | 107 | private JSONObject file(String name, JSONObject src) { 108 | return src.element("name",name).element("size",(Object)null); 109 | } 110 | 111 | private JSONObject parse(String url) throws IOException { 112 | HtmlPage p = getPage(url); 113 | 114 | Object results = p.executeJavaScript("downloads").getJavaScriptResult() 115 | if (results == null) { 116 | // no downloads/legacy method on page 117 | return new JSONObject() 118 | } 119 | return (JSONObject)results; 120 | } 121 | 122 | private JSONObject combine(JSONObject... args) { 123 | JSONObject o = new JSONObject(); 124 | args.each { a -> o.putAll(a); } 125 | return o; 126 | } 127 | 128 | private HtmlPage getPage(String url) throws IOException { 129 | // System.out.println("Fetching ${url} ...") 130 | long start = System.currentTimeMillis(); 131 | HtmlPage p = wc.getPage(url); 132 | long end = System.currentTimeMillis(); 133 | // println("done (took ${end - start} msec)") 134 | return p; 135 | } 136 | } 137 | 138 | new ListJDK().main(); 139 | -------------------------------------------------------------------------------- /leiningen.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Leiningen auto-installation 3 | import org.htmlunit.html.*; 4 | 5 | import net.sf.json.* 6 | import org.htmlunit.WebClient 7 | 8 | def wc = new WebClient() 9 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 10 | wc.getOptions().setJavaScriptEnabled(false); 11 | wc.getOptions().setThrowExceptionOnScriptError(false); 12 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 13 | 14 | def baseUrl = 'https://github.com' 15 | HtmlPage p = wc.getPage(baseUrl + '/technomancy/leiningen/releases'); 16 | // curl 'https://api.github.com/repos/technomancy/leiningen/releases' | jq '.[].assets[].name,.[].assets[].browser_download_url' 17 | 18 | 19 | def json = []; 20 | 21 | p.getByXPath("//a[@href]").reverse().collect { HtmlAnchor e -> 22 | def url = baseUrl + e.getHrefAttribute() 23 | def m = (url =~ /leiningen-(.*)-standalone.jar$/) 24 | if (m) { 25 | json << ["id":m[0][1], "name": "Leiningen ${m[0][1]}".toString(), "url":url]; 26 | } 27 | } 28 | 29 | lib.DataWriter.write("org.jenkins-ci.plugins.leiningen.LeinInstaller",JSONObject.fromObject([list:json])); 30 | -------------------------------------------------------------------------------- /lib/DataWriter.groovy: -------------------------------------------------------------------------------- 1 | package lib; 2 | 3 | import net.sf.json.JSONObject 4 | import org.jvnet.hudson.update_center.Signer 5 | 6 | /** 7 | * Writes out the JSON data file. 8 | */ 9 | public class DataWriter { 10 | public static void write(String key,JSONObject envelope) { 11 | println envelope.toString(2) 12 | 13 | // write unsigned data to *.json because of JENKINS-15105 14 | File d = new File("target") 15 | d.mkdirs() 16 | new File(d,"${key}.json").write("downloadService.post('${key}',${envelope.toString(2)})","UTF-8"); 17 | 18 | // then signed data to *.json.html 19 | if (System.getenv("JENKINS_SIGNER")!=null) 20 | new Signer().configureFromEnvironment().sign(envelope); 21 | new File(d,"${key}.json.html").write("\uFEFF","UTF-8"); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /lib/init.groovy: -------------------------------------------------------------------------------- 1 | // code to load all the necessary dependencies 2 | // placed in a separate file to load this lazily after we set the necessary system property to work around Maven resolution 3 | package lib; 4 | import java.util.logging.*; 5 | 6 | @GrabResolver(name="repo.jenkins-ci.org",root='https://repo.jenkins-ci.org/public/') 7 | @GrabExclude('nekohtml:nekohtml') 8 | @Grapes([ 9 | @Grab("net.sourceforge.nekohtml:nekohtml:1.9.21"), 10 | @Grab("org.htmlunit:htmlunit:3.9.0"), 11 | @Grab("org.jenkins-ci:update-center2:2.0") 12 | ]) 13 | class init { 14 | static { 15 | println "done" 16 | which org.apache.xerces.parsers.AbstractSAXParser.class 17 | which org.htmlunit.html.parser.HTMLParser.class 18 | which org.cyberneko.html.HTMLConfiguration.class 19 | 20 | Logger.getLogger("org.htmlunit").setLevel(Level.OFF); 21 | } 22 | 23 | static void which(Class c) { 24 | // println c.classLoader.getResource(c.name.replace(".","/")+".class") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lib/runner.groovy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env groovy 2 | System.setProperty("mavenVersion","3.0") 3 | print "loading dependencies..." 4 | Class.forName("lib.init",true,this.class.classLoader) 5 | 6 | if (args.length==0) { 7 | println "No script specified"; 8 | System.exit(-1); 9 | } else { 10 | args.each { arg -> 11 | // if (arg.endsWith(".groovy")) arg = arg.substring(0,arg.length()-7); 12 | // Class.forName(arg,true,this.class.classLoader).newInstance().run(); 13 | evaluate(new File(arg)) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lib/signer.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | import net.sf.json.* 3 | import org.jvnet.hudson.update_center.Signer 4 | import groovy.io.FileType 5 | 6 | def list = [] 7 | 8 | def dir = new File('./target/') 9 | dir.eachFileRecurse (FileType.FILES) { file -> 10 | if(file.getPath().endsWith('.json')) 11 | list << file; 12 | } 13 | 14 | list.each { 15 | String file = it.path 16 | println "== Checking file: ${file}" 17 | 18 | // Load JSON file content to be signed 19 | File inputfile = new File(file) 20 | String utf8Content = inputfile.getText("UTF-8") 21 | if(utf8Content.length() < 1) { 22 | println "ERROR: the file ${file} is empty. Exiting." 23 | System.exit(1) 24 | } 25 | 26 | // There is JSONp callback wrapping the envelope. Separator is the first comma (','). 27 | int commaIndex = utf8Content.indexOf(',') 28 | // Do not forget to remove the closing parenthesis char at the end 29 | String jsonBody = utf8Content.substring(commaIndex + 1, utf8Content.length() - 1) 30 | if(!jsonBody.contains(':') || jsonBody.length() < 100 ) { 31 | println "ERROR: the JSON body in the ${file} is empty or with harmful content. Exiting." 32 | System.exit(1) 33 | } 34 | 35 | JSONObject jsonContent = JSONObject.fromObject(jsonBody) 36 | 37 | // Sign the content (it adds JSON fields at the end) 38 | if (System.getenv("JENKINS_SIGNER")!=null) { 39 | println 'The environment variable "JENKINS_SIGNER" is defined: let\'s sign' 40 | new Signer().configureFromEnvironment().sign(jsonContent) 41 | } 42 | 43 | // Write HTML file (with signed content) 44 | new File("${file}.html").write("\uFEFF","UTF-8"); 45 | println "== Writing file: ${file}.html" 46 | } 47 | -------------------------------------------------------------------------------- /maven.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Maven 3 | import org.htmlunit.WebClient 4 | import org.htmlunit.html.HtmlAnchor 5 | import org.htmlunit.html.HtmlPage 6 | import java.util.regex.Pattern 7 | import net.sf.json.JSONObject 8 | import hudson.util.VersionNumber 9 | 10 | def getHtmlPage(url) { 11 | def wc = new WebClient() 12 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 13 | wc.getOptions().setJavaScriptEnabled(false); 14 | wc.getOptions().setThrowExceptionOnScriptError(false); 15 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 16 | return wc.getPage(url) 17 | } 18 | 19 | def listFromURL(url) { 20 | def HtmlPage p = getHtmlPage(url) 21 | def pattern = Pattern.compile("maven-([0-9\\.]+)(-bin)?.zip\$") 22 | 23 | return p.getAnchors().collect { HtmlAnchor a -> 24 | def m = pattern.matcher(a.hrefAttribute) 25 | if (m.find()) { 26 | def ver = m.group(1) 27 | def fqUrl = p.getFullyQualifiedUrl(a.hrefAttribute).toExternalForm() 28 | return ["id": ver, "name": ver, "url": fqUrl] 29 | } 30 | return null 31 | }.findAll { it != null } 32 | } 33 | 34 | def listFromUrl(url, pattern) { 35 | wc = new WebClient(); 36 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 37 | wc.getOptions().setJavaScriptEnabled(false); 38 | wc.getOptions().setThrowExceptionOnScriptError(false); 39 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 40 | 41 | HtmlPage p = wc.getPage(url); 42 | initialurl = url; 43 | // Ignore message digest files 44 | ignorePattern = Pattern.compile("\\.sha([0-9]+)\$"); 45 | versionpattern = Pattern.compile("[0-9]([0-9.]+)"); 46 | pattern=Pattern.compile(pattern); 47 | 48 | return p.getAnchors().collect { HtmlAnchor a -> 49 | m = versionpattern.matcher(a.hrefAttribute) 50 | if(m.find() ) { 51 | l=a.hrefAttribute.length()-2; 52 | ver=a.hrefAttribute.getAt(0..l); 53 | url = p.getFullyQualifiedUrl(a.hrefAttribute); 54 | // Skip files matching the ignore pattern 55 | ignoreMatcher = ignorePattern.matcher(url.toString()); 56 | if(ignoreMatcher.find()) { 57 | return null; 58 | } 59 | HtmlPage pb = wc.getPage(url); 60 | return pb.getAnchors().collect { HtmlAnchor a1 -> 61 | n = pattern.matcher(a1.hrefAttribute) 62 | if (n.find()) { 63 | urld = pb.getFullyQualifiedUrl(a1.hrefAttribute); 64 | return ["id":ver, "name":ver, "url":urld.toExternalForm()] 65 | } 66 | return null; 67 | } 68 | } 69 | return null; 70 | }.flatten().findAll { it!=null }; 71 | } 72 | 73 | // Archives are coming from Apache 74 | def listFromOldURL() { 75 | return listFromURL("https://archive.apache.org/dist/maven/binaries/") 76 | } 77 | 78 | // Recent releases are coming from Maven central 79 | // Discussed with the Maven team here 80 | // http://mail-archives.apache.org/mod_mbox/maven-dev/201505.mbox/%3cCAFNCU--iq2nYb3wnO715CbcXN+S8umRTyRnfk4_JSZ2qCR+1fg@mail.gmail.com%3e 81 | def listFromNewUrl() { 82 | return listFromUrl("https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/", "maven-([0-9.]+)(-bin)?.zip\$") 83 | } 84 | 85 | def listAll() { 86 | return (listFromNewUrl() + listFromOldURL()) 87 | .unique { o1, o2 -> o1.id <=> o2.id } 88 | .sort { o1, o2 -> 89 | try { 90 | def v1 = new VersionNumber(o1.id) 91 | try { 92 | return new VersionNumber(o2.id).compareTo(v1) 93 | } catch (IllegalArgumentException _2) { 94 | return -1 95 | } 96 | } catch (IllegalArgumentException _1) { 97 | try { 98 | new VersionNumber(o2.id) 99 | return 1 100 | } catch (IllegalArgumentException _2) { 101 | return o2.id.compareTo(o1.id) 102 | } 103 | } 104 | } 105 | } 106 | 107 | def store(key, o) { 108 | JSONObject envelope = JSONObject.fromObject(["list": o]) 109 | lib.DataWriter.write(key, envelope) 110 | } 111 | 112 | store("hudson.tasks.Maven.MavenInstaller", listAll()) 113 | -------------------------------------------------------------------------------- /mongodb.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for MongoDB auto-installation 3 | import org.htmlunit.html.*; 4 | import net.sf.json.* 5 | import org.htmlunit.WebClient 6 | 7 | def wc = new WebClient() 8 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 9 | wc.getOptions().setJavaScriptEnabled(false); 10 | wc.getOptions().setThrowExceptionOnScriptError(false); 11 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 12 | 13 | def json = []; 14 | [osx: ['i386', 'x86_64'], 15 | linux: ['i686', 'x86_64'], 16 | win32: ['i386', 'x86_64'], 17 | sunos5: ['i86pc', 'x86_64'] 18 | ].each { osname, archs -> archs.each { arch -> 19 | HtmlPage p = wc.getPage("http://dl.mongodb.org/dl/$osname/$arch") 20 | p.getByXPath("//a[@href]").reverse().collect { HtmlAnchor e -> 21 | def m = e.getHrefAttribute() =~ /^.*mongodb-$osname-$arch-(.*?)\.(tgz|zip)$/ 22 | if (m) { 23 | String version = "${osname}-${arch}-${m[0][1]}" 24 | json << [id:version, name:version, url:m[0][0].replace('http://', 'https://')] 25 | } 26 | } 27 | }} 28 | 29 | lib.DataWriter.write("org.jenkinsci.plugins.mongodb.MongoDBInstaller",JSONObject.fromObject([list:json])); 30 | -------------------------------------------------------------------------------- /nodejs.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Gradle auto-installation 3 | import org.htmlunit.html.*; 4 | import org.htmlunit.WebClient 5 | 6 | import net.sf.json.* 7 | 8 | def wc = new WebClient() 9 | def baseUrl = 'https://nodejs.org/dist' 10 | HtmlPage p = wc.getPage(baseUrl); 11 | 12 | def json = []; 13 | 14 | p.getByXPath("//a[@href]").reverse().collect { HtmlAnchor e -> 15 | def url = baseUrl + "/" + e.getHrefAttribute() 16 | println url 17 | String versionRegex = "v(\\d+(?:\\.\\d+)*)/"; 18 | def m = (url =~ versionRegex) 19 | if (m) { 20 | json << ["id": m[0][1], "name": "NodeJS ${m[0][1]}".toString(), "url": url]; 21 | } 22 | } 23 | 24 | lib.DataWriter.write("hudson.plugins.nodejs.tools.NodeJSInstaller",JSONObject.fromObject([list:json])); 25 | -------------------------------------------------------------------------------- /packer.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Packer auto-installation 3 | import org.htmlunit.html.*; 4 | import net.sf.json.* 5 | import org.htmlunit.WebClient 6 | 7 | def baseUrl = 'https://releases.hashicorp.com' 8 | def json = [] 9 | def wc = new WebClient() 10 | HtmlPage p = wc.getPage("${baseUrl}/packer/") 11 | p.getByXPath("//a[@href]").grep { it.hrefAttribute =~ /\/packer\/.*/ }.each { 12 | wc.getPage("${baseUrl}${it.hrefAttribute}").getByXPath("//a[@href]").each { 13 | def m = (it.textContent =~ /packer_.*(\d+.\d+.\d+)_(.*)_(.*).zip/) 14 | if (m) { 15 | def verId = "${m[0][1]}-${m[0][2]}-${m[0][3]}".toString() 16 | json << ["id": verId, "name": "Packer ${m[0][1]} ${m[0][2]} (${m[0][3]})".toString(), "url": "${it.hrefAttribute}".toString()]; 17 | } 18 | } 19 | } 20 | 21 | lib.DataWriter.write("biz.neustar.jenkins.plugins.packer.PackerInstaller",JSONObject.fromObject([list:json])); 22 | -------------------------------------------------------------------------------- /play.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Play auto-installation 3 | import org.htmlunit.html.*; 4 | 5 | import net.sf.json.* 6 | import org.htmlunit.WebClient 7 | 8 | def wc = new WebClient() 9 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 10 | wc.getOptions().setJavaScriptEnabled(false); 11 | wc.getOptions().setThrowExceptionOnScriptError(false); 12 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 13 | 14 | def baseUrl = 'https://www.playframework.com/releases' 15 | HtmlPage p = wc.getPage(baseUrl); 16 | 17 | def json = []; 18 | 19 | p.getByXPath("//a[@href]").reverse().collect { HtmlAnchor e -> 20 | def url = e.getHrefAttribute() 21 | def m = (url =~ /play-(.*).zip$/) 22 | if (m) { 23 | json << ["id":m[0][1], "name": "Play ${m[0][1]}".toString(), "url":url]; 24 | } 25 | } 26 | 27 | lib.DataWriter.write("hudson.plugins.play.PlayInstaller",JSONObject.fromObject([list:json])); 28 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | org.jenkins-ci 6 | jenkins 7 | 1.108 8 | 9 | 10 | crawler 11 | 1.0-SNAPSHOT 12 | pom 13 | Tool Crawler 14 | Code that we use behind the scene. 15 | 16 | 17 | false 18 | 19 | 20 | 21 | 22 | jakarta.servlet 23 | jakarta.servlet-api 24 | 4.0.4 25 | 26 | 27 | org.codehaus.groovy 28 | groovy-all 29 | 2.4.21 30 | 31 | 32 | org.codehaus.groovy.modules.http-builder 33 | http-builder 34 | 0.7.1 35 | 36 | 37 | org.htmlunit 38 | htmlunit 39 | 3.9.0 40 | 41 | 42 | org.jenkins-ci 43 | update-center2 44 | 2.0 45 | 46 | 47 | log4j 48 | log4j 49 | 50 | 51 | 52 | 53 | org.jenkins-ci 54 | version-number 55 | 1.11 56 | 57 | 58 | org.kohsuke.stapler 59 | json-lib 60 | 2.4-jenkins-3 61 | 62 | 63 | 64 | 65 | 66 | repo.jenkins-ci.org 67 | https://repo.jenkins-ci.org/public/ 68 | 69 | 70 | 71 | 72 | 73 | 74 | org.apache.maven.plugins 75 | maven-enforcer-plugin 76 | 77 | 78 | display-info 79 | 80 | 81 | 82 | 83 | org.apache.httpcomponents:httpclient 84 | org.apache.maven:maven-model 85 | xerces:xercesImpl 86 | xml-apis:xml-apis 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /recipe.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // generate metadata for recipes 3 | import net.sf.json.* 4 | import java.util.zip.* 5 | 6 | def branch = "inbound"; 7 | 8 | // def zip = new File("recipes.zip"); 9 | def zip = File.createTempFile("recipes","zip"); 10 | zip.deleteOnExit() 11 | zip.withOutputStream { o -> 12 | new URL("https://github.com/jenkinsci/submitted-recipes/archive/${branch}.zip").withInputStream { i -> 13 | o << i; 14 | } 15 | } 16 | 17 | def json = []; 18 | 19 | def z = new ZipFile(zip) 20 | z.entries().each { e -> 21 | if (e.name.endsWith(".jrcp")) { 22 | def xml = new XmlSlurper().parse(z.getInputStream(e)); 23 | def o = [:]; 24 | ["id","version","displayName","description","author"].each { p -> 25 | o[p] = xml[p].toString(); 26 | } 27 | o.timestamp = e.time 28 | // TODO: copy it to our server 29 | o.url = "https://raw.github.com/jenkinsci/submitted-recipes/${branch}/${o.id}.jrcp".toString() 30 | json << o; 31 | } 32 | } 33 | 34 | lib.DataWriter.write("org.jenkinsci.plugins.recipe.RecipeCatalog",JSONObject.fromObject([list:json])); 35 | -------------------------------------------------------------------------------- /sbt.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for sbt-launch auto-installation 3 | import org.htmlunit.html.*; 4 | import org.htmlunit.WebClient 5 | import org.htmlunit.xml.XmlPage 6 | import hudson.util.VersionNumber 7 | import net.sf.json.* 8 | 9 | def listFromMaven() { 10 | String baseUrl = 'https://repo1.maven.org/maven2/org/scala-sbt/sbt' 11 | URL metaUrl = new URL("$baseUrl/maven-metadata.xml") 12 | 13 | WebClient wc = new WebClient() 14 | XmlPage meta = wc.getPage(metaUrl) 15 | 16 | List versions = meta.getByXPath("//metadata/versioning/versions/version") 17 | .collect() { DomElement e -> e.getTextContent() } 18 | .findAll() { e -> !e.contains('RC') } 19 | .reverse() 20 | 21 | return versions.collect() { version -> 22 | return ["id" : version, 23 | "name": version, 24 | "url" : getGithubArtifactUrl(version) 25 | ] 26 | } 27 | } 28 | 29 | def getGithubArtifactUrl(String version) { 30 | return String.format("https://github.com/sbt/sbt/releases/download/v%s/sbt-%s.zip", version, version) 31 | } 32 | 33 | def listAll() { 34 | List versions = new ArrayList() 35 | versions.addAll(listFromMaven()) 36 | 37 | return versions 38 | .findAll { it != null } 39 | .sort { o1,o2 -> 40 | try { 41 | def v1 = new VersionNumber(o1.id) 42 | try { 43 | new VersionNumber(o2.id).compareTo(v1) 44 | } catch (IllegalArgumentException _2) { 45 | -1 46 | } 47 | } catch (IllegalArgumentException _1) { 48 | try { 49 | new VersionNumber(o2.id) 50 | 1 51 | } catch (IllegalArgumentException _2) { 52 | o2.id.compareTo(o1.id) 53 | } 54 | } 55 | } 56 | } 57 | 58 | lib.DataWriter.write("org.jvnet.hudson.plugins.SbtPluginBuilder.SbtInstaller",JSONObject.fromObject([list:listAll()])); 59 | -------------------------------------------------------------------------------- /sbuild.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for SBuild auto-installation 3 | 4 | import net.sf.json.* 5 | import groovy.util.ConfigSlurper 6 | 7 | def url = "http://sbuild.org/download/sbuild-jenkins-plugin/sbuild-releases.properties".toURL() 8 | def text = url.text 9 | def json = [] 10 | 11 | text.split("\n").each { line -> 12 | def pair = line.split("=", 2) 13 | def version = pair[0] 14 | def dist = pair[1] 15 | json << ["id":version, "name":"SBuild ${version}".toString(), "url":dist]; 16 | } 17 | 18 | def result = JSONObject.fromObject([list:json]) 19 | 20 | lib.DataWriter.write("org.sbuild.jenkins.plugin.SBuildInstaller", result); 21 | -------------------------------------------------------------------------------- /scala.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Scala auto-installation 3 | import java.net.URI 4 | import net.sf.json.* 5 | import org.htmlunit.WebClient 6 | import org.htmlunit.html.* 7 | 8 | def wc = new WebClient() 9 | 10 | //Unfortunately we cannot get an index page from http://www.scala-lang.org/downloads/distrib/files/ so instead 11 | //we need to extract content from the HTML downloads page 12 | def baseUrl = 'http://www.scala-lang.org/files/archive/' 13 | def pathAndVersionRegex = /^(scala-\d+\.\d+(?:\.\d+)?(?:(?:-RC\d)|(?:\.final))?.tgz)$/ 14 | // def pathAndVersionRegex = /^\/downloads\/distrib\/files\/(scala-\d+\.\d+(?:\.\d+)?(?:(?:-RC\d)|(?:\.final))?.tgz)$/ 15 | 16 | HtmlPage p = wc.getPage(baseUrl) 17 | 18 | def json = []; 19 | 20 | p 21 | .getByXPath("//a[@href]") 22 | .grep({ 23 | it.getHrefAttribute() ==~ pathAndVersionRegex 24 | }) 25 | .collect { HtmlAnchor e -> 26 | def file = e.getHrefAttribute() 27 | def url = new URI(baseUrl).resolve(new URI(file)).toString() 28 | //println url 29 | json << ["id": file, "name": file.replaceFirst("scala-", "Scala ").replaceFirst(".tgz", ""), "url": url] 30 | } 31 | 32 | lib.DataWriter.write("hudson.plugins.scala.ScalaInstaller", JSONObject.fromObject([list: json])) 33 | -------------------------------------------------------------------------------- /scriptler.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates index for scriptler scripts 3 | import net.sf.json.* 4 | 5 | def repository = 'https://github.com/jenkinsci/jenkins-scripts' 6 | def branch = 'main' 7 | 8 | def json = []; 9 | 10 | def dir = new File("./jenkins-scripts"); 11 | 12 | void git(File workingDir, String... commands) { 13 | commands.each { cmd -> 14 | def fullCommand = "git -C $workingDir.canonicalPath $cmd" 15 | print "Executing \"${fullCommand}\"..." 16 | fullCommand.execute().waitFor() 17 | println ' done' 18 | } 19 | } 20 | 21 | if (dir.isDirectory() && new File(dir, ".git").isDirectory()) { 22 | git dir, 23 | "fetch origin $branch", 24 | "checkout origin/$branch" 25 | } else { 26 | dir.mkdirs() 27 | git dir, "clone $repository . -b $branch" 28 | } 29 | 30 | def scriptlerDir = new File(dir, "scriptler") 31 | 32 | scriptlerDir.eachFileMatch(~/.+\.groovy/) { File f -> 33 | if(f.name.equals('testMetaFormat.groovy')) { 34 | return 35 | } 36 | def m = (f.text =~ /(?ms)BEGIN META(.+?)END META/) 37 | if (m) { 38 | try { 39 | def metadata = JSONObject.fromObject(m[0][1]); 40 | metadata['script'] = f.name 41 | json << metadata 42 | } catch (Exception e) { 43 | e.printStackTrace(System.err); 44 | } 45 | } 46 | } 47 | 48 | lib.DataWriter.write("org.jenkinsci.plugins.scriptler.CentralScriptJsonCatalog",JSONObject.fromObject([list:json])); 49 | -------------------------------------------------------------------------------- /sonarqubescanner.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for Sonar Runner 3 | import org.htmlunit.WebClient 4 | import org.htmlunit.html.HtmlAnchor 5 | import org.htmlunit.html.HtmlPage 6 | import java.util.regex.Pattern 7 | import net.sf.json.JSONObject 8 | import hudson.util.VersionNumber 9 | 10 | def listFromOldCodehausURL() { 11 | def url = "https://repo1.maven.org/maven2/org/codehaus/sonar-plugins/sonar-runner/"; 12 | def wc = new WebClient() 13 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 14 | wc.getOptions().setJavaScriptEnabled(false); 15 | wc.getOptions().setThrowExceptionOnScriptError(false); 16 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 17 | HtmlPage p = wc.getPage(url); 18 | def pattern=Pattern.compile("^([0-9][0-9\\.]+)/\$"); 19 | 20 | return p.getAnchors().collect { HtmlAnchor a -> 21 | m = pattern.matcher(a.hrefAttribute) 22 | println(a.hrefAttribute) 23 | if(m.find()) { 24 | ver=m.group(1) 25 | url = p.getFullyQualifiedUrl(a.hrefAttribute + "sonar-runner-" + ver + ".zip"); 26 | return ["id":ver, "name": "SonarQube Scanner " + ver, "url":url.toExternalForm()] 27 | } 28 | return null; 29 | } 30 | } 31 | 32 | def listFromNewCodehausUrl() { 33 | def url = "https://repo1.maven.org/maven2/org/codehaus/sonar/runner/sonar-runner-dist/"; 34 | def wc = new WebClient() 35 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 36 | wc.getOptions().setJavaScriptEnabled(false); 37 | wc.getOptions().setThrowExceptionOnScriptError(false); 38 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 39 | HtmlPage p = wc.getPage(url); 40 | def pattern = Pattern.compile("^([0-9][0-9\\.]+)/"); 41 | 42 | return p.getAnchors().collect { HtmlAnchor a -> 43 | m = pattern.matcher(a.hrefAttribute) 44 | println(a.hrefAttribute) 45 | if(m.find()) { 46 | ver=m.group(1) 47 | url = p.getFullyQualifiedUrl(a.hrefAttribute + "sonar-runner-dist-" + ver + ".zip"); 48 | return ["id":ver, "name": "SonarQube Scanner " + ver, "url":url.toExternalForm()] 49 | } 50 | return null; 51 | } 52 | } 53 | 54 | def listFromNewSonarSourceUrl() { 55 | def url = "https://repo1.maven.org/maven2/org/sonarsource/scanner/cli/sonar-scanner-cli/"; 56 | def wc = new WebClient() 57 | wc.setCssErrorHandler(new org.htmlunit.SilentCssErrorHandler()); 58 | wc.getOptions().setJavaScriptEnabled(false); 59 | wc.getOptions().setThrowExceptionOnScriptError(false); 60 | wc.getOptions().setThrowExceptionOnFailingStatusCode(false); 61 | HtmlPage p = wc.getPage(url); 62 | def pattern = Pattern.compile("^([0-9][0-9\\.]+)/"); 63 | 64 | return p.getAnchors().collect { HtmlAnchor a -> 65 | m = pattern.matcher(a.hrefAttribute) 66 | println(a.hrefAttribute) 67 | if(m.find()) { 68 | ver=m.group(1) 69 | url = p.getFullyQualifiedUrl(a.hrefAttribute + "sonar-scanner-cli-" + ver + ".zip"); 70 | return ["id":ver, "name": "SonarQube Scanner " + ver, "url":url.toExternalForm()] 71 | } 72 | return null; 73 | } 74 | } 75 | 76 | def listAll() { 77 | return (listFromOldCodehausURL() + listFromNewCodehausUrl() + listFromNewSonarSourceUrl()) 78 | .findAll { it!=null }.sort { o1,o2 -> 79 | try { 80 | def v1 = new VersionNumber(o1.id) 81 | try { 82 | new VersionNumber(o2.id).compareTo(v1) 83 | } catch (IllegalArgumentException _2) { 84 | -1 85 | } 86 | } catch (IllegalArgumentException _1) { 87 | try { 88 | new VersionNumber(o2.id) 89 | 1 90 | } catch (IllegalArgumentException _2) { 91 | o2.id.compareTo(o1.id) 92 | } 93 | } 94 | } 95 | } 96 | 97 | def store(key,o) { 98 | JSONObject envelope = JSONObject.fromObject(["list": o]); 99 | lib.DataWriter.write(key,envelope); 100 | } 101 | 102 | store("hudson.plugins.sonar.SonarRunnerInstaller", listAll()) 103 | 104 | -------------------------------------------------------------------------------- /sonarqubescannermsbuild.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | // Generates server-side metadata for SonarScanner for MSBuild 3 | import net.sf.json.* 4 | 5 | def url = "https://api.github.com/repos/SonarSource/sonar-scanner-msbuild/releases?per_page=100".toURL() 6 | def releases = JSONArray.fromObject(url.text) 7 | 8 | def json = [] 9 | 10 | for (JSONObject release : releases) { 11 | def tagName = release.get("tag_name") 12 | if (!release.get("draft") && !release.get("prerelease") && !tagName.toLowerCase().contains("vsts")) { 13 | 14 | if (tagName.startsWith("1.") || tagName.equals("2.0") || tagName.equals("2.1")) { 15 | json << ["id": tagName, 16 | "name": "SonarScanner for MSBuild ${tagName}".toString(), 17 | "url": "https://github.com/SonarSource/sonar-scanner-msbuild/releases/download/${tagName}/MSBuild.SonarQube.Runner-${tagName}.zip".toString()]; 18 | 19 | } else if (tagName.startsWith("2.") || tagName.startsWith("3.") || tagName.startsWith("4.0.")) { 20 | json << ["id": tagName, 21 | "name": "SonarScanner for MSBuild ${tagName}".toString(), 22 | "url": "https://github.com/SonarSource/sonar-scanner-msbuild/releases/download/${tagName}/sonar-scanner-msbuild-${tagName}.zip".toString()]; 23 | 24 | } else if (tagName.startsWith("5.") || tagName.startsWith("4.")) { 25 | json << ["id": tagName, 26 | "name": "SonarScanner for MSBuild ${tagName} - .NET Fwk 4.6".toString(), 27 | "url": "https://github.com/SonarSource/sonar-scanner-msbuild/releases/download/${tagName}/sonar-scanner-msbuild-${tagName}-net46.zip".toString()]; 28 | 29 | json << ["id": "${tagName}-net5".toString(), 30 | "name": "SonarScanner for MSBuild ${tagName} - .NET 5.0".toString(), 31 | "url": "https://github.com/SonarSource/sonar-scanner-msbuild/releases/download/${tagName}/sonar-scanner-msbuild-${tagName}-net5.0.zip".toString()]; 32 | 33 | json << ["id": "${tagName}-netcore".toString(), 34 | "name": "SonarScanner for MSBuild ${tagName} - .NET Core 2.0".toString(), 35 | "url": "https://github.com/SonarSource/sonar-scanner-msbuild/releases/download/${tagName}/sonar-scanner-msbuild-${tagName}-netcoreapp2.0.zip".toString()]; 36 | 37 | json << ["id": "${tagName}-netcore3".toString(), 38 | "name": "SonarScanner for MSBuild ${tagName} - .NET Core 3.0".toString(), 39 | "url": "https://github.com/SonarSource/sonar-scanner-msbuild/releases/download/${tagName}/sonar-scanner-msbuild-${tagName}-netcoreapp3.0.zip".toString()]; 40 | 41 | } else { 42 | json << ["id": "${tagName}-net-framework".toString(), 43 | "name": "SonarScanner for .NET Framework ${tagName}".toString(), 44 | "url": "https://github.com/SonarSource/sonar-scanner-msbuild/releases/download/${tagName}/sonar-scanner-${tagName}-net-framework.zip".toString()]; 45 | 46 | json << ["id": "${tagName}-net".toString(), 47 | "name": "SonarScanner for .NET ${tagName}".toString(), 48 | "url": "https://github.com/SonarSource/sonar-scanner-msbuild/releases/download/${tagName}/sonar-scanner-${tagName}-net.zip".toString()]; 49 | 50 | } 51 | } 52 | } 53 | 54 | lib.DataWriter.write("hudson.plugins.sonar.MsBuildSonarQubeRunnerInstaller",JSONObject.fromObject([list:json])); 55 | -------------------------------------------------------------------------------- /terraform.groovy: -------------------------------------------------------------------------------- 1 | #!./lib/runner.groovy 2 | import org.htmlunit.html.*; 3 | import net.sf.json.* 4 | import org.htmlunit.WebClient 5 | 6 | def baseUrl = 'https://releases.hashicorp.com' 7 | def json = [] 8 | def wc = new WebClient() 9 | HtmlPage p = wc.getPage("${baseUrl}/terraform/") 10 | p.getByXPath("//a[@href]").grep { it.hrefAttribute =~ /\/terraform\/.*/ }.each { 11 | wc.getPage("${baseUrl}${it.hrefAttribute}").getByXPath("//a[@href]").each { 12 | def m = (it.textContent =~ /terraform.*(\d+.\d+.\d+)_(.*)_(.*).zip/) 13 | if (m) { 14 | def verId = "${m[0][1]}-${m[0][2]}-${m[0][3]}".toString() 15 | json << ["id": verId, "name": "Terraform ${m[0][1]} ${m[0][2]} (${m[0][3]})".toString(), "url": "${it.hrefAttribute}".toString()]; 16 | } 17 | } 18 | } 19 | 20 | lib.DataWriter.write("org.jenkinsci.plugins.terraform.TerraformInstaller", JSONObject.fromObject([list:json])); 21 | --------------------------------------------------------------------------------