names = new ArrayList<>();
87 | while (!element.getKind().equals(ElementKind.PACKAGE)) {
88 | names.add(0, element.getSimpleName().toString());
89 | element = element.getEnclosingElement();
90 | }
91 | if (names.contains(null)) {
92 | return null;
93 | }
94 |
95 | return String.join(".", names);
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | 4.0.0
4 |
5 | org.jenkins-ci
6 | jenkins
7 | 1.142
8 |
9 |
10 | extension-indexer
11 |
12 | ${revision}${changelist}
13 | Extension Indexer
14 | List extension points and their implementations.
15 |
16 |
17 | 1.0
18 | -SNAPSHOT
19 | jenkins-infra/backend-${project.artifactId}
20 | ${project.basedir}/src/spotbugs/spotbugs-excludes.xml
21 |
22 |
23 |
24 |
25 |
26 | maven-jar-plugin
27 |
28 |
29 |
30 | true
31 | org.jenkinsci.extension_indexer.ExtensionPointListGenerator
32 |
33 |
34 |
35 |
36 |
37 | maven-assembly-plugin
38 |
39 |
40 |
41 | make-assembly
42 |
43 | single
44 |
45 |
46 | package
47 |
48 |
49 | src/assembly.xml
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 | args4j
62 | args4j
63 | 2.37
64 |
65 |
66 | org.jsoup
67 | jsoup
68 | 1.21.2
69 |
70 |
71 | org.kohsuke.stapler
72 | json-lib
73 | 2.4-jenkins-15
74 |
75 |
76 | commons-io
77 | commons-io
78 | 2.21.0
79 |
80 |
81 | org.apache.commons
82 | commons-lang3
83 | 3.20.0
84 |
85 |
86 | org.apache.maven.resolver
87 | maven-resolver-api
88 | 1.9.20
89 |
90 |
91 | com.github.spotbugs
92 | spotbugs-annotations
93 | true
94 |
95 |
96 |
97 |
98 |
99 | repo.jenkins-ci.org
100 | https://repo.jenkins-ci.org/public/
101 |
102 |
103 |
104 |
105 |
106 | repo.jenkins-ci.org
107 | https://repo.jenkins-ci.org/public/
108 |
109 |
110 |
111 |
112 | scm:git:https://github.com/${gitHubRepo}.git
113 | scm:git:git@github.com:${gitHubRepo}.git
114 | ${scmTag}
115 | https://github.com/${gitHubRepo}
116 |
117 |
118 |
--------------------------------------------------------------------------------
/src/main/java/org/jenkinsci/extension_indexer/ClassOfInterest.java:
--------------------------------------------------------------------------------
1 | package org.jenkinsci.extension_indexer;
2 |
3 | import com.sun.source.tree.ClassTree;
4 | import com.sun.source.tree.CompilationUnitTree;
5 | import com.sun.source.tree.ExpressionTree;
6 | import com.sun.source.util.JavacTask;
7 | import com.sun.source.util.TreePath;
8 | import com.sun.source.util.Trees;
9 | import net.sf.json.JSONObject;
10 | import org.jsoup.Jsoup;
11 | import org.jsoup.safety.Safelist;
12 |
13 | import javax.lang.model.element.TypeElement;
14 | import java.io.File;
15 | import java.util.HashMap;
16 | import java.util.HashSet;
17 | import java.util.Map;
18 | import java.util.Set;
19 | import java.util.regex.Matcher;
20 | import java.util.regex.Pattern;
21 |
22 | /**
23 | * Interesting thing that we pick up from plugin source code.
24 | * Common parts between {@link Extension} and {@link Action}.
25 | *
26 | * @author Vivek Pandey
27 | */
28 | public abstract class ClassOfInterest {
29 | /**
30 | * Back reference to the module where this implementation was found.
31 | */
32 | public final Module module;
33 |
34 | /**
35 | * The compiler session where this information was determined.
36 | */
37 | public final JavacTask javac;
38 |
39 | /**
40 | * {@link TreePath} that leads to {@link #implementation}
41 | */
42 | public final TreePath implPath;
43 |
44 | /**
45 | * Class of interest whose metadata is described in this object.
46 | *
47 | * This is an implementation of an extension point with {@code @Extension} on (for {@link Extension}
48 | * or it is a class that implements {@code Action}.
49 | */
50 | public final TypeElement implementation;
51 |
52 | /**
53 | * {@link Trees} object for {@link #javac}
54 | */
55 | public final Trees trees;
56 |
57 | /**
58 | * Jelly/groovy views associated to this class, including those defined for the ancestor types.
59 | *
60 | * Keyed by the view name (which is the base portion of the view file name). The value is the fully qualified
61 | * resource name.
62 | */
63 | public final Map views;
64 |
65 | ClassOfInterest(Module module, JavacTask javac, Trees trees, TypeElement implementation, TreePath implPath, Map views) {
66 | this.module = module;
67 | this.javac = javac;
68 | this.implPath = implPath;
69 | this.implementation = implementation;
70 | this.trees = trees;
71 | this.views = views;
72 | }
73 |
74 |
75 | /**
76 | * Returns the {@link ClassTree} representation of {@link #implementation}.
77 | */
78 | public ClassTree getClassTree() {
79 | return (ClassTree) implPath.getLeaf();
80 | }
81 |
82 | public CompilationUnitTree getCompilationUnit() {
83 | return implPath.getCompilationUnit();
84 | }
85 |
86 | /**
87 | * Gets the source file name that contains this definition, including directories
88 | * that match the package name portion.
89 | */
90 | public String getSourceFile() {
91 | ExpressionTree packageName = getCompilationUnit().getPackageName();
92 | String pkg = packageName == null ? "" : packageName.toString().replace('.', '/') + '/';
93 |
94 | String name = new File(getCompilationUnit().getSourceFile().getName()).getName();
95 | return pkg + name;
96 | }
97 |
98 | /**
99 | * Gets the line number in the source file where this implementation was defined.
100 | */
101 | public long getLineNumber() {
102 | return getCompilationUnit().getLineMap().getLineNumber(
103 | trees.getSourcePositions().getStartPosition(getCompilationUnit(), getClassTree()));
104 | }
105 |
106 | public String getJavadoc() {
107 | return javac.getElements().getDocComment(implementation);
108 | }
109 |
110 | /**
111 | * Javadoc excerpt converted to jenkins.io flavored Asciidoc markup.
112 | */
113 | public String getDocumentation() {
114 | String javadoc = getJavadoc();
115 | if (javadoc == null) return null;
116 |
117 | StringBuilder output = new StringBuilder(javadoc.length());
118 | for (String line : javadoc.split("\n")) {
119 | if (line.trim().length() == 0) break;
120 |
121 | if (line.trim().startsWith("@")) continue;
122 |
123 | {// replace @link
124 | Matcher m = LINK.matcher(line);
125 | StringBuilder sb = new StringBuilder();
126 | sb.append("+++");
127 | while (m.find()) {
128 | String simpleName = m.group(1);
129 | m.appendReplacement(sb, "+++ jenkinsdoc:"+simpleName+"[] +++");
130 | }
131 | m.appendTail(sb);
132 | sb.append("+++");
133 | line = sb.toString();
134 | }
135 |
136 | output.append(line).append(' ');
137 | }
138 |
139 | return Jsoup.clean(output.toString(), Safelist.basic());
140 | }
141 |
142 | /**
143 | * Returns the module Id of the plugin that it came from.
144 | */
145 | public String getArtifactId() {
146 | return module.artifactId;
147 | }
148 |
149 | private static final Pattern LINK = Pattern.compile("\\s*\\{@link ([^}]+)}\\s*");
150 |
151 | public String getImplementationName(){
152 | return implementation.getQualifiedName().toString();
153 | }
154 |
155 | public void addViews(Map views){
156 | this.views.putAll(views);
157 | }
158 |
159 | /**
160 | * Returns true if there are jelly files
161 | */
162 | public boolean hasView(){
163 | return views.size() > 0;
164 | }
165 |
166 |
167 | public JSONObject toJSON(){
168 | JSONObject i = new JSONObject();
169 | i.put("className",getImplementationName());
170 | i.put("module",module.gav);
171 | i.put("javadoc",getJavadoc());
172 | i.put("documentation", getDocumentation());
173 | i.put("sourceFile",getSourceFile());
174 | i.put("lineNumber",getLineNumber());
175 | i.put("hasView", hasView());
176 | Set> vs = new HashSet<>();
177 | for (Map.Entry entry : views.entrySet()) {
178 | Map v = new HashMap<>();
179 | v.put("name", entry.getKey());
180 | v.put("source", entry.getValue());
181 | vs.add(v);
182 | }
183 | i.put("views",vs);
184 | return i;
185 |
186 | }
187 | }
188 |
--------------------------------------------------------------------------------
/src/main/java/org/jenkinsci/extension_indexer/Module.java:
--------------------------------------------------------------------------------
1 | package org.jenkinsci.extension_indexer;
2 |
3 | import java.net.MalformedURLException;
4 | import java.net.URL;
5 | import java.util.ArrayList;
6 | import java.util.HashSet;
7 | import java.util.List;
8 | import java.util.Set;
9 |
10 | import org.apache.commons.lang3.StringUtils;
11 |
12 | import net.sf.json.JSONArray;
13 | import net.sf.json.JSONObject;
14 |
15 | /**
16 | * Information about the module that we scanned extensions.
17 | */
18 | abstract class Module implements Comparable {
19 | final String gav;
20 |
21 | final String artifactId;
22 | final String version;
23 | final String group;
24 |
25 | final String url;
26 | final String displayName;
27 | /**
28 | * Extension point or extensions that are found inside this module.
29 | */
30 | final List extensions = new ArrayList<>();
31 | /**
32 | * Actions that are found inside this module.
33 | */
34 | final List actions = new ArrayList<>();
35 |
36 | private static final String JENKINS_CORE_URL_NAME = "jenkins-core";
37 |
38 | private static String repositoryOrigin = "";
39 |
40 | public static String getRepositoryOrigin() {
41 | if (repositoryOrigin.isEmpty()){
42 | // Retrieve the env var containing the artifact caching proxy origin
43 | // or use the default https://repo.jenkins-ci.org origin
44 | repositoryOrigin = (System.getenv("ARTIFACT_CACHING_PROXY_ORIGIN") != null) ? System.getenv("ARTIFACT_CACHING_PROXY_ORIGIN") : "https://repo.jenkins-ci.org";
45 | }
46 | return repositoryOrigin;
47 | }
48 |
49 | protected Module(String gav, String url, String displayName) {
50 | this.gav = gav;
51 | this.url = url;
52 | this.displayName = simplifyDisplayName(displayName);
53 |
54 | String[] splitGav = gav.split(":", 3);
55 | this.group = splitGav[0];
56 | this.artifactId = splitGav[1];
57 | this.version = splitGav[2];
58 | }
59 |
60 | protected String simplifyDisplayName(String displayName) {
61 | if (displayName.equals("Jenkins Core")) {
62 | return displayName;
63 | }
64 | displayName = StringUtils.removeStartIgnoreCase(displayName, "Jenkins ");
65 | displayName = StringUtils.removeStartIgnoreCase(displayName, "Hudson ");
66 | displayName = StringUtils.removeEndIgnoreCase(displayName, " for Jenkins");
67 | displayName = StringUtils.removeEndIgnoreCase(displayName, " Plugin");
68 | displayName = StringUtils.removeEndIgnoreCase(displayName, " Plug-In");
69 | displayName = displayName + " Plugin"; // standardize spelling
70 | return displayName;
71 | }
72 |
73 | /**
74 | * Returns an Asciidoc (jenkins.io flavor) formatted link to point to this module.
75 | */
76 | abstract String getFormattedLink();
77 |
78 | abstract String getUrlName();
79 |
80 | public URL getSourcesUrl() throws MalformedURLException {
81 | return new URL(getRepositoryOrigin() + "/releases/" + group.replaceAll("\\.", "/") + "/" + artifactId + "/" + version + "/" + artifactId + "-" + version + "-sources.jar");
82 | }
83 |
84 | public URL getResolvedPomUrl() throws MalformedURLException {
85 | return new URL(getRepositoryOrigin() + "/releases/" + group.replaceAll("\\.", "/") + "/" + artifactId + "/" + version + "/" + artifactId + "-" + version + ".pom");
86 | }
87 |
88 | JSONObject toJSON() {
89 | JSONObject o = new JSONObject();
90 | o.put("gav",gav);
91 | o.put("url",url);
92 | o.put("displayName",displayName);
93 |
94 | Set defs = new HashSet<>();
95 |
96 | JSONArray extensions = new JSONArray();
97 | JSONArray actions = new JSONArray();
98 | JSONArray extensionPoints = new JSONArray();
99 | int viewCount=0;
100 | for (ExtensionSummary es : this.extensions) {
101 | (es.isDefinition ? extensionPoints : extensions).add(es.json);
102 | defs.add(es.family.definition);
103 |
104 | if(es.hasView){
105 | viewCount++;
106 | }
107 | }
108 |
109 | for(ActionSummary action:this.actions){
110 | JSONObject jsonObject = action.json;
111 | actions.add(jsonObject);
112 | if(action.hasView){
113 | viewCount++;
114 | }
115 | }
116 |
117 | if(actions.size() > 0 || extensions.size() > 0) {
118 | double viewScore = (double) viewCount / (extensions.size() + actions.size());
119 | o.put("viewScore", Double.parseDouble(String.format("%.2f", viewScore)));
120 | }
121 |
122 | o.put("extensions",extensions); // extensions defined in this module
123 | o.put("extensionPoints",extensionPoints); // extension points defined in this module
124 | o.put("actions", actions); // actions implemented in this module
125 |
126 | JSONArray uses = new JSONArray();
127 | for (ExtensionSummary es : defs) {
128 | if (es==null) continue;
129 | uses.add(es.json);
130 | }
131 | o.put("uses", uses); // extension points that this module consumes
132 |
133 | return o;
134 | }
135 |
136 | @Override
137 | public int compareTo(Module o) {
138 | String self = this.getUrlName();
139 | String other = o.getUrlName();
140 |
141 | if (other.equals(JENKINS_CORE_URL_NAME) || self.equals(JENKINS_CORE_URL_NAME)) {
142 | return self.equals(JENKINS_CORE_URL_NAME) ? (other.equals(JENKINS_CORE_URL_NAME) ? 0 : -1 ) : 1;
143 | } else {
144 | return this.displayName.compareToIgnoreCase(o.displayName);
145 | }
146 | }
147 |
148 |
149 | public static class PluginModule extends Module {
150 | public final String scm;
151 | protected PluginModule(String gav, String url, String displayName, String scm) {
152 | super(gav, url, displayName);
153 | this.scm = scm;
154 | }
155 |
156 | @Override
157 | String getFormattedLink() {
158 | return "plugin:" + artifactId + "[" + displayName + "]";
159 | }
160 |
161 | @Override
162 | String getUrlName() {
163 | return artifactId;
164 | }
165 | }
166 |
167 | public static class CoreModule extends Module {
168 | public CoreModule(String version) {
169 | super("org.jenkins-ci.main:jenkins-core:" + version, "http://github.com/jenkinsci/jenkins/", "Jenkins Core");
170 | }
171 |
172 | @Override
173 | String getFormattedLink() {
174 | // TODO different target
175 | return "link:https://github.com/jenkinsci/jenkins/[Jenkins Core]";
176 | }
177 |
178 | @Override
179 | String getUrlName() {
180 | return JENKINS_CORE_URL_NAME;
181 | }
182 | }
183 |
184 | }
185 |
--------------------------------------------------------------------------------
/src/main/java/org/jenkinsci/extension_indexer/ExtensionPointsExtractor.java:
--------------------------------------------------------------------------------
1 | package org.jenkinsci.extension_indexer;
2 |
3 | import com.sun.source.tree.ClassTree;
4 | import com.sun.source.tree.CompilationUnitTree;
5 | import com.sun.source.util.JavacTask;
6 | import com.sun.source.util.TreePath;
7 | import com.sun.source.util.TreePathScanner;
8 | import com.sun.source.util.Trees;
9 | import org.apache.commons.io.FilenameUtils;
10 |
11 | import javax.lang.model.element.TypeElement;
12 | import javax.lang.model.type.NoType;
13 | import javax.lang.model.type.TypeMirror;
14 | import javax.lang.model.util.Elements;
15 | import javax.lang.model.util.Types;
16 | import javax.tools.DiagnosticListener;
17 | import javax.tools.JavaCompiler;
18 | import javax.tools.JavaFileObject;
19 | import javax.tools.StandardJavaFileManager;
20 | import javax.tools.StandardLocation;
21 | import javax.tools.ToolProvider;
22 | import java.io.File;
23 | import java.io.IOException;
24 | import java.nio.charset.Charset;
25 | import java.util.ArrayList;
26 | import java.util.Collections;
27 | import java.util.HashMap;
28 | import java.util.List;
29 | import java.util.Locale;
30 | import java.util.Map;
31 |
32 | /**
33 | * Finds the defined extension points in a HPI.
34 | *
35 | * @author Robert Sandell
36 | * @author Kohsuke Kawaguchi
37 | */
38 | public class ExtensionPointsExtractor {
39 | public List extract(Module module, Module.CoreModule core) throws IOException, InterruptedException {
40 | return extract(module,SourceAndLibs.create(module, core));
41 | }
42 |
43 | public List extract(final Module module, final SourceAndLibs sal) throws IOException {
44 | StandardJavaFileManager fileManager = null;
45 | try {
46 | JavaCompiler javac1 = ToolProvider.getSystemJavaCompiler();
47 | DiagnosticListener errorListener = createErrorListener();
48 | fileManager = javac1.getStandardFileManager(errorListener, Locale.getDefault(), Charset.defaultCharset());
49 |
50 |
51 | fileManager.setLocation(StandardLocation.CLASS_PATH, sal.getClassPath());
52 |
53 | // annotation processing appears to cause the source files to be reparsed
54 | // (even though I couldn't find exactly where it's done), which causes
55 | // Tree symbols created by the original JavacTask.parse() call to be thrown away,
56 | // which breaks later processing.
57 | // So for now, don't perform annotation processing
58 | List options = List.of("-proc:none");
59 |
60 | Iterable files = fileManager.getJavaFileObjectsFromFiles(sal.getSourceFiles());
61 | JavaCompiler.CompilationTask task = javac1.getTask(null, fileManager, errorListener, options, null, files);
62 | final JavacTask javac = (JavacTask) task;
63 | final Trees trees = Trees.instance(javac);
64 | final Elements elements = javac.getElements();
65 | final Types types = javac.getTypes();
66 |
67 | Iterable parsed = javac.parse();
68 | javac.analyze();
69 |
70 | final List r = new ArrayList<>();
71 |
72 | // discover all compiled types
73 | TreePathScanner classScanner = new TreePathScanner() {
74 | final TypeElement extensionPoint = elements.getTypeElement("hudson.ExtensionPoint");
75 | final TypeElement action = elements.getTypeElement("hudson.model.Action");
76 |
77 | @Override
78 | public Void visitClass(ClassTree ct, Void ignored) {
79 | TreePath path = getCurrentPath();
80 | TypeElement e = (TypeElement) trees.getElement(path);
81 | if (e != null) {
82 | checkIfExtension(path, e, e);
83 | checkIfAction(path, e);
84 | }
85 | return super.visitClass(ct, ignored);
86 | }
87 |
88 | /**
89 | * If the class is an action, create a record for it.
90 | */
91 | private void checkIfAction(TreePath path, TypeElement e) {
92 | if (types.isSubtype(e.asType(), action.asType())) {
93 | r.add(new Action(module, javac, trees, e, path, collectViews(e)));
94 | }
95 | }
96 |
97 | /**
98 | * Recursively ascend the type hierarchy toward {@link Object} and find all extension points
99 | * {@code root} implement.
100 | */
101 | private void checkIfExtension(TreePath pathToRoot, TypeElement root, TypeElement e) {
102 | if (e==null) return; // if the compilation fails, this can happen
103 |
104 | for (TypeMirror i : e.getInterfaces()) {
105 | if (types.asElement(i).equals(extensionPoint)){
106 | r.add(new Extension(module, javac, trees, root, pathToRoot, e, collectViews(e)));
107 | }
108 | checkIfExtension(pathToRoot,root,(TypeElement)types.asElement(i));
109 | }
110 | TypeMirror s = e.getSuperclass();
111 | if (!(s instanceof NoType))
112 | checkIfExtension(pathToRoot,root,(TypeElement)types.asElement(s));
113 | }
114 |
115 | /**
116 | * Collect views recursively going up the ancestors.
117 | */
118 | private Map collectViews(TypeElement clazz) {
119 | Map views;
120 |
121 | TypeMirror s = clazz.getSuperclass();
122 | if (!(s instanceof NoType))
123 | views = collectViews((TypeElement)types.asElement(s));
124 | else
125 | views = new HashMap<>();
126 |
127 | for (String v : sal.getViewFiles(clazz.getQualifiedName().toString())) {
128 | // views defined in subtypes override those defined in the base type
129 | views.put(FilenameUtils.getBaseName(v),v);
130 | }
131 |
132 | return views;
133 | }
134 | };
135 |
136 | for( CompilationUnitTree u : parsed )
137 | classScanner.scan(u,null);
138 |
139 | return r;
140 | } catch (AssertionError e) {
141 | // javac has thrown this exception for some input
142 | System.err.println("Failed to analyze "+module.gav);
143 | e.printStackTrace();
144 | return Collections.emptyList();
145 | } finally {
146 | if (fileManager!=null)
147 | fileManager.close();
148 | sal.close();
149 | }
150 | }
151 |
152 | private void populateViewMap(List files, Map views){
153 | for(File f: files) {
154 | String fqName = f.getAbsolutePath();
155 | int loc = fqName.indexOf("src");
156 | if (loc > 0) {
157 | String path = fqName.substring(loc + 4);
158 | String[] a = path.split("/");
159 | String name = a[a.length - 1];
160 | int i = name.lastIndexOf(".");
161 | if(i > 0){
162 | name = name.substring(0,i);
163 | }
164 |
165 | views.putIfAbsent(name, path);
166 | } else {
167 | //We can't get here as jelly files are always stored inside src root
168 | }
169 | }
170 | }
171 |
172 | protected DiagnosticListener createErrorListener() {
173 | //TODO report
174 | return System.out::println;
175 | }
176 | }
177 |
--------------------------------------------------------------------------------
/src/main/java/org/jenkinsci/extension_indexer/SourceAndLibs.java:
--------------------------------------------------------------------------------
1 | package org.jenkinsci.extension_indexer;
2 |
3 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
4 |
5 | import org.apache.commons.io.FileUtils;
6 | import org.apache.commons.io.FilenameUtils;
7 | import org.apache.commons.io.IOUtils;
8 |
9 | import java.io.ByteArrayOutputStream;
10 | import java.io.Closeable;
11 | import java.io.File;
12 | import java.io.IOException;
13 | import java.io.InputStream;
14 | import java.io.OutputStream;
15 | import java.io.UnsupportedEncodingException;
16 | import java.nio.file.Files;
17 | import java.util.ArrayList;
18 | import java.util.Arrays;
19 | import java.util.Enumeration;
20 | import java.util.List;
21 | import java.util.Set;
22 | import java.util.jar.JarEntry;
23 | import java.util.jar.JarFile;
24 | import java.util.Base64;
25 | import java.net.URL;
26 | import java.net.URLConnection;
27 | import java.net.MalformedURLException;
28 |
29 | /**
30 | * Extracted source files and dependency jar files for a Maven project.
31 | *
32 | * @author Kohsuke Kawaguchi
33 | */
34 | public class SourceAndLibs implements Closeable {
35 | public final File srcDir;
36 | public final File libDir;
37 |
38 | /**
39 | * Lazily built list of all views in classpath.
40 | */
41 | private List allViews;
42 |
43 | public SourceAndLibs(File srcDir, File libDir) {
44 | this.srcDir = srcDir;
45 | this.libDir = libDir;
46 | }
47 |
48 | /**
49 | * Frees any resources allocated for this.
50 | * In particular, delete the files if they are temporarily extracted.
51 | */
52 | @Override
53 | public void close() throws IOException {
54 | }
55 |
56 | public List getClassPath() {
57 | return FileUtilsExt.getFileIterator(libDir, "jar");
58 | }
59 |
60 | public List getSourceFiles() {
61 | return FileUtilsExt.getFileIterator(srcDir, "java");
62 | }
63 |
64 | /**
65 | * Give list of view files in a given package.
66 | *
67 | * @param pkg
68 | * Package name like 'foo.bar'
69 | * @return
70 | * All view files in the qualified form, such as 'foo/bar/abc.groovy'
71 | */
72 | public List getViewFiles(String pkg) {
73 | List views = new ArrayList<>();
74 |
75 | pkg = pkg.replace('.', '/');
76 |
77 | // views in source files
78 | File[] srcViews = new File(srcDir,pkg).listFiles();
79 | if (srcViews!=null) {
80 | for (File f : srcViews) {
81 | if (VIEW_EXTENSIONS.contains(FilenameUtils.getExtension(f.getPath()))) {
82 | views.add(f.getName());
83 | }
84 | }
85 | }
86 |
87 | // views from dependencies
88 | if (allViews==null) {
89 | allViews = new ArrayList<>();
90 | for (File jar : getClassPath()) {
91 | try (JarFile jf = new JarFile(jar)) {
92 | Enumeration e = jf.entries();
93 | while (e.hasMoreElements()) {
94 | JarEntry je = e.nextElement();
95 | String n = je.getName();
96 | if (VIEW_EXTENSIONS.contains(FilenameUtils.getExtension(n))) {
97 | allViews.add(n);
98 | }
99 | }
100 | } catch (IOException x) {
101 | System.err.println("Failed to open "+jar);
102 | x.printStackTrace();
103 | }
104 | }
105 | }
106 |
107 | // 'foo/bar/zot.jelly' is a view but 'foo/bar/xxx/yyy.jelly' is NOT a view for 'foo/bar'
108 | String prefix = pkg+'/';
109 | for (String v : allViews) {
110 | if (v.startsWith(prefix)) {
111 | String rest = v.substring(prefix.length());
112 | if (!rest.contains("/"))
113 | views.add(v);
114 | }
115 | }
116 |
117 | return views;
118 | }
119 |
120 | private static byte[] auth;
121 |
122 | // Retrieve the auth from the artifact caching proxy Maven settings file
123 | private static byte[] getAuth() {
124 | if (auth == null && System.getenv("ARTIFACT_CACHING_PROXY_USERNAME") != null && System.getenv("ARTIFACT_CACHING_PROXY_PASSWORD") != null) {
125 | try {
126 | auth = Base64.getEncoder().encode((System.getenv("ARTIFACT_CACHING_PROXY_USERNAME") + ':' + System.getenv("ARTIFACT_CACHING_PROXY_PASSWORD")).getBytes("UTF-8"));
127 | } catch(UnsupportedEncodingException uee) {
128 | uee.printStackTrace();
129 | }
130 | }
131 | return auth;
132 | }
133 |
134 | private static URLConnection getURLConnection(URL url) throws MalformedURLException, IOException {
135 | String urlString = url.toString();
136 | URLConnection conn = new URL(urlString).openConnection();
137 | // If we're querying one of the artifact caching proxies we need to add authentication
138 | if (!urlString.startsWith("https://repo.jenkins-ci.org")) {
139 | conn.setRequestProperty("Accept-Charset", "UTF-8");
140 | conn.setRequestProperty("Accept-Encoding", "identity");
141 | conn.setRequestProperty("User-Agent", "backend-extension-indexer/0.1");
142 | conn.setRequestProperty("Authorization", "Basic " + new String(getAuth(), "UTF-8"));
143 | }
144 | return conn;
145 | }
146 |
147 | public static SourceAndLibs create(Module module, Module.CoreModule core) throws IOException, InterruptedException {
148 | final File tempDir = Files.createTempDirectory("jenkins-extPoint").toFile();
149 | File srcdir = new File(tempDir,"src");
150 | File libdir = new File(tempDir,"lib");
151 |
152 | System.out.println("Fetching " + module.getSourcesUrl());
153 |
154 | File sourcesJar = File.createTempFile("extension-indexer-" + module.artifactId, "-sources.jar");
155 | try (InputStream is = getURLConnection(module.getSourcesUrl()).getInputStream(); OutputStream os = Files.newOutputStream(sourcesJar.toPath())) {
156 | IOUtils.copy(is, os);
157 | }
158 | FileUtilsExt.unzip(sourcesJar, srcdir);
159 |
160 | System.out.println("Fetching " + module.getResolvedPomUrl());
161 | try (InputStream is = getURLConnection(module.getResolvedPomUrl()).getInputStream(); OutputStream os = Files.newOutputStream(new File(srcdir, "pom.xml").toPath())) {
162 | IOUtils.copy(is, os);
163 | }
164 |
165 | System.out.println("Downloading Dependencies");
166 | downloadDependencies(srcdir, libdir, core);
167 |
168 | return new SourceAndLibs(srcdir, libdir) {
169 | @Override
170 | public void close() throws IOException {
171 | FileUtils.deleteDirectory(tempDir);
172 | }
173 | };
174 | }
175 |
176 | @SuppressFBWarnings(value = "COMMAND_INJECTION", justification = "Command injection is not a viable risk here")
177 | private static void downloadDependencies(File pomDir, File destDir, Module.CoreModule core) throws IOException, InterruptedException {
178 | Files.createDirectories(destDir.toPath());
179 | String process = "mvn";
180 | if (System.getenv("M2_HOME") != null) {
181 | process = System.getenv("M2_HOME") + "/bin/mvn";
182 | }
183 | List command = new ArrayList<>();
184 | command.add(process);
185 | command.addAll(Arrays.asList("--settings", (System.getenv("MAVEN_SETTINGS") != null) ? System.getenv("MAVEN_SETTINGS") : new File("maven-settings.xml").getAbsolutePath()));
186 |
187 | command.addAll(Arrays.asList("--update-snapshots",
188 | "--batch-mode",
189 | "org.apache.maven.plugins:maven-dependency-plugin:3.8.0:copy-dependencies",
190 | "org.apache.maven.plugins:maven-dependency-plugin:3.8.0:copy",
191 | "-Dartifact=" + core.gav,
192 | "-DincludeScope=compile",
193 | "-DoutputDirectory=" + destDir.getAbsolutePath()));
194 |
195 | ProcessBuilder builder = new ProcessBuilder(command);
196 | builder.environment().put("JAVA_HOME",System.getProperty("java.home"));
197 | builder.directory(pomDir);
198 | builder.redirectErrorStream(true);
199 | Process proc = builder.start();
200 |
201 | // capture the output, but only report it in case of an error
202 | ByteArrayOutputStream output = new ByteArrayOutputStream();
203 | proc.getOutputStream().close();
204 | IOUtils.copy(proc.getInputStream(), output);
205 | proc.getErrorStream().close();
206 | proc.getInputStream().close();
207 |
208 | int result = proc.waitFor();
209 | if (result != 0) {
210 | System.out.write(output.toByteArray());
211 | throw new IOException("Maven didn't like this (exit code=" + result + ")! " + pomDir.getAbsolutePath());
212 | }
213 | }
214 |
215 | private static final Set VIEW_EXTENSIONS = Set.of("jelly", "groovy");
216 | }
217 |
--------------------------------------------------------------------------------
/src/main/java/org/jenkinsci/extension_indexer/ExtensionPointListGenerator.java:
--------------------------------------------------------------------------------
1 | package org.jenkinsci.extension_indexer;
2 |
3 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
4 |
5 | import java.io.BufferedReader;
6 | import java.io.File;
7 | import java.io.IOException;
8 | import java.io.InputStream;
9 | import java.io.InputStreamReader;
10 | import java.io.PrintWriter;
11 | import java.io.Reader;
12 | import java.nio.charset.StandardCharsets;
13 | import java.net.URL;
14 | import java.nio.file.Files;
15 | import java.util.ArrayList;
16 | import java.util.Collection;
17 | import java.util.Collections;
18 | import java.util.Comparator;
19 | import java.util.HashMap;
20 | import java.util.HashSet;
21 | import java.util.List;
22 | import java.util.Map;
23 | import java.util.Map.Entry;
24 | import java.util.Set;
25 | import java.util.SortedSet;
26 | import java.util.TreeMap;
27 | import java.util.TreeSet;
28 | import java.util.concurrent.ExecutorService;
29 | import java.util.concurrent.Executors;
30 | import java.util.concurrent.Future;
31 |
32 | import org.apache.commons.io.FileUtils;
33 | import org.apache.commons.io.FilenameUtils;
34 | import org.apache.commons.io.IOUtils;
35 | import org.kohsuke.args4j.Argument;
36 | import org.kohsuke.args4j.CmdLineParser;
37 | import org.kohsuke.args4j.Option;
38 |
39 | import net.sf.json.JSONArray;
40 | import net.sf.json.JSONObject;
41 |
42 | /**
43 | * Command-line tool to list up extension points and their implementations into a JSON file.
44 | *
45 | * @author Kohsuke Kawaguchi
46 | */
47 | public class ExtensionPointListGenerator {
48 | /**
49 | * All known {@link Family}s keyed by {@link Family#definition}'s FQCN.
50 | */
51 | private final Map families = new HashMap<>();
52 | /**
53 | * All the modules we scanned keyed by its {@link Module#artifactId}
54 | */
55 | private final Map modules = Collections.synchronizedMap(new HashMap<>());
56 |
57 | @Option(name="-adoc",usage="Generate the extension list index and write it out to the specified directory.")
58 | public File asciidocOutputDir;
59 |
60 | @Option(name="-json",usage="Generate extension points, implementatoins, and their relationships in JSON")
61 | public File jsonFile;
62 |
63 | @Option(name="-plugins",usage="Collect *.hpi/jpi into this directory")
64 | public File pluginsDir;
65 |
66 | @Option(name="-updateCenterJson",usage="Update center's json")
67 | public String updateCenterJsonFile = "https://updates.jenkins.io/current/update-center.actual.json";
68 |
69 | @Argument
70 | public List args = new ArrayList<>();
71 |
72 | private ExtensionPointsExtractor extractor = new ExtensionPointsExtractor();
73 |
74 | private Comparator IMPLEMENTATION_SORTER = new Comparator<>() {
75 | @Override
76 | public int compare(ExtensionSummary o1, ExtensionSummary o2) {
77 | int moduleOrder = o1.module.compareTo(o2.module);
78 | if (moduleOrder != 0) {
79 | return moduleOrder;
80 | }
81 | if (o1.className == null || o2.className == null) {
82 | return o1.className == null ? (o2.className == null ? 0 : 1) : -1;
83 | }
84 | return o1.className.compareTo(o2.className);
85 | }
86 | };
87 |
88 | /**
89 | * Relationship between definition and implementations of the extension points.
90 | */
91 | public class Family implements Comparable {
92 | // from definition
93 | ExtensionSummary definition;
94 | private final SortedSet implementations = new TreeSet<>(IMPLEMENTATION_SORTER);
95 |
96 | public String getName() {
97 | return definition.extensionPoint;
98 | }
99 |
100 | public String getShortName() {
101 | return definition.className;
102 | }
103 |
104 | void formatAsAsciidoc(PrintWriter w) {
105 | w.println();
106 | w.println("## " + getShortName().replace(".", ".++++++"));
107 | if ("jenkins-core".equals(definition.module.artifactId)) {
108 | w.println("`jenkinsdoc:" + definition.extensionPoint + "[]`");
109 | } else {
110 | w.println("`jenkinsdoc:" + definition.module.artifactId + ":" + definition.extensionPoint + "[]`");
111 | }
112 | w.println();
113 | w.println(definition.documentation == null || formatJavadoc(definition.documentation).trim().isEmpty() ? "_This extension point has no Javadoc documentation._" : formatJavadoc(definition.documentation));
114 | w.println();
115 | w.println("**Implementations:**");
116 | w.println();
117 | for (ExtensionSummary e : implementations) {
118 | w.print("* " + e.module.getFormattedLink() + ": ");
119 | if (e.implementation == null || e.implementation.trim().isEmpty()) {
120 | w.print("Anonymous class in " + (e.packageName + ".**" + e.topLevelClassName).replace(".", ".++++++") + "**");
121 | } else {
122 | w.print((e.packageName + ".**" + e.className + "**").replace(".", ".++++++"));
123 | }
124 | w.println(" " + getSourceReference(e));
125 | }
126 | if (implementations.isEmpty())
127 | w.println("_(no known implementations)_");
128 | w.println();
129 | }
130 |
131 | public String getSourceReference(ExtensionSummary e) {
132 | String artifactId = e.module.artifactId;
133 | if (artifactId.equals("jenkins-core")) {
134 | return "(link:https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/" + e.packageName.replace(".", "/") + "/" + e.topLevelClassName + ".java" + "[view on GitHub])";
135 | } else if (e.module instanceof Module.PluginModule) {
136 | String scmUrl = ((Module.PluginModule) e.module).scm;
137 | if (scmUrl != null) {
138 | if (scmUrl.contains("github.com")) { // should be limited to GitHub URLs, but best to be safe
139 | return "(link:" + scmUrl + "/search?q=" + e.className + "&type=Code[view on GitHub])";
140 | }
141 | }
142 | }
143 | return "";
144 | }
145 |
146 | private String formatJavadoc(String javadoc) {
147 | if (javadoc == null || javadoc.trim().isEmpty()) {
148 | return "";
149 | }
150 | StringBuilder formatted = new StringBuilder();
151 |
152 | for (String line : javadoc.split("\n")) {
153 | line = line.trim();
154 | if (line.startsWith("@author")) {
155 | continue;
156 | }
157 | if (line.startsWith("@since")) {
158 | continue;
159 | }
160 | formatted.append(line + "\n");
161 | }
162 |
163 | return formatted.toString();
164 | }
165 |
166 | private String getModuleLink(ExtensionSummary e) {
167 | final Module m = e.module;
168 | if (m==null)
169 | throw new IllegalStateException("Unable to find module for "+e.module.artifactId);
170 | return m.getFormattedLink();
171 | }
172 |
173 | @Override
174 | public int compareTo(Family that) {
175 | return this.getShortName().compareTo(that.getShortName());
176 | }
177 | }
178 |
179 | private Module addModule(Module m) {
180 | modules.put(m.artifactId,m);
181 | return m;
182 | }
183 |
184 | public static void main(String[] args) throws Exception {
185 | ExtensionPointListGenerator app = new ExtensionPointListGenerator();
186 | CmdLineParser p = new CmdLineParser(app);
187 | p.parseArgument(args);
188 | app.run();
189 | }
190 |
191 | public JSONObject getJsonUrl(String url) throws IOException {
192 | try (
193 | InputStream is = new URL(url).openStream();
194 | InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
195 | BufferedReader bufferedReader = new BufferedReader(isr)
196 | ) {
197 | String readLine;
198 | StringBuilder sb = new StringBuilder();
199 | while ((readLine = bufferedReader.readLine()) != null) {
200 | sb.append(readLine);
201 | }
202 | return JSONObject.fromObject(sb.toString());
203 | }
204 | }
205 |
206 | public void run() throws Exception {
207 | JSONObject updateCenterJson = getJsonUrl(updateCenterJsonFile);
208 |
209 | if (asciidocOutputDir ==null && jsonFile==null && pluginsDir ==null)
210 | throw new IllegalStateException("Nothing to do. Either -adoc, -json, or -pipeline is needed");
211 |
212 | Module.CoreModule coreModule = new Module.CoreModule(updateCenterJson.getJSONObject("core").getString("version"));
213 | discover(addModule(coreModule), coreModule);
214 |
215 | processPlugins(updateCenterJson.getJSONObject("plugins").values(), coreModule);
216 |
217 | if (jsonFile!=null) {
218 | JSONObject all = new JSONObject();
219 | for (Family f : families.values()) {
220 | if (f.definition==null) continue; // skip undefined extension points
221 | JSONObject o = JSONObject.fromObject(f.definition.json);
222 |
223 | JSONArray use = new JSONArray();
224 | for (ExtensionSummary impl : f.implementations)
225 | use.add(impl.json);
226 | o.put("implementations", use);
227 |
228 | all.put(f.getName(),o);
229 | }
230 |
231 | // this object captures information about modules where extensions are defined/found.
232 | final JSONObject artifacts = new JSONObject();
233 | for (Module m : modules.values()) {
234 | artifacts.put(m.gav, m.toJSON());
235 | }
236 |
237 | JSONObject container = new JSONObject();
238 | container.put("extensionPoints",all);
239 | container.put("artifacts",artifacts);
240 |
241 | Files.writeString(jsonFile.toPath(), container.toString(2), StandardCharsets.UTF_8);
242 | }
243 |
244 | if (asciidocOutputDir !=null) {
245 | generateAsciidocReport();
246 | }
247 | }
248 |
249 | /**
250 | * Walks over the plugins, record {@link #modules} and call {@link #discover(Module, Module.CoreModule)}.
251 | * @param plugins
252 | */
253 | private void processPlugins(Collection plugins, Module.CoreModule core) throws Exception {
254 | int availableProcessors = Runtime.getRuntime().availableProcessors();
255 | int nThreads = availableProcessors * 3;
256 | System.out.printf("Running with %d threads%n", nThreads);
257 | ExecutorService svc = Executors.newFixedThreadPool(nThreads);
258 | try {
259 | Set> futures = new HashSet<>();
260 | for (final JSONObject plugin : plugins) {
261 | final String artifactId = plugin.getString("name");
262 | if (!args.isEmpty() && !args.contains(artifactId)) {
263 | continue; // skip
264 | }
265 |
266 | futures.add(svc.submit(new Runnable() {
267 | @Override
268 | public void run() {
269 | try {
270 | System.out.println(artifactId);
271 | if (asciidocOutputDir !=null || jsonFile!=null) {
272 | Module pluginModule = addModule(new Module.PluginModule(plugin.getString("gav"), plugin.getString("url"), plugin.getString("title"), plugin.optString("scm")));
273 | discover(pluginModule, core);
274 | }
275 | if (pluginsDir!=null) {
276 | FileUtils.copyURLToFile(
277 | new URL(plugin.getString("url")),
278 | new File(pluginsDir, FilenameUtils.getName(plugin.getString("url")))
279 | );
280 | }
281 | } catch (Exception e) {
282 | System.err.println("Failed to process "+artifactId);
283 | // TODO record problem with this plugin so we can report on it
284 | e.printStackTrace();
285 | }
286 | }
287 | }));
288 | }
289 | for (Future f : futures) {
290 | f.get();
291 | }
292 | } finally {
293 | svc.shutdown();
294 | }
295 | }
296 |
297 | private void generateAsciidocReport() throws IOException {
298 | Map> byModule = new TreeMap<>();
299 | for (Family f : families.values()) {
300 | if (f.definition==null) continue; // skip undefined extension points
301 |
302 | Module key = f.definition.module;
303 | List value = byModule.computeIfAbsent(key, unused -> new ArrayList<>());
304 | value.add(f);
305 | }
306 |
307 | Files.createDirectories(asciidocOutputDir.toPath());
308 |
309 | try (Reader r = new InputStreamReader(getClass().getResourceAsStream("index-preamble.txt"), StandardCharsets.UTF_8);
310 | PrintWriter w = new PrintWriter(new File(asciidocOutputDir, "index.adoc"), StandardCharsets.UTF_8)) {
311 | IOUtils.copy(r, w);
312 | for (Entry> e : byModule.entrySet()) {
313 | w.println();
314 | w.println("* link:" + e.getKey().getUrlName() + "[Extension points defined in " + e.getKey().displayName + "]");
315 | }
316 | }
317 |
318 | for (Entry> e : byModule.entrySet()) {
319 | List fam = e.getValue();
320 | Module m = e.getKey();
321 | Collections.sort(fam);
322 | try (Reader r = new InputStreamReader(getClass().getResourceAsStream("component-preamble.txt"), StandardCharsets.UTF_8);
323 | PrintWriter w = new PrintWriter(new File(asciidocOutputDir, m.getUrlName() + ".adoc"), StandardCharsets.UTF_8)) {
324 | IOUtils.copy(r, w);
325 | w.println("# Extension Points defined in " + m.displayName);
326 | w.println();
327 | w.println(m.getFormattedLink());
328 | for (Family f : fam) {
329 | f.formatAsAsciidoc(w);
330 | }
331 | }
332 | }
333 | }
334 |
335 | private void discover(Module m, Module.CoreModule core) throws IOException, InterruptedException {
336 | if (asciidocOutputDir !=null || jsonFile!=null) {
337 | for (ClassOfInterest e : extractor.extract(m, core)) {
338 | synchronized (families) {
339 | System.out.println("Found "+e);
340 |
341 | if (e instanceof Extension) {
342 | Extension ee = (Extension) e;
343 | String key = ee.extensionPoint.getQualifiedName().toString();
344 |
345 | Family f = families.get(key);
346 | if (f==null) families.put(key,f=new Family());
347 |
348 | ExtensionSummary es = new ExtensionSummary(f, ee);
349 | m.extensions.add(es);
350 | if (ee.isDefinition()) {
351 | assert f.definition == null;
352 | f.definition = es;
353 | } else {
354 | f.implementations.add(es);
355 | }
356 | }else if(e instanceof Action){
357 | m.actions.add(new ActionSummary((Action)e));
358 | }
359 | }
360 | }
361 | }
362 | }
363 | }
364 |
--------------------------------------------------------------------------------