package aQute.maven;
import java.io.*;
import java.lang.reflect.*;
import java.net.*;
import java.security.*;
import java.util.*;
import java.util.regex.*;
import javax.xml.stream.*;
import aQute.lib.hex.*;
import aQute.lib.io.*;
import aQute.lib.struct.*;
import aQute.service.reporter.*;
import aQute.struct.*;
/**
* Abstraction of a maven repository. A repository is based on a URL, the
* contents is addressed with the 4 coordinates:
*
* <pre>
* groupId an identifier for the group (. are replaced with /)
* artifactId an identifier for the project within the group
* version the version and qualifier
* classifier optional extension
* packaging the type of packaging
* </pre>
*
* Unfortunately, the path names are not self-identifying since there is no
* clear separator for the coordinates. Therefore, the pom plays the anchor. The
* pom is always identified with
* {@code groupPath/artifactId/version/artifactId-version.pom}. Removing the
* .pom part and matching the resulting stem the other files in the same
* directory allows one to find the other contents.
*/
public class MavenRepository {
public static Pattern MACRO = Pattern.compile("\\$\\{([-\\w\\d_.]+)\\}");
public static Pattern VALIDID = Pattern.compile("[-_\\d\\w.]+");
public static Pattern VALIDJARNAME = Pattern
.compile("([-_\\d\\w.]+)-([-_\\d\\w.]+)(:?-([_\\d\\w.]+))?\\.(jar|wab|war)$");
public static Pattern VALIDPOMNAME = Pattern.compile("([-_\\d\\w.]+)-([-_\\d\\w.]+)\\.pom$");
final LinkedHashMap<URI,PomImpl> parentPoms = new LinkedHashMap<URI,MavenRepository.PomImpl>();
public static class Content extends struct {
public String classifier;
public String packaging;
public URI url;
public byte[] md5;
public byte[] sha;
public long size;
}
public static class Project extends struct {
public String groupId;
public String artifactId;
public List<Release> releases = list();
}
public static class Release extends struct {
public String version;
public List<Content> content = list();
}
final String base;
HtmlPage root;
public MavenRepository(String base) throws Exception {
if (!base.endsWith("/"))
base += "/";
URI uri = new URI(base);
if (uri.isAbsolute())
this.base = base;
else
this.base = IO.work.toURI().toString() + uri;
}
public MavenRepository(URI uri) throws Exception {
this(uri.toString());
}
public Iterable<String> getGroupIds(String beginGroupId) {
return null;
// root.iterator();
}
Iterable<String> getArtifactIds(String groupId) {
return null;
}
Project getProject(String groupId, String artifactId) throws Exception {
return null;
}
Release getRelease(String groupId, String artifactId, String version) {
return null;
}
static public class PomImpl extends Pom implements Report {
void resolve() throws Exception {
resolve(this);
//
// Now handle dependencyManagement. We look through our dependencies
// and fill in any versions from the dependencyManagement field.
// Why on earth they do not flatten this when they write the repo
// is way beyond me :-(
//
if (dependencies == null || dependencies.isEmpty())
return;
if (dependencyManagement == null || dependencyManagement.dependencies == null
|| dependencyManagement.dependencies.isEmpty())
return;
Map<String,String> versions = new HashMap<String,String>();
for (Dependency d : dependencyManagement.dependencies) {
versions.put(d.groupId + ":" + d.artifactId, d.version);
}
for (Dependency d : dependencies) {
if (d.version == null || d.version.isEmpty()) {
d.version = versions.get(d.groupId + ":" + d.artifactId);
if (d.version == null)
errors.add("no version for dependency " + d.groupId + ":" + d.artifactId);
}
}
}
@SuppressWarnings({
"unchecked", "rawtypes"
})
void resolve(Object rover) throws Exception {
if (rover == null)
return;
if (rover instanceof struct) {
struct struct = (struct) rover;
for (Field f : struct.fields()) {
Object o = f.get(struct);
if (o == null)
continue;
if (o instanceof URI) {
String p = replace(((URI) o).toString().replaceAll("~~~~([^~]+)~~~~", "\\${$1}"));
f.set(struct, new URI(p));
} else if (o instanceof String) {
o = replace((String) o);
f.set(struct, o);
} else {
resolve(o);
}
}
return;
}
if (rover instanceof Collection) {
List<String> s = new ArrayList<String>();
boolean replaced = false;
Collection<String> c = (Collection<String>) rover;
for (Object oo : c) {
if (oo instanceof String) {
String r = replace((String) oo);
replaced |= r != null;
s.add(r);
} else {
resolve(oo);
}
}
if (replaced) {
c.clear();
c.addAll(s);
}
return;
}
if (rover instanceof Map) {
Map<String,Object> m = (Map) rover;
for (Map.Entry<String,Object> e : m.entrySet()) {
if (e.getValue() == null)
continue;
if (e.getValue() instanceof String) {
String r = replace((String) e.getValue());
if (r != null)
e.setValue(r);
} else {
resolve(e.getValue());
}
}
return;
}
}
private String replace(String o) throws Exception {
StringBuilder sb = new StringBuilder(o);
Matcher m = MACRO.matcher(sb);
int n = 0;
boolean found = false;
while (m.find(n)) {
found = true;
String key = m.group(1);
sb.delete(m.start(), m.end());
String r = find(key);
if (r != null) {
sb.insert(m.start(), r);
n = m.start() + r.length();
} else
n = m.start();
}
if (found)
return sb.toString();
return o;
}
private String find(String key) throws Exception {
if (key.startsWith("env."))
return null;
// Is system dependent, dangerous
if (key.startsWith("settings."))
return null;
// Some legacy shit that was found
if (key.equals("version") || key.equals("pom.version") || key.equals("pkgVersion"))
key = "project.version";
else if (key.equals("groupId") || key.equals("pom.groupId") || key.equals("pkgGroupId"))
key = "project.groupId";
else if (key.equals("artifactId") || key.equals("pom.artifactId") || key.equals("pkgArtifactId"))
key = "project.artifactId";
if (key.startsWith("pom."))
key = "project." + key.substring(4);
// Is system dependent, dangerous
if (key.startsWith("project.")) {
String ks = key.substring(8);
Field f = getField(ks);
if (f == null)
return null;
if (f.getType() != String.class)
return null;
String s = (String) f.get(this);
if (s == null || s.isEmpty())
return null;
if (s.indexOf("${") >= 0) {
f.set(this, ""); // cycles!
s = replace(s);
f.set(this, s);
}
return s;
}
if (properties == null)
return null;
String s = properties.get(key);
if (s == null)
return null;
properties.put(key, ""); // cycles
s = replace(s);
properties.put(key, s);
return s;
}
@Override
public List<String> getWarnings() {
return warnings;
}
@Override
public List<String> getErrors() {
return errors;
}
@Override
public Location getLocation(String msg) {
return null;
}
@Override
public boolean isOk() {
return errors.isEmpty();
}
}
public PomImpl getPom(String groupId, String artifactId, String version) throws Exception {
PomImpl pom = readEffective(getUrl(groupId, artifactId, null, version, "pom"));
pom.resolve();
return pom;
}
static PomImpl readPom(URI url) throws Exception {
try {
return readPom(url, "UTF-8");
}
catch (Exception e) {
return readPom(url, "ISO-8859-1");
}
}
static PomImpl readPom(URI url, String encoding) throws Exception {
int retries = 0;
while (true)
try {
URLConnection connection = url.toURL().openConnection();
InputStream in = connection.getInputStream();
long created = connection.getLastModified();
try {
DigestInputStream dis = new DigestInputStream(in, MessageDigest.getInstance("SHA-1"));
PomImpl pom = new XmlCodec().parse(IO.reader(dis, encoding));
pom.sha = dis.getMessageDigest().digest();
byte[] sha = getSha(url);
if (sha == null) {
pom.warnings.add("No sha found");
} else if (!Arrays.equals(sha, pom.sha)) {
pom.errors.add("SHA mismatch, read " + Hex.toHexString(pom.sha) + " found "
+ Hex.toHexString(sha));
}
pom.created = created;
if (pom.parent != null) {
if (pom.version == null && pom.parent.version != null)
pom.version = pom.parent.version;
if (pom.groupId == null && pom.parent.groupId != null)
pom.groupId = pom.parent.groupId;
}
pom.url = url;
return pom;
}
finally {
in.close();
}
}
catch (FileNotFoundException e) {
return null;
}
catch (XMLStreamException xse) {
return null;
}
catch (Exception e) {
if (retries++ < 2)
throw e;
Thread.sleep(1000);
}
}
private PomImpl readEffective(URI url) throws IOException, Exception {
PomImpl pom = readPom(url);
makeEffective(pom);
return pom;
}
private void makeEffective(Pom pom) throws Exception {
if (pom.parent != null && pom.parent.groupId != null && pom.parent.artifactId != null) {
URI url = getUrl(pom.parent.groupId, pom.parent.artifactId, null, pom.parent.version, "pom");
PomImpl parent = parentPoms.get(url);
if (parent == null) {
parent = readPom(url);
if (parent != null) {
makeEffective(parent);
if (parent.distributionManagement != null)
parent.distributionManagement.relocation = null;
parentPoms.put(url, parent);
if (parentPoms.size() > 200) {
URI first = parentPoms.keySet().iterator().next();
parentPoms.remove(first);
}
} else {
pom.errors.add("Could not read parent");
return;
}
}
StructUtil.merge(parent, pom, "parent", "packaging");
}
}
/**
* Read the corresponding SHA file
*
* @param url
* @return
* @throws URISyntaxException
*/
static Pattern SHA_MATCHER = Pattern.compile("([a-f0-9A-F]{40,40})");
public static byte[] getSha(URI url) {
try {
String u = url.toString() + ".sha1";
return getSha(new URL(u).openStream());
}
catch (Exception e) {
// we allow the caller to fallback
}
return null;
}
public static byte[] getSha(InputStream in) throws Exception {
String sha1 = IO.collect(in);
Matcher m = SHA_MATCHER.matcher(sha1);
if (m.find()) {
byte[] digest = Hex.toByteArray(m.group(1));
return digest;
} else {
return null;
}
}
public URI getUrl(String groupId, String artifactId, String classifier, String version, String packaging)
throws Exception {
String bad = validate(groupId, artifactId, classifier, version, packaging);
if (bad != null)
throw new IllegalArgumentException(bad);
StringBuilder sb = new StringBuilder(base);
sb.append(groupId.replace('.', '/')).append('/');
sb.append(artifactId).append('/');
sb.append(version).append('/');
sb.append(artifactId).append("-").append(version);
if (classifier != null) {
sb.append('-').append(classifier);
}
// hmm, bundle packaging = actually stored as jar file
// guess I am the cause since the bundle plugin is
// delivering it as a JAR file
if (packaging != null && !"bundle".equals(packaging)) {
sb.append('.');
sb.append(packaging);
} else {
sb.append(".jar");
}
return new URI(sb.toString());
}
public static String validate(String groupId, String artifactId, String classifier, String version, String packaging) {
if (groupId == null || artifactId == null || version == null)
return "Must have groupId, artifactId, and version non-null";
if (!isGroupId(groupId))
return "Not a valid groupId: " + groupId;
if (!isArtifactId(artifactId))
return "Not a valid artifactId: " + artifactId;
if (!isVersion(version))
return "Not a valid version: " + version;
if (classifier != null && !isClassifier(classifier))
return "Not a valid classifier: " + classifier;
return null;
}
public static boolean isGroupId(String groupId) {
return VALIDID.matcher(groupId).matches();
}
public static boolean isArtifactId(String artifactId) {
return VALIDID.matcher(artifactId).matches();
}
public static boolean isClassifier(String classifier) {
return VALIDID.matcher(classifier).matches();
}
public static boolean isVersion(String version) {
return VALIDID.matcher(version).matches();
}
public String getBase() {
return base;
}
/**
* Scan the maven index
*/
public Properties scanIndex(IndexEntryScanner scanner, Properties current) throws Exception {
Properties fresh = new Properties();
URL url = new URL(base + ".index/nexus-maven-repository-index.properties");
InputStream in = url.openStream();
try {
fresh.load(in);
}
finally {
in.close();
}
// nexus.index.chain-id: this is the chain-id of the current
// incremental items. If at any time this value changes from
// what
// the consumer has in its local properties file, the consumer
// should trigger a full .gz index download (and of course the
// properties file, to keep up to date)
if (current != null) {
String cChainId = current.getProperty("nexus.index.chain-id");
if (cChainId != null) {
String fChainId = fresh.getProperty("nexus.index.chain-id");
if (fChainId != null) {
if (fChainId.equals(cChainId)) {
// nexus.index.last-incremental: This is the
// last
// incremental
// item available, simply an integer that gets
// inserted
// into the
// download file name. If consumer has the same
// value in
// its
// local properties file, no need to download
// anything.
String cIncrement = current.getProperty("nexus.index.last-incremental");
String fIncrement = current.getProperty("nexus.index.last-incremental");
if (cIncrement != null && fIncrement != null) {
if (cIncrement.equals(fIncrement))
return null;
// nexus.index.incremental-X: These are the
// properties that list each incremental
// item
// available. The first item (where X = 0)
// is the
// oldest incremental piece that the
// provider still
// maintains. If the consumer’s local
// properties
// file contains a last-incremental value
// less than
// this, then need to download full .gz
// index (and
// properties file) and continue on.
// Otherwise,
// simply need to grab every
// nexus-maven-repository-index.X.gz file
// (where x
// is greater than your local
// last-incremental and
// less than or equal to the remote
// last-incremental) available from the
// provider.
int cInc = Integer.parseInt(cIncrement);
int fInc = Integer.parseInt(fIncrement);
for (int nInc = cInc + 1; nInc <= fInc; nInc++) {
if (!parse(new URI(base + ".index/nexus-maven-repository-index." + nInc + ".gz"),
scanner))
return null;
}
return fresh;
}
}
}
}
}
// Parse full
if (parse(new URI(base + ".index/nexus-maven-repository-index.gz"), scanner))
return fresh;
return null;
}
private boolean parse(URI index, IndexEntryScanner scanner) throws Exception {
InputStream in = index.toURL().openStream();
try {
NexusIndexReader nir = new NexusIndexReader(in);
while (nir.hasNext()) {
if (Thread.currentThread().isInterrupted())
return false;
IndexEntry entry = nir.next();
scanner.scan(entry);
}
return true;
}
finally {
in.close();
}
}
public static String getCoordinates(String groupId, String artifactId, String classifier, String version) {
StringBuilder sb = new StringBuilder(groupId).append(":").append(artifactId).append(":").append(version);
if (classifier != null)
sb.append(":").append(classifier);
return sb.toString();
}
/**
* Sometimes we only have a URL to an artifact and we want to find out if
* there is an associated POM. Since the layout group id is translated to an
* arbitrary number of segments it is impossible to decide what the root of
* the repository is from a URL. We there have to read the pom first, find
* out the groupId, and then calculate the root so we can find information
* about parent poms.
*
* @param url
* @return
*/
static Pattern MAVEN_URL = Pattern
.compile("(.*/([-.\\w\\d_]+)/([-.\\w\\d_]+))/([^/]+)");
static Pattern MAVEN_CLASSIFIER_URL = Pattern.compile("-([^-.]+)\\.pom$");
static LinkedHashMap<String,MavenRepository> repositories = new LinkedHashMap<String,MavenRepository>();
public static Pom getPomFromArtifactUrl(URI url) throws Exception {
String classifier = null;
String stem = url.toString();
int n = stem.lastIndexOf('.');
if (n > 0) {
stem = stem.substring(0, n);
String local[] = stem.split("/");
String prefix = local[local.length - 3] + "-" + local[local.length - 2] + "-";
String name = local[local.length - 1];
if (name.startsWith(prefix)) {
classifier = name.substring(prefix.length(), name.length());
stem = stem.substring(0, stem.length() - classifier.length() - 1);
}
}
URI uri = new URI(stem + ".pom");
Pom pom = getPom(uri);
if (pom != null)
pom.classifier = classifier;
return pom;
}
/**
* Keep a cache of repositories (which cache poms)
*
* @throws Exception
*/
public static MavenRepository getRepository(String base) throws Exception {
if (!base.endsWith("/"))
base = base + "/";
synchronized (repositories) {
MavenRepository mr = repositories.get(base);
if (mr == null) {
mr = new MavenRepository(base);
repositories.put(base, mr);
if (repositories.size() > 200) {
repositories.entrySet().iterator().remove();
}
}
return mr;
}
}
public static PomImpl getPom(URI pomUrl) throws Exception {
PomImpl pom = readPom(pomUrl);
if (pom != null) {
Matcher m = MAVEN_URL.matcher(pomUrl.toString());
if (m.matches()) {
String dir = m.group(1);
String groupId = pom.groupId;
if (groupId == null && pom.parent != null) {
groupId = pom.parent.groupId;
pom.errors.add("no groupId, trying parent groupId");
}
if (groupId == null) {
pom.errors.add("no parent, giving up, not resolved");
pom.groupId = "unknown";
} else {
String suffix = groupId.replace('.', '/') + "/" + pom.artifactId + "/" + pom.version;
if (dir.endsWith(suffix)) {
MavenRepository mr = getRepository(dir.substring(0, dir.length() - suffix.length()));
mr.makeEffective(pom);
pom.repository = new URI(mr.getBase());
}
}
pom.resolve();
}
}
return pom;
}
public enum VERIFY {
ACTUAL, SHA, MD5, ASC;
}
public static class Coordinates {
public String groupId;
public String artifactId;
public String classifier;
public String version;
public String extension;
public VERIFY verify = VERIFY.ACTUAL;
public String getPath() {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < groupId.length(); i++) {
char c = groupId.charAt(i);
if (c == '.')
sb.append('/');
else
sb.append(c);
}
sb.append('/').append(artifactId);
if (version == null)
sb.append("metadata.xml");
else
sb.append(version);
if (classifier != null) {
sb.append('-').append(classifier);
}
sb.append('.').append(extension);
switch (verify) {
case ACTUAL :
break;
case ASC :
sb.append(".asc");
break;
case MD5 :
sb.append(".md5");
break;
case SHA :
sb.append(".sha1");
break;
}
return sb.toString();
}
public String toString() {
StringBuilder sb = new StringBuilder();
if (groupId != null)
sb.append(groupId);
sb.append(":");
if (artifactId != null)
sb.append(artifactId);
if (classifier != null)
sb.append(":").append(classifier);
if (version != null)
sb.append("@").append(version);
return sb.toString();
}
}
/**
* Answer the coordinates of a file entry in a maven repository. It parses
* the last part of the URL and return an exact coordinate of that file
* entry or it throws an exception.
*
* @param path
* The path minus the repository part.
* @return a Coordinates identifying the file
* @throws Exception
* if anything is not right
*/
public static Coordinates getCoordinates(String path) throws Exception {
Coordinates coordinates = new Coordinates();
String parts[] = path.split("/");
if (parts.length < 3)
throw new FileNotFoundException();
int last = parts.length - 1;
String fname = parts[last];
if (fname.endsWith(".sha1")) {
coordinates.verify = VERIFY.SHA;
fname = fname.substring(0, fname.length() - 5);
} else if (fname.endsWith(".md5")) {
coordinates.verify = VERIFY.MD5;
fname = fname.substring(0, fname.length() - 4);
} else if (fname.endsWith(".asc")) {
coordinates.verify = VERIFY.ASC;
fname = fname.substring(0, fname.length() - 4);
}
if (fname.startsWith("metadata.xml")) {
coordinates.artifactId = parts[last - 1];
coordinates.groupId = getGroupId(parts, 0, last - 1);
return coordinates;
} else {
coordinates.version = parts[last - 1];
coordinates.artifactId = parts[last - 2];
coordinates.groupId = getGroupId(parts, 0, last - 2);
String prefix = coordinates.artifactId + "-" + coordinates.version;
if (!fname.startsWith(prefix))
throw new FileNotFoundException("prefix mismatch: " + path);
String suffix = fname.substring(prefix.length());
if (suffix.startsWith("-")) {
int n = suffix.indexOf('.');
if (n < 0)
throw new FileNotFoundException("classifier but no '.': " + path);
coordinates.classifier = suffix.substring(1, n);
suffix = suffix.substring(n);
}
if (!suffix.startsWith("."))
throw new FileNotFoundException("no '.' at the expected place: " + path);
coordinates.extension = suffix.substring(1);
return coordinates;
}
}
private static String getGroupId(String[] parts, int i, int j) {
String del = "";
StringBuilder sb = new StringBuilder();
for (; i < j; i++) {
sb.append(del).append(parts[i]);
del = ".";
}
return sb.toString();
}
/**
* Return the classifier from a jar name given the artifactId and version.
* This is a bit tricky because the encoding of the name is not canonical
* since a version could contain a pattern that would match a classifier
* pattern. So we we need to check based on the artifactId and version.
*
* @param jar
* the name of the binary
* @param artifactId
* @param version
* @return the classifier or null if no classifier embedded
*/
public static String getClassifier(String jar, String artifactId, String version) {
Matcher m = VALIDJARNAME.matcher(jar);
if (!m.matches())
throw new IllegalArgumentException("not a valid jar name '" + jar + "', should match "
+ VALIDJARNAME.pattern());
// Cannot use the regex to decide because the classifier
// cannot be discriminated from a version with an embedded
// -
int start = artifactId.length() + 1 + version.length();
if (jar.charAt(start) == '.')
return null;
int end = jar.lastIndexOf('.');
return jar.substring(start + 1, end);
}
public static String getPackaging(String jar) {
Matcher m = VALIDJARNAME.matcher(jar);
if (m.matches())
return m.group(4);
else
return null;
}
}