package aQute.library.bnd2;
import java.awt.datatransfer.*;
import java.io.*;
import java.lang.reflect.*;
import java.net.*;
import java.security.*;
import java.util.*;
import java.util.Map.Entry;
import java.util.jar.*;
import java.util.regex.*;
import aQute.bnd.header.*;
import aQute.bnd.osgi.*;
import aQute.bnd.osgi.Descriptors.PackageRef;
import aQute.bnd.service.*;
import aQute.bnd.version.*;
import aQute.lib.collections.*;
import aQute.lib.converter.*;
import aQute.lib.deployer.*;
import aQute.lib.hex.*;
import aQute.lib.io.*;
import aQute.lib.json.*;
import aQute.lib.justif.*;
import aQute.lib.settings.*;
import aQute.libg.command.*;
import aQute.libg.cryptography.*;
import aQute.library.remote.*;
import aQute.service.library.*;
import aQute.service.library.Library.Deposit;
import aQute.service.library.Library.Phase;
import aQute.service.library.Library.Program;
import aQute.service.library.Library.Requirement;
import aQute.service.library.Library.Revision;
import aQute.service.library.Library.RevisionRef;
import aQute.service.library.SharedTypes.Contributor;
import aQute.service.library.SharedTypes.Developer;
import aQute.service.library.SharedTypes.License;
import aQute.service.reporter.*;
import aQute.struct.struct.Error;
/**
* A bnd repository based upon the Remote Library. It extends the FileRepo and
* will use its super class to do the normal file stuff.
*/
public class Repository extends FileRepo implements Plugin, Closeable, Refreshable, Actionable {
static Pattern SHA = Pattern.compile("([A-F0-9][a-fA-F0-9]){20,20}",
Pattern.CASE_INSENSITIVE);
final Set<String> notfound = new HashSet<String>();
static Pattern COORDINATE_P = Pattern
.compile("([-\\w_.\\d]+):([-\\w_.\\d]+)[:@]([-\\w_.\\d]+)");
RemoteLibrary library;
File dir = null;
final static JSONCodec codec = new JSONCodec();
final static Comparator<Version> REVERSE_COMPARATOR = Collections.reverseOrder();
final Justif j = new Justif(40, new int[] {
20, 28, 36, 44
});
final Settings settings = new Settings();
// If exists, is the refresh time
File refresh;
boolean canwrite;
static {
// Hmm, some idiot (that was me) though it a good idea
// to allo hex/base64 and made it optional. :-( This
// bad idea was justly removed but now sometimes
// we link to an old bndlib ... so we ensure it is
// hex. If this fails, then we have a modern bndlib
// and that always uses hex.
try {
Method m = JSONCodec.class.getMethod("setHex", boolean.class);
m.invoke(codec, true);
}
catch (Exception e) {
// ignore
}
}
interface Options {
URL url();
String domain();
String depository();
String email();
}
Options options;
Reporter reporter;
private Map<String,Program> programCache = new WeakHashMap<String,Library.Program>();
@Override
public File get(String bsn, Version version, Map<String,String> attrs, final DownloadListener... downloadListeners)
throws Exception {
// Check if we already have it in our cache/file repo
File file = super.get(bsn, version, attrs, downloadListeners);
if (file != null)
return file;
// Get the file name of the file when it is downloaded
final File f = super.getLocal(bsn, version, attrs);
// Find the revision for this bsn+version combination
RevisionRef ref = getRevisionRef(bsn, version);
if (ref == null)
return null;
download(ref.url.toURL(), f, ref, downloadListeners);
return f;
}
void download(final URL url, final File f, final RevisionRef r, final DownloadListener... listeners)
throws Exception {
if (listeners == null || listeners.length == 0) {
download(f, r);
} else {
Thread t = new Thread("downloading " + url) {
public void run() {
try {
download(f, r);
for (DownloadListener d : listeners) {
try {
d.success(f);
}
catch (Exception ee) {
// ignore
}
}
}
catch (Throwable e) {
for (DownloadListener d : listeners) {
try {
d.failure(f, "Downloading " + url + " : " + e.toString());
}
catch (Exception ee) {
// ignore
}
}
}
}
};
t.start();
}
}
private void download(final File f, final RevisionRef r) throws IOException, NoSuchAlgorithmException, Exception {
f.getParentFile().mkdirs();
File tmp = IO.createTempFile(f.getParentFile(), "tmp", ".tmp");
try {
IO.copy(r.url.toURL(), tmp);
if (r.revision != null) {
byte[] digest = SHA1.digest(tmp).digest();
if (!Arrays.equals(r.revision, digest))
throw new IllegalStateException("Shas did not match for " + r.url + "("
+ Hex.toHexString(r.revision) + ") and " + tmp + " (" + Hex.toHexString(digest) + ")");
}
IO.rename(tmp, f);
File meta = new File(f.getAbsolutePath() + ".json");
codec.enc().to(meta).put(r);
}
finally {
// tmp.delete();
}
}
@Override
public boolean canWrite() {
return canwrite;
}
@Override
public PutResult put(InputStream in, PutOptions options) throws Exception {
try {
Deposit depo = new Library.Deposit();
depo.depository = this.options.depository();
depo.domain = this.options.domain();
depo.email = settings.getEmail();
depo.type = options.type;
List<Error> s = depo.validate();
if (!s.isEmpty())
throw new IllegalArgumentException("Invalid request: " + s.toString());
File f = File.createTempFile("depo-", ".jar");
try {
MessageDigest md = MessageDigest.getInstance("SHA1");
DigestInputStream din = new DigestInputStream(in, md);
IO.copy(din, f);
byte[] sha = md.digest();
if (options.digest != null && !Arrays.equals(sha, depo.sha))
throw new IllegalArgumentException("Indicated SHA does not match input stream");
depo.sha = sha;
FileInputStream fin = new FileInputStream(f);
try {
Revision r = library.deposit(depo, fin);
if (r == null)
throw new IllegalArgumentException("No result returned");
if (!Arrays.equals(r._id, sha))
throw new IllegalArgumentException("Mismatch in shas " + Hex.toHexString(r._id) + "!="
+ Hex.toHexString(options.digest));
PutResult putr = new PutResult();
putr.artifact = r.url;
putr.digest = r._id;
return putr;
}
finally {
fin.close();
}
}
finally {
f.delete();
}
}
finally {
in.close();
}
}
@Override
public List<String> list(String regex) throws Exception {
if (regex == null || regex.trim().isEmpty()) {
List<String> list = super.list(regex);
for (Iterator<String> i = list.iterator(); i.hasNext();) {
// Remove the sha's we've download from the list
// they look ugly and it is not clear what they are anyway
String bsn = i.next();
if (isSha(bsn))
i.remove();
else {
File f = getLocal(bsn, Version.emptyVersion, null).getParentFile();
File pf = new File(f, "program.json");
if (!pf.isFile() || pf.length() == 0)
i.remove();
}
}
return list;
}
while (regex.startsWith("*")) {
regex = regex.substring(1);
}
while (regex.endsWith("*")) {
regex = regex.substring(0, regex.length() - 1);
}
trace("finding %s", regex);
TreeSet<String> bsns = new TreeSet<String>();
for (Program p : library.findProgram().query(regex)) {
programCache.put(p.last.bsn, p);
for (RevisionRef r : p.revisions) {
bsns.add(r.bsn);
}
}
return new ArrayList<String>(bsns);
}
private void trace(String string, Object... args) {
if (reporter != null)
reporter.trace(string, args);
else
System.out.printf(string, args);
}
/**
* This function must list all the revisions for this program. This function
* will return ALL master revisions but only the latest version of staged
* versions. The reason is that bnd will try to compile against the lowest
* version it is allowed to use (this increases backward compatibility) but
* that means that during staging you never overwrite the latest versions.
* <p/>
* Since we have the stage of a revision we always add the revisions that
* are marked MASTER. Then for the STAGING we create a map that links the
* baseline to a version. During the traversal of the revisions we keep the
* map with the highest version. Since the ordering is undefined, any MASTER
* for a baseline will ignore any baselined versions.
*/
@Override
public SortedSet<Version> versions(String bsn) throws Exception {
try {
List<Version> ls = new ArrayList<Version>();
if (isSha(bsn)) {
ls.add(Version.LOWEST);
} else {
// Maintains the highest staging revision's version
Map<String,Version> vtoq = new HashMap<String,Version>();
for (RevisionRef r : getRevisionRefs(bsn)) {
// Construct the version carefully
Version v = null;
if (r.qualifier == null || r.qualifier.isEmpty())
v = new Version(r.baseline);
else
v = new Version(r.baseline + "." + r.qualifier);
// MASTERs are easy
if (r.phase == Phase.MASTER) {
vtoq.put(r.baseline, null); // reserve spot
ls.add(v);
} else if (r.phase == Phase.STAGING) {
// get previous baseline version with qualifier
Version previous = vtoq.get(r.baseline);
// Do we already have a master for this version?
// this is indicated with a null value in the map
if (previous == null && vtoq.containsKey(r.baseline))
continue;
// If we are later than the previously staged
// version than we override
if (previous == null || previous.compareTo(v) < 0)
vtoq.put(r.baseline, v);
} else {
// we ignore RETIRED and EXPIRED here
}
}
// Get the latest versions for staging revisions
for (Map.Entry<String,Version> e : vtoq.entrySet())
if (e.getValue() != null)
ls.add(e.getValue());
}
return new SortedList<Version>(ls, 0, ls.size(), REVERSE_COMPARATOR);
}
catch (Exception e) {
e.printStackTrace();
throw e;
}
}
private boolean isSha(String bsn) {
return SHA.matcher(bsn).matches();
}
Program getProgram(String bsn) throws Exception {
if (refresh == null) {
refresh = new File(root, "refresh");
if (!refresh.isFile())
IO.store(new byte[0], refresh);
}
Program p = programCache.get(bsn);
if (p == null) {
File f = getLocal(bsn, Version.emptyVersion, null);
File meta = new File(f.getParentFile(), "program.json");
if (meta.exists() && meta.lastModified() >= refresh.lastModified()) {
// Empty file signifies we checked but it was not there
// so we will no reseek all the time
if (meta.length() == 0)
return null;
p = codec.dec().from(meta).get(Program.class);
} else {
p = library.getProgram(Library.OSGI_GROUP, bsn);
meta.getParentFile().mkdirs();
if (p != null) {
codec.enc().to(meta).put(p);
} else {
// We did not find it, so we create
// an empty meta file.
meta.delete();
meta.createNewFile();
return null;
}
}
programCache.put(bsn, p);
}
return p._id == null ? null : p;
}
@Override
public String getName() {
return "jpm4j";
}
@Override
public void setProperties(Map<String,String> map) {
try {
super.setProperties(map);
options = Converter.cnv(Options.class, map);
canwrite = options.domain() != null && options.depository() != null;
library = new RemoteLibrary(options.url().toString());
String email = options.email();
if (email == null)
email = settings.getEmail();
if (options.domain() != null) {
if (email != null)
library.credentials(email, InetAddress.getLocalHost().getHostName(), settings.getPublicKey(),
settings.getPrivateKey());
}
}
catch (Exception e) {
if (reporter != null)
reporter.exception(e, "Creating options");
else
e.printStackTrace();
}
}
@Override
public void setReporter(Reporter processor) {
reporter = processor;
}
@Override
public boolean refresh() {
try {
notfound.clear();
if (refresh != null)
refresh.delete();
return super.refresh();
}
catch (Exception e) {
if (reporter != null)
reporter.exception(e, "Trying to synchronize");
throw new RuntimeException(e);
}
}
@Override
public Map<String,Runnable> actions(Object... target) throws Exception {
if (target != null && target.length >= 2) {
final String bsn = (String) target[0];
final Version version = (Version) target[1];
Map<String,Runnable> map = new TreeMap<String,Runnable>();
map.put("Inspect Program", new Runnable() {
public void run() {
Command c = new Command("open https://www.jpm4j.org#/p/osgi/" + bsn);
try {
c.execute(null, null);
}
catch (Exception e) {
e.printStackTrace();
}
}
});
map.put("Inspect Revision", new Runnable() {
public void run() {
Command c = new Command("open https://www.jpm4j.org#/p/osgi/" + bsn + "//" + version);
try {
c.execute(null, null);
}
catch (Exception e) {
e.printStackTrace();
}
}
});
map.put("Copy Metadata", new Runnable() {
public void run() {
try {
wrap(bsn, version);
}
catch (Exception e) {
e.printStackTrace();
}
}
});
return map;
}
// TODO Auto-generated method stub
return null;
}
@Override
public String tooltip(Object... target) throws Exception {
if (target != null && target.length >= 2)
try {
String bsn = (String) target[0];
Version version = null;
if (target.length == 1) {
Program p = programCache.get(bsn);
if (p != null) {
StringBuilder sb = new StringBuilder();
if (p.wiki != null) {
sb.append("Description\n").append(p.wiki.text).append("\n");
}
j.wrap(sb);
return sb.toString();
}
} else {
try {
version = (Version) target[1];
RevisionRef r = getRevisionRef(bsn, version);
if (r == null)
return "No revision found for " + version;
Formatter sb = new Formatter();
try {
sb.format("[%s:%s", r.groupId, r.artifactId);
if (r.classifier != null) {
sb.format(":%s", r.classifier);
}
sb.format("@%s\n\n", r.version);
if (r.description != null)
sb.format("%s\n\n", r.description);
sb.format("%s, %s, %s\n\n", size(r.size, 0), Hex.toHexString(r.revision), age(r.created));
return sb.toString();
}
finally {
sb.close();
}
}
finally {
// System.out.println("exit");
}
}
}
catch (Exception e) {
e.printStackTrace();
}
else
; // System.out.println("no target or length is not 2");
return super.tooltip(target);
}
private List<RevisionRef> getRevisionRefs(String bsn) throws Exception {
String classifier = null;
String parts[] = bsn.split("__");
if (parts.length == 3) {
bsn = parts[0] + "__" + parts[1];
classifier = parts[2];
}
Program program = getProgram(bsn);
if (program != null) {
List<RevisionRef> refs = new ArrayList<Library.RevisionRef>();
for (RevisionRef r : program.revisions) {
if (eq(classifier, r.classifier))
refs.add(r);
}
return refs;
}
return Collections.emptyList();
}
private RevisionRef getRevisionRef(String bsn, Version version) throws Exception {
// Handle when we have a sha reference
if (isSha(bsn) && version.equals(Version.LOWEST)) {
Revision r = library.getRevision(Library.SHA_GROUP + ":" + bsn + "@0.0.0");
if (r == null)
return null;
return new RevisionRef(r);
}
String baseline = version.getWithoutQualifier().toString();
String qualifier = version.getQualifier();
for (RevisionRef r : getRevisionRefs(bsn)) {
if (baseline.equals(r.baseline) && eq(qualifier, r.qualifier))
return r;
}
return null;
}
private boolean eq(String a, String b) {
if (a == null)
a = "";
if (b == null)
b = "";
return a.equals(b);
}
private String age(long created) {
if (created == 0)
return "unknown";
long diff = (System.currentTimeMillis() - created) / (1000 * 60 * 60);
if (diff < 48)
return diff + " hours";
diff /= 24;
if (diff < 14)
return diff + " days";
diff /= 7;
if (diff < 8)
return diff + " weeks";
diff /= 4;
if (diff < 24)
return diff + " months";
diff /= 12;
return diff + " years";
}
static String[] sizes = {
"bytes", "Kb", "Mb", "Gb", "Tb", "Pb", "Showing off?"
};
private String size(long size, int power) {
if (power >= sizes.length)
return size + " Pb";
if (size < 1000)
return size + sizes[power];
return size(size / 1000, power + 1);
}
public void setLibrary(RemoteLibrary library) {
this.library = library;
}
/**
* Create a bnd recipe for this element from the library data that wraps the
* JAR with some proper metadata. This is used to easily wrap jars from JPM
* into real bundles.
*
* @param bsn
* @param version
* @throws Exception
*/
private void wrap(String bsn, Version version) throws Exception {
Revision r = library.getRevision(Library.OSGI_GROUP + ":" + bsn + "@" + version.toString());
if (r == null)
return;
Jar jar = new Jar(get(bsn, version, null));
Formatter f = new Formatter();
Analyzer an = new Analyzer();
try {
an.setJar(jar);
an.analyze();
f.format("# Generated bnd wrapper for %s:%s@%s\n", r.groupId, r.artifactId, r.version);
f.format("# Recipe by ${global;name}\n\n");
if (an.getBundleSymbolicName() != null) {
f.format("#\n# ALREADY AN OSGI BUNDLE!!!\n#\n");
}
Manifest m = jar.getManifest();
if (m != null) {
String del = "# From original manifest\n";
boolean yes = false;
for (Entry<Object,Object> e : m.getMainAttributes().entrySet()) {
yes = true;
String key = e.getKey().toString();
if ("Manifest-Version".equalsIgnoreCase(key) || "Bundle-ManifestVersion".equalsIgnoreCase(key)
|| key.startsWith("Bnd") || key.startsWith("Created"))
continue;
f.format("%s%-40s: %s", del, key, e.getValue());
del = "\n";
}
if (yes)
f.format("\n\n# End of original manifest");
f.format("\n\n");
}
if (r.description != null)
f.format("Bundle-Description: %s %s\\\n ${disclaimer}\n", r.title == null ? ""
: r.title, r.description);
else
f.format("Bundle-Description: ##########################################################");
String v = r.baseline;
if (r.qualifier != null)
v += "." + r.qualifier;
f.format("Bundle-Version: %s\n", v);
if (r.icon != null)
f.format("Bundle-Icon: %s\n", r.icon);
f.format("\n#\n# Coordinates\n#\n");
f.format("JPM-From: sha:%s@0.0.0;coordinates=%s;bsn=%s\n",
Hex.toHexString(r._id), getCoordinates(r), r.bsn);
f.format("JPM-URL: https://jpm4j.org/#/p/%s/%s/%s/%s\n", r.groupId,
r.artifactId, r.classifier == null ? "" : r.classifier, r.version);
f.format("Include-Resource: @${repo;%s;0.0.0}\n", Hex.toHexString(r._id));
if (r.docUrl != null)
f.format("Bundle-DocURL: %s\n", r.docUrl);
if (r.organization != null && r.organization.name != null) {
f.format("Bundle-Vendor: %s\n", r.organization.name);
}
if (r.organization != null && r.organization.url != null) {
f.format("Bundle-ContactAddress: %s\n", r.organization.url);
}
String del = "Bundle-License: ";
for (License license : r.licenses) {
f.format("%s %s", del, license.name);
if (license.url != null)
f.format(";url='%s'", license.url);
if (license.comments != null)
f.format(";description='%s'", license.comments);
del = ", \\\n ";
}
f.format("\n\n");
del = "Bundle-Developers: \\\n ";
for (Developer dev : r.developers) {
f.format("%s '%s'", del, dev.name);
if (dev.email != null)
f.format(";email='%s'", dev.email);
if (dev.roles != null) {
ExtList<String> el = new ExtList<String>(dev.roles);
if (!el.isEmpty())
f.format(";roles='%s'", el.join());
}
del = ", \\\n ";
}
f.format("\n\n");
del = "Bundle-Contributors: \\\n ";
for (Contributor dev : r.contributors) {
f.format("%s %s", del, dev.name);
if (dev.email != null)
f.format(";email='%s'", dev.email);
if (dev.roles != null) {
ExtList<String> el = new ExtList<String>(dev.roles);
if (!el.isEmpty())
f.format(";roles='%s'", el.join());
}
del = ", \\\n ";
}
f.format("\n\n");
f.format("Export-Package: ");
del = "";
for (Entry<PackageRef,Attrs> pr : an.getContained().entrySet()) {
f.format("%s\\\n %s", del, pr.getKey().getFQN());
if (pr.getValue().containsKey("version")) {
f.format(";version=%s", pr.getValue().get("version"));
} else {
f.format(";version=100.0.0;provide:=true");
}
del = ",";
}
f.format("\n\n");
f.format("#\n# Remove after inspection:\n\nImport-Package: ");
del = "";
for (Entry<PackageRef,Attrs> pr : an.getReferred().entrySet()) {
if (pr.getKey().isJava() || pr.getKey().isMetaData())
continue;
f.format("%s\\\n %s", del, pr.getKey().getFQN());
if (pr.getValue().containsKey("version")) {
f.format(";version='%s'", pr.getValue().get("version"));
}
del = ",";
}
f.format("\n\n");
f.format("-buildpath:");
del = " \\\n ";
for (Requirement req : r.requirements) {
String name = req.name == null ? (String) req.ps.get("name:") : req.name;
if ("x-maven".equals(req.ns) && name != null) {
Matcher matcher = COORDINATE_P.matcher(name);
if (matcher.matches()) {
String g = matcher.group(1);
String a = matcher.group(2);
String vv = matcher.group(3);
Revision dep = library.getRevision(g + ":" + a + "@" + vv);
if (dep == null) {
f.format("%s%s.%s;version=%s", del, g, a, vv);
} else {
f.format("%s%s;version=%s", del, r.bsn, vv);
}
del = ", \\\n";
} else {
f.format("\n#### %s\n", req.name);
}
}
}
f.format("\n\n");
}
catch (Exception e) {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
pw.close();
f.format("Sorry, should not have happend :-(\n\n%s\n", sw);
}
finally {
jar.close();
f.close();
an.close();
}
clipboard(f.toString());
}
private Object getCoordinates(Revision r) {
return r.groupId + ":" + r.artifactId + (r.classifier == null ? "" : ":" + r.classifier) + "@" + r.version;
}
private void clipboard(final String string) {
Clipboard clipboard = java.awt.Toolkit.getDefaultToolkit().getSystemClipboard();
clipboard.setContents(new Transferable() {
@Override
public boolean isDataFlavorSupported(DataFlavor flavor) {
return false;
}
@Override
public DataFlavor[] getTransferDataFlavors() {
return new DataFlavor[] {
DataFlavor.stringFlavor
};
}
@Override
public Object getTransferData(DataFlavor flavor) throws UnsupportedFlavorException, IOException {
return string;
}
}, new ClipboardOwner() {
@Override
public void lostOwnership(Clipboard clipboard, Transferable contents) {
// what me worry?
}
});
}
}