@Override
public final DoubleVector minimize(CostFunction f, DoubleVector theta,
int length, boolean verbose) {
DoubleVector input = theta;
int M = 0;
int i = 0; // zero the run length counter
int red = 1; // starting point
int ls_failed = 0; // no previous line search has failed
final CostGradientTuple evaluateCost = f.evaluateCost(input);
double f1 = evaluateCost.getCost();
DoubleVector df1 = evaluateCost.getGradient();
i = i + (length < 0 ? 1 : 0);
// search direction is steepest
DoubleVector s = df1.multiply(-1.0d);
double d1 = s.multiply(-1.0d).dot(s); // this is the slope
double z1 = red / (1.0 - d1); // initial step is red/(|s|+1)
while (i < Math.abs(length)) {// while not finished
i = i + (length > 0 ? 1 : 0);// count iterations?!
// make a copy of current values
DoubleVector X0 = input.deepCopy();
double f0 = f1;
DoubleVector df0 = df1.deepCopy();
// begin line search
input = input.add(s.multiply(z1));
final CostGradientTuple evaluateCost2 = f.evaluateCost(input);
double f2 = evaluateCost2.getCost();
DoubleVector df2 = evaluateCost2.getGradient();
i = i + (length < 0 ? 1 : 0); // count epochs
double d2 = df2.dot(s);
// initialize point 3 equal to point 1
double f3 = f1;
double d3 = d1;
double z3 = -z1;
if (length > 0) {
M = MAX;
} else {
M = Math.min(MAX, -length - i);
}
// initialize quanteties
int success = 0;
double limit = -1;
while (true) {
while (((f2 > f1 + z1 * RHO * d1) | (d2 > -SIG * d1)) && (M > 0)) {
// tighten the bracket
limit = z1;
double z2 = 0.0d;
double A = 0.0d;
double B = 0.0d;
if (f2 > f1) {
// quadratic fit
z2 = z3 - (0.5 * d3 * z3 * z3) / (d3 * z3 + f2 - f3);
} else {
// cubic fit
A = 6 * (f2 - f3) / z3 + 3 * (d2 + d3);
B = 3 * (f3 - f2) - z3 * (d3 + 2 * d2);
// numerical error possible - ok!
z2 = (Math.sqrt(B * B - A * d2 * z3 * z3) - B) / A;
}
if (Double.isNaN(z2) || Double.isInfinite(z2)) {
// if we had a numerical problem then bisect
z2 = z3 / 2.0d;
}
// don't accept too close to limits
z2 = Math.max(Math.min(z2, INT * z3), (1 - INT) * z3);
// update the step
z1 = z1 + z2;
input = input.add(s.multiply(z2));
final CostGradientTuple evaluateCost3 = f.evaluateCost(input);
f2 = evaluateCost3.getCost();
df2 = evaluateCost3.getGradient();
M = M - 1;
i = i + (length < 0 ? 1 : 0); // count epochs
d2 = df2.dot(s);
// z3 is now relative to the location of z2
z3 = z3 - z2;
}
if (f2 > f1 + z1 * RHO * d1 || d2 > -SIG * d1) {
break; // this is a failure
} else if (d2 > SIG * d1) {
success = 1;
break; // success
} else if (M == 0) {
break; // failure
}
// make cubic extrapolation
double A = 6 * (f2 - f3) / z3 + 3 * (d2 + d3);
double B = 3 * (f3 - f2) - z3 * (d3 + 2 * d2);
double z2 = -d2 * z3 * z3 / (B + Math.sqrt(B * B - A * d2 * z3 * z3));
// num prob or wrong sign?
if (Double.isNaN(z2) || Double.isInfinite(z2) || z2 < 0)
// if we have no upper limit
if (limit < -0.5) {
// the extrapolate the maximum amount
z2 = z1 * (EXT - 1);
} else {
// otherwise bisect
z2 = (limit - z1) / 2;
}
else if ((limit > -0.5) && (z2 + z1 > limit)) {
// extraplation beyond max?
z2 = (limit - z1) / 2; // bisect
} else if ((limit < -0.5) && (z2 + z1 > z1 * EXT)) {
// extrapolationbeyond limit
z2 = z1 * (EXT - 1.0); // set to extrapolation limit
} else if (z2 < -z3 * INT) {
z2 = -z3 * INT;
} else if ((limit > -0.5) && (z2 < (limit - z1) * (1.0 - INT))) {
// too close to the limit
z2 = (limit - z1) * (1.0 - INT);
}
// set point 3 equal to point 2
f3 = f2;
d3 = d2;
z3 = -z2;
z1 = z1 + z2;
// update current estimates
input = input.add(s.multiply(z2));
final CostGradientTuple evaluateCost3 = f.evaluateCost(input);
f2 = evaluateCost3.getCost();
df2 = evaluateCost3.getGradient();
M = M - 1;
i = i + (length < 0 ? 1 : 0); // count epochs?!
d2 = df2.dot(s);
}// end of line search
DoubleVector tmp = null;
if (success == 1) { // if line search succeeded
f1 = f2;
if (verbose) {
LOG.info("Iteration " + i + " | Cost: " + f1);