package util;
//----- JDK Imports ------------------------------------------------------------
import java.awt.image.WritableRaster;
import java.util.Vector;
import java.lang.reflect.Array;
//----- Quicktime Imports ------------------------------------------------------
import quicktime.qd.Pict;
import quicktime.std.StdQTConstants;
import quicktime.std.StdQTException;
import quicktime.std.clocks.TimeRecord;
import quicktime.std.movies.Movie;
import quicktime.std.movies.TimeInfo;
import quicktime.std.movies.Track;
//----- SISC Imports -----------------------------------------------------------
import sisc.data.Pair;
import sisc.interpreter.SchemeException;
//----- Phoenix Imports --------------------------------------------------------
import controller.PhoenixController;
/**
* Video Phoenix
* Version 0.2.0
* Copyright (c) 2007 Lunderskov, Ian; Pan, Jiabei; Rebelsky, Samuel;
* Whisenhunt, Heather; Young, Ian; Zuleta Benavides, Luis.
* All rights reserved.
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
* @author Pan, Jiabei; Rebelsky, Samuel; Whisenhunt, Heather
* @author Glimmer Labs
* @version 0.2.0
*/
public class CutDetector
{
/*--------*-------------------------------------------------------------------
* Fields *
*--------*/
// veritcal and horizontal divisions on a frame for region comparisons
final static int divisionVertical = 4;
final static int divisionHorizontal = 4;
// number of bins for histograms (hue on 360 degrees)
final static int bins = 360;
// cache the last computed histogram to avoid computing again
int[][] cache = new int[divisionHorizontal * divisionVertical][bins];
// these two variables need to be reset every time cut detection is called:
// location of the last cut
private int lastCut = 0;
// final result containing all cuts found
private Vector<Float> cuts = new Vector<Float>(0);;
// sensitivity parameters
private float interval;
private float threshold;
private float blockRange;
/*--------------*-------------------------------------------------------------
* Constructors *
*--------------*/
public CutDetector()
{
// no initialization needed
} // CutDetector()
/*---------*------------------------------------------------------------------
* Methods *
*---------*/
/**
* Find the mode of an array
*
* @param array Array to find mode of
* @return mode Mode of array
*/
public float mode(int[] array)
{
int length = Array.getLength(array);
float sumOfSquares = 0;
for (int i = 0; i < length; i++)
{
sumOfSquares = sumOfSquares + array[i] * array[i];
} // for
return (float) Math.sqrt(sumOfSquares);
} // mode(int[])
/**
* Find the dot products of two arrays
*
* @param array1 Array to find dot product of
* @param array2 Array to find dot product of
* @return product Dot product of array1 and array2
*/
public int dotProduct(int[] array1, int[] array2)
{
int length = Array.getLength(array1);
int sum = 0;
for (int i = 0; i < length; i++)
{
sum = sum + array1[i] * array2[i];
} // for
return sum;
} // dotProduct(int[], int[])
/**
* Find the difference between two histograms
*
* @param hist1 Histogram to compare
* @param hist2 Histogram to compare
* @return diff Difference between hist1 and hist2
*/
public float histogramDifference(int[] hist1, int[] hist2)
{
double alpha = dotProduct(hist1, hist2) / (mode(hist1) * mode(hist2));
// identical vectors may result in an alpha of 1.00000000000000xxx
// with arc cosine of NaN
if ((alpha > 1) || (alpha < -1))
{
alpha = 1;
} // if ((alpha > 1) || (alpha < -1))
return (float)java.lang.Math.acos(alpha);
} // histogramDifference(int[], int[])
/**
* Divide a Pict into divisionHorizontal by divisionVertical equal blocks,
* and calculate the hue histogram of each
*
* @param pic Pict to calculate region histograms for
* @return hists Histograms of the regions of pic
*/
public int[][] chopFrame(Pict pic)
{
int[][] result = new int[CutDetector.divisionHorizontal
* CutDetector.divisionVertical][CutDetector.bins];
WritableRaster raster = ImageUtils.makeRaster(pic);
int height = raster.getHeight() / CutDetector.divisionHorizontal;
int width = raster.getWidth() / CutDetector.divisionVertical;
// for each region, calculate the histogram
for (int i = 0; i < CutDetector.divisionHorizontal; i++)
{
for (int j = 0; j < CutDetector.divisionVertical; j++)
{
int[] hist = new int[CutDetector.bins];
hist = PhoenixController.getHistogramController().getHistogramData(
raster, "h", i * width, (i + 1) * width, j * height,
(j + 1) * height);
result[i * CutDetector.divisionHorizontal + j] = hist;
} // for
} //for
return result;
} // chopFrame(Pict)
/**
* Find the difference score of two sets of region histograms
*
* @param hist1 Region histograms to find difference score
* @param hist2 Region histograms to find difference score
* @return diff Difference score of hist1 and hist2
*/
public float frameDifference(int[][] hist1, int[][] hist2)
{
float grandDiff = 0;
for (int i = 0; i < (CutDetector.divisionHorizontal
* CutDetector.divisionVertical); i++)
{
grandDiff = grandDiff + histogramDifference(hist1[i], hist2[i]);
} // for
return grandDiff;
} // frameDifference(int[][], int[][])
/**
* Detect cuts in a section of a movie
*
* @param mov Movie to detect cuts in
* @param start Start time in seconds of the movie section
* @param end End time in seconds of the movie section
* @param threshold Minimum frame difference score to register a cut
* @param blockRange Seconds after a cut to block other cuts from registering
* @return cuts Cuts found in the section of mov between start and end
* @throws SchemeException
*/
public Float[] detectFrameByFrame(Movie mov, float start, float end,
float threshold, float blockRange)
throws SchemeException
{
this.cuts = new Vector<Float>(0);
try
{
int timeScale = mov.getTimeScale();
int begin = (int)Math.floor(start * timeScale);
int finish = (int)Math.floor(end * timeScale);
Pict oldPic = mov.getPict(begin);
this.cache = chopFrame(oldPic);
Track visualTrack = mov.getIndTrackType(1,
StdQTConstants.visualMediaCharacteristic,
StdQTConstants.movieTrackCharacteristic);
mov.setTime(new TimeRecord(timeScale, (int) begin));
Boolean go = true;
int lastTime = 0;
while (go)
{
// user terminated execution
if (Thread.interrupted())
{
throw new SchemeException(new Pair(), null, null);
} // if (Thread.interrupted())
// find the next frame
TimeInfo ti = visualTrack.getNextInterestingTime(
StdQTConstants.nextTimeMediaSample, mov.getTime(), 1);
// if looped to earlier frame or finished selected section
if ((lastTime > ti.time) || (ti.time >= finish))
{
go = false;
} // if ((lastTime > ti.time) || (ti.time >= finish))
else
{
// advance to the next frame
mov.setTime(new TimeRecord(timeScale, ti.time));
Pict newPic = mov.getPict(mov.getTime());
float score = this.frameDifference(newPic);
if ((score > threshold)
&& (mov.getTime() - lastCut > blockRange * mov.getTimeScale()))
{
this.cuts.add((float)mov.getTime() / mov.getTimeScale());
this.lastCut = mov.getTime();
} // if ((score > threshold) ...
// update oldPic to avoid recalculating
oldPic = newPic;
} // else
} // while (go)
} // try
catch (Exception e)
{
e.printStackTrace();
} // catch (Exception)
Float[] t = new Float[0];
return this.cuts.toArray(t);
} // detectFrameByFrame(Movie, float, float, float, float)
/**
* Find the difference score of two frames
*
* @param pic1 Pict to find difference score
* @param pic2 Pict to find difference score
* @return diff Difference score of hist1 and hist2
*/
public float frameDifference(Pict pic1, Pict pic2)
{
float diff = 0;
// make region histograms for the picts
int[][] chopped1 = new int[CutDetector.divisionHorizontal
* CutDetector.divisionVertical][CutDetector.bins];
int[][] chopped2 = new int[CutDetector.divisionHorizontal
* CutDetector.divisionVertical][CutDetector.bins];
chopped1 = chopFrame(pic1);
chopped2 = chopFrame(pic2);
for (int i = 0; i < (CutDetector.divisionHorizontal
* CutDetector.divisionVertical); i++)
{
diff = diff + histogramDifference(chopped1[i], chopped2[i]);
} // for
return diff;
} // frameDifference(Pict, Pict)
/* calculate the frame difference of all neighboring frames using caching */
private float frameDifference(Pict pic)
{
float diff = 0;
// make region histogram for pic
int[][] chopped = new int[CutDetector.divisionHorizontal
* CutDetector.divisionVertical][CutDetector.bins];
chopped = chopFrame(pic);
for (int i = 0; i < (CutDetector.divisionHorizontal
* CutDetector.divisionVertical); i++)
{
diff = diff + histogramDifference(this.cache[i], chopped[i]);
} // for
this.cache = chopped;
return diff;
} // frameDifference(Pict)
/* find cuts in a section of <mov> */
private void doCutDetection(Movie mov, float start, float end)
throws SchemeException
{
try
{
int timeScale = mov.getTimeScale();
int begin = (int)Math.floor(start * timeScale);
int finish = (int)Math.floor(end * timeScale);
Pict oldPic = mov.getPict(begin);
this.cache = chopFrame(oldPic);
// get the first visual track from mov
Track visualTrack = mov.getIndTrackType(1,
StdQTConstants.visualMediaCharacteristic,
StdQTConstants.movieTrackCharacteristic);
mov.setTime(new TimeRecord(timeScale, (int) begin));
Boolean go = true;
int lastTime = 0;
while (go)
{
// user terminated execution
if (Thread.interrupted())
{
throw new SchemeException(new Pair(), null, null);
} // if (Thread.interrupted())
// find next frame
TimeInfo ti = visualTrack.getNextInterestingTime(
StdQTConstants.nextTimeMediaSample, mov.getTime(), 1);
// if looped to earlier frame or finished selected section
if ((lastTime > ti.time) || (ti.time >= finish))
{
go = false;
} // if ((lastTime > ti.time) || (ti.time >= finish))
else
{
mov.setTime(new TimeRecord(timeScale, ti.time));
Pict newPic = mov.getPict(mov.getTime());
float score = frameDifference(newPic);
if ((score > this.threshold)
&& (mov.getTime() - this.lastCut
> this.blockRange * mov.getTimeScale()))
{
this.cuts.add((float)mov.getTime() / mov.getTimeScale());
this.lastCut = mov.getTime();
} // if ((score > threshold) ...
oldPic = newPic;
} // else
} // while (go)
} // try
catch (Exception e)
{
e.printStackTrace();
} // catch (Exception)
} // doCutDetection(Movie, float, float)
/**
* Detect cuts in a movie
*
* @param mov Movie to detect cuts
* @param interval Initial interval to check for cuts
* @param threshold Minimum frame difference score to register a cut
* @param blockRange Seconds after a cut to block other cuts from registering
* @return cuts Cuts found in the section of mov between start and end
* @throws SchemeException
*/
public Float[] detectCuts(Movie mov, float interval, float threshold,
float blockRange)
throws SchemeException
{
this.cuts = new Vector<Float>(0);
this.interval = interval;
this.threshold = threshold;
this.blockRange = blockRange;
try
{
this.lastCut = 0 - ((int)this.blockRange * mov.getTimeScale());
float end = (float) mov.getDuration() / (float) mov.getTimeScale();
float now = 0;
Pict oldPic = MovieUtils.getMovieFrame(mov, 0);
Pict newPic;
while (now < end)
{
// user terminated execution
if (Thread.interrupted())
{
throw new SchemeException(new Pair(), null, null);
} // if (Thread.interrupted())
now = now + this.interval;
if (now > end)
{
break;
} // if (now > end)
newPic = MovieUtils.getMovieFrame(mov, now);
float diff = frameDifference(oldPic, newPic);
// if a cut is found
if (diff > this.threshold)
{
if ((this.lastCut / mov.getTimeScale()) + this.blockRange < now)
{
// start frame-by-frame detection to determine exact loccation
doCutDetection(mov, now - this.interval, now);
this.lastCut = Math.round(now * mov.getTimeScale());
} // if (this.lastCut + this.blockRange < now)
} // if (diff > this.threshold)
// cache pic
oldPic = newPic;
} // while (now < end)
} // try
catch (StdQTException stdqte)
{
stdqte.printStackTrace();
} // catch (StdQTException)
Float[] t = new Float[0];
return this.cuts.toArray(t);
} // detectCuts(Movie, float, float, float)
public Float[] coarseDetectCuts(Movie mov, float interval, float threshold,
float blockRange)
throws SchemeException
{
this.cuts = new Vector<Float>(0);
this.interval = interval;
this.threshold = threshold;
this.blockRange = blockRange;
try
{
this.lastCut = 0 - ((int)this.blockRange * mov.getTimeScale());
float end = (float) mov.getDuration() / (float) mov.getTimeScale();
float now = 0;
Pict oldPic = MovieUtils.getMovieFrame(mov, 0);
Pict newPic;
while (now < end)
{
// user terminated execution
if (Thread.interrupted())
{
throw new SchemeException(new Pair(), null, null);
} // if (Thread.interrupted())
now = now + this.interval;
if (now > end)
{
break;
} // if (now > end)
newPic = MovieUtils.getMovieFrame(mov, now);
float diff = frameDifference(oldPic, newPic);
// if a cut is found
if (diff > this.threshold)
{
if ((this.lastCut / mov.getTimeScale()) + this.blockRange < now)
{
this.cuts.add(now - this.interval);
this.lastCut = Math.round((now - this.interval)
* mov.getTimeScale());
} // if (this.lastCut + this.blockRange < now)
} // if (diff > this.threshold)
// cache pic
oldPic = newPic;
} // while (now < end)
} // try
catch (StdQTException stdqte)
{
stdqte.printStackTrace();
} // catch (StdQTException)
Float[] t = new Float[0];
return this.cuts.toArray(t);
} // coarseDetectCuts(Movie, float, float, float)
} // class CutDetector