Examples of org.encog.ml.data.buffer.BufferedNeuralDataSet

org.encog.ml.data.buffer.BufferedNeuralDataSet
This class is not memory based, so very long files can be used, without running out of memory. This dataset uses a Encog binary training file as a buffer. When used with a slower access dataset, such as CSV, XML or SQL, where parsing must occur, this dataset can be used to load from the slower dataset and train at much higher speeds. This class makes use of Java file channels for maximum file access performance. If you are going to create a binary file, by using the add methods, you must call beginLoad to cause Encog to open an output file. Once the data has been loaded, call endLoad. You can also use the BinaryDataLoader class, with a CODEC, to load many other popular external formats. The binary files produced by this class are in the Encog binary training format, and can be used with any Encog platform. Encog binary files are stored using "little endian" numbers.

   */
  @Override
  public final void performJobUnit(final JobUnitContext context) {


    final BasicNetwork network = (BasicNetwork) context.getJobUnit();
    BufferedNeuralDataSet buffer = null;
    MLDataSet useTraining = this.training;


    if (this.training instanceof BufferedNeuralDataSet) {
      buffer = (BufferedNeuralDataSet) this.training;
      useTraining = buffer.openAdditional();
    }


    // train the neural network


    double error = Double.POSITIVE_INFINITY;
    for (int z = 0; z < this.weightTries; z++) {
      network.reset();
      final Propagation train = new ResilientPropagation(network,
          useTraining);
      final StopTrainingStrategy strat = new StopTrainingStrategy(0.001,
          5);


      train.addStrategy(strat);
      train.setNumThreads(1); // force single thread mode


      for (int i = 0; (i < this.iterations) && !getShouldStop()
          && !strat.shouldStop(); i++) {
        train.iteration();
      }


      error = Math.min(error, train.getError());
    }


    if (buffer != null) {
      buffer.close();
    }


    if (!getShouldStop()) {
      // update min and max

View Full Code Here

  private BufferedNeuralDataSet data;
  
  public BinaryDataTab(ProjectFile file) {
    super(file);
    
    this.data = new BufferedNeuralDataSet(file.getFile());
  
    setLayout(new BorderLayout());
    this.toolbar = new JToolBar();
    this.toolbar.setFloatable(false);
    this.toolbar.add(this.addInputColumn = new JButton("Add Input Column"));

View Full Code Here

      final File binFile, final int inputCount, final int outputCount,
      final boolean headers) {
    binFile.delete();
    final CSVNeuralDataSet csv = new CSVNeuralDataSet(csvFile.toString(),
        inputCount, outputCount, false);
    final BufferedNeuralDataSet buffer = new BufferedNeuralDataSet(binFile);
    buffer.beginLoad(inputCount, outputCount);
    for (final MLDataPair pair : csv) {
      buffer.add(pair);
    }
    buffer.endLoad();
  }

View Full Code Here

  private EncogUtility() {


  }


  public static MLDataSet loadEGB2Memory(File filename) {
    BufferedNeuralDataSet buffer = new BufferedNeuralDataSet(filename);
    return buffer.loadToMemory();
  }

View Full Code Here

    {


        (new File(binFile)).delete();
        CSVNeuralDataSet csv = new CSVNeuralDataSet(csvFile.toString(),
               inputCount, outputCount, headers);
        BufferedNeuralDataSet buffer = new BufferedNeuralDataSet(new File(binFile));
        buffer.beginLoad(inputCount, outputCount);
        for(MLDataPair pair : csv)
        {
            buffer.add(pair);
        }
        buffer.endLoad();
    }

View Full Code Here

   {


       binFile.delete();
       ReadCSV csv = new ReadCSV(csvFile.toString(), headers, format);
       
       BufferedNeuralDataSet buffer = new BufferedNeuralDataSet(binFile);
       buffer.beginLoad(input.length, ideal.length);
       while(csv.next())
       {
         BasicMLData inputData = new BasicMLData(input.length);
         BasicMLData idealData = new BasicMLData(ideal.length);
         
         // handle input data
         for(int i=0;i<input.length;i++) {
           inputData.setData(i, csv.getDouble(input[i]));
         }
         
         // handle input data
         for(int i=0;i<ideal.length;i++) {
           idealData.setData(i, csv.getDouble(ideal[i]));
         }
         
         // add to dataset
         
           buffer.add(inputData,idealData);
       }
       buffer.endLoad();
   }

View Full Code Here

        1000, 10000, 10, 10, -1, 1);


    // create the binary file


    file.delete();
    BufferedNeuralDataSet training2 = new BufferedNeuralDataSet(file);
    training2.load(training);


    final long start = System.currentTimeMillis();
    final long stop = start + (10 * Evaluate.MILIS);
    int record = 0;


    MLDataPair pair = BasicMLDataPair.createPair(10, 10);


    int iterations = 0;
    while (System.currentTimeMillis() < stop) {
      iterations++;
      training2.getRecord(record++, pair);
      if (record >= training2.getRecordCount())
        record = 0;
    }


    training.close();
    iterations /= 100000;

View Full Code Here

        10000, 10, 10, -1, 1);
    
    // create the binary file
    
    file.delete();
    BufferedNeuralDataSet training2 = new BufferedNeuralDataSet(file);
    training2.load(training);
    
    final long start = System.currentTimeMillis();
    final long stop = start + (10*Evaluate.MILIS);
    int record = 0;
    
    MLDataPair pair = BasicMLDataPair.createPair(10, 10);
    
    int iterations = 0;
    while( System.currentTimeMillis()<stop ) {
      iterations++;
      training2.getRecord(record++, pair);  
      if( record>=training2.getRecordCount() )
        record = 0;
    }
    
    System.out.println("In 10 seconds, the disk(binary) dataset read " + 
        Format.formatInteger( iterations) + " records.");

View Full Code Here

  
  
  public void testBufferData() throws Exception 
  {
    new File(FILENAME).delete();
    BufferedNeuralDataSet set = new BufferedNeuralDataSet(new File(FILENAME));
    set.beginLoad(2, 1);
    for(int i=0;i<XOR.XOR_INPUT.length;i++) {
      BasicMLData input = new BasicMLData(XOR.XOR_INPUT[i]);
      BasicMLData ideal = new BasicMLData(XOR.XOR_IDEAL[i]);
      set.add(input,ideal);
    }
    set.endLoad();
    
    XOR.testXORDataSet(set);
    
  }

View Full Code Here

   */
  public MLDataSet getTrainingSet() {
    if( this.comboTraining.getSelectedValue()==null )      
      return null;
    File file = ((ProjectTraining)this.comboTraining.getSelectedValue()).getFile();
    BufferedNeuralDataSet result = new BufferedNeuralDataSet(file);
    return result;
  }

View Full Code Here

0 1

TOP

Related Classes of org.encog.ml.data.buffer.BufferedNeuralDataSet

org.encog.examples.neural.benchmark.BinaryVsMemory

org.encog.ml.data.basic.BasicMLDataSet

org.encog.ml.data.MLlDataError

org.encog.neural.data.buffer.TestBufferedNeuralDataSet

org.encog.neural.prune.PruneIncremental

org.encog.util.benchmark.EncogBenchmark

org.encog.util.simple.EncogUtility

org.encog.workbench.dialogs.training.methods.InputNEAT

org.encog.workbench.dialogs.training.NetworkAndTrainingDialog

org.encog.workbench.dialogs.training.TrainDialog

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.