Package com.jpetrak.gate.stringannotation.extendedgazetteer2

Examples of com.jpetrak.gate.stringannotation.extendedgazetteer2.ExtendedGazetteer2


    eg = null;
    System.gc();
    System.out.println("Saving completed, trying to load into a new gaz store");
    before = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
    startTime = System.currentTimeMillis();
    GazStoreTrie3 gs = new GazStoreTrie3();
    gs = (GazStoreTrie3)gs.load(save);
    endTime = System.currentTimeMillis();
    after = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
    System.out.println("Loading from GAZBIN completed");
    System.out.println("Elapsed time for loading from gazbin: "+((endTime-startTime)/1000.0));
    System.out.println("Memory used up for cache loading: "+(after-before));
View Full Code Here


      throw new GateRuntimeException("Config file must have def or defyaml extension");
    }
    File gazbinFile = new File(gazbinFileName);
  
   if(backendNr == 3 && useCache && gazbinFile.exists()) {
     gazStore = new GazStoreTrie3();
     gazStore = gazStore.load(gazbinFile);
  
   } else {
    if(backendNr == 1) {
      gazStore = new GazStoreTrie1();
    } else if (backendNr == 2) {
      gazStore = new GazStoreTrie2();     
    } else if (backendNr == 3) {
      gazStore = new GazStoreTrie3();     
    } else {
      throw new GateRuntimeException("Invalid backend number: "+backendNr);
    }
    BufferedReader defReader =
      new BomStrippingInputStreamReader((configFileURL).openStream(), encoding);
View Full Code Here

      throw new GateRuntimeException("Strange YAML format for the defyaml file "+configFileURL);
    }
   
    // if we want to load the cache and it exists, load it
    if(backendNr == 3 && useCache && gazbinFile.exists()) {
      gazStore = new GazStoreTrie3();
      gazStore = gazStore.load(gazbinFile);  
   } else {
     // otherwise process the files listed in the yaml file
    if(backendNr == 1) {
      gazStore = new GazStoreTrie1();
    } else if (backendNr == 2) {
      gazStore = new GazStoreTrie2();     
    } else if (backendNr == 3) {
      gazStore = new GazStoreTrie3();     
    } else {
      throw new GateRuntimeException("Invalid backend number: "+backendNr);
    }
    // go through all the list and tsv files to load and load them
    for(Map configListFile : configListFiles) {
View Full Code Here

    GazStoreTrie2 gs = new GazStoreTrie2();
    gs.runImplementationTests();
  }
  @Test
  public void testLookupTrie3() {
    GazStoreTrie3 gs = new GazStoreTrie3();
    gs.runImplementationTests();
  }
View Full Code Here

    Lookup l1 = it1.next();
    */
  }
  @Test
  public void testTrie3() {
    GazStoreTrie3 gs = new GazStoreTrie3();
    FeatureMap fm = Factory.newFeatureMap();
    fm.put("listFeature1","value1");
    fm.put("listFeature2","value2");
    int info1 = gs.addListInfo("Type1", "URL1", fm);
    String[] keyvals1 = new String[]{"key1","val1","key2","","k3","valuenumberthree"};
   
    gs.addLookup("asdf", info1, keyvals1);
    gs.addLookup("asdf", info1, keyvals1);
    gs.addLookup("asdf", info1, keyvals1);
    // TODO: check we have only added the lookup once!
    Iterator<Lookup> it = gs.match("asdf");
    assertNotNull(it);
    int nrLookups = 0;
    while(it.hasNext()) {
      it.next();
      nrLookups++;
View Full Code Here

  }

  @Test
  public void testGazStoreTrie3() throws IOException {
    System.out.println("GazStoreTrie3: *******************************");
    GazStoreTrie3 gs = new GazStoreTrie3();
    FeatureMap fm1 = Factory.newFeatureMap();
    fm1.put("feature1","value1");
    int info1 = gs.addListInfo("Type1", "TheFile", fm1);
    String[] kv1 = new String[2];
    kv1[0] = "key1of1";
    kv1[1] = "value1of1";
    gs.addLookup("entry", info1, kv1);
    String[] kv2 = new String[4];
    kv2[0] = "KEY1of2";
    kv2[1] = "VALUE1of2";
    kv2[2] = "KEY2of2";
    kv2[3] = "VALUE2of2";
    gs.addLookup("as", info1, kv2);
    State init = gs.getInitialState();
    System.out.println("Initial State: "+init);
    State s1 = init.next('a');
    System.out.println("State after a: "+s1);
    System.out.println("isFinal: "+s1.isFinal());
    State s2 = s1.next('s');
    System.out.println("State after s: "+s2);
    System.out.println("isFinal: "+s2.isFinal());
    Iterator<Lookup> lookupIter = gs.getLookups(s2);
    System.out.println("Have lookups: "+lookupIter.hasNext());
    while(lookupIter.hasNext()) {
      Lookup l = lookupIter.next();
      System.out.println("Have a lookup"+l);
    }
    File someFile = new File("tmp.gazbin");
    try {
      gs.save(someFile);
    } catch (FileNotFoundException e) {
      e.printStackTrace();
      assertTrue("could not save trie", false);
      return;
    }
    GazStoreTrie3 gs2 = new GazStoreTrie3();
    try {
      gs2 = (GazStoreTrie3)gs2.load(someFile);
    } catch (FileNotFoundException e) {
      e.printStackTrace();
      assertTrue("could not load trie",false);
      return;
    }
    State init_2 = gs2.getInitialState();
    System.out.println("Initial State: "+init_2);
    State s1_2 = init_2.next('a');
    System.out.println("State after a: "+s1_2);
    System.out.println("isFinal: "+s1_2.isFinal());
    State s2_2 = s1_2.next('s');
    System.out.println("State after s: "+s2_2);
    System.out.println("isFinal: "+s2_2.isFinal());
    Iterator<Lookup> lookupIter_2 = gs2.getLookups(s2_2);
    System.out.println("Have lookups: "+lookupIter_2.hasNext());
    while(lookupIter_2.hasNext()) {
      Lookup l = lookupIter_2.next();
      System.out.println("Have a lookup"+l);
    }
View Full Code Here

    eg = null;
    System.gc();
    System.out.println("Saving completed, trying to load into a new gaz store");
    before = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
    startTime = System.currentTimeMillis();
    GazStoreTrie3 gs = new GazStoreTrie3();
    gs = (GazStoreTrie3)gs.load(save);
    endTime = System.currentTimeMillis();
    after = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
    System.out.println("Loading completed");
    System.out.println("Elapsed time for cache loading: "+((endTime-startTime)/1000.0));
    System.out.println("Memory used up for cache loading: "+(after-before));
View Full Code Here

    return in;
  }
 
  @Test
  public void testPersistLargeArray() throws IOException, ClassNotFoundException {
    StoreArrayOfCharArrays as = new StoreArrayOfCharArrays();
    char[] data = new char[]{'a', 's', 'd', 'f', 'g' };
    for(int i = 0; i< 20000000; i++) {
      as.addData(data);
    }
    ObjectOutputStream os = new ObjectOutputStream(openOutputStream("testout1.gazbin",true));
    long startTime = System.currentTimeMillis();
    os.writeObject(as);
    os.flush();
    os.close();
    long endTime = System.currentTimeMillis();
    System.out.println("Elapsed time for save: "+((endTime-startTime)/1000.0));
    System.gc();
   
    as = null;
    ObjectInputStream is = new ObjectInputStream(openInputStream("testout1.gazbin",true));
    startTime = System.currentTimeMillis();
    Object object = is.readObject();
    is.close();
    endTime = System.currentTimeMillis();
    System.out.println("Elapsed time for load: "+((endTime-startTime)/1000.0));
    assertTrue(object instanceof StoreArrayOfCharArrays);
    as = (StoreArrayOfCharArrays)object;
    char[] back = as.getData(0);
    assertEquals("asdfg",new String(back));
  }
View Full Code Here

  }
 

  @Test
  public void testStore()  {
    StoreArrayOfCharArrays as = new StoreArrayOfCharArrays();
   
    int i;
    char[] r;
    i = as.addData("asdfjk".toCharArray());
    assertEquals(0,i);
    i = as.addData("qwertyqwerty".toCharArray());
    assertEquals(8,i);
    r = as.getData(0);
    assertEquals("asdfjk",new String(r));
    r = as.getData(8);
    assertEquals("qwertyqwerty",new String(r));
   
    // test the list methods
    i = as.addListData("l1:4567890".toCharArray());
    assertEquals(22,i);
    int size = as.getListSize(i);
    assertEquals(1,size);
    r = as.getListData(i, 0);
    assertEquals("l1:4567890",new String(r));
   
    i = as.addListData(i,"l2:4567890".toCharArray());
    assertEquals(22,i);
    size = as.getListSize(i);
    assertEquals(2,size);
    r = as.getListData(i,1);
    assertEquals("l2:4567890",new String(r));
       
    // check if the first element (element 0) is still the same!
    r = as.getListData(i,0);
    assertEquals("l1:4567890",new String(r));
   
    i = as.addListData(i,"l3:4567890".toCharArray());
    assertEquals(22,i);
    size = as.getListSize(i);
    assertEquals(3,size);
    r = as.getListData(i,2);
    assertEquals("l3:4567890",new String(r));
   
    i = as.addListData("another".toCharArray());
    size = as.getListSize(i);
    assertEquals(1,size);
    r = as.getListData(i,0);
    assertEquals("another",new String(r));

    char[] f1 = "fixed data 01".toCharArray();
    int i1 = as.addFixedLengthData(f1);
    int i2 = as.addData("last".toCharArray());
    r = as.getFixedLengthData(i1, f1.length);
    assertNotNull(r);
    assertEquals(r.length,f1.length);
    assertEquals(new String(f1),new String(r));
    r = as.getData(i2);
    assertNotNull(r);
    assertEquals("last",new String(r));
   
    i = as.addListData("element1".toCharArray());
    as.addListData(i,"element2".toCharArray());
    int j = as.findListData(i, "element1".toCharArray());
    assertEquals(0,j);
    j = as.findListData(i, "element2".toCharArray());
    assertEquals(1,j);
  }
View Full Code Here

  }
 

  @Test
  public void testStoreCharMapPhase1() {
    StoreCharMapPhase1 store = new StoreCharMapPhase1(new StoreArrayOfCharArrays());
    System.out.println("Adding first:1");
    int i = store.put(-1, 'a', 1);
    assertEquals(0,i);
    System.out.println("Adding first:2");
    i = store.put(0,'b', 2);
View Full Code Here

TOP

Related Classes of com.jpetrak.gate.stringannotation.extendedgazetteer2.ExtendedGazetteer2

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.