Package edu.isi.karma.kr2rml.writer

Examples of edu.isi.karma.kr2rml.writer.KR2RMLBloomFilter


      });
    }
    HNode hnode = factory.getHNode(hNodeId);
    List<Table> dataTables = new ArrayList<Table>();
    CloneTableUtils.getDatatable(worksheet.getDataTable(), factory.getHTable(hnode.getHTableId()), dataTables, selection);
    KR2RMLBloomFilter uris = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
    Set<String> uriSet = new HashSet<String>();
    for(Table t : dataTables) {
      for(Row r : t.getRows(0, t.getNumRows(), selection)) {
        Node n = r.getNode(hNodeId);
        if(n != null && n.getValue() != null && !n.getValue().isEmptyValue() && n.getValue().asString() != null && !n.getValue().asString().trim().isEmpty() ) {
          String value = n.getValue().asString().trim().replace(" ", "");
          String baseURI = worksheet.getMetadataContainer().getWorksheetProperties().getPropertyValue(Property.baseURI);
          try {
            URI uri = new URI(value);
            if (!uri.isAbsolute() && baseURI != null) {
              value = baseURI + value;
            }
          } catch (URISyntaxException e) {
          }
          builder = new StringBuilder();
          value = builder.append("<").append(value).append(">").toString(); //String builder
          uriSet.add(value);
          uris.add(new Key(value.getBytes(UTF8_CHARSET)));
        }
      }
    }
    Set<String> maps = new HashSet<String>();
    Map<String, String> bloomfilterMapping = new HashMap<String, String>();
    try{
      for (String concatenatedPredicateObjectMaps : concatenatedPredicateObjectMapsList) {
        List<String> predicateObjectMaps = new ArrayList<String>(Arrays.asList(concatenatedPredicateObjectMaps.split(",")));
        maps.addAll(predicateObjectMaps);
        if (maps.size() > limit) {
          bloomfilterMapping.putAll(util.getBloomFiltersForMaps(tripleStoreUrl, null, maps));
          maps = new HashSet<String>();
        }
      }
      if (maps.size() > 0)
        bloomfilterMapping.putAll(util.getBloomFiltersForMaps(tripleStoreUrl, null, maps));
    } catch (KarmaException e1) {
      e1.printStackTrace();
    }
    while(concatenatedPredicateObjectMapsListItr.hasNext() && predicatesItr.hasNext() && otherClassesItr.hasNext())
    {

      String concatenatedPredicateObjectMaps = concatenatedPredicateObjectMapsListItr.next();
      List<String> predicateObjectMaps = new ArrayList<String>(Arrays.asList(concatenatedPredicateObjectMaps.split(",")));
      String predicate =  predicatesItr.next();
      String otherClass = otherClassesItr.next();
      try {
        KR2RMLBloomFilter intersectionBF = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
        for (String triplemap : predicateObjectMaps) {
          String serializedBloomFilter = bloomfilterMapping.get(triplemap);
          if (serializedBloomFilter != null) {
            KR2RMLBloomFilter bf = new KR2RMLBloomFilter();
            bf.populateFromCompressedAndBase64EncodedString(serializedBloomFilter);
            intersectionBF.or(bf);
          }
        }
        System.out.println(predicate + " " + intersectionBF.estimateNumberOfHashedValues());
        intersectionBF.and(uris);
View Full Code Here


  @Override
  public void run() {

    while(!isDone || !bloomfilters.isEmpty()) {
      KR2RMLBloomFilter bf = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
      String tmp = getBloomfilter();
      try {
        if (tmp != null)
          bf.populateFromCompressedAndBase64EncodedString(tmp);
      } catch (IOException e) {
       
      }
      this.bf.or(bf);
    }
View Full Code Here

      verification.putAll(utilObj.getBloomFiltersForMaps(triplestoreURL, context, triplemaps));
      boolean verify = true;
      for (Entry<String, String> entry : verification.entrySet()) {
        String key = entry.getKey();
        String value = entry.getValue();
        KR2RMLBloomFilter bf2 = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
        KR2RMLBloomFilter bf = bfs.get(key);
        bf2.populateFromCompressedAndBase64EncodedString(value);
        bf2.and(bf);
        bf2.xor(bf);
        try {
          Field f = BloomFilter.class.getDeclaredField("bits");
View Full Code Here

    Iterator<String> otherClassesItr = otherClasses.iterator();
    String hNodeId = FetchHNodeIdFromAlignmentCommand.gethNodeId(AlignmentManager.Instance().constructAlignmentId(workspace.getId(), worksheetId), columnUri);
    HNode hnode = factory.getHNode(hNodeId);
    List<Table> dataTables = new ArrayList<Table>();
    CloneTableUtils.getDatatable(worksheet.getDataTable(), factory.getHTable(hnode.getHTableId()), dataTables, selection);
    KR2RMLBloomFilter uris = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
    Set<String> uriSet = new HashSet<String>();
    for(Table t : dataTables) {
      for(Row r : t.getRows(0, t.getNumRows(), selection)) {
        Node n = r.getNode(hNodeId);
        if(n != null && n.getValue() != null && !n.getValue().isEmptyValue() && n.getValue().asString() != null && !n.getValue().asString().trim().isEmpty() ) {
          String value = n.getValue().asString().trim().replace(" ", "");;
          builder = new StringBuilder();
          String baseURI = worksheet.getMetadataContainer().getWorksheetProperties().getPropertyValue(Property.baseURI);
          try {
            URI uri = new URI(value);
            if (!uri.isAbsolute() && baseURI != null) {
              value = baseURI + value;
            }
          } catch (URISyntaxException e) {
          }
          value = builder.append("<").append(value).append(">").toString(); //String builder
          uriSet.add(value);
          uris.add(new Key(value.getBytes(UTF8_CHARSET)));
        }
      }
    }
    Set<String> maps = new HashSet<String>();
    Map<String, String> bloomfilterMapping = new HashMap<String, String>();
    try{
      for (String concatenatedPredicateObjectMaps : concatenatedPredicateObjectMapsList) {
        List<String> predicateObjectMaps = new ArrayList<String>(Arrays.asList(concatenatedPredicateObjectMaps.split(",")));
        maps.addAll(predicateObjectMaps);
        if (maps.size() > limit) {
          bloomfilterMapping.putAll(util.getBloomFiltersForMaps(tripleStoreUrl, null, maps));
          maps = new HashSet<String>();
        }
      }
      if (maps.size() > 0)
        bloomfilterMapping.putAll(util.getBloomFiltersForMaps(tripleStoreUrl, null, maps));
    } catch (KarmaException e1) {
      e1.printStackTrace();
    }
    while(concatenatedPredicateObjectMapsListItr.hasNext() && predicatesItr.hasNext() && otherClassesItr.hasNext())
    {
      String concatenatedPredicateObjectMaps = concatenatedPredicateObjectMapsListItr.next();
      List<String> predicateObjectMaps = new ArrayList<String>(Arrays.asList(concatenatedPredicateObjectMaps.split(",")));
      String predicate =  predicatesItr.next();
      String otherClass = otherClassesItr.next();
      try {
        KR2RMLBloomFilter intersectionBF = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
        for (String triplemap : predicateObjectMaps) {
          String serializedBloomFilter = bloomfilterMapping.get(triplemap);
          if (serializedBloomFilter != null) {
            KR2RMLBloomFilter bf = new KR2RMLBloomFilter();
            bf.populateFromCompressedAndBase64EncodedString(serializedBloomFilter);
            intersectionBF.or(bf);
          }
        }
        intersectionBF.and(uris);
        int estimate = intersectionBF.estimateNumberOfHashedValues();
View Full Code Here

  public void testGenerateRDF1() {
    try {

      KR2RMLBloomFilterManager peopleBloomFilterManager = getBloomFilterManagerForSource("people.json", InputType.JSON, "people-model");
      KR2RMLBloomFilterManager scheduleBloomFilterManager = getBloomFilterManagerForSource("schedule.csv", InputType.CSV, "schedule-model");
      KR2RMLBloomFilter peoplePersonWithTwitterIdBF = peopleBloomFilterManager.getBloomFilter("http://isi.edu/integration/karma/dev#PredicateObjectMap_5fcf2d39-f62b-4cdd-863e-bde21493e1bd");
      Key k = new Key(("<http://lod.isi.edu/cs548/person/Slepicka>").getBytes());
      assertFalse(peoplePersonWithTwitterIdBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Taheriyan>").getBytes());
      assertTrue(peoplePersonWithTwitterIdBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Kozareva>").getBytes());
      assertFalse(peoplePersonWithTwitterIdBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Ambite>").getBytes());
      assertFalse(peoplePersonWithTwitterIdBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Szekely>").getBytes());
      assertTrue(peoplePersonWithTwitterIdBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Knoblock>").getBytes());
      assertTrue(peoplePersonWithTwitterIdBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Wu>").getBytes());
      assertFalse(peoplePersonWithTwitterIdBF.membershipTest(k));
      assertEquals(3, peoplePersonWithTwitterIdBF.estimateNumberOfHashedValues());
      KR2RMLBloomFilter schedulePersonBF = scheduleBloomFilterManager.getBloomFilter("http://isi.edu/integration/karma/dev#TriplesMap_413a6176-d893-45aa-b1c2-6661b5c491ab");
      k = new Key(("<http://lod.isi.edu/cs548/person/Slepicka>").getBytes());
      assertTrue(schedulePersonBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Taheriyan>").getBytes());
      assertTrue(schedulePersonBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Ambite>").getBytes());
      assertTrue(schedulePersonBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Szekely>").getBytes());
      assertTrue(schedulePersonBF.membershipTest(k));
      k = new Key(("<http://lod.isi.edu/cs548/person/Knoblock>").getBytes());
      assertTrue(schedulePersonBF.membershipTest(k));
      assertEquals(5, schedulePersonBF.estimateNumberOfHashedValues());
     
      KR2RMLBloomFilter intersectionBF = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
      intersectionBF.or(peoplePersonWithTwitterIdBF);
      intersectionBF.and(schedulePersonBF);
      assertEquals(3, intersectionBF.estimateNumberOfHashedValues());
     
      KR2RMLBloomFilter hasInstructorBF = scheduleBloomFilterManager.getBloomFilter("http://isi.edu/integration/karma/dev#RefObjectMap_bb82f923-2953-4bd4-bc7b-d1196e05dbf6");
     
      k = new Key(("<http://lod.isi.edu/cs548/person/Szekely>").getBytes());
      assertTrue(hasInstructorBF.membershipTest(k));
      intersectionBF = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
      intersectionBF.or(hasInstructorBF);
      intersectionBF.and(peoplePersonWithTwitterIdBF);
      assertEquals(3, intersectionBF.estimateNumberOfHashedValues());
     
    } catch (Exception e) {
View Full Code Here

        verification.putAll(utilObj.getBloomFiltersForMaps(modelRepoUrl, modelContext, triplemaps));
        boolean verify = true;
        for (Entry<String, String> entry : verification.entrySet()) {
          String key = entry.getKey();
          String value = entry.getValue();
          KR2RMLBloomFilter bf2 = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
          KR2RMLBloomFilter bf = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
          bf2.populateFromCompressedAndBase64EncodedString(value);
          bf.populateFromCompressedAndBase64EncodedString(obj.getString(key));
          bf2.and(bf);
          bf2.xor(bf);
          try {
            Field f1 = BloomFilter.class.getDeclaredField("bits");
            f1.setAccessible(true);
View Full Code Here

    Set<String> triplemaps = new HashSet<String>(Arrays.asList(obj.getString("ids").split(",")));
    bloomfilterMapping.putAll(utilObj.getBloomFiltersForMaps(modelRepoUrl, modelContext, triplemaps));
    Map<String, KR2RMLBloomFilter> bfs = new HashMap<String, KR2RMLBloomFilter>();
    for (String tripleUri : triplemaps) {
      String serializedBloomFilter = obj.getString(tripleUri);
      KR2RMLBloomFilter bf = new KR2RMLBloomFilter();
      bf.populateFromCompressedAndBase64EncodedString(serializedBloomFilter);
      bfs.put(tripleUri, bf);
    }
    return utilObj.updateTripleStoreWithBloomFilters(bfs, bloomfilterMapping, modelRepoUrl, modelContext);
  }
View Full Code Here

        Map<String, KR2RMLBloomFilter> mapping = new HashMap<String, KR2RMLBloomFilter>();
       
        for (Entry<String, String> entry : serializedmapping.entrySet()) {
          String key = entry.getKey();
          String value = entry.getValue();
          KR2RMLBloomFilter bf = new KR2RMLBloomFilter(KR2RMLBloomFilter.defaultVectorSize, KR2RMLBloomFilter.defaultnbHash, Hash.JENKINS_HASH);
          bf.populateFromCompressedAndBase64EncodedString(value);
          mapping.put(key, bf);
        }
       
        Model model = ModelFactory.createDefaultModel();
        InputStream s = new FileInputStream(new File(filepath));
    model.read(s, null, "TURTLE");
    StmtIterator iterator = model.listStatements();
    while(iterator.hasNext()) {
      Statement st = iterator.next();
      String subject = "<" + st.getSubject().toString() + ">";
      String object = "<" + st.getObject().toString() + ">";
      for (Entry<String, KR2RMLBloomFilter> entry : mapping.entrySet()) {
        KR2RMLBloomFilter bf = entry.getValue();
        if (bf.membershipTest(new Key(subject.getBytes("UTF-8"))))
          bf.add(new Key(object.getBytes("UTF-8")));
        if (bf.membershipTest(new Key(object.getBytes("UTF-8"))))
          bf.add(new Key(subject.getBytes("UTF-8")));
      }
    }
   
    utilObj.updateTripleStoreWithBloomFilters(mapping, serializedmapping, triplestoreURL, null);
   
View Full Code Here

  }
 
  public boolean updateTripleStoreWithBloomFilters(Map<String, KR2RMLBloomFilter> bfs, Map<String, String> bloomfilterMapping, String modelurl, String context) throws KarmaException, IOException {
    Set<String> triplemaps = bfs.keySet();
    for (String tripleUri : triplemaps) {
      KR2RMLBloomFilter bf = bfs.get(tripleUri);
      String oldserializedBloomFilter = bloomfilterMapping.get(tripleUri);
      if (oldserializedBloomFilter != null) {
        KR2RMLBloomFilter bf2 = new KR2RMLBloomFilter();
        bf2.populateFromCompressedAndBase64EncodedString(oldserializedBloomFilter);
        bf.or(bf2);
      }
      bfs.put(tripleUri, bf);
    }
    deleteBloomFiltersForMaps(modelurl, null, triplemaps);
View Full Code Here

TOP

Related Classes of edu.isi.karma.kr2rml.writer.KR2RMLBloomFilter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.