Examples of com.hp.hpl.jena.tdb.base.file.MetaFile

com.hp.hpl.jena.tdb.base.file.MetaFile
Abstraction and many convenience operations on metadata. Metadata is recorded in Java properties style - not RDF - because it's relative to the file or context used. Keys and values are always strings.

         * tdb.cache.blockread.size=10000
         * tdb.synctick=100000
         */
        
        String propertiesFile = "tdb.properties" ; 
        MetaFile metafile = locationMetadata(location) ;
        // dupulicate
        Properties config = new Properties(globalConfig) ;
        boolean localProperties = false ;
        
        if ( location.exists(propertiesFile) )
        {
            // Load now to test for errors.
            localProperties = true ;
            try { PropertyUtils.loadFromFile(config, propertiesFile) ; }
            catch (IOException ex) { throw new TDBException(ex) ; }
        }
        
        // Only support this so far.
        if ( ! metafile.propertyEquals("tdb.layout", "v1") )
            SetupTDB_OLD.error(log, "Excepted 'v1': Wrong layout: "+metafile.getProperty("tdb.layout")) ;
            
        if ( ! metafile.propertyEquals("tdb.type", "standalone") )
            SetupTDB_OLD.error(log, "Not marked as a standalone type: "+metafile.getProperty("tdb.type")) ;


        // Check expectations.
        
        metafile.checkOrSetMetadata("tdb.nodeid.size", Integer.toString(SizeOfNodeId)) ;
        metafile.checkOrSetMetadata("tdb.node.hashsize", Integer.toString(LenNodeHash)) ;
        
        metafile.checkOrSetMetadata("tdb.record.triple", Integer.toString(LenIndexTripleRecord)) ;
        metafile.checkOrSetMetadata("tdb.record.quad",   Integer.toString(LenIndexQuadRecord)) ;
        
        // ---------------------
        
        // ---- Logical structure


        // -- Node Table.
        
        String indexNode2Id = metafile.getProperty("tdb.nodetable.mapping.node2id") ;
        String indexId2Node = metafile.getProperty("tdb.nodetable.mapping.id2node") ;
        
        String nodesdata = metafile.getProperty("tdb.nodetable.mapping.data") ;
        
        log.debug("Object table: "+indexNode2Id+" - "+indexId2Node) ;
        
        int n2idCacheSize = PropertyUtils.getPropertyAsInteger(config, Names.pNode2NodeIdCacheSize) ;
        int id2nCacheSize = PropertyUtils.getPropertyAsInteger(config, Names.pNodeId2NodeCacheSize) ;
        int nodeMissCacheSize = PropertyUtils.getPropertyAsInteger(config, Names.pNodeMissesCacheSize) ;
        
        // Cache sizes should come from this.info.
        NodeTable nodeTable = makeNodeTable(location, 
                                            indexNode2Id, n2idCacheSize,
                                            indexId2Node, id2nCacheSize,
                                            nodeMissCacheSize) ;


        DatasetControl policy = createConcurrencyPolicy() ;
        
        TripleTable tripleTable = makeTripleTable(location, config, nodeTable, 
                                                  Names.primaryIndexTriples, Names.tripleIndexes, policy) ;
        QuadTable quadTable = makeQuadTable(location, config, nodeTable,
                                            Names.primaryIndexQuads, Names.quadIndexes, policy) ;


        DatasetPrefixesTDB prefixes = makePrefixes(location, config, policy) ;


        // ---- Create the DatasetGraph object
        StoreConfig storeConfig = new StoreConfig(location, null, null, null, null) ;
        DatasetGraphTDB dsg = new DatasetGraphTDB(tripleTable, quadTable, prefixes, chooseOptimizer(location), storeConfig) ;


        // Finalize
        metafile.flush() ;
        
        // Set TDB features.
        if ( localProperties )
        {
            /*

View Full Code Here


    protected static DatasetControl createConcurrencyPolicy() { return new DatasetControlMRSW() ; }
    
    public static TripleTable makeTripleTable(Location location, Properties config, NodeTable nodeTable, String dftPrimary, String[] dftIndexes, DatasetControl policy)
    {
        MetaFile metafile = location.getMetaFile() ;
        String primary = metafile.getOrSetDefault("tdb.indexes.triples.primary", dftPrimary) ;
        String x = metafile.getOrSetDefault("tdb.indexes.triples", StrUtils.strjoin(",",dftIndexes)) ;
        String indexes[] = x.split(",") ;
        
        if ( indexes.length != 3 )
            SetupTDB_OLD.error(log, "Wrong number of triple table indexes: "+StrUtils.strjoin(",", indexes)) ;
        log.debug("Triple table: "+primary+" :: "+StrUtils.strjoin(",", indexes)) ;
        
        TupleIndex tripleIndexes[] = makeTupleIndexes(location, config, primary, indexes, indexes) ;
        if ( tripleIndexes.length != indexes.length )
            SetupTDB_OLD.error(log, "Wrong number of triple table tuples indexes: "+tripleIndexes.length) ;
        TripleTable tripleTable = new TripleTable(tripleIndexes, nodeTable, policy) ;
        metafile.flush() ;
        return tripleTable ;
    }

View Full Code Here

        return tripleTable ;
    }
    
    public static QuadTable makeQuadTable(Location location, Properties config, NodeTable nodeTable, String dftPrimary, String[] dftIndexes, DatasetControl policy)
    {
        MetaFile metafile = location.getMetaFile() ; 
        String primary = metafile.getOrSetDefault("tdb.indexes.quads.primary", dftPrimary) ;
        String x = metafile.getOrSetDefault("tdb.indexes.quads", StrUtils.strjoin(",",dftIndexes)) ;
        String indexes[] = x.split(",") ;


        if ( indexes.length != 6 )
            SetupTDB_OLD.error(log, "Wrong number of quad table indexes: "+StrUtils.strjoin(",", indexes)) ;
        log.debug("Quad table: "+primary+" :: "+StrUtils.strjoin(",", indexes)) ;
        
        TupleIndex quadIndexes[] = makeTupleIndexes(location, config, primary, indexes, indexes) ;
        if ( quadIndexes.length != indexes.length )
            SetupTDB_OLD.error(log, "Wrong number of quad table tuples indexes: "+quadIndexes.length) ;
        QuadTable quadTable = new QuadTable(quadIndexes, nodeTable, policy) ;
        metafile.flush() ;
        return quadTable ;
    }

View Full Code Here

         * 
         */


        // Some of this is also in locationMetadata.
        
        MetaFile metafile = location.getMetaFile() ;
    
        // The index using for Graph+Prefix => URI
        String indexPrefixes = metafile.getOrSetDefault("tdb.prefixes.index.file", Names.indexPrefix) ;
        String primary = metafile.getOrSetDefault("tdb.prefixes.primary", Names.primaryIndexPrefix) ;
        String x = metafile.getOrSetDefault("tdb.prefixes.indexes", StrUtils.strjoin(",",Names.prefixIndexes)) ;
        String indexes[] = x.split(",") ;
        
        TupleIndex prefixIndexes[] = makeTupleIndexes(location, config, primary, indexes, new String[]{indexPrefixes}) ;
        if ( prefixIndexes.length != indexes.length )
            SetupTDB_OLD.error(log, "Wrong number of triple table tuples indexes: "+prefixIndexes.length) ;
        
        // The nodetable.
        String pnNode2Id = metafile.getOrSetDefault("tdb.prefixes.nodetable.mapping.node2id", Names.prefixNode2Id) ;
        String pnId2Node = metafile.getOrSetDefault("tdb.prefixes.nodetable.mapping.id2node", Names.prefixId2Node) ;
        
        // No cache - the prefix mapping is a cache
        NodeTable prefixNodes = makeNodeTable(location, pnNode2Id, -1, pnId2Node, -1, -1)  ;
        
        DatasetPrefixesTDB prefixes = new DatasetPrefixesTDB(prefixIndexes, prefixNodes, policy) ;

View Full Code Here

        * tdb.file.impl.version=bplustree-v1          
        */


        FileSet fs = new FileSet(location, indexName) ;
        // Physical
        MetaFile metafile = fs.getMetaFile() ;
        
        metafile.checkOrSetMetadata("tdb.file.type", "rangeindex") ;
        metafile.checkOrSetMetadata("tdb.file.indexorder", indexOrder) ;
        
        int readCacheSize = PropertyUtils.getPropertyAsInteger(config, Names.pBlockReadCacheSize) ;
        int writeCacheSize = PropertyUtils.getPropertyAsInteger(config, Names.pBlockWriteCacheSize) ;
        
        // Value part is null (zero length)
        RangeIndex rIndex = makeRangeIndex(location, indexName, keyLength, 0, readCacheSize, writeCacheSize) ;
        TupleIndex tupleIndex = new TupleIndexRecord(primary.length(), new ColumnMap(primary, indexOrder), rIndex.getRecordFactory(), rIndex) ;
        metafile.flush() ;
        return tupleIndex ;
    }

View Full Code Here

         * tdb.file.impl=bplustree         # Implementation
         * tdb.file.impl.version=bplustree-v1          
         */
         FileSet fs = new FileSet(location, indexName) ;
         // Physical
         MetaFile metafile = fs.getMetaFile() ;
         metafile.checkOrSetMetadata("tdb.file.type", "rangeindex") ;
         String indexType = metafile.getOrSetDefault("tdb.file.impl", "bplustree") ;
         if ( ! indexType.equals("bplustree") )
         {
             log.error("Unknown index type: "+indexType) ;
             throw new TDBException("Unknown index type: "+indexType) ;
         }
         metafile.checkOrSetMetadata("tdb.file.impl.version", "bplustree-v1") ;
         
         RangeIndex rIndex =  makeBPlusTree(fs, readCacheSize, writeCacheSize, dftKeyLength, dftValueLength) ;
         metafile.flush();
         return rIndex ;
    }

View Full Code Here

         * tdb.bplustree.record=24,0
         * tdb.bplustree.blksize=
         * tdb.bplustree.order=
         */
        
        MetaFile metafile = fs.getMetaFile() ;
        RecordFactory recordFactory = makeRecordFactory(metafile, "tdb.bplustree.record", dftKeyLength, dftValueLength) ;
        
        String blkSizeStr = metafile.getOrSetDefault("tdb.bplustree.blksize", Integer.toString(SystemTDB.BlockSize)) ; 
        int blkSize = SetupTDB_OLD.parseInt(blkSizeStr, "Bad block size") ;
        
        // IndexBuilder.getBPlusTree().newRangeIndex(fs, recordFactory) ;
        // Does not set order.
        
        int calcOrder = BPlusTreeParams.calcOrder(blkSize, recordFactory.recordLength()) ;
        String orderStr = metafile.getOrSetDefault("tdb.bplustree.order", Integer.toString(calcOrder)) ;
        int order = SetupTDB_OLD.parseInt(orderStr, "Bad order for B+Tree") ;
        if ( order != calcOrder )
            SetupTDB_OLD.error(log, "Wrong order (" + order + "), calculated = "+calcOrder) ;


        RangeIndex rIndex = createBPTree(fs, order, blkSize, readCacheSize, writeCacheSize, recordFactory) ;
        metafile.flush() ;
        return rIndex ;
    }

View Full Code Here

         * tdb.file.impl.version=dat-v1
         *
         * tdb.object.encoding=sse 
         */
        
        MetaFile metafile = fsIdToNode.getMetaFile() ;
        metafile.checkOrSetMetadata("tdb.file.type", ObjectFile.type) ;
        metafile.checkOrSetMetadata("tdb.file.impl", "dat") ;
        metafile.checkOrSetMetadata("tdb.file.impl.version", "dat-v1") ;
        metafile.checkOrSetMetadata("tdb.object.encoding", "sse") ;
        
        String filename = fsIdToNode.filename(Names.extNodeData) ;
        ObjectFile objFile = FileFactory.createObjectFileDisk(filename);
        metafile.flush();
        return objFile ;
    }

View Full Code Here

    /** Check and set default for the dataset design */
    public static MetaFile locationMetadata(Location location)
    {
        boolean newDataset = location.isMem() || ! FileOps.existsAnyFiles(location.getDirectoryPath()) ; 


        MetaFile metafile = location.getMetaFile() ;
        boolean isPreMetadata = false ;
        
        if (!newDataset && metafile.existsMetaData())
        {
            // Existing metadata
            String verString = metafile.getProperty("tdb.create.version", "unknown") ;
            TDB.logInfo.debug("Location: " + location.toString()) ;
            TDB.logInfo.debug("Version:  " + verString) ;
        }
        else
        {
            // Not new ?, no metadata
            // Either it's brand new (so set the defaults)
            // or it's a pre-0.9 dataset (files exists)


            if ( ! newDataset )
            {
                // Well-known name of the primary triples index.
                isPreMetadata = FileOps.exists(location.getPath("SPO.idn")) ;
                // PROBLEM.
//                boolean b = FileOps.exists(location.getPath("SPO.idn")) ;
//                if ( !b )
//                {
//                    log.error("Existing files but no metadata and not old-style fixed layout: "+location.getDirectoryPath()) ;
//                    File d = new File(location.getDirectoryPath()) ;
//                    File[] entries = d.listFiles() ;
//                    for ( File f : d.listFiles()  )
//                        log.error("File: "+f.getName()) ;
//                    throw new TDBException("Can't build dataset: "+location) ;
//                }
//                isPreMetadata = true ;
            }
        }
            
        // Ensure defaults.
        
        if ( newDataset )
        {
            metafile.ensurePropertySet("tdb.create.version", TDB.VERSION) ;
            metafile.ensurePropertySet("tdb.created", Utils.nowAsXSDDateTimeString()) ;
        }
        
        if ( isPreMetadata )
        {
            // Existing location (has some files in it) but no metadata.
            // Fake it as TDB 0.8.1 (which did not have metafiles)
            // If it's the wrong file format, things do badly wrong later.
            metafile.ensurePropertySet("tdb.create.version", "0.8") ;
            metafile.setProperty(Names.kCreatedDate, Utils.nowAsXSDDateTimeString()) ;
        }
            
        metafile.ensurePropertySet("tdb.layout", "v1") ;
        metafile.ensurePropertySet("tdb.type", "standalone") ;
        
        String layout = metafile.getProperty("tdb.layout") ;
        
        if ( layout.equals("v1") )
        {
            metafile.ensurePropertySet("tdb.indexes.triples.primary", Names.primaryIndexTriples) ;
            metafile.ensurePropertySet("tdb.indexes.triples", StrUtils.strjoin(",", Names.tripleIndexes)) ;


            metafile.ensurePropertySet("tdb.indexes.quads.primary", Names.primaryIndexQuads) ;
            metafile.ensurePropertySet("tdb.indexes.quads", StrUtils.strjoin(",", Names.quadIndexes)) ;
            
            metafile.ensurePropertySet("tdb.nodetable.mapping.node2id", Names.indexNode2Id) ;
            metafile.ensurePropertySet("tdb.nodetable.mapping.id2node", Names.indexId2Node) ;
            
            metafile.ensurePropertySet("tdb.prefixes.index.file", Names.indexPrefix) ;
            metafile.ensurePropertySet("tdb.prefixes.nodetable.mapping.node2id", Names.prefixNode2Id) ;
            metafile.ensurePropertySet("tdb.prefixes.nodetable.mapping.id2node", Names.prefixId2Node) ;
            
        }
        else
            SetupTDB_OLD.error(log, "tdb.layout: expected v1") ;
            
        
        metafile.flush() ;
        return metafile ; 
    }

View Full Code Here

    /** Check and set default for the dataset design */
    public static MetaFile locationMetadata(Location location)
    {
        boolean newDataset = location.isMem() || ! FileOps.existsAnyFiles(location.getDirectoryPath()) ; 


        MetaFile metafile = location.getMetaFile() ;
        boolean isPreMetadata = false ;
        
        if (!newDataset && metafile.existsMetaData())
        {
            // Existing metadata
            String verString = metafile.getProperty("tdb.create.version", "unknown") ;
            TDB.logInfo.debug("Location: " + location.toString()) ;
            TDB.logInfo.debug("Version:  " + verString) ;
        }
        else
        {
            // Not new ?, no metadata
            // Either it's brand new (so set the defaults)
            // or it's a pre-0.9 dataset (files exists)


            if ( ! newDataset )
            {
                // Well-known name of the primary triples index.
                isPreMetadata = FileOps.exists(location.getPath("SPO.idn")) ;
                // PROBLEM.
//                boolean b = FileOps.exists(location.getPath("SPO.idn")) ;
//                if ( !b )
//                {
//                    log.error("Existing files but no metadata and not old-style fixed layout: "+location.getDirectoryPath()) ;
//                    File d = new File(location.getDirectoryPath()) ;
//                    File[] entries = d.listFiles() ;
//                    for ( File f : d.listFiles()  )
//                        log.error("File: "+f.getName()) ;
//                    throw new TDBException("Can't build dataset: "+location) ;
//                }
//                isPreMetadata = true ;
            }
        }
            
        // Ensure defaults.
        
        if ( newDataset )
        {
            metafile.ensurePropertySet("tdb.create.version", TDB.VERSION) ;
            metafile.ensurePropertySet("tdb.created", Utils.nowAsXSDDateTimeString()) ;
        }
        
        if ( isPreMetadata )
        {
            // Existing location (has some files in it) but no metadata.
            // Fake it as TDB 0.8.1 (which did not have metafiles)
            // If it's the wrong file format, things do badly wrong later.
            metafile.ensurePropertySet("tdb.create.version", "0.8") ;
            metafile.setProperty(Names.kCreatedDate, Utils.nowAsXSDDateTimeString()) ;
        }
            
        metafile.ensurePropertySet("tdb.layout", "v1") ;
        metafile.ensurePropertySet("tdb.type", "standalone") ;
        
        String layout = metafile.getProperty("tdb.layout") ;
        
        if ( layout.equals("v1") )
        {
            metafile.ensurePropertySet("tdb.indexes.triples.primary", Names.primaryIndexTriples) ;
            metafile.ensurePropertySet("tdb.indexes.triples", StrUtils.strjoin(",", Names.tripleIndexes)) ;


            metafile.ensurePropertySet("tdb.indexes.quads.primary", Names.primaryIndexQuads) ;
            metafile.ensurePropertySet("tdb.indexes.quads", StrUtils.strjoin(",", Names.quadIndexes)) ;
            
            metafile.ensurePropertySet("tdb.nodetable.mapping.node2id", Names.indexNode2Id) ;
            metafile.ensurePropertySet("tdb.nodetable.mapping.id2node", Names.indexId2Node) ;
            
            metafile.ensurePropertySet("tdb.prefixes.index.file", Names.indexPrefix) ;
            metafile.ensurePropertySet("tdb.prefixes.nodetable.mapping.node2id", Names.prefixNode2Id) ;
            metafile.ensurePropertySet("tdb.prefixes.nodetable.mapping.id2node", Names.prefixId2Node) ;
            
        }
        else
            SetupTDB_OLD.error(log, "tdb.layout: expected v1") ;
            
        
        metafile.flush() ;
        return metafile ; 
    }

View Full Code Here

0 1

TOP

Related Classes of com.hp.hpl.jena.tdb.base.file.MetaFile

com.hp.hpl.jena.tdb.sys.SetupTDB_OLD

org.openjena.atlas.lib.PropertiesSorted

java.io.InputStream

java.io.ByteArrayOutputStream

java.util.Properties

java.io.PrintStream

java.io.File

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.