/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.util.Time.now;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutFlags;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.StringUtils;
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
/**
* This class loads and stores the FSImage of the NameNode. The file
* src/main/proto/fsimage.proto describes the on-disk layout of the FSImage.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class FSImageFormat {
private static final Log LOG = FSImage.LOG;
// Static-only class
private FSImageFormat() {}
interface AbstractLoader {
MD5Hash getLoadedImageMd5();
long getLoadedImageTxId();
}
static class LoaderDelegator implements AbstractLoader {
private AbstractLoader impl;
private final Configuration conf;
private final FSNamesystem fsn;
LoaderDelegator(Configuration conf, FSNamesystem fsn) {
this.conf = conf;
this.fsn = fsn;
}
@Override
public MD5Hash getLoadedImageMd5() {
return impl.getLoadedImageMd5();
}
@Override
public long getLoadedImageTxId() {
return impl.getLoadedImageTxId();
}
public void load(File file) throws IOException {
Preconditions.checkState(impl == null, "Image already loaded!");
FileInputStream is = null;
try {
is = new FileInputStream(file);
byte[] magic = new byte[FSImageUtil.MAGIC_HEADER.length];
IOUtils.readFully(is, magic, 0, magic.length);
if (Arrays.equals(magic, FSImageUtil.MAGIC_HEADER)) {
FSImageFormatProtobuf.Loader loader = new FSImageFormatProtobuf.Loader(
conf, fsn);
impl = loader;
loader.load(file);
} else {
Loader loader = new Loader(conf, fsn);
impl = loader;
loader.load(file);
}
} finally {
IOUtils.cleanup(LOG, is);
}
}
}
/**
* Construct a loader class to load the image. It chooses the loader based on
* the layout version.
*/
public static LoaderDelegator newLoader(Configuration conf, FSNamesystem fsn) {
return new LoaderDelegator(conf, fsn);
}
/**
* A one-shot class responsible for loading an image. The load() function
* should be called once, after which the getter methods may be used to retrieve
* information about the image that was loaded, if loading was successful.
*/
public static class Loader implements AbstractLoader {
private final Configuration conf;
/** which namesystem this loader is working for */
private final FSNamesystem namesystem;
/** Set to true once a file has been loaded using this loader. */
private boolean loaded = false;
/** The transaction ID of the last edit represented by the loaded file */
private long imgTxId;
/** The MD5 sum of the loaded file */
private MD5Hash imgDigest;
private Map<Integer, Snapshot> snapshotMap = null;
private final ReferenceMap referenceMap = new ReferenceMap();
Loader(Configuration conf, FSNamesystem namesystem) {
this.conf = conf;
this.namesystem = namesystem;
}
/**
* Return the MD5 checksum of the image that has been loaded.
* @throws IllegalStateException if load() has not yet been called.
*/
@Override
public MD5Hash getLoadedImageMd5() {
checkLoaded();
return imgDigest;
}
@Override
public long getLoadedImageTxId() {
checkLoaded();
return imgTxId;
}
/**
* Throw IllegalStateException if load() has not yet been called.
*/
private void checkLoaded() {
if (!loaded) {
throw new IllegalStateException("Image not yet loaded!");
}
}
/**
* Throw IllegalStateException if load() has already been called.
*/
private void checkNotLoaded() {
if (loaded) {
throw new IllegalStateException("Image already loaded!");
}
}
public void load(File curFile) throws IOException {
checkNotLoaded();
assert curFile != null : "curFile is null";
StartupProgress prog = NameNode.getStartupProgress();
Step step = new Step(StepType.INODES);
prog.beginStep(Phase.LOADING_FSIMAGE, step);
long startTime = now();
//
// Load in bits
//
MessageDigest digester = MD5Hash.getDigester();
DigestInputStream fin = new DigestInputStream(
new FileInputStream(curFile), digester);
DataInputStream in = new DataInputStream(fin);
try {
// read image version: first appeared in version -1
int imgVersion = in.readInt();
if (getLayoutVersion() != imgVersion) {
throw new InconsistentFSStateException(curFile,
"imgVersion " + imgVersion +
" expected to be " + getLayoutVersion());
}
boolean supportSnapshot = NameNodeLayoutVersion.supports(
LayoutVersion.Feature.SNAPSHOT, imgVersion);
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.ADD_LAYOUT_FLAGS, imgVersion)) {
LayoutFlags.read(in);
}
// read namespaceID: first appeared in version -2
in.readInt();
long numFiles = in.readLong();
// read in the last generation stamp for legacy blocks.
long genstamp = in.readLong();
namesystem.setGenerationStampV1(genstamp);
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.SEQUENTIAL_BLOCK_ID, imgVersion)) {
// read the starting generation stamp for sequential block IDs
genstamp = in.readLong();
namesystem.setGenerationStampV2(genstamp);
// read the last generation stamp for blocks created after
// the switch to sequential block IDs.
long stampAtIdSwitch = in.readLong();
namesystem.setGenerationStampV1Limit(stampAtIdSwitch);
// read the max sequential block ID.
long maxSequentialBlockId = in.readLong();
namesystem.setLastAllocatedBlockId(maxSequentialBlockId);
} else {
long startingGenStamp = namesystem.upgradeGenerationStampToV2();
// This is an upgrade.
LOG.info("Upgrading to sequential block IDs. Generation stamp " +
"for new blocks set to " + startingGenStamp);
}
// read the transaction ID of the last edit represented by
// this image
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.STORED_TXIDS, imgVersion)) {
imgTxId = in.readLong();
} else {
imgTxId = 0;
}
// read the last allocated inode id in the fsimage
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.ADD_INODE_ID, imgVersion)) {
long lastInodeId = in.readLong();
namesystem.resetLastInodeId(lastInodeId);
if (LOG.isDebugEnabled()) {
LOG.debug("load last allocated InodeId from fsimage:" + lastInodeId);
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Old layout version doesn't have inode id."
+ " Will assign new id for each inode.");
}
}
if (supportSnapshot) {
snapshotMap = namesystem.getSnapshotManager().read(in, this);
}
// read compression related info
FSImageCompression compression;
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.FSIMAGE_COMPRESSION, imgVersion)) {
compression = FSImageCompression.readCompressionHeader(conf, in);
} else {
compression = FSImageCompression.createNoopCompression();
}
in = compression.unwrapInputStream(fin);
LOG.info("Loading image file " + curFile + " using " + compression);
// load all inodes
LOG.info("Number of files = " + numFiles);
prog.setTotal(Phase.LOADING_FSIMAGE, step, numFiles);
Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step);
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.FSIMAGE_NAME_OPTIMIZATION, imgVersion)) {
if (supportSnapshot) {
loadLocalNameINodesWithSnapshot(numFiles, in, counter);
} else {
loadLocalNameINodes(numFiles, in, counter);
}
} else {
loadFullNameINodes(numFiles, in, counter);
}
loadFilesUnderConstruction(in, supportSnapshot, counter);
prog.endStep(Phase.LOADING_FSIMAGE, step);
// Now that the step is finished, set counter equal to total to adjust
// for possible under-counting due to reference inodes.
prog.setCount(Phase.LOADING_FSIMAGE, step, numFiles);
loadSecretManagerState(in);
loadCacheManagerState(in);
// make sure to read to the end of file
boolean eof = (in.read() == -1);
assert eof : "Should have reached the end of image file " + curFile;
} finally {
in.close();
}
imgDigest = new MD5Hash(digester.digest());
loaded = true;
LOG.info("Image file " + curFile + " of size " + curFile.length() +
" bytes loaded in " + (now() - startTime)/1000 + " seconds.");
}
/** Update the root node's attributes */
private void updateRootAttr(INodeWithAdditionalFields root) {
final Quota.Counts q = root.getQuotaCounts();
final long nsQuota = q.get(Quota.NAMESPACE);
final long dsQuota = q.get(Quota.DISKSPACE);
FSDirectory fsDir = namesystem.dir;
if (nsQuota != -1 || dsQuota != -1) {
fsDir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
}
fsDir.rootDir.cloneModificationTime(root);
fsDir.rootDir.clonePermissionStatus(root);
}
/**
* Load fsimage files when 1) only local names are stored,
* and 2) snapshot is supported.
*
* @param numFiles number of files expected to be read
* @param in Image input stream
* @param counter Counter to increment for namenode startup progress
*/
private void loadLocalNameINodesWithSnapshot(long numFiles, DataInput in,
Counter counter) throws IOException {
assert NameNodeLayoutVersion.supports(
LayoutVersion.Feature.FSIMAGE_NAME_OPTIMIZATION, getLayoutVersion());
assert NameNodeLayoutVersion.supports(
LayoutVersion.Feature.SNAPSHOT, getLayoutVersion());
// load root
loadRoot(in, counter);
// load rest of the nodes recursively
loadDirectoryWithSnapshot(in, counter);
}
/**
* load fsimage files assuming only local names are stored. Used when
* snapshots are not supported by the layout version.
*
* @param numFiles number of files expected to be read
* @param in image input stream
* @param counter Counter to increment for namenode startup progress
* @throws IOException
*/
private void loadLocalNameINodes(long numFiles, DataInput in, Counter counter)
throws IOException {
assert NameNodeLayoutVersion.supports(
LayoutVersion.Feature.FSIMAGE_NAME_OPTIMIZATION, getLayoutVersion());
assert numFiles > 0;
// load root
loadRoot(in, counter);
// have loaded the first file (the root)
numFiles--;
// load rest of the nodes directory by directory
while (numFiles > 0) {
numFiles -= loadDirectory(in, counter);
}
if (numFiles != 0) {
throw new IOException("Read unexpect number of files: " + -numFiles);
}
}
/**
* Load information about root, and use the information to update the root
* directory of NameSystem.
* @param in The {@link DataInput} instance to read.
* @param counter Counter to increment for namenode startup progress
*/
private void loadRoot(DataInput in, Counter counter)
throws IOException {
// load root
if (in.readShort() != 0) {
throw new IOException("First node is not root");
}
final INodeDirectory root = loadINode(null, false, in, counter)
.asDirectory();
// update the root's attributes
updateRootAttr(root);
}
/** Load children nodes for the parent directory. */
private int loadChildren(INodeDirectory parent, DataInput in,
Counter counter) throws IOException {
int numChildren = in.readInt();
for (int i = 0; i < numChildren; i++) {
// load single inode
INode newNode = loadINodeWithLocalName(false, in, true, counter);
addToParent(parent, newNode);
}
return numChildren;
}
/**
* Load a directory when snapshot is supported.
* @param in The {@link DataInput} instance to read.
* @param counter Counter to increment for namenode startup progress
*/
private void loadDirectoryWithSnapshot(DataInput in, Counter counter)
throws IOException {
// Step 1. Identify the parent INode
long inodeId = in.readLong();
final INodeDirectory parent = this.namesystem.dir.getInode(inodeId)
.asDirectory();
// Check if the whole subtree has been saved (for reference nodes)
boolean toLoadSubtree = referenceMap.toProcessSubtree(parent.getId());
if (!toLoadSubtree) {
return;
}
// Step 2. Load snapshots if parent is snapshottable
int numSnapshots = in.readInt();
if (numSnapshots >= 0) {
final INodeDirectorySnapshottable snapshottableParent
= INodeDirectorySnapshottable.valueOf(parent, parent.getLocalName());
// load snapshots and snapshotQuota
SnapshotFSImageFormat.loadSnapshotList(snapshottableParent,
numSnapshots, in, this);
if (snapshottableParent.getSnapshotQuota() > 0) {
// add the directory to the snapshottable directory list in
// SnapshotManager. Note that we only add root when its snapshot quota
// is positive.
this.namesystem.getSnapshotManager().addSnapshottable(
snapshottableParent);
}
}
// Step 3. Load children nodes under parent
loadChildren(parent, in, counter);
// Step 4. load Directory Diff List
SnapshotFSImageFormat.loadDirectoryDiffList(parent, in, this);
// Recursively load sub-directories, including snapshot copies of deleted
// directories
int numSubTree = in.readInt();
for (int i = 0; i < numSubTree; i++) {
loadDirectoryWithSnapshot(in, counter);
}
}
/**
* Load all children of a directory
*
* @param in
* @param counter Counter to increment for namenode startup progress
* @return number of child inodes read
* @throws IOException
*/
private int loadDirectory(DataInput in, Counter counter) throws IOException {
String parentPath = FSImageSerialization.readString(in);
// Rename .snapshot paths if we're doing an upgrade
parentPath = renameReservedPathsOnUpgrade(parentPath, getLayoutVersion());
final INodeDirectory parent = INodeDirectory.valueOf(
namesystem.dir.rootDir.getNode(parentPath, true), parentPath);
return loadChildren(parent, in, counter);
}
/**
* load fsimage files assuming full path names are stored
*
* @param numFiles total number of files to load
* @param in data input stream
* @param counter Counter to increment for namenode startup progress
* @throws IOException if any error occurs
*/
private void loadFullNameINodes(long numFiles, DataInput in, Counter counter)
throws IOException {
byte[][] pathComponents;
byte[][] parentPath = {{}};
FSDirectory fsDir = namesystem.dir;
INodeDirectory parentINode = fsDir.rootDir;
for (long i = 0; i < numFiles; i++) {
pathComponents = FSImageSerialization.readPathComponents(in);
final INode newNode = loadINode(
pathComponents[pathComponents.length-1], false, in, counter);
if (isRoot(pathComponents)) { // it is the root
// update the root's attributes
updateRootAttr(newNode.asDirectory());
continue;
}
namesystem.dir.addToInodeMap(newNode);
// check if the new inode belongs to the same parent
if(!isParent(pathComponents, parentPath)) {
parentINode = getParentINodeDirectory(pathComponents);
parentPath = getParent(pathComponents);
}
// add new inode
addToParent(parentINode, newNode);
}
}
private INodeDirectory getParentINodeDirectory(byte[][] pathComponents
) throws FileNotFoundException, PathIsNotDirectoryException,
UnresolvedLinkException {
if (pathComponents.length < 2) { // root
return null;
}
// Gets the parent INode
final INodesInPath inodes = namesystem.dir.getExistingPathINodes(
pathComponents);
return INodeDirectory.valueOf(inodes.getINode(-2), pathComponents);
}
/**
* Add the child node to parent and, if child is a file, update block map.
* This method is only used for image loading so that synchronization,
* modification time update and space count update are not needed.
*/
private void addToParent(INodeDirectory parent, INode child) {
FSDirectory fsDir = namesystem.dir;
if (parent == fsDir.rootDir) {
child.setLocalName(renameReservedRootComponentOnUpgrade(
child.getLocalNameBytes(), getLayoutVersion()));
}
// NOTE: This does not update space counts for parents
if (!parent.addChild(child)) {
return;
}
namesystem.dir.cacheName(child);
if (child.isFile()) {
updateBlocksMap(child.asFile());
}
}
public void updateBlocksMap(INodeFile file) {
// Add file->block mapping
final BlockInfo[] blocks = file.getBlocks();
if (blocks != null) {
final BlockManager bm = namesystem.getBlockManager();
for (int i = 0; i < blocks.length; i++) {
file.setBlock(i, bm.addBlockCollection(blocks[i], file));
}
}
}
public INode loadINodeWithLocalName(boolean isSnapshotINode, DataInput in,
boolean updateINodeMap) throws IOException {
return loadINodeWithLocalName(isSnapshotINode, in, updateINodeMap, null);
}
public INode loadINodeWithLocalName(boolean isSnapshotINode,
DataInput in, boolean updateINodeMap, Counter counter)
throws IOException {
byte[] localName = FSImageSerialization.readLocalName(in);
localName =
renameReservedComponentOnUpgrade(localName, getLayoutVersion());
INode inode = loadINode(localName, isSnapshotINode, in, counter);
if (updateINodeMap) {
namesystem.dir.addToInodeMap(inode);
}
return inode;
}
/**
* load an inode from fsimage except for its name
*
* @param in data input stream from which image is read
* @param counter Counter to increment for namenode startup progress
* @return an inode
*/
@SuppressWarnings("deprecation")
INode loadINode(final byte[] localName, boolean isSnapshotINode,
DataInput in, Counter counter) throws IOException {
final int imgVersion = getLayoutVersion();
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.SNAPSHOT, imgVersion)) {
namesystem.getFSDirectory().verifyINodeName(localName);
}
long inodeId = NameNodeLayoutVersion.supports(
LayoutVersion.Feature.ADD_INODE_ID, imgVersion) ? in.readLong()
: namesystem.allocateNewInodeId();
final short replication = namesystem.getBlockManager().adjustReplication(
in.readShort());
final long modificationTime = in.readLong();
long atime = 0;
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.FILE_ACCESS_TIME, imgVersion)) {
atime = in.readLong();
}
final long blockSize = in.readLong();
final int numBlocks = in.readInt();
if (numBlocks >= 0) {
// file
// read blocks
BlockInfo[] blocks = new BlockInfo[numBlocks];
for (int j = 0; j < numBlocks; j++) {
blocks[j] = new BlockInfo(replication);
blocks[j].readFields(in);
}
String clientName = "";
String clientMachine = "";
boolean underConstruction = false;
FileDiffList fileDiffs = null;
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.SNAPSHOT, imgVersion)) {
// read diffs
fileDiffs = SnapshotFSImageFormat.loadFileDiffList(in, this);
if (isSnapshotINode) {
underConstruction = in.readBoolean();
if (underConstruction) {
clientName = FSImageSerialization.readString(in);
clientMachine = FSImageSerialization.readString(in);
// convert the last block to BlockUC
if (blocks != null && blocks.length > 0) {
BlockInfo lastBlk = blocks[blocks.length - 1];
blocks[blocks.length - 1] = new BlockInfoUnderConstruction(
lastBlk, replication);
}
}
}
}
final PermissionStatus permissions = PermissionStatus.read(in);
// return
if (counter != null) {
counter.increment();
}
final INodeFile file = new INodeFile(inodeId, localName, permissions,
modificationTime, atime, blocks, replication, blockSize);
if (underConstruction) {
file.toUnderConstruction(clientName, clientMachine, null);
}
return fileDiffs == null ? file : new INodeFile(file, fileDiffs);
} else if (numBlocks == -1) {
//directory
//read quotas
final long nsQuota = in.readLong();
long dsQuota = -1L;
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.DISKSPACE_QUOTA, imgVersion)) {
dsQuota = in.readLong();
}
//read snapshot info
boolean snapshottable = false;
boolean withSnapshot = false;
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.SNAPSHOT, imgVersion)) {
snapshottable = in.readBoolean();
if (!snapshottable) {
withSnapshot = in.readBoolean();
}
}
final PermissionStatus permissions = PermissionStatus.read(in);
//return
if (counter != null) {
counter.increment();
}
final INodeDirectory dir = new INodeDirectory(inodeId, localName,
permissions, modificationTime);
if (nsQuota >= 0 || dsQuota >= 0) {
dir.addDirectoryWithQuotaFeature(nsQuota, dsQuota);
}
if (withSnapshot) {
dir.addSnapshotFeature(null);
}
return snapshottable ? new INodeDirectorySnapshottable(dir) : dir;
} else if (numBlocks == -2) {
//symlink
if (!FileSystem.areSymlinksEnabled()) {
throw new IOException("Symlinks not supported - please remove symlink before upgrading to this version of HDFS");
}
final String symlink = Text.readString(in);
final PermissionStatus permissions = PermissionStatus.read(in);
if (counter != null) {
counter.increment();
}
return new INodeSymlink(inodeId, localName, permissions,
modificationTime, atime, symlink);
} else if (numBlocks == -3) {
//reference
// Intentionally do not increment counter, because it is too difficult at
// this point to assess whether or not this is a reference that counts
// toward quota.
final boolean isWithName = in.readBoolean();
// lastSnapshotId for WithName node, dstSnapshotId for DstReference node
int snapshotId = in.readInt();
final INodeReference.WithCount withCount
= referenceMap.loadINodeReferenceWithCount(isSnapshotINode, in, this);
if (isWithName) {
return new INodeReference.WithName(null, withCount, localName,
snapshotId);
} else {
final INodeReference ref = new INodeReference.DstReference(null,
withCount, snapshotId);
return ref;
}
}
throw new IOException("Unknown inode type: numBlocks=" + numBlocks);
}
/** Load {@link INodeFileAttributes}. */
public INodeFileAttributes loadINodeFileAttributes(DataInput in)
throws IOException {
final int layoutVersion = getLayoutVersion();
if (!NameNodeLayoutVersion.supports(
LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
return loadINodeWithLocalName(true, in, false).asFile();
}
final byte[] name = FSImageSerialization.readLocalName(in);
final PermissionStatus permissions = PermissionStatus.read(in);
final long modificationTime = in.readLong();
final long accessTime = in.readLong();
final short replication = namesystem.getBlockManager().adjustReplication(
in.readShort());
final long preferredBlockSize = in.readLong();
return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
accessTime, replication, preferredBlockSize);
}
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
throws IOException {
final int layoutVersion = getLayoutVersion();
if (!NameNodeLayoutVersion.supports(
LayoutVersion.Feature.OPTIMIZE_SNAPSHOT_INODES, layoutVersion)) {
return loadINodeWithLocalName(true, in, false).asDirectory();
}
final byte[] name = FSImageSerialization.readLocalName(in);
final PermissionStatus permissions = PermissionStatus.read(in);
final long modificationTime = in.readLong();
//read quotas
final long nsQuota = in.readLong();
final long dsQuota = in.readLong();
return nsQuota == -1L && dsQuota == -1L?
new INodeDirectoryAttributes.SnapshotCopy(name, permissions, null, modificationTime)
: new INodeDirectoryAttributes.CopyWithQuota(name, permissions,
null, modificationTime, nsQuota, dsQuota);
}
private void loadFilesUnderConstruction(DataInput in,
boolean supportSnapshot, Counter counter) throws IOException {
FSDirectory fsDir = namesystem.dir;
int size = in.readInt();
LOG.info("Number of files under construction = " + size);
for (int i = 0; i < size; i++) {
INodeFile cons = FSImageSerialization.readINodeUnderConstruction(in,
namesystem, getLayoutVersion());
counter.increment();
// verify that file exists in namespace
String path = cons.getLocalName();
INodeFile oldnode = null;
boolean inSnapshot = false;
if (path != null && FSDirectory.isReservedName(path) &&
NameNodeLayoutVersion.supports(
LayoutVersion.Feature.ADD_INODE_ID, getLayoutVersion())) {
// TODO: for HDFS-5428, we use reserved path for those INodeFileUC in
// snapshot. If we support INode ID in the layout version, we can use
// the inode id to find the oldnode.
oldnode = namesystem.dir.getInode(cons.getId()).asFile();
inSnapshot = true;
} else {
final INodesInPath iip = fsDir.getLastINodeInPath(path);
oldnode = INodeFile.valueOf(iip.getINode(0), path);
}
FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature();
oldnode.toUnderConstruction(uc.getClientName(), uc.getClientMachine(),
uc.getClientNode());
if (oldnode.numBlocks() > 0) {
BlockInfo ucBlock = cons.getLastBlock();
// we do not replace the inode, just replace the last block of oldnode
BlockInfo info = namesystem.getBlockManager().addBlockCollection(
ucBlock, oldnode);
oldnode.setBlock(oldnode.numBlocks() - 1, info);
}
if (!inSnapshot) {
namesystem.leaseManager.addLease(cons
.getFileUnderConstructionFeature().getClientName(), path);
}
}
}
private void loadSecretManagerState(DataInput in)
throws IOException {
int imgVersion = getLayoutVersion();
if (!NameNodeLayoutVersion.supports(
LayoutVersion.Feature.DELEGATION_TOKEN, imgVersion)) {
//SecretManagerState is not available.
//This must not happen if security is turned on.
return;
}
namesystem.loadSecretManagerStateCompat(in);
}
private void loadCacheManagerState(DataInput in) throws IOException {
int imgVersion = getLayoutVersion();
if (!NameNodeLayoutVersion.supports(
LayoutVersion.Feature.CACHING, imgVersion)) {
return;
}
namesystem.getCacheManager().loadStateCompat(in);
}
private int getLayoutVersion() {
return namesystem.getFSImage().getStorage().getLayoutVersion();
}
private boolean isRoot(byte[][] path) {
return path.length == 1 &&
path[0] == null;
}
private boolean isParent(byte[][] path, byte[][] parent) {
if (path == null || parent == null)
return false;
if (parent.length == 0 || path.length != parent.length + 1)
return false;
boolean isParent = true;
for (int i = 0; i < parent.length; i++) {
isParent = isParent && Arrays.equals(path[i], parent[i]);
}
return isParent;
}
/**
* Return string representing the parent of the given path.
*/
String getParent(String path) {
return path.substring(0, path.lastIndexOf(Path.SEPARATOR));
}
byte[][] getParent(byte[][] path) {
byte[][] result = new byte[path.length - 1][];
for (int i = 0; i < result.length; i++) {
result[i] = new byte[path[i].length];
System.arraycopy(path[i], 0, result[i], 0, path[i].length);
}
return result;
}
public Snapshot getSnapshot(DataInput in) throws IOException {
return snapshotMap.get(in.readInt());
}
}
@VisibleForTesting
public static final TreeMap<String, String> renameReservedMap =
new TreeMap<String, String>();
/**
* Use the default key-value pairs that will be used to determine how to
* rename reserved paths on upgrade.
*/
@VisibleForTesting
public static void useDefaultRenameReservedPairs() {
renameReservedMap.clear();
for (String key: HdfsConstants.RESERVED_PATH_COMPONENTS) {
renameReservedMap.put(
key,
key + "." + HdfsConstants.NAMENODE_LAYOUT_VERSION + "."
+ "UPGRADE_RENAMED");
}
}
/**
* Set the key-value pairs that will be used to determine how to rename
* reserved paths on upgrade.
*/
@VisibleForTesting
public static void setRenameReservedPairs(String renameReserved) {
// Clear and set the default values
useDefaultRenameReservedPairs();
// Overwrite with provided values
setRenameReservedMapInternal(renameReserved);
}
private static void setRenameReservedMapInternal(String renameReserved) {
Collection<String> pairs =
StringUtils.getTrimmedStringCollection(renameReserved);
for (String p : pairs) {
String[] pair = StringUtils.split(p, '/', '=');
Preconditions.checkArgument(pair.length == 2,
"Could not parse key-value pair " + p);
String key = pair[0];
String value = pair[1];
Preconditions.checkArgument(DFSUtil.isReservedPathComponent(key),
"Unknown reserved path " + key);
Preconditions.checkArgument(DFSUtil.isValidNameForComponent(value),
"Invalid rename path for " + key + ": " + value);
LOG.info("Will rename reserved path " + key + " to " + value);
renameReservedMap.put(key, value);
}
}
/**
* When upgrading from an old version, the filesystem could contain paths
* that are now reserved in the new version (e.g. .snapshot). This renames
* these new reserved paths to a user-specified value to avoid collisions
* with the reserved name.
*
* @param path Old path potentially containing a reserved path
* @return New path with reserved path components renamed to user value
*/
static String renameReservedPathsOnUpgrade(String path,
final int layoutVersion) {
final String oldPath = path;
// If any known LVs aren't supported, we're doing an upgrade
if (!NameNodeLayoutVersion.supports(Feature.ADD_INODE_ID, layoutVersion)) {
String[] components = INode.getPathNames(path);
// Only need to worry about the root directory
if (components.length > 1) {
components[1] = DFSUtil.bytes2String(
renameReservedRootComponentOnUpgrade(
DFSUtil.string2Bytes(components[1]),
layoutVersion));
path = DFSUtil.strings2PathString(components);
}
}
if (!NameNodeLayoutVersion.supports(Feature.SNAPSHOT, layoutVersion)) {
String[] components = INode.getPathNames(path);
// Special case the root path
if (components.length == 0) {
return path;
}
for (int i=0; i<components.length; i++) {
components[i] = DFSUtil.bytes2String(
renameReservedComponentOnUpgrade(
DFSUtil.string2Bytes(components[i]),
layoutVersion));
}
path = DFSUtil.strings2PathString(components);
}
if (!path.equals(oldPath)) {
LOG.info("Upgrade process renamed reserved path " + oldPath + " to "
+ path);
}
return path;
}
private final static String RESERVED_ERROR_MSG =
FSDirectory.DOT_RESERVED_PATH_PREFIX + " is a reserved path and "
+ HdfsConstants.DOT_SNAPSHOT_DIR + " is a reserved path component in"
+ " this version of HDFS. Please rollback and delete or rename"
+ " this path, or upgrade with the "
+ StartupOption.RENAMERESERVED.getName()
+ " [key-value pairs]"
+ " option to automatically rename these paths during upgrade.";
/**
* Same as {@link #renameReservedPathsOnUpgrade}, but for a single
* byte array path component.
*/
private static byte[] renameReservedComponentOnUpgrade(byte[] component,
final int layoutVersion) {
// If the LV doesn't support snapshots, we're doing an upgrade
if (!NameNodeLayoutVersion.supports(Feature.SNAPSHOT, layoutVersion)) {
if (Arrays.equals(component, HdfsConstants.DOT_SNAPSHOT_DIR_BYTES)) {
Preconditions.checkArgument(
renameReservedMap != null &&
renameReservedMap.containsKey(HdfsConstants.DOT_SNAPSHOT_DIR),
RESERVED_ERROR_MSG);
component =
DFSUtil.string2Bytes(renameReservedMap
.get(HdfsConstants.DOT_SNAPSHOT_DIR));
}
}
return component;
}
/**
* Same as {@link #renameReservedPathsOnUpgrade}, but for a single
* byte array path component.
*/
private static byte[] renameReservedRootComponentOnUpgrade(byte[] component,
final int layoutVersion) {
// If the LV doesn't support inode IDs, we're doing an upgrade
if (!NameNodeLayoutVersion.supports(Feature.ADD_INODE_ID, layoutVersion)) {
if (Arrays.equals(component, FSDirectory.DOT_RESERVED)) {
Preconditions.checkArgument(
renameReservedMap != null &&
renameReservedMap.containsKey(FSDirectory.DOT_RESERVED_STRING),
RESERVED_ERROR_MSG);
final String renameString = renameReservedMap
.get(FSDirectory.DOT_RESERVED_STRING);
component =
DFSUtil.string2Bytes(renameString);
LOG.info("Renamed root path " + FSDirectory.DOT_RESERVED_STRING
+ " to " + renameString);
}
}
return component;
}
}