@Deprecated public class RateControlledFileSystem extends ThrottledFileSystem
FileSystem that wraps with a Limiter
to control HDFS call rate.
This classes uses Guava's Cache for storing FileSystem URI to
Limiter mapping.
For methods that require HDFS calls, this class will first acquire a permit using Limiter,
to make sure HDFS call rate is allowed by the uppper limit.
ThrottledFileSystem.Factory<S extends ScopeType<S>>FileSystem.Statistics| Modifier and Type | Field and Description |
|---|---|
protected Configuration |
conf |
protected String |
replacementScheme |
protected FileSystem |
underlyingFs |
protected String |
underlyingScheme |
LISTING_FILES_PER_PERMITclosedDEFAULT_FS, FS_DEFAULT_NAME_KEY, LOG, SHUTDOWN_HOOK_PRIORITY, statistics| Constructor and Description |
|---|
RateControlledFileSystem(FileSystem fs,
long limitPerSecond)
Deprecated.
|
append, close, create, delete, delete, exists, getFileStatus, globStatus, listStatus, listStatus, mkdirs, open, renameonCloseareSymlinksEnabled, canonicalizeUri, checkPath, clearStatistics, closeAll, closeAllForUGI, create, createSnapshot, enableSymlinks, fixRelativePart, get, get, get, getAllStatistics, getCanonicalUri, getDefaultPort, getDefaultUri, getFileSystemClass, getFSofPath, getInitialWorkingDirectory, getLocal, getNamed, getStatistics, getStatistics, listLocatedStatus, mkdirs, newInstance, newInstance, newInstance, newInstanceLocal, primitiveCreate, primitiveMkdir, primitiveMkdir, printStatistics, processDeleteOnExit, rename, resolveLink, setDefaultUri, setDefaultUriprotected String replacementScheme
protected String underlyingScheme
protected Configuration conf
protected FileSystem underlyingFs
public RateControlledFileSystem(FileSystem fs, long limitPerSecond)
public static Optional<Long> getRateIfRateControlled(FileSystem fs)
fs - FileSystem to check for rate control.Optional.absent() if file system is not rate controlled, otherwise, the rate in operations per second.public void startRateControl()
throws ExecutionException
ExecutionExceptionprotected Limiter getRateLimiter()
getRateLimiter in class ThrottledFileSystempublic Object getDecoratedObject()
getDecoratedObject in interface Decoratorpublic String getScheme()
getScheme in class FileSystempublic void initialize(URI uri, Configuration conf) throws IOException
initialize in class FileSystemIOExceptionpublic void setConf(Configuration conf)
setConf in interface ConfigurablesetConf in class Configuredpublic URI getUri()
getUri in class FileSystempublic FileStatus getFileLinkStatus(Path f) throws IOException
getFileLinkStatus in class FileSystemIOExceptionpublic FsStatus getStatus() throws IOException
getStatus in class FileSystemIOExceptionpublic FSDataOutputStream append(Path f) throws IOException
append in class FileSystemIOExceptionpublic short getReplication(Path src) throws IOException
getReplication in class FileSystemIOExceptionpublic void setWriteChecksum(boolean writeChecksum)
setWriteChecksum in class FileSystempublic FileChecksum getFileChecksum(Path f) throws IOException
getFileChecksum in class FileSystemIOExceptionpublic boolean isDirectory(Path f) throws IOException
isDirectory in class FileSystemIOExceptionpublic void createSymlink(Path target, Path link, boolean createParent) throws IOException
createSymlink in class FileSystemIOExceptionpublic Path createSnapshot(Path path, String snapshotName) throws IOException
createSnapshot in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress, Options.ChecksumOpt checksumOpt) throws IOException
create in class FileSystemIOExceptionpublic Path resolvePath(Path p) throws IOException
resolvePath in class FileSystemIOExceptionpublic long getUsed()
throws IOException
getUsed in class FileSystemIOExceptionpublic Configuration getConf()
getConf in interface ConfigurablegetConf in class Configuredpublic FSDataOutputStream create(Path f, Progressable progress) throws IOException
create in class FileSystemIOExceptionpublic boolean isFile(Path f) throws IOException
isFile in class FileSystemIOExceptionpublic Path getWorkingDirectory()
getWorkingDirectory in class FileSystempublic FsServerDefaults getServerDefaults() throws IOException
getServerDefaults in class FileSystemIOExceptionpublic void copyToLocalFile(boolean delSrc,
Path src,
Path dst,
boolean useRawLocalFileSystem)
throws IOException
copyToLocalFile in class FileSystemIOExceptionpublic void setWorkingDirectory(Path new_dir)
setWorkingDirectory in class FileSystempublic String getName()
getName in class FileSystempublic boolean createNewFile(Path f) throws IOException
createNewFile in class FileSystemIOExceptionpublic FileStatus[] listStatus(Path[] files) throws IOException
listStatus in class FileSystemIOExceptionpublic Path getLinkTarget(Path f) throws IOException
getLinkTarget in class FileSystemIOExceptionpublic void copyToLocalFile(boolean delSrc,
Path src,
Path dst)
throws IOException
copyToLocalFile in class FileSystemIOExceptionpublic short getDefaultReplication()
getDefaultReplication in class FileSystempublic org.apache.hadoop.security.token.Token<?> getDelegationToken(String renewer) throws IOException
getDelegationToken in class FileSystemIOExceptionpublic FsServerDefaults getServerDefaults(Path p) throws IOException
getServerDefaults in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f, short replication) throws IOException
create in class FileSystemIOExceptionpublic BlockLocation[] getFileBlockLocations(Path p, long start, long len) throws IOException
getFileBlockLocations in class FileSystemIOExceptionpublic void concat(Path trg, Path[] psrcs) throws IOException
concat in class FileSystemIOExceptionpublic Path getHomeDirectory()
getHomeDirectory in class FileSystempublic boolean supportsSymlinks()
supportsSymlinks in class FileSystempublic RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f) throws FileNotFoundException, IOException
listLocatedStatus in class FileSystemFileNotFoundExceptionIOExceptionpublic RemoteIterator<Path> listCorruptFileBlocks(Path path) throws IOException
listCorruptFileBlocks in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f, short replication, Progressable progress) throws IOException
create in class FileSystemIOExceptionpublic boolean setReplication(Path src, short replication) throws IOException
setReplication in class FileSystemIOExceptionpublic Path makeQualified(Path path)
makeQualified in class FileSystempublic FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
createNonRecursive in class FileSystemIOExceptionpublic long getBlockSize(Path f) throws IOException
getBlockSize in class FileSystemIOExceptionpublic short getDefaultReplication(Path path)
getDefaultReplication in class FileSystempublic org.apache.hadoop.security.token.Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException
addDelegationTokens in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
create in class FileSystemIOExceptionpublic void copyToLocalFile(Path src, Path dst) throws IOException
copyToLocalFile in class FileSystemIOExceptionpublic FileStatus[] listStatus(Path[] files, PathFilter filter) throws IOException
listStatus in class FileSystemIOExceptionpublic long getLength(Path f) throws IOException
getLength in class FileSystemIOExceptionpublic void moveToLocalFile(Path src, Path dst) throws IOException
moveToLocalFile in class FileSystemIOExceptionpublic FsStatus getStatus(Path p) throws IOException
getStatus in class FileSystemIOExceptionpublic FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
createNonRecursive in class FileSystemIOExceptionpublic String getCanonicalServiceName()
getCanonicalServiceName in class FileSystempublic boolean cancelDeleteOnExit(Path f)
cancelDeleteOnExit in class FileSystempublic void copyFromLocalFile(boolean delSrc,
boolean overwrite,
Path[] srcs,
Path dst)
throws IOException
copyFromLocalFile in class FileSystemIOExceptionpublic void setPermission(Path p, FsPermission permission) throws IOException
setPermission in class FileSystemIOExceptionpublic Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException
startLocalOutput in class FileSystemIOExceptionpublic void copyFromLocalFile(Path src, Path dst) throws IOException
copyFromLocalFile in class FileSystemIOExceptionpublic FileSystem[] getChildFileSystems()
getChildFileSystems in class FileSystempublic void copyFromLocalFile(boolean delSrc,
Path src,
Path dst)
throws IOException
copyFromLocalFile in class FileSystemIOExceptionpublic void setTimes(Path p, long mtime, long atime) throws IOException
setTimes in class FileSystemIOExceptionpublic BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException
getFileBlockLocations in class FileSystemIOExceptionpublic ContentSummary getContentSummary(Path f) throws IOException
getContentSummary in class FileSystemIOExceptionpublic void renameSnapshot(Path path, String snapshotOldName, String snapshotNewName) throws IOException
renameSnapshot in class FileSystemIOExceptionpublic void moveFromLocalFile(Path[] srcs, Path dst) throws IOException
moveFromLocalFile in class FileSystemIOExceptionpublic FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
createNonRecursive in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f) throws IOException
create in class FileSystemIOExceptionpublic boolean mkdirs(Path f) throws IOException
mkdirs in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException
create in class FileSystemIOExceptionpublic void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException
completeLocalOutput in class FileSystemIOExceptionpublic FSDataInputStream open(Path f) throws IOException
open in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, Progressable progress) throws IOException
create in class FileSystemIOExceptionpublic void setVerifyChecksum(boolean verifyChecksum)
setVerifyChecksum in class FileSystempublic FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException
create in class FileSystemIOExceptionpublic long getDefaultBlockSize()
getDefaultBlockSize in class FileSystempublic void copyFromLocalFile(boolean delSrc,
boolean overwrite,
Path src,
Path dst)
throws IOException
copyFromLocalFile in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f, boolean overwrite) throws IOException
create in class FileSystemIOExceptionpublic long getDefaultBlockSize(Path f)
getDefaultBlockSize in class FileSystempublic void deleteSnapshot(Path path, String snapshotName) throws IOException
deleteSnapshot in class FileSystemIOExceptionpublic FSDataOutputStream create(Path f, boolean overwrite, int bufferSize) throws IOException
create in class FileSystemIOExceptionpublic void setOwner(Path p, String username, String groupname) throws IOException
setOwner in class FileSystemIOExceptionpublic RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive) throws IOException
listFiles in class FileSystemIOExceptionpublic FSDataOutputStream append(Path f, int bufferSize) throws IOException
append in class FileSystemIOExceptionpublic boolean deleteOnExit(Path f) throws IOException
deleteOnExit in class FileSystemIOExceptionpublic void moveFromLocalFile(Path src, Path dst) throws IOException
moveFromLocalFile in class FileSystemIOExceptionpublic FileStatus[] globStatus(Path pathPattern, PathFilter filter) throws IOException
globStatus in class FileSystemIOException