@InterfaceAudience.Private @InterfaceStability.Unstable public class JobHistoryUtils extends Object
Modifier and Type | Field and Description |
---|---|
static String |
CONF_FILE_NAME_SUFFIX
Suffix for configuration files.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_DONE_DIR_PERMISSION
Permissions for the history done dir and derivatives.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_DONE_DIR_UMASK
Umask for the done dir and derivatives.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_DONE_FILE_PERMISSION |
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS
Permissions for the intermediate done directory.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_INTERMEDIATE_FILE_PERMISSIONS |
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_STAGING_DIR_PERMISSIONS
Permissions for the history staging dir while JobInProgress.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_STAGING_USER_DIR_PERMISSIONS
Permissions for the user directory under the staging directory.
|
static String |
JOB_HISTORY_FILE_EXTENSION
Job History File extension.
|
static int |
SERIAL_NUMBER_DIRECTORY_DIGITS |
static String |
SUMMARY_FILE_NAME_SUFFIX
Suffix for summary files.
|
static Pattern |
TIMESTAMP_DIR_PATTERN |
static String |
TIMESTAMP_DIR_REGEX |
static int |
VERSION |
Constructor and Description |
---|
JobHistoryUtils() |
Modifier and Type | Method and Description |
---|---|
static String |
doneSubdirsBeforeSerialTail() |
static org.apache.hadoop.fs.PathFilter |
getConfFileFilter()
Gets a PathFilter which would match configuration files.
|
static String |
getConfiguredHistoryIntermediateDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
Gets the configured directory prefix for intermediate done history files.
|
static org.apache.hadoop.fs.permission.FsPermission |
getConfiguredHistoryIntermediateUserDoneDirPermissions(org.apache.hadoop.conf.Configuration conf)
Gets the configured directory permissions for the user directories in the
directory of the intermediate done history files.
|
static String |
getConfiguredHistoryServerDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
Gets the configured directory prefix for Done history files.
|
static String |
getConfiguredHistoryStagingDirPrefix(org.apache.hadoop.conf.Configuration conf,
String jobId)
Gets the configured directory prefix for In Progress history files.
|
static List<org.apache.hadoop.fs.FileStatus> |
getHistoryDirsForCleaning(org.apache.hadoop.fs.FileContext fc,
org.apache.hadoop.fs.Path root,
long cutoff)
Looks for the dirs to clean.
|
static org.apache.hadoop.fs.PathFilter |
getHistoryFileFilter()
Gets a PathFilter which would match job history file names.
|
static String |
getHistoryIntermediateDoneDirForUser(org.apache.hadoop.conf.Configuration conf)
Gets the user directory for intermediate done history files.
|
static String |
getIntermediateConfFileName(JobId jobId)
Get the done configuration file name for a job.
|
static String |
getIntermediateSummaryFileName(JobId jobId)
Get the done summary file name for a job.
|
static org.apache.hadoop.mapreduce.JobID |
getJobIDFromHistoryFilePath(String pathString)
Returns the jobId from a job history file name.
|
static org.apache.hadoop.fs.Path |
getPreviousJobHistoryPath(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId) |
static org.apache.hadoop.fs.Path |
getStagingConfFile(org.apache.hadoop.fs.Path logDir,
JobId jobId,
int attempt)
Gets the conf file path for jobs in progress.
|
static org.apache.hadoop.fs.Path |
getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir,
JobId jobId,
int attempt)
Get the job history file path for non Done history files.
|
static org.apache.hadoop.fs.Path |
getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir,
String jobId,
int attempt)
Get the job history file path for non Done history files.
|
static String |
getTimestampPartFromPath(String path)
Extracts the timstamp component from the path.
|
static String |
historyLogSubdirectory(JobId id,
String timestampComponent,
String serialNumberFormat)
Gets the history subdirectory based on the jobId, timestamp and serial number format.
|
static boolean |
isValidJobHistoryFileName(String pathString)
Checks whether the provided path string is a valid job history file.
|
static int |
jobSerialNumber(JobId id)
Computes a serial number used as part of directory naming for the given jobId.
|
static List<org.apache.hadoop.fs.FileStatus> |
localGlobber(org.apache.hadoop.fs.FileContext fc,
org.apache.hadoop.fs.Path root,
String tail) |
static List<org.apache.hadoop.fs.FileStatus> |
localGlobber(org.apache.hadoop.fs.FileContext fc,
org.apache.hadoop.fs.Path root,
String tail,
org.apache.hadoop.fs.PathFilter filter) |
static List<org.apache.hadoop.fs.FileStatus> |
localGlobber(org.apache.hadoop.fs.FileContext fc,
org.apache.hadoop.fs.Path root,
String tail,
org.apache.hadoop.fs.PathFilter filter,
AtomicBoolean hasFlatFiles) |
static String |
serialNumberDirectoryComponent(JobId id,
String serialNumberFormat)
Gets the serial number part of the path based on the jobId and serialNumber format.
|
static boolean |
shouldCreateNonUserDirectory(org.apache.hadoop.conf.Configuration conf) |
static String |
timestampDirectoryComponent(long millisecondTime)
Gets the timestamp component based on millisecond time.
|
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_STAGING_DIR_PERMISSIONS
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_STAGING_USER_DIR_PERMISSIONS
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_DIR_PERMISSION
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_FILE_PERMISSION
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_DIR_UMASK
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS
public static final String CONF_FILE_NAME_SUFFIX
public static final String SUMMARY_FILE_NAME_SUFFIX
public static final String JOB_HISTORY_FILE_EXTENSION
public static final int VERSION
public static final int SERIAL_NUMBER_DIRECTORY_DIGITS
public static final String TIMESTAMP_DIR_REGEX
public static final Pattern TIMESTAMP_DIR_PATTERN
public static boolean isValidJobHistoryFileName(String pathString)
pathString
- the path to be checked.public static org.apache.hadoop.mapreduce.JobID getJobIDFromHistoryFilePath(String pathString) throws IOException
pathString
- the path string.IOException
- if the filename format is invalid.public static org.apache.hadoop.fs.PathFilter getConfFileFilter()
PathFilter
for matching conf files.public static org.apache.hadoop.fs.PathFilter getHistoryFileFilter()
PathFilter
matching job history files.public static String getConfiguredHistoryStagingDirPrefix(org.apache.hadoop.conf.Configuration conf, String jobId) throws IOException
conf
- the configuration for hte jobjobId
- the id of the job the history file is for.IOException
public static String getConfiguredHistoryIntermediateDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
conf
- public static org.apache.hadoop.fs.permission.FsPermission getConfiguredHistoryIntermediateUserDoneDirPermissions(org.apache.hadoop.conf.Configuration conf)
conf
- The configuration objectpublic static String getConfiguredHistoryServerDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
conf
- the configuration objectpublic static String getHistoryIntermediateDoneDirForUser(org.apache.hadoop.conf.Configuration conf) throws IOException
conf
- the configuration objectIOException
public static boolean shouldCreateNonUserDirectory(org.apache.hadoop.conf.Configuration conf)
public static org.apache.hadoop.fs.Path getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir, JobId jobId, int attempt)
public static org.apache.hadoop.fs.Path getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir, String jobId, int attempt)
public static String getIntermediateConfFileName(JobId jobId)
jobId
- the jobId.public static String getIntermediateSummaryFileName(JobId jobId)
jobId
- the jobId.public static org.apache.hadoop.fs.Path getStagingConfFile(org.apache.hadoop.fs.Path logDir, JobId jobId, int attempt)
logDir
- the log directory prefix.jobId
- the jobId.attempt
- attempt number for this job.public static String serialNumberDirectoryComponent(JobId id, String serialNumberFormat)
id
- serialNumberFormat
- public static String getTimestampPartFromPath(String path)
path
- public static String historyLogSubdirectory(JobId id, String timestampComponent, String serialNumberFormat)
id
- timestampComponent
- serialNumberFormat
- public static String timestampDirectoryComponent(long millisecondTime)
millisecondTime
- public static String doneSubdirsBeforeSerialTail()
public static int jobSerialNumber(JobId id)
id
- the jobId.public static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail) throws IOException
IOException
public static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail, org.apache.hadoop.fs.PathFilter filter) throws IOException
IOException
public static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail, org.apache.hadoop.fs.PathFilter filter, AtomicBoolean hasFlatFiles) throws IOException
IOException
public static org.apache.hadoop.fs.Path getPreviousJobHistoryPath(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId) throws IOException
IOException
public static List<org.apache.hadoop.fs.FileStatus> getHistoryDirsForCleaning(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, long cutoff) throws IOException
fc
- done dir FileContextroot
- folder for completed jobscutoff
- The cutoff for the max history ageIOException
Copyright © 2008–2020 Apache Software Foundation. All rights reserved.