@InterfaceAudience.LimitedPrivate(value="yarn") @InterfaceStability.Unstable public class FairScheduler extends AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Modifier and Type | Field and Description |
---|---|
protected boolean |
assignMultiple |
static org.apache.hadoop.yarn.api.records.Resource |
CONTAINER_RESERVED |
protected boolean |
continuousSchedulingEnabled |
protected int |
continuousSchedulingSleepMs |
protected int |
maxAssign |
protected long |
nodeLocalityDelayMs |
protected double |
nodeLocalityThreshold |
protected long |
rackLocalityDelayMs |
protected double |
rackLocalityThreshold |
protected boolean |
sizeBasedWeight |
activitiesManager, applications, EMPTY_ALLOCATION, lastNodeUpdateTime, minimumAllocation, nmExpireInterval, nmHeartbeatInterval, nodeTracker, readLock, rmContext, schedulerHealth, THREAD_JOIN_TIMEOUT_MS, updateInterval, writeLock
Constructor and Description |
---|
FairScheduler() |
Modifier and Type | Method and Description |
---|---|
protected void |
addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String queueName,
String user,
boolean isAppRecovering)
Add a new application to the scheduler, with a given id, queue name, and
user.
|
protected void |
addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
boolean transferStateFromPreviousAttempt,
boolean isAttemptRecovering)
Add a new application attempt to the scheduler.
|
Allocation |
allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId,
List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask,
List<org.apache.hadoop.yarn.api.records.ContainerId> release,
List<String> blacklistAdditions,
List<String> blacklistRemovals,
ContainerUpdates updateRequests)
The main api between the ApplicationMaster and the Scheduler.
|
boolean |
checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
org.apache.hadoop.yarn.api.records.QueueACL acl,
String queueName)
Check if the user has permission to perform the operation.
|
long |
checkAndGetApplicationLifetime(String queueName,
long lifetime)
Verify whether a submitted application lifetime is valid as per configured
Queue lifetime.
|
protected void |
completedContainerInternal(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event)
Clean up a completed container.
|
protected void |
createPreemptionThread() |
AllocationConfiguration |
getAllocationConfiguration() |
List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> |
getAppsInQueue(String queueName)
Gets the apps under a given queue
|
ResourceWeights |
getAppWeight(FSAppAttempt app) |
FairSchedulerConfiguration |
getConf() |
RMContainerTokenSecretManager |
getContainerTokenSecretManager() |
FSContext |
getContext() |
int |
getContinuousSchedulingSleepMs() |
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog |
getEventLog() |
org.apache.hadoop.yarn.api.records.Resource |
getIncrementResourceCapability() |
long |
getNodeLocalityDelayMs() |
double |
getNodeLocalityThreshold() |
org.apache.hadoop.yarn.api.records.Resource |
getNormalizedResource(org.apache.hadoop.yarn.api.records.Resource requestedResource)
Normalize a resource request.
|
int |
getNumClusterNodes()
Get the number of nodes available in the cluster.
|
int |
getNumNodesInRack(String rackName) |
Set<String> |
getPlanQueues()
Gets the list of names for queues managed by the Reservation System
|
org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo(String queueName,
boolean includeChildQueues,
boolean recursive)
Get queue information
|
QueueManager |
getQueueManager() |
List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> |
getQueueUserAclInfo()
Get acls for queues for current user.
|
long |
getRackLocalityDelayMs() |
double |
getRackLocalityThreshold() |
float |
getReservableNodesRatio() |
org.apache.hadoop.yarn.util.resource.ResourceCalculator |
getResourceCalculator() |
QueueMetrics |
getRootQueueMetrics()
Get the root queue for the scheduler.
|
FSAppAttempt |
getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId) |
EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> |
getSchedulingResourceTypes()
Return a collection of the resource types that are considered when
scheduling
|
void |
handle(SchedulerEvent event) |
boolean |
isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator,
org.apache.hadoop.yarn.api.records.Resource resource) |
boolean |
isContinuousSchedulingEnabled() |
void |
killContainer(RMContainer container)
Kill a RMContainer.
|
String |
moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String queueName)
Moves the given application to the given queue
|
protected void |
nodeUpdate(RMNode nm)
Process a heartbeat update from a node.
|
void |
preValidateMoveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String newQueue) |
void |
recover(RMStateStore.RMState state) |
void |
reinitialize(org.apache.hadoop.conf.Configuration conf,
RMContext rmContext)
Re-initialize the
ResourceScheduler . |
void |
removeQueue(String queueName)
Only supports removing empty leaf queues
|
void |
serviceInit(org.apache.hadoop.conf.Configuration conf) |
void |
serviceStart() |
void |
serviceStop() |
void |
setEntitlement(String queueName,
QueueEntitlement entitlement)
This method increase the entitlement for current queue (must respect
invariants, e.g., no overcommit of parents, non negative, etc.).
|
void |
setRMContext(RMContext rmContext)
Set RMContext for
ResourceScheduler . |
void |
update()
Recompute the internal variables used by the scheduler - per-job weights,
fair shares, deficits, minimum slot allocations, and amount of used and
required resources per job.
|
void |
updateNodeResource(RMNode nm,
org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
Process resource update on a node and update Queue.
|
addQueue, asyncContainerRelease, checkAndGetApplicationPriority, clearPendingContainerCache, completedContainer, containerIncreasedOnNode, containerLaunchedOnNode, createReleaseCache, createSchedContainerChangeRequests, getActivitiesManager, getApplicationAttempt, getAppResourceUsageReport, getBlacklistedNodes, getClock, getClusterResource, getCurrentAttemptForContainer, getLastNodeUpdateTime, getMaxClusterLevelAppPriority, getMaximumAllocation, getMaximumApplicationLifetime, getMaximumResourceCapability, getMaximumResourceCapability, getMinimumAllocation, getMinimumResourceCapability, getNode, getNodeIds, getNodeReport, getNodeTracker, getPendingResourceRequestsForAttempt, getRMContainer, getSchedulerAppInfo, getSchedulerApplications, getSchedulerHealth, getSchedulerNode, getTransferredContainers, handleContainerUpdates, initMaximumResourceCapability, killAllAppsInQueue, moveAllApps, normalizeRequests, recoverContainersOnNode, refreshMaximumAllocation, releaseContainers, resetSchedulerMetrics, rollbackContainerUpdate, setClock, setClusterMaxPriority, setLastNodeUpdateTime, shouldContainersBeAutoUpdated, triggerUpdate, updateApplicationPriority, updateCompletedContainers, updateNewContainerInfo, updateNodeResourceUtilization, updateSchedulerHealthInformation
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop
public static final org.apache.hadoop.yarn.api.records.Resource CONTAINER_RESERVED
protected boolean sizeBasedWeight
protected boolean continuousSchedulingEnabled
protected volatile int continuousSchedulingSleepMs
protected double nodeLocalityThreshold
protected double rackLocalityThreshold
protected long nodeLocalityDelayMs
protected long rackLocalityDelayMs
protected boolean assignMultiple
protected int maxAssign
public FSContext getContext()
public boolean isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator, org.apache.hadoop.yarn.api.records.Resource resource)
public FairSchedulerConfiguration getConf()
public int getNumNodesInRack(String rackName)
public QueueManager getQueueManager()
public void update()
update
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public RMContainerTokenSecretManager getContainerTokenSecretManager()
public ResourceWeights getAppWeight(FSAppAttempt app)
public org.apache.hadoop.yarn.api.records.Resource getIncrementResourceCapability()
public double getNodeLocalityThreshold()
public double getRackLocalityThreshold()
public long getNodeLocalityDelayMs()
public long getRackLocalityDelayMs()
public boolean isContinuousSchedulingEnabled()
public int getContinuousSchedulingSleepMs()
public org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog getEventLog()
protected void addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId, String queueName, String user, boolean isAppRecovering)
protected void addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering)
protected void completedContainerInternal(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
completedContainerInternal
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public org.apache.hadoop.yarn.api.records.Resource getNormalizedResource(org.apache.hadoop.yarn.api.records.Resource requestedResource)
YarnScheduler
getNormalizedResource
in interface YarnScheduler
getNormalizedResource
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
requestedResource
- the resource to be normalizedpublic void killContainer(RMContainer container)
AbstractYarnScheduler
killContainer
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
container
- the container to killpublic Allocation allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId, List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask, List<org.apache.hadoop.yarn.api.records.ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests)
YarnScheduler
Allocation
for the applicationprotected void nodeUpdate(RMNode nm)
AbstractYarnScheduler
nodeUpdate
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
nm
- The RMNode corresponding to the NodeManagerpublic FSAppAttempt getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
public org.apache.hadoop.yarn.util.resource.ResourceCalculator getResourceCalculator()
public QueueMetrics getRootQueueMetrics()
YarnScheduler
public void handle(SchedulerEvent event)
public void recover(RMStateStore.RMState state) throws Exception
Exception
public void setRMContext(RMContext rmContext)
ResourceScheduler
ResourceScheduler
.
This method should be called immediately after instantiating
a scheduler once.rmContext
- created by ResourceManagerprotected void createPreemptionThread()
public void serviceInit(org.apache.hadoop.conf.Configuration conf) throws Exception
serviceInit
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void serviceStart() throws Exception
serviceStart
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void serviceStop() throws Exception
serviceStop
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void reinitialize(org.apache.hadoop.conf.Configuration conf, RMContext rmContext) throws IOException
ResourceScheduler
ResourceScheduler
.conf
- configurationIOException
public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, boolean recursive) throws IOException
YarnScheduler
queueName
- queue nameincludeChildQueues
- include child queues?recursive
- get children queues?IOException
public List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> getQueueUserAclInfo()
YarnScheduler
public int getNumClusterNodes()
YarnScheduler
public boolean checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI, org.apache.hadoop.yarn.api.records.QueueACL acl, String queueName)
YarnScheduler
QueueACL.ADMINISTER_QUEUE
permission,
this user can view/modify the applications in this queuetrue
if the user has the permission,
false
otherwisepublic AllocationConfiguration getAllocationConfiguration()
public List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> getAppsInQueue(String queueName)
YarnScheduler
queueName
- the name of the queue.public String moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveApplication
in interface YarnScheduler
moveApplication
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
org.apache.hadoop.yarn.exceptions.YarnException
- if the move cannot be carried outpublic void preValidateMoveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String newQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
preValidateMoveApplication
in interface YarnScheduler
preValidateMoveApplication
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
appId
- Application IDnewQueue
- Target QueueNameorg.apache.hadoop.yarn.exceptions.YarnException
- if the pre-validation for move cannot be carried outpublic void updateNodeResource(RMNode nm, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
updateNodeResource
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> getSchedulingResourceTypes()
getSchedulingResourceTypes
in interface YarnScheduler
getSchedulingResourceTypes
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public Set<String> getPlanQueues() throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
getPlanQueues
in interface YarnScheduler
getPlanQueues
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
org.apache.hadoop.yarn.exceptions.YarnException
public void setEntitlement(String queueName, QueueEntitlement entitlement) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
setEntitlement
in interface YarnScheduler
setEntitlement
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- the queue for which we change entitlemententitlement
- the new entitlement for the queue (capacity,
maxCapacity, etc..)org.apache.hadoop.yarn.exceptions.YarnException
public void removeQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
removeQueue
in interface YarnScheduler
removeQueue
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- name of queue to removeorg.apache.hadoop.yarn.exceptions.YarnException
- if queue to remove is either not a leaf or if its
not emptypublic float getReservableNodesRatio()
public long checkAndGetApplicationLifetime(String queueName, long lifetime)
YarnScheduler
checkAndGetApplicationLifetime
in interface YarnScheduler
checkAndGetApplicationLifetime
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- Name of the Queuelifetime
- configured application lifetimeCopyright © 2022 Apache Software Foundation. All rights reserved.