@NotThreadSafe public class DefaultBlockMaster extends CoreMaster implements BlockMaster
Modifier and Type | Class and Description |
---|---|
class |
DefaultBlockMaster.LostWorkerDetectionHeartbeatExecutor
Lost worker periodic check.
|
static class |
DefaultBlockMaster.Metrics
Class that contains metrics related to BlockMaster.
|
class |
DefaultBlockMaster.WorkerRegisterStreamGCExecutor
Periodically checks the open worker register streams.
|
mBackupManager, mJournalSystem, mPort, mPrimarySelector, mSafeModeManager, mStartTimeMs
mClock, mJournal, mMasterContext
Constructor and Description |
---|
DefaultBlockMaster(MetricsMaster metricsMaster,
CoreMasterContext masterContext,
java.time.Clock clock,
ExecutorServiceFactory executorServiceFactory)
Creates a new instance of
DefaultBlockMaster . |
Modifier and Type | Method and Description |
---|---|
void |
close()
Closes the server.
|
void |
commitBlock(long workerId,
long usedBytesOnTier,
String tierAlias,
String mediumType,
long blockId,
long length)
Marks a block as committed on a specific worker.
|
void |
commitBlockInUFS(long blockId,
long length,
JournalContext journalContext)
Marks a block as committed, but without a worker location.
|
void |
decommissionWorker(DecommissionWorkerPOptions requestOptions)
Decommission a worker.
|
void |
forgetAllWorkers()
Forces all workers to be lost.
|
BlockInfo |
getBlockInfo(long blockId) |
List<BlockInfo> |
getBlockInfoList(List<Long> blockIds)
Retrieves information for the given list of block ids.
|
BlockMetaStore |
getBlockMetaStore() |
long |
getBlockReplicaCount() |
long |
getCapacityBytes() |
CheckpointName |
getCheckpointName() |
java.time.Clock |
getClock()
Returns the BlockMaster's clock so other components can align with
the BlockMaster's time.
|
int |
getDecommissionedWorkerCount() |
Set<Class<? extends Server>> |
getDependencies() |
StorageTierAssoc |
getGlobalStorageTierAssoc() |
long |
getJournaledNextContainerId()
Returns the internal JournaledNextContainerId.
|
CloseableIterator<Journal.JournalEntry> |
getJournalEntryIterator() |
int |
getLostBlocksCount() |
Iterator<Long> |
getLostBlocksIterator()
Returns an
Iterator over the lost blocks. |
int |
getLostWorkerCount() |
List<WorkerInfo> |
getLostWorkersInfoList() |
String |
getName() |
long |
getNewContainerId() |
Map<ServiceType,GrpcService> |
getServices() |
Map<ServiceType,GrpcService> |
getStandbyServices() |
Map<String,Long> |
getTotalBytesOnTiers() |
long |
getUniqueBlockCount() |
long |
getUsedBytes() |
Map<String,Long> |
getUsedBytesOnTiers() |
MasterWorkerInfo |
getWorker(long workerId)
Returns the internal
MasterWorkerInfo object to the caller. |
Set<WorkerNetAddress> |
getWorkerAddresses() |
int |
getWorkerCount() |
long |
getWorkerId(WorkerNetAddress workerNetAddress)
Returns a worker id for the given worker, creating one if the worker is new.
|
List<WorkerInfo> |
getWorkerInfoList() |
List<WorkerLostStorageInfo> |
getWorkerLostStorage() |
List<WorkerInfo> |
getWorkerReport(GetWorkerReportOptions options)
Gets the worker information list for report CLI.
|
boolean |
hasRegisterLease(long workerId)
Verifies if the worker currently holds a
RegisterLease . |
boolean |
isBlockLost(long blockId) |
boolean |
isRejected(WorkerNetAddress address) |
void |
notifyWorkerId(long workerId,
WorkerNetAddress workerNetAddress)
Notify the worker id to a master.
|
boolean |
processJournalEntry(Journal.JournalEntry entry)
Attempts to apply a journal entry.
|
protected MasterWorkerInfo |
recordWorkerRegistration(long workerId)
Re-register a lost worker or complete registration after getting a worker id.
|
void |
registerLostWorkerFoundListener(java.util.function.Consumer<Address> function)
Registers callback functions to use when lost workers become alive.
|
void |
registerNewWorkerConfListener(java.util.function.BiConsumer<Address,List<ConfigProperty>> function)
Registers callback functions to use when workers register with configuration.
|
void |
registerWorkerDeleteListener(java.util.function.Consumer<Address> function)
Registers callback functions to use when detecting lost workers.
|
void |
registerWorkerLostListener(java.util.function.Consumer<Address> function)
Registers callback functions to use when detecting lost workers.
|
void |
releaseRegisterLease(long workerId)
Releases the
RegisterLease for the specified worker. |
void |
removeBlocks(Collection<Long> blockIds,
boolean delete)
Removes blocks from workers.
|
void |
removeDisabledWorker(RemoveDisabledWorkerPOptions requestOptions)
Revert disabling a worker, enabling it to register to the cluster.
|
void |
reportLostBlocks(List<Long> blockIds)
Reports the ids of the blocks lost on workers.
|
void |
resetState()
Resets the object's journaled state.
|
CompletableFuture<Void> |
restoreFromCheckpoint(File directory,
ExecutorService executorService)
Restores state from a checkpoint asynchronously.
|
void |
start(Boolean isLeader)
Starts the Alluxio server.
|
void |
stop()
Stops the Alluxio server.
|
Optional<RegisterLease> |
tryAcquireRegisterLease(GetRegisterLeasePRequest request)
Try to acquire a
RegisterLease for the worker. |
Command |
workerHeartbeat(long workerId,
Map<String,Long> capacityBytesOnTiers,
Map<String,Long> usedBytesOnTiers,
List<Long> removedBlockIds,
Map<Block.BlockLocation,List<Long>> addedBlocks,
Map<String,StorageList> lostStorage,
List<Metric> metrics)
Updates metadata when a worker periodically heartbeats with the master.
|
void |
workerRegister(long workerId,
List<String> storageTiers,
Map<String,Long> totalBytesOnTiers,
Map<String,Long> usedBytesOnTiers,
Map<Block.BlockLocation,List<Long>> currentBlocksOnLocation,
Map<String,StorageList> lostStorage,
RegisterWorkerPOptions options)
Updates metadata when a worker registers with the master.
|
protected void |
workerRegisterBatch(WorkerRegisterContext context,
RegisterWorkerPRequest chunk) |
void |
workerRegisterFinish(WorkerRegisterContext context)
Completes the worker registration stream.
|
protected void |
workerRegisterStart(WorkerRegisterContext context,
RegisterWorkerPRequest chunk) |
void |
workerRegisterStream(WorkerRegisterContext context,
RegisterWorkerPRequest chunk,
boolean isFirstMsg)
Handles messages in a register stream.
|
CompletableFuture<Void> |
writeToCheckpoint(File directory,
ExecutorService executorService)
Writes a checkpoint to the specified directory asynchronously using the provided executor.
|
createJournalContext, getExecutorService, getMasterContext
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
commitBlockInUFS
createJournalContext, getMasterContext
applyAndJournal, restoreFromCheckpoint, writeToCheckpoint
public DefaultBlockMaster(MetricsMaster metricsMaster, CoreMasterContext masterContext, java.time.Clock clock, ExecutorServiceFactory executorServiceFactory)
DefaultBlockMaster
.
Used for tests where we manually control the clock.metricsMaster
- the metrics mastermasterContext
- the context for Alluxio masterclock
- the clock to use for determining the timeexecutorServiceFactory
- a factory for creating the executor service to use for running
maintenance threadspublic String getName()
public Map<ServiceType,GrpcService> getServices()
getServices
in interface Server<Boolean>
public Map<ServiceType,GrpcService> getStandbyServices()
getStandbyServices
in interface Master
public boolean processJournalEntry(Journal.JournalEntry entry)
Journaled
processJournalEntry
in interface Journaled
entry
- the entry to applypublic void resetState()
Journaled
resetState
in interface Journaled
public CheckpointName getCheckpointName()
getCheckpointName
in interface Checkpointed
public CompletableFuture<Void> writeToCheckpoint(File directory, ExecutorService executorService)
Checkpointed
writeToCheckpoint
in interface Checkpointed
directory
- where the checkpoint will be writtenexecutorService
- to use when running tasks asynchronouslypublic CompletableFuture<Void> restoreFromCheckpoint(File directory, ExecutorService executorService)
Checkpointed
restoreFromCheckpoint
in interface Checkpointed
directory
- where the checkpoint will be locatedexecutorService
- to use when running asynchronous taskspublic CloseableIterator<Journal.JournalEntry> getJournalEntryIterator()
getJournalEntryIterator
in interface JournalEntryIterable
CloseableIterator
that iterates all the journal entriespublic void start(Boolean isLeader) throws IOException
Server
start
in interface Server<Boolean>
start
in class AbstractMaster
isLeader
- the start optionsIOException
public void stop() throws IOException
Server
#start(T)
should be
cleaned up and shutdown.stop
in interface Server<Boolean>
stop
in class AbstractMaster
IOException
public void close() throws IOException
Server
close
in interface Server<Boolean>
close
in class AbstractMaster
IOException
public int getWorkerCount()
getWorkerCount
in interface BlockMaster
public int getLostWorkerCount()
getLostWorkerCount
in interface BlockMaster
public int getDecommissionedWorkerCount()
getDecommissionedWorkerCount
in interface BlockMaster
public long getCapacityBytes()
getCapacityBytes
in interface BlockMaster
public long getUniqueBlockCount()
getUniqueBlockCount
in interface BlockMaster
public long getBlockReplicaCount()
getBlockReplicaCount
in interface BlockMaster
public StorageTierAssoc getGlobalStorageTierAssoc()
getGlobalStorageTierAssoc
in interface BlockMaster
public long getUsedBytes()
getUsedBytes
in interface BlockMaster
public List<WorkerInfo> getWorkerInfoList() throws UnavailableException
getWorkerInfoList
in interface BlockMaster
WorkerInfo
objects representing the live workers in AlluxioUnavailableException
public List<WorkerInfo> getLostWorkersInfoList() throws UnavailableException
getLostWorkersInfoList
in interface BlockMaster
WorkerInfo
s of lost workersUnavailableException
public void removeDisabledWorker(RemoveDisabledWorkerPOptions requestOptions) throws NotFoundException
BlockMaster
removeDisabledWorker
in interface BlockMaster
requestOptions
- the requestNotFoundException
public Set<WorkerNetAddress> getWorkerAddresses() throws UnavailableException
getWorkerAddresses
in interface BlockMaster
UnavailableException
public List<WorkerInfo> getWorkerReport(GetWorkerReportOptions options) throws UnavailableException, InvalidArgumentException
BlockMaster
getWorkerReport
in interface BlockMaster
options
- the GetWorkerReportOptions defines the info rangeWorkerInfo
objects representing the workers in AlluxioUnavailableException
InvalidArgumentException
public List<WorkerLostStorageInfo> getWorkerLostStorage()
getWorkerLostStorage
in interface BlockMaster
public void removeBlocks(Collection<Long> blockIds, boolean delete) throws UnavailableException
BlockMaster
removeBlocks
in interface BlockMaster
blockIds
- a list of block ids to remove from Alluxio spacedelete
- whether to delete blocks' metadata in MasterUnavailableException
public boolean isRejected(WorkerNetAddress address)
isRejected
in interface BlockMaster
address
- worker address to checkpublic void decommissionWorker(DecommissionWorkerPOptions requestOptions) throws NotFoundException
BlockMaster
decommissionWorker
in interface BlockMaster
requestOptions
- the requestNotFoundException
public long getJournaledNextContainerId()
BlockMaster
getJournaledNextContainerId
in interface BlockMaster
public long getNewContainerId() throws UnavailableException
getNewContainerId
in interface ContainerIdGenerable
UnavailableException
public void commitBlock(long workerId, long usedBytesOnTier, String tierAlias, String mediumType, long blockId, long length) throws NotFoundException, UnavailableException
BlockMaster
commitBlock
in interface BlockMaster
workerId
- the worker id committing the blockusedBytesOnTier
- the updated used bytes on the tier of the workertierAlias
- the alias of the storage tier where the worker is committing the block tomediumType
- the medium type where the worker is committing the block toblockId
- the committing block idlength
- the length of the blockNotFoundException
- if the workerId is not activeUnavailableException
public void commitBlockInUFS(long blockId, long length, JournalContext journalContext)
BlockMaster
commitBlockInUFS
in interface BlockMaster
blockId
- the id of the block to commitlength
- the length of the blockjournalContext
- the journal contextpublic BlockInfo getBlockInfo(long blockId) throws BlockInfoException, UnavailableException
getBlockInfo
in interface BlockMaster
blockId
- the block id to get information forBlockInfo
for the given block idBlockInfoException
- if the block info is not foundUnavailableException
public List<BlockInfo> getBlockInfoList(List<Long> blockIds) throws UnavailableException
BlockMaster
getBlockInfoList
in interface BlockMaster
blockIds
- A list of block ids to retrieve the information forBlockInfo
objects corresponding to the input list of block ids. The
list is in the same order as the input listUnavailableException
public Map<String,Long> getTotalBytesOnTiers()
getTotalBytesOnTiers
in interface BlockMaster
public Map<String,Long> getUsedBytesOnTiers()
getUsedBytesOnTiers
in interface BlockMaster
@Nullable protected MasterWorkerInfo recordWorkerRegistration(long workerId)
MasterWorkerInfo
because it is only
reading final fields.workerId
- the worker id to registerpublic long getWorkerId(WorkerNetAddress workerNetAddress)
BlockMaster
getWorkerId
in interface BlockMaster
workerNetAddress
- the worker WorkerNetAddress
public void notifyWorkerId(long workerId, WorkerNetAddress workerNetAddress)
BlockMaster
notifyWorkerId
in interface BlockMaster
workerId
- the worker idworkerNetAddress
- the worker addresspublic Optional<RegisterLease> tryAcquireRegisterLease(GetRegisterLeasePRequest request)
BlockMaster
RegisterLease
for the worker.
If the lease is not granted, this will return empty immediately rather than blocking.tryAcquireRegisterLease
in interface BlockMaster
request
- the request with all information for the master to make a decision withpublic boolean hasRegisterLease(long workerId)
BlockMaster
RegisterLease
.hasRegisterLease
in interface BlockMaster
workerId
- the worker IDpublic void releaseRegisterLease(long workerId)
BlockMaster
RegisterLease
for the specified worker.
If the worker currently does not hold a lease, return without throwing an error.
The lease may have been recycled already.releaseRegisterLease
in interface BlockMaster
workerId
- the worker IDpublic void workerRegister(long workerId, List<String> storageTiers, Map<String,Long> totalBytesOnTiers, Map<String,Long> usedBytesOnTiers, Map<Block.BlockLocation,List<Long>> currentBlocksOnLocation, Map<String,StorageList> lostStorage, RegisterWorkerPOptions options) throws NotFoundException
BlockMaster
workerRegister
in interface BlockMaster
workerId
- the worker id of the worker registeringstorageTiers
- a list of storage tier aliases in order of their position in the worker's
hierarchytotalBytesOnTiers
- a mapping from storage tier alias to total bytesusedBytesOnTiers
- a mapping from storage tier alias to the used byescurrentBlocksOnLocation
- a mapping from storage tier alias to a list of blockslostStorage
- a mapping from storage tier alias to a list of lost storage pathsoptions
- the options that may contain worker configurationNotFoundException
- if workerId cannot be foundpublic MasterWorkerInfo getWorker(long workerId) throws NotFoundException
BlockMaster
MasterWorkerInfo
object to the caller.
This is specifically for the tests and the WorkerRegisterContext
.
Note that this operations on the object requires locking.
See the javadoc of MasterWorkerInfo
for how the locking should be carefully done.
When in doubt, do not use this API. Find other methods in this class that exposes
necessary information.getWorker
in interface BlockMaster
workerId
- the worker IDMasterWorkerInfo
for the workerNotFoundException
public void workerRegisterStream(WorkerRegisterContext context, RegisterWorkerPRequest chunk, boolean isFirstMsg)
BlockMaster
workerRegisterStream
in interface BlockMaster
context
- the stream context that contains the worker informationchunk
- the message in a streamisFirstMsg
- whether the message is the 1st in a streamprotected void workerRegisterStart(WorkerRegisterContext context, RegisterWorkerPRequest chunk)
protected void workerRegisterBatch(WorkerRegisterContext context, RegisterWorkerPRequest chunk)
public void workerRegisterFinish(WorkerRegisterContext context)
BlockMaster
workerRegisterFinish
in interface BlockMaster
context
- the stream context to be closedpublic Command workerHeartbeat(long workerId, Map<String,Long> capacityBytesOnTiers, Map<String,Long> usedBytesOnTiers, List<Long> removedBlockIds, Map<Block.BlockLocation,List<Long>> addedBlocks, Map<String,StorageList> lostStorage, List<Metric> metrics)
BlockMaster
workerHeartbeat
in interface BlockMaster
workerId
- the worker idcapacityBytesOnTiers
- a mapping from tier alias to the capacity bytesusedBytesOnTiers
- a mapping from tier alias to the used bytesremovedBlockIds
- a list of block ids removed from this workeraddedBlocks
- a mapping from tier alias to the added blockslostStorage
- a mapping from tier alias to lost storage pathsmetrics
- worker metricspublic java.time.Clock getClock()
BlockMaster
getClock
in interface BlockMaster
public boolean isBlockLost(long blockId)
isBlockLost
in interface BlockMaster
blockId
- the block IDpublic Iterator<Long> getLostBlocksIterator()
BlockMaster
Iterator
over the lost blocks.
Note that the iterator should not be shared across threads.getLostBlocksIterator
in interface BlockMaster
public int getLostBlocksCount()
getLostBlocksCount
in interface BlockMaster
public void reportLostBlocks(List<Long> blockIds)
BlockMaster
reportLostBlocks
in interface BlockMaster
blockIds
- the ids of the lost blockspublic Set<Class<? extends Server>> getDependencies()
getDependencies
in interface Server<Boolean>
getDependencies
in class AbstractMaster
public void forgetAllWorkers()
public void registerLostWorkerFoundListener(java.util.function.Consumer<Address> function)
BlockMaster
registerLostWorkerFoundListener
in interface BlockMaster
function
- the function to registerpublic void registerWorkerLostListener(java.util.function.Consumer<Address> function)
BlockMaster
registerWorkerLostListener
in interface BlockMaster
function
- the function to registerpublic void registerWorkerDeleteListener(java.util.function.Consumer<Address> function)
BlockMaster
registerWorkerDeleteListener
in interface BlockMaster
function
- the function to registerpublic void registerNewWorkerConfListener(java.util.function.BiConsumer<Address,List<ConfigProperty>> function)
BlockMaster
registerNewWorkerConfListener
in interface BlockMaster
function
- the function to registerpublic BlockMetaStore getBlockMetaStore()
Copyright © 2023. All Rights Reserved.