<?php
abstract class DrydockWorker extends PhabricatorWorker {
protected function getViewer() {
return PhabricatorUser::getOmnipotentUser();
}
protected function loadLease($lease_phid) {
$viewer = $this->getViewer();
$lease = id(new DrydockLeaseQuery())
->setViewer($viewer)
->withPHIDs(array($lease_phid))
->executeOne();
if (!$lease) {
throw new PhabricatorWorkerPermanentFailureException(
pht('No such lease "%s"!', $lease_phid));
}
return $lease;
}
protected function loadResource($resource_phid) {
$viewer = $this->getViewer();
$resource = id(new DrydockResourceQuery())
->setViewer($viewer)
->withPHIDs(array($resource_phid))
->executeOne();
if (!$resource) {
throw new PhabricatorWorkerPermanentFailureException(
pht('No such resource "%s"!', $resource_phid));
}
return $resource;
}
protected function loadOperation($operation_phid) {
$viewer = $this->getViewer();
$operation = id(new DrydockRepositoryOperationQuery())
->setViewer($viewer)
->withPHIDs(array($operation_phid))
->executeOne();
if (!$operation) {
throw new PhabricatorWorkerPermanentFailureException(
pht('No such operation "%s"!', $operation_phid));
}
return $operation;
}
protected function loadCommands($target_phid) {
$viewer = $this->getViewer();
$commands = id(new DrydockCommandQuery())
->setViewer($viewer)
->withTargetPHIDs(array($target_phid))
->withConsumed(false)
->execute();
$commands = msort($commands, 'getID');
return $commands;
}
protected function checkLeaseExpiration(DrydockLease $lease) {
$this->checkObjectExpiration($lease);
}
protected function checkResourceExpiration(DrydockResource $resource) {
$this->checkObjectExpiration($resource);
}
private function checkObjectExpiration($object) {
// Check if the resource or lease has expired. If it has, we're going to
// send it a release command.
// This command is sent from within the update worker so it is handled
// immediately, but doing this generates a log and improves consistency.
$expires = $object->getUntil();
if (!$expires) {
return;
}
$now = PhabricatorTime::getNow();
if ($expires > $now) {
return;
}
$viewer = $this->getViewer();
$drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
$command = DrydockCommand::initializeNewCommand($viewer)
->setTargetPHID($object->getPHID())
->setAuthorPHID($drydock_phid)
->setCommand(DrydockCommand::COMMAND_RELEASE)
->save();
}
protected function yieldIfExpiringLease(DrydockLease $lease) {
if (!$lease->canReceiveCommands()) {
return;
}
$this->yieldIfExpiring($lease->getUntil());
}
protected function yieldIfExpiringResource(DrydockResource $resource) {
if (!$resource->canReceiveCommands()) {
return;
}
$this->yieldIfExpiring($resource->getUntil());
}
private function yieldIfExpiring($expires) {
if (!$expires) {
return;
}
if (!$this->getTaskDataValue('isExpireTask')) {
return;
}
$now = PhabricatorTime::getNow();
throw new PhabricatorWorkerYieldException($expires - $now);
}
protected function isTemporaryException(Exception $ex) {
if ($ex instanceof PhabricatorWorkerYieldException) {
return true;
}
if ($ex instanceof DrydockSlotLockException) {
return true;
}
if ($ex instanceof PhutilAggregateException) {
$any_temporary = false;
foreach ($ex->getExceptions() as $sub) {
if ($this->isTemporaryException($sub)) {
$any_temporary = true;
break;
}
}
if ($any_temporary) {
return true;
}
}
if ($ex instanceof PhutilProxyException) {
return $this->isTemporaryException($ex->getPreviousException());
}
return false;
}
protected function getYieldDurationFromException(Exception $ex) {
if ($ex instanceof PhabricatorWorkerYieldException) {
return $ex->getDuration();
}
if ($ex instanceof DrydockSlotLockException) {
return 5;
}
return 15;
}
protected function flushDrydockTaskQueue() {
// NOTE: By default, queued tasks are not scheduled if the current task
// fails. This is a good, safe default behavior. For example, it can
// protect us from executing side effect tasks too many times, like
// sending extra email.
// However, it is not the behavior we want in Drydock, because we queue
// followup tasks after lease and resource failures and want them to
// execute in order to clean things up.
// At least for now, we just explicitly flush the queue before exiting
// with a failure to make sure tasks get queued up properly.
try {
$this->flushTaskQueue();
} catch (Exception $ex) {
// If this fails, we want to swallow the exception so the caller throws
// the original error, since we're more likely to be able to understand
// and fix the problem if we have the original error than if we replace
// it with this one.
phlog($ex);
}
return $this;
}
protected function canReclaimResource(DrydockResource $resource) {
$viewer = $this->getViewer();
// Don't reclaim a resource if it has been updated recently. If two
// leases are fighting, we don't want them to keep reclaiming resources
// from one another forever without making progress, so make resources
// immune to reclamation for a little while after they activate or update.
$now = PhabricatorTime::getNow();
$max_epoch = ($now - phutil_units('3 minutes in seconds'));
// TODO: It would be nice to use a more narrow time here, like "last
// activation or lease release", but we don't currently store that
// anywhere.
$updated = $resource->getDateModified();
if ($updated > $max_epoch) {
return false;
}
$statuses = array(
DrydockLeaseStatus::STATUS_PENDING,
DrydockLeaseStatus::STATUS_ACQUIRED,
DrydockLeaseStatus::STATUS_ACTIVE,
DrydockLeaseStatus::STATUS_RELEASED,
DrydockLeaseStatus::STATUS_BROKEN,
);
// Don't reclaim resources that have any active leases.
$leases = id(new DrydockLeaseQuery())
->setViewer($viewer)
->withResourcePHIDs(array($resource->getPHID()))
->withStatuses($statuses)
->setLimit(1)
->execute();
if ($leases) {
return false;
}
// See T13676. Don't reclaim a resource if a lease recently released.
$leases = id(new DrydockLeaseQuery())
->setViewer($viewer)
->withResourcePHIDs(array($resource->getPHID()))
->withStatuses(
array(
DrydockLeaseStatus::STATUS_DESTROYED,
))
->withDateModifiedBetween($max_epoch, null)
->setLimit(1)
->execute();
if ($leases) {
return false;
}
return true;
}
protected function reclaimResource(
DrydockResource $resource,
DrydockLease $lease) {
$viewer = $this->getViewer();
// Mark the lease as reclaiming this resource. It won't be allowed to start
// another reclaim as long as this resource is still in the process of
// being reclaimed.
$lease->addReclaimedResourcePHIDs(array($resource->getPHID()));
// When the resource releases, we we want to reawaken this task since it
// should (usually) be able to start building a new resource right away.
$worker_task_id = $this->getCurrentWorkerTaskID();
$command = DrydockCommand::initializeNewCommand($viewer)
->setTargetPHID($resource->getPHID())
->setAuthorPHID($lease->getPHID())
->setCommand(DrydockCommand::COMMAND_RECLAIM)
->setProperty('awakenTaskIDs', array($worker_task_id));
$lease->openTransaction();
$lease->save();
$command->save();
$lease->saveTransaction();
$resource->scheduleUpdate();
return $this;
}
}