<?php
final class PhutilDaemonPool extends Phobject {
private $properties = array();
private $commandLineArguments;
private $overseer;
private $daemons = array();
private $argv;
private $lastAutoscaleUpdate;
private $inShutdown;
private function __construct() {
// <empty>
}
public static function newFromConfig(array $config) {
PhutilTypeSpec::checkMap(
$config,
array(
'class' => 'string',
'label' => 'string',
'argv' => 'optional list<string>',
'load' => 'optional list<string>',
'log' => 'optional string|null',
'pool' => 'optional int',
'up' => 'optional int',
'down' => 'optional int',
'reserve' => 'optional int|float',
));
$config = $config + array(
'argv' => array(),
'load' => array(),
'log' => null,
'pool' => 1,
'up' => 2,
'down' => 15,
'reserve' => 0,
);
$pool = new self();
$pool->properties = $config;
return $pool;
}
public function setOverseer(PhutilDaemonOverseer $overseer) {
$this->overseer = $overseer;
return $this;
}
public function getOverseer() {
return $this->overseer;
}
public function setCommandLineArguments(array $arguments) {
$this->commandLineArguments = $arguments;
return $this;
}
public function getCommandLineArguments() {
return $this->commandLineArguments;
}
private function shouldShutdown() {
return $this->inShutdown;
}
private function newDaemon() {
$config = $this->properties;
if (count($this->daemons)) {
$down_duration = $this->getPoolScaledownDuration();
} else {
// TODO: For now, never scale pools down to 0.
$down_duration = 0;
}
$forced_config = array(
'down' => $down_duration,
);
$config = $forced_config + $config;
$config = array_select_keys(
$config,
array(
'class',
'log',
'load',
'argv',
'down',
));
$daemon = PhutilDaemonHandle::newFromConfig($config)
->setDaemonPool($this)
->setCommandLineArguments($this->getCommandLineArguments());
$daemon_id = $daemon->getDaemonID();
$this->daemons[$daemon_id] = $daemon;
$daemon->didLaunch();
return $daemon;
}
public function getDaemons() {
return $this->daemons;
}
public function didReceiveSignal($signal, $signo) {
switch ($signal) {
case PhutilDaemonOverseer::SIGNAL_GRACEFUL:
case PhutilDaemonOverseer::SIGNAL_TERMINATE:
$this->inShutdown = true;
break;
}
foreach ($this->getDaemons() as $daemon) {
switch ($signal) {
case PhutilDaemonOverseer::SIGNAL_NOTIFY:
$daemon->didReceiveNotifySignal($signo);
break;
case PhutilDaemonOverseer::SIGNAL_RELOAD:
$daemon->didReceiveReloadSignal($signo);
break;
case PhutilDaemonOverseer::SIGNAL_GRACEFUL:
$daemon->didReceiveGracefulSignal($signo);
break;
case PhutilDaemonOverseer::SIGNAL_TERMINATE:
$daemon->didReceiveTerminateSignal($signo);
break;
default:
throw new Exception(
pht(
'Unknown signal "%s" ("%d").',
$signal,
$signo));
}
}
}
public function getPoolLabel() {
return $this->getPoolProperty('label');
}
public function getPoolMaximumSize() {
return $this->getPoolProperty('pool');
}
public function getPoolScaleupDuration() {
return $this->getPoolProperty('up');
}
public function getPoolScaledownDuration() {
return $this->getPoolProperty('down');
}
public function getPoolMemoryReserve() {
return $this->getPoolProperty('reserve');
}
public function getPoolDaemonClass() {
return $this->getPoolProperty('class');
}
private function getPoolProperty($key) {
return idx($this->properties, $key);
}
public function updatePool() {
$daemons = $this->getDaemons();
foreach ($daemons as $key => $daemon) {
$daemon->update();
if ($daemon->isDone()) {
$daemon->didExit();
unset($this->daemons[$key]);
if ($this->shouldShutdown()) {
$this->logMessage(
'DOWN',
pht(
'Pool "%s" is exiting, with %s daemon(s) remaining.',
$this->getPoolLabel(),
new PhutilNumber(count($this->daemons))));
} else {
$this->logMessage(
'POOL',
pht(
'Autoscale pool "%s" scaled down to %s daemon(s).',
$this->getPoolLabel(),
new PhutilNumber(count($this->daemons))));
}
}
}
$this->updateAutoscale();
}
public function isHibernating() {
foreach ($this->getDaemons() as $daemon) {
if (!$daemon->isHibernating()) {
return false;
}
}
return true;
}
public function wakeFromHibernation() {
if (!$this->isHibernating()) {
return $this;
}
$this->logMessage(
'WAKE',
pht(
'Autoscale pool "%s" is being awakened from hibernation.',
$this->getPoolLabel()));
$did_wake_daemons = false;
foreach ($this->getDaemons() as $daemon) {
if ($daemon->isHibernating()) {
$daemon->wakeFromHibernation();
$did_wake_daemons = true;
}
}
if (!$did_wake_daemons) {
// TODO: Pools currently can't scale down to 0 daemons, but we should
// scale up immediately here once they can.
}
$this->updatePool();
return $this;
}
private function updateAutoscale() {
if ($this->shouldShutdown()) {
return;
}
// Don't try to autoscale more than once per second. This mostly stops the
// logs from getting flooded in verbose mode.
$now = time();
if ($this->lastAutoscaleUpdate >= $now) {
return;
}
$this->lastAutoscaleUpdate = $now;
$daemons = $this->getDaemons();
// If this pool is already at the maximum size, we can't launch any new
// daemons.
$max_size = $this->getPoolMaximumSize();
if (count($daemons) >= $max_size) {
$this->logMessage(
'POOL',
pht(
'Autoscale pool "%s" already at maximum size (%s of %s).',
$this->getPoolLabel(),
new PhutilNumber(count($daemons)),
new PhutilNumber($max_size)));
return;
}
$scaleup_duration = $this->getPoolScaleupDuration();
foreach ($daemons as $daemon) {
$busy_epoch = $daemon->getBusyEpoch();
// If any daemons haven't started work yet, don't scale the pool up.
if (!$busy_epoch) {
$this->logMessage(
'POOL',
pht(
'Autoscale pool "%s" has an idle daemon, declining to scale.',
$this->getPoolLabel()));
return;
}
// If any daemons started work very recently, wait a little while
// to scale the pool up.
$busy_for = ($now - $busy_epoch);
if ($busy_for < $scaleup_duration) {
$this->logMessage(
'POOL',
pht(
'Autoscale pool "%s" has not been busy long enough to scale up '.
'(busy for %s of %s seconds).',
$this->getPoolLabel(),
new PhutilNumber($busy_for),
new PhutilNumber($scaleup_duration)));
return;
}
}
// If we have a configured memory reserve for this pool, it tells us that
// we should not scale up unless there's at least that much memory left
// on the system (for example, a reserve of 0.25 means that 25% of system
// memory must be free to autoscale).
// Note that the first daemon is exempt: we'll always launch at least one
// daemon, regardless of any memory reservation.
if (count($daemons)) {
$reserve = $this->getPoolMemoryReserve();
if ($reserve) {
// On some systems this may be slightly more expensive than other
// checks, so we only do it once we're prepared to scale up.
$memory = PhutilSystem::getSystemMemoryInformation();
$free_ratio = ($memory['free'] / $memory['total']);
// If we don't have enough free memory, don't scale.
if ($free_ratio <= $reserve) {
$this->logMessage(
'POOL',
pht(
'Autoscale pool "%s" does not have enough free memory to '.
'scale up (%s free of %s reserved).',
$this->getPoolLabel(),
new PhutilNumber($free_ratio, 3),
new PhutilNumber($reserve, 3)));
return;
}
}
}
$this->logMessage(
'AUTO',
pht(
'Scaling pool "%s" up to %s daemon(s).',
$this->getPoolLabel(),
new PhutilNumber(count($daemons) + 1)));
$this->newDaemon();
}
public function logMessage($type, $message, $context = null) {
return $this->getOverseer()->logMessage($type, $message, $context);
}
}