<?php
/**
* Global, MySQL-backed lock. This is a high-reliability, low-performance
* global lock.
*
* The lock is maintained by using GET_LOCK() in MySQL, and automatically
* released when the connection terminates. Thus, this lock can safely be used
* to control access to shared resources without implementing any sort of
* timeout or override logic: the lock can't normally be stuck in a locked state
* with no process actually holding the lock.
*
* However, acquiring the lock is moderately expensive (several network
* roundtrips). This makes it unsuitable for tasks where lock performance is
* important.
*
* $lock = PhabricatorGlobalLock::newLock('example');
* $lock->lock();
* do_contentious_things();
* $lock->unlock();
*
* NOTE: This lock is not completely global; it is namespaced to the active
* storage namespace so that unit tests running in separate table namespaces
* are isolated from one another.
*
* @task construct Constructing Locks
* @task impl Implementation
*/
final class PhabricatorGlobalLock extends PhutilLock {
private $parameters;
private $conn;
private $externalConnection;
private $log;
private $disableLogging;
private static $pool = array();
/* -( Constructing Locks )------------------------------------------------- */
public static function newLock($name, $parameters = array()) {
$namespace = PhabricatorLiskDAO::getStorageNamespace();
$namespace = PhabricatorHash::digestToLength($namespace, 20);
$parts = array();
ksort($parameters);
foreach ($parameters as $key => $parameter) {
if (!preg_match('/^[a-zA-Z0-9]+\z/', $key)) {
throw new Exception(
pht(
'Lock parameter key "%s" must be alphanumeric.',
$key));
}
if (!is_scalar($parameter) && !is_null($parameter)) {
throw new Exception(
pht(
'Lock parameter for key "%s" must be a scalar.',
$key));
}
$value = phutil_json_encode($parameter);
$parts[] = "{$key}={$value}";
}
$parts = implode(', ', $parts);
$local = "{$name}({$parts})";
$local = PhabricatorHash::digestToLength($local, 20);
$full_name = "ph:{$namespace}:{$local}";
$lock = self::getLock($full_name);
if (!$lock) {
$lock = new PhabricatorGlobalLock($full_name);
self::registerLock($lock);
$lock->parameters = $parameters;
}
return $lock;
}
/**
* Use a specific database connection for locking.
*
* By default, `PhabricatorGlobalLock` will lock on the "repository" database
* (somewhat arbitrarily). In most cases this is fine, but this method can
* be used to lock on a specific connection.
*
* @param AphrontDatabaseConnection
* @return this
*/
public function setExternalConnection(AphrontDatabaseConnection $conn) {
if ($this->conn) {
throw new Exception(
pht(
'Lock is already held, and must be released before the '.
'connection may be changed.'));
}
$this->externalConnection = $conn;
return $this;
}
public function setDisableLogging($disable) {
$this->disableLogging = $disable;
return $this;
}
/* -( Connection Pool )---------------------------------------------------- */
public static function getConnectionPoolSize() {
return count(self::$pool);
}
public static function clearConnectionPool() {
self::$pool = array();
}
public static function newConnection() {
// NOTE: Use of the "repository" database is somewhat arbitrary, mostly
// because the first client of locks was the repository daemons.
// We must always use the same database for all locks, because different
// databases may be on different hosts if the database is partitioned.
// However, we don't access any tables so we could use any valid database.
// We could build a database-free connection instead, but that's kind of
// messy and unusual.
$dao = new PhabricatorRepository();
// NOTE: Using "force_new" to make sure each lock is on its own connection.
// See T13627. This is critically important in versions of MySQL older
// than MySQL 5.7, because they can not hold more than one lock per
// connection simultaneously.
return $dao->establishConnection('w', $force_new = true);
}
/* -( Implementation )----------------------------------------------------- */
protected function doLock($wait) {
$conn = $this->conn;
if (!$conn) {
if ($this->externalConnection) {
$conn = $this->externalConnection;
}
}
if (!$conn) {
// Try to reuse a connection from the connection pool.
$conn = array_pop(self::$pool);
}
if (!$conn) {
$conn = self::newConnection();
}
// See T13627. We must never hold more than one lock per connection, so
// make sure this connection has no existing locks. (Normally, we should
// only be able to get here if callers explicitly provide the same external
// connection to multiple locks.)
if ($conn->isHoldingAnyLock()) {
throw new Exception(
pht(
'Unable to establish lock on connection: this connection is '.
'already holding a lock. Acquiring a second lock on the same '.
'connection would release the first lock in MySQL versions '.
'older than 5.7.'));
}
// NOTE: Since MySQL will disconnect us if we're idle for too long, we set
// the wait_timeout to an enormous value, to allow us to hold the
// connection open indefinitely (or, at least, for 24 days).
$max_allowed_timeout = 2147483;
queryfx($conn, 'SET wait_timeout = %d', $max_allowed_timeout);
$lock_name = $this->getName();
$result = queryfx_one(
$conn,
'SELECT GET_LOCK(%s, %f)',
$lock_name,
$wait);
$ok = head($result);
if (!$ok) {
// See PHI1794. We failed to acquire the lock, but the connection itself
// is still good. We're done with it, so add it to the pool, just as we
// would if we were releasing the lock.
// If we don't do this, we may establish a huge number of connections
// very rapidly if many workers try to acquire a lock at once. For
// example, this can happen if there are a large number of webhook tasks
// in the queue.
// See T13627. If this is an external connection, don't put it into
// the shared connection pool.
if (!$this->externalConnection) {
self::$pool[] = $conn;
}
throw id(new PhutilLockException($lock_name))
->setHint($this->newHint($lock_name, $wait));
}
$conn->rememberLock($lock_name);
$this->conn = $conn;
if ($this->shouldLogLock()) {
$lock_context = $this->newLockContext();
$log = id(new PhabricatorDaemonLockLog())
->setLockName($lock_name)
->setLockParameters($this->parameters)
->setLockContext($lock_context)
->save();
$this->log = $log;
}
}
protected function doUnlock() {
$lock_name = $this->getName();
$conn = $this->conn;
try {
$result = queryfx_one(
$conn,
'SELECT RELEASE_LOCK(%s)',
$lock_name);
$conn->forgetLock($lock_name);
} catch (Exception $ex) {
$result = array(null);
}
$ok = head($result);
if (!$ok) {
// TODO: We could throw here, but then this lock doesn't get marked
// unlocked and we throw again later when exiting. It also doesn't
// particularly matter for any current applications. For now, just
// swallow the error.
}
$this->conn = null;
if (!$this->externalConnection) {
$conn->close();
self::$pool[] = $conn;
}
if ($this->log) {
$log = $this->log;
$this->log = null;
$conn = $log->establishConnection('w');
queryfx(
$conn,
'UPDATE %T SET lockReleased = UNIX_TIMESTAMP() WHERE id = %d',
$log->getTableName(),
$log->getID());
}
}
private function shouldLogLock() {
if ($this->disableLogging) {
return false;
}
$policy = id(new PhabricatorDaemonLockLogGarbageCollector())
->getRetentionPolicy();
if (!$policy) {
return false;
}
return true;
}
private function newLockContext() {
$context = array(
'pid' => getmypid(),
'host' => php_uname('n'),
'sapi' => php_sapi_name(),
);
global $argv;
if ($argv) {
$context['argv'] = $argv;
}
$access_log = null;
// TODO: There's currently no cohesive way to get the parameterized access
// log for the current request across different request types. Web requests
// have an "AccessLog", SSH requests have an "SSHLog", and other processes
// (like scripts) have no log. But there's no method to say "give me any
// log you've got". For now, just test if we have a web request and use the
// "AccessLog" if we do, since that's the only one we actually read any
// parameters from.
// NOTE: "PhabricatorStartup" is only available from web requests, not
// from CLI scripts.
if (class_exists('PhabricatorStartup', false)) {
$access_log = PhabricatorAccessLog::getLog();
}
if ($access_log) {
$controller = $access_log->getData('C');
if ($controller) {
$context['controller'] = $controller;
}
$method = $access_log->getData('m');
if ($method) {
$context['method'] = $method;
}
}
return $context;
}
private function newHint($lock_name, $wait) {
if (!$this->shouldLogLock()) {
return pht(
'Enable the lock log for more detailed information about '.
'which process is holding this lock.');
}
$now = PhabricatorTime::getNow();
// First, look for recent logs. If other processes have been acquiring and
// releasing this lock while we've been waiting, this is more likely to be
// a contention/throughput issue than an issue with something hung while
// holding the lock.
$limit = 100;
$logs = id(new PhabricatorDaemonLockLog())->loadAllWhere(
'lockName = %s AND dateCreated >= %d ORDER BY id ASC LIMIT %d',
$lock_name,
($now - $wait),
$limit);
if ($logs) {
if (count($logs) === $limit) {
return pht(
'During the last %s second(s) spent waiting for the lock, more '.
'than %s other process(es) acquired it, so this is likely a '.
'bottleneck. Use "bin/lock log --name %s" to review log activity.',
new PhutilNumber($wait),
new PhutilNumber($limit),
$lock_name);
} else {
return pht(
'During the last %s second(s) spent waiting for the lock, %s '.
'other process(es) acquired it, so this is likely a '.
'bottleneck. Use "bin/lock log --name %s" to review log activity.',
new PhutilNumber($wait),
phutil_count($logs),
$lock_name);
}
}
$last_log = id(new PhabricatorDaemonLockLog())->loadOneWhere(
'lockName = %s ORDER BY id DESC LIMIT 1',
$lock_name);
if ($last_log) {
$info = array();
$acquired = $last_log->getDateCreated();
$context = $last_log->getLockContext();
$process_info = array();
$pid = idx($context, 'pid');
if ($pid) {
$process_info[] = 'pid='.$pid;
}
$host = idx($context, 'host');
if ($host) {
$process_info[] = 'host='.$host;
}
$sapi = idx($context, 'sapi');
if ($sapi) {
$process_info[] = 'sapi='.$sapi;
}
$argv = idx($context, 'argv');
if ($argv) {
$process_info[] = 'argv='.(string)csprintf('%LR', $argv);
}
$controller = idx($context, 'controller');
if ($controller) {
$process_info[] = 'controller='.$controller;
}
$method = idx($context, 'method');
if ($method) {
$process_info[] = 'method='.$method;
}
$process_info = implode(', ', $process_info);
$info[] = pht(
'This lock was most recently acquired by a process (%s) '.
'%s second(s) ago.',
$process_info,
new PhutilNumber($now - $acquired));
$released = $last_log->getLockReleased();
if ($released) {
$info[] = pht(
'This lock was released %s second(s) ago.',
new PhutilNumber($now - $released));
} else {
$info[] = pht('There is no record of this lock being released.');
}
return implode(' ', $info);
}
return pht(
'Found no records of processes acquiring or releasing this lock.');
}
}