<?php
/**
* Execute system commands in parallel using futures.
*
* ExecFuture is a future, which means it runs asynchronously and represents
* a value which may not exist yet. See @{article:Using Futures} for an
* explanation of futures. When an ExecFuture resolves, it returns the exit
* code, stdout and stderr of the process it executed.
*
* ExecFuture is the core command execution implementation in libphutil, but is
* exposed through a number of APIs. See @{article:Command Execution} for more
* discussion about executing system commands.
*
* @task create Creating ExecFutures
* @task resolve Resolving Execution
* @task config Configuring Execution
* @task info Command Information
* @task interact Interacting With Commands
* @task internal Internals
*/
final class ExecFuture extends PhutilExecutableFuture {
private $pipes = array();
private $proc = null;
private $start = null;
private $procStatus = null;
private $stdout = null;
private $stderr = null;
private $stdin = null;
private $closePipe = true;
private $stdoutPos = 0;
private $stderrPos = 0;
private $readBufferSize;
private $stdoutSizeLimit = PHP_INT_MAX;
private $stderrSizeLimit = PHP_INT_MAX;
private $profilerCallID;
private $killedByTimeout;
private $windowsStdoutTempFile = null;
private $windowsStderrTempFile = null;
private $terminateTimeout;
private $didTerminate;
private $killTimeout;
private static $descriptorSpec = array(
0 => array('pipe', 'r'), // stdin
1 => array('pipe', 'w'), // stdout
2 => array('pipe', 'w'), // stderr
);
protected function didConstruct() {
$this->stdin = new PhutilRope();
}
/* -( Command Information )------------------------------------------------ */
/**
* Retrieve the byte limit for the stderr buffer.
*
* @return int Maximum buffer size, in bytes.
* @task info
*/
public function getStderrSizeLimit() {
return $this->stderrSizeLimit;
}
/**
* Retrieve the byte limit for the stdout buffer.
*
* @return int Maximum buffer size, in bytes.
* @task info
*/
public function getStdoutSizeLimit() {
return $this->stdoutSizeLimit;
}
/**
* Get the process's pid. This only works after execution is initiated, e.g.
* by a call to start().
*
* @return int Process ID of the executing process.
* @task info
*/
public function getPID() {
$status = $this->procGetStatus();
return $status['pid'];
}
public function hasPID() {
if ($this->procStatus) {
return true;
}
if ($this->proc) {
return true;
}
return false;
}
/* -( Configuring Execution )---------------------------------------------- */
/**
* Set a maximum size for the stdout read buffer. To limit stderr, see
* @{method:setStderrSizeLimit}. The major use of these methods is to use less
* memory if you are running a command which sometimes produces huge volumes
* of output that you don't really care about.
*
* NOTE: Setting this to 0 means "no buffer", not "unlimited buffer".
*
* @param int Maximum size of the stdout read buffer.
* @return this
* @task config
*/
public function setStdoutSizeLimit($limit) {
$this->stdoutSizeLimit = $limit;
return $this;
}
/**
* Set a maximum size for the stderr read buffer.
* See @{method:setStdoutSizeLimit} for discussion.
*
* @param int Maximum size of the stderr read buffer.
* @return this
* @task config
*/
public function setStderrSizeLimit($limit) {
$this->stderrSizeLimit = $limit;
return $this;
}
/**
* Set the maximum internal read buffer size this future. The future will
* block reads once the internal stdout or stderr buffer exceeds this size.
*
* NOTE: If you @{method:resolve} a future with a read buffer limit, you may
* block forever!
*
* TODO: We should probably release the read buffer limit during
* @{method:resolve}, or otherwise detect this. For now, be careful.
*
* @param int|null Maximum buffer size, or `null` for unlimited.
* @return this
*/
public function setReadBufferSize($read_buffer_size) {
$this->readBufferSize = $read_buffer_size;
return $this;
}
/* -( Interacting With Commands )------------------------------------------ */
/**
* Read and return output from stdout and stderr, if any is available. This
* method keeps a read cursor on each stream, but the entire streams are
* still returned when the future resolves. You can call read() again after
* resolving the future to retrieve only the parts of the streams you did not
* previously read:
*
* $future = new ExecFuture('...');
* // ...
* list($stdout) = $future->read(); // Returns output so far
* list($stdout) = $future->read(); // Returns new output since first call
* // ...
* list($stdout) = $future->resolvex(); // Returns ALL output
* list($stdout) = $future->read(); // Returns unread output
*
* NOTE: If you set a limit with @{method:setStdoutSizeLimit} or
* @{method:setStderrSizeLimit}, this method will not be able to read data
* past the limit.
*
* NOTE: If you call @{method:discardBuffers}, all the stdout/stderr data
* will be thrown away and the cursors will be reset.
*
* @return pair <$stdout, $stderr> pair with new output since the last call
* to this method.
* @task interact
*/
public function read() {
$stdout = $this->readStdout();
$result = array(
$stdout,
(string)substr($this->stderr, $this->stderrPos),
);
$this->stderrPos = strlen($this->stderr);
return $result;
}
public function readStdout() {
if ($this->start) {
$this->updateFuture(); // Sync
}
$result = (string)substr($this->stdout, $this->stdoutPos);
$this->stdoutPos = strlen($this->stdout);
return $result;
}
/**
* Write data to stdin of the command.
*
* @param string Data to write.
* @param bool If true, keep the pipe open for writing. By default, the pipe
* will be closed as soon as possible so that commands which
* listen for EOF will execute. If you want to keep the pipe open
* past the start of command execution, do an empty write with
* `$keep_pipe = true` first.
* @return this
* @task interact
*/
public function write($data, $keep_pipe = false) {
if (strlen($data)) {
if (!$this->stdin) {
throw new Exception(pht('Writing to a closed pipe!'));
}
$this->stdin->append($data);
}
$this->closePipe = !$keep_pipe;
return $this;
}
/**
* Permanently discard the stdout and stderr buffers and reset the read
* cursors. This is basically useful only if you are streaming a large amount
* of data from some process.
*
* Conceivably you might also need to do this if you're writing a client using
* @{class:ExecFuture} and `netcat`, but you probably should not do that.
*
* NOTE: This completely discards the data. It won't be available when the
* future resolves. This is almost certainly only useful if you need the
* buffer memory for some reason.
*
* @return this
* @task interact
*/
public function discardBuffers() {
$this->discardStdoutBuffer();
$this->stderr = '';
$this->stderrPos = 0;
return $this;
}
public function discardStdoutBuffer() {
$this->stdout = '';
$this->stdoutPos = 0;
return $this;
}
/**
* Returns true if this future was killed by a timeout configured with
* @{method:setTimeout}.
*
* @return bool True if the future was killed for exceeding its time limit.
*/
public function getWasKilledByTimeout() {
return $this->killedByTimeout;
}
/* -( Configuring Execution )---------------------------------------------- */
/**
* Set a hard limit on execution time. If the command runs longer, it will
* be terminated and the future will resolve with an error code. You can test
* if a future was killed by a timeout with @{method:getWasKilledByTimeout}.
*
* The subprocess will be sent a `TERM` signal, and then a `KILL` signal a
* short while later if it fails to exit.
*
* @param int Maximum number of seconds this command may execute for before
* it is signaled.
* @return this
* @task config
*/
public function setTimeout($seconds) {
$this->terminateTimeout = $seconds;
$this->killTimeout = $seconds + min($seconds, 60);
return $this;
}
/* -( Resolving Execution )------------------------------------------------ */
/**
* Resolve a command you expect to exit with return code 0. Works like
* @{method:resolve}, but throws if $err is nonempty. Returns only
* $stdout and $stderr. See also @{function:execx}.
*
* list($stdout, $stderr) = $future->resolvex();
*
* @param float Optional timeout after which resolution will pause and
* execution will return to the caller.
* @return pair <$stdout, $stderr> pair.
* @task resolve
*/
public function resolvex() {
$result = $this->resolve();
return $this->raiseResultError($result);
}
/**
* Resolve a command you expect to return valid JSON. Works like
* @{method:resolvex}, but also throws if stderr is nonempty, or stdout is not
* valid JSON. Returns a PHP array, decoded from the JSON command output.
*
* @param float Optional timeout after which resolution will pause and
* execution will return to the caller.
* @return array PHP array, decoded from JSON command output.
* @task resolve
*/
public function resolveJSON() {
list($stdout, $stderr) = $this->resolvex();
if (strlen($stderr)) {
$cmd = $this->getCommand();
throw new CommandException(
pht(
"JSON command '%s' emitted text to stderr when none was expected: %d",
$cmd,
$stderr),
$cmd,
0,
$stdout,
$stderr);
}
try {
return phutil_json_decode($stdout);
} catch (PhutilJSONParserException $ex) {
$cmd = $this->getCommand();
throw new CommandException(
pht(
"JSON command '%s' did not produce a valid JSON object on stdout: %s",
$cmd,
$stdout),
$cmd,
0,
$stdout,
$stderr);
}
}
/**
* Resolve the process by abruptly terminating it.
*
* @return list List of <err, stdout, stderr> results.
* @task resolve
*/
public function resolveKill() {
if (!$this->hasResult()) {
$signal = 9;
if ($this->proc) {
proc_terminate($this->proc, $signal);
}
$this->closeProcess();
$result = array(
128 + $signal,
$this->stdout,
$this->stderr,
);
$this->recordResult($result);
}
return $this->getResult();
}
private function recordResult(array $result) {
$resolve_on_error = $this->getResolveOnError();
if (!$resolve_on_error) {
$result = $this->raiseResultError($result);
}
$this->setResult($result);
}
private function raiseResultError($result) {
list($err, $stdout, $stderr) = $result;
if ($err) {
$cmd = $this->getCommand();
if ($this->getWasKilledByTimeout()) {
// NOTE: The timeout can be a float and PhutilNumber only handles
// integers, so just use "%s" to render it.
$message = pht(
'Command killed by timeout after running for more than %s seconds.',
$this->terminateTimeout);
} else {
$message = pht('Command failed with error #%d!', $err);
}
throw new CommandException(
$message,
$cmd,
$err,
$stdout,
$stderr);
}
return array($stdout, $stderr);
}
/* -( Internals )---------------------------------------------------------- */
/**
* Provides read sockets to the future core.
*
* @return list List of read sockets.
* @task internal
*/
public function getReadSockets() {
list($stdin, $stdout, $stderr) = $this->pipes;
$sockets = array();
if (isset($stdout) && !feof($stdout)) {
$sockets[] = $stdout;
}
if (isset($stderr) && !feof($stderr)) {
$sockets[] = $stderr;
}
return $sockets;
}
/**
* Provides write sockets to the future core.
*
* @return list List of write sockets.
* @task internal
*/
public function getWriteSockets() {
list($stdin, $stdout, $stderr) = $this->pipes;
$sockets = array();
if (isset($stdin) && $this->stdin->getByteLength() && !feof($stdin)) {
$sockets[] = $stdin;
}
return $sockets;
}
/**
* Determine if the read buffer is empty.
*
* @return bool True if the read buffer is empty.
* @task internal
*/
public function isReadBufferEmpty() {
return !strlen($this->stdout);
}
/**
* Determine if the write buffer is empty.
*
* @return bool True if the write buffer is empty.
* @task internal
*/
public function isWriteBufferEmpty() {
return !$this->getWriteBufferSize();
}
/**
* Determine the number of bytes in the write buffer.
*
* @return int Number of bytes in the write buffer.
* @task internal
*/
public function getWriteBufferSize() {
if (!$this->stdin) {
return 0;
}
return $this->stdin->getByteLength();
}
/**
* Reads some bytes from a stream, discarding output once a certain amount
* has been accumulated.
*
* @param resource Stream to read from.
* @param int Maximum number of bytes to return from $stream. If
* additional bytes are available, they will be read and
* discarded.
* @param string Human-readable description of stream, for exception
* message.
* @param int Maximum number of bytes to read.
* @return string The data read from the stream.
* @task internal
*/
private function readAndDiscard($stream, $limit, $description, $length) {
$output = '';
if ($length <= 0) {
return '';
}
do {
$data = fread($stream, min($length, 64 * 1024));
if (false === $data) {
throw new Exception(pht('Failed to read from %s', $description));
}
$read_bytes = strlen($data);
if ($read_bytes > 0 && $limit > 0) {
if ($read_bytes > $limit) {
$data = substr($data, 0, $limit);
}
$output .= $data;
$limit -= strlen($data);
}
if (strlen($output) >= $length) {
break;
}
} while ($read_bytes > 0);
return $output;
}
/**
* Begin or continue command execution.
*
* @return bool True if future has resolved.
* @task internal
*/
public function isReady() {
// NOTE: We have a soft dependencies on PhutilErrorTrap here, to avoid
// the need to build it into the Phage agent. Under normal circumstances,
// this class are always available.
if (!$this->pipes) {
$is_windows = phutil_is_windows();
if (!$this->start) {
// We might already have started the timer via initiating resolution.
$this->start = microtime(true);
}
$unmasked_command = $this->getCommand();
$unmasked_command = $unmasked_command->getUnmaskedString();
$pipes = array();
if ($this->hasEnv()) {
$env = $this->getEnv();
} else {
$env = null;
}
$cwd = $this->getCWD();
// NOTE: See note above about Phage.
if (class_exists('PhutilErrorTrap')) {
$trap = new PhutilErrorTrap();
} else {
$trap = null;
}
$spec = self::$descriptorSpec;
if ($is_windows) {
$stdout_file = new TempFile();
$stderr_file = new TempFile();
$stdout_handle = fopen($stdout_file, 'wb');
if (!$stdout_handle) {
throw new Exception(
pht(
'Unable to open stdout temporary file ("%s") for writing.',
$stdout_file));
}
$stderr_handle = fopen($stderr_file, 'wb');
if (!$stderr_handle) {
throw new Exception(
pht(
'Unable to open stderr temporary file ("%s") for writing.',
$stderr_file));
}
$spec = array(
0 => self::$descriptorSpec[0],
1 => $stdout_handle,
2 => $stderr_handle,
);
}
$proc = @proc_open(
$unmasked_command,
$spec,
$pipes,
$cwd,
$env,
array(
'bypass_shell' => true,
));
if ($trap) {
$err = $trap->getErrorsAsString();
$trap->destroy();
} else {
$err = error_get_last();
if ($err) {
$err = $err['message'];
}
}
if ($is_windows) {
fclose($stdout_handle);
fclose($stderr_handle);
}
if (!is_resource($proc)) {
// When you run an invalid command on a Linux system, the "proc_open()"
// works and then the process (really a "/bin/sh -c ...") exits after
// it fails to resolve the command.
// When you run an invalid command on a Windows system, we bypass the
// shell and the "proc_open()" itself fails. See also T13504. Fail the
// future immediately, acting as though it exited with an error code
// for consistency with Linux.
$result = array(
1,
'',
pht(
'Call to "proc_open()" to open a subprocess failed: %s',
$err),
);
$this->recordResult($result);
return true;
}
if ($is_windows) {
$stdout_handle = fopen($stdout_file, 'rb');
if (!$stdout_handle) {
throw new Exception(
pht(
'Unable to open stdout temporary file ("%s") for reading.',
$stdout_file));
}
$stderr_handle = fopen($stderr_file, 'rb');
if (!$stderr_handle) {
throw new Exception(
pht(
'Unable to open stderr temporary file ("%s") for reading.',
$stderr_file));
}
$pipes = array(
0 => $pipes[0],
1 => $stdout_handle,
2 => $stderr_handle,
);
$this->windowsStdoutTempFile = $stdout_file;
$this->windowsStderrTempFile = $stderr_file;
}
$this->pipes = $pipes;
$this->proc = $proc;
list($stdin, $stdout, $stderr) = $pipes;
if (!$is_windows) {
// On Windows, we redirect process standard output and standard error
// through temporary files. Files don't block, so we don't need to make
// these streams nonblocking.
if ((!stream_set_blocking($stdout, false)) ||
(!stream_set_blocking($stderr, false)) ||
(!stream_set_blocking($stdin, false))) {
$this->__destruct();
throw new Exception(pht('Failed to set streams nonblocking.'));
}
}
$this->tryToCloseStdin();
return false;
}
if (!$this->proc) {
return true;
}
list($stdin, $stdout, $stderr) = $this->pipes;
while (isset($this->stdin) && $this->stdin->getByteLength()) {
$write_segment = $this->stdin->getAnyPrefix();
try {
$bytes = fwrite($stdin, $write_segment);
} catch (RuntimeException $ex) {
// If the subprocess has exited, we may get a broken pipe error here
// in recent versions of PHP. There does not seem to be any way to
// get the actual error code other than reading the exception string.
// For now, treat this as if writes are blocked.
break;
}
if ($bytes === false) {
throw new Exception(pht('Unable to write to stdin!'));
} else if ($bytes) {
$this->stdin->removeBytesFromHead($bytes);
} else {
// Writes are blocked for now.
break;
}
}
$this->tryToCloseStdin();
// Read status before reading pipes so that we can never miss data that
// arrives between our last read and the process exiting.
$status = $this->procGetStatus();
$read_buffer_size = $this->readBufferSize;
$max_stdout_read_bytes = PHP_INT_MAX;
$max_stderr_read_bytes = PHP_INT_MAX;
if ($read_buffer_size !== null) {
$max_stdout_read_bytes = $read_buffer_size - strlen($this->stdout);
$max_stderr_read_bytes = $read_buffer_size - strlen($this->stderr);
}
if ($max_stdout_read_bytes > 0) {
$this->stdout .= $this->readAndDiscard(
$stdout,
$this->getStdoutSizeLimit() - strlen($this->stdout),
'stdout',
$max_stdout_read_bytes);
}
if ($max_stderr_read_bytes > 0) {
$this->stderr .= $this->readAndDiscard(
$stderr,
$this->getStderrSizeLimit() - strlen($this->stderr),
'stderr',
$max_stderr_read_bytes);
}
$is_done = false;
if (!$status['running']) {
// We may still have unread bytes on stdout or stderr, particularly if
// this future is being buffered and streamed. If we do, we don't want to
// consider the subprocess to have exited until we've read everything.
// See T9724 for context.
if (feof($stdout) && feof($stderr)) {
$is_done = true;
}
}
if ($is_done) {
$signal_info = null;
// If the subprocess got nuked with `kill -9`, we get a -1 exitcode.
// Upgrade this to a slightly more informative value by examining the
// terminating signal code.
$err = $status['exitcode'];
if ($err == -1) {
if ($status['signaled']) {
$signo = $status['termsig'];
$err = 128 + $signo;
$signal_info = pht(
"<Process was terminated by signal %s (%d).>\n\n",
phutil_get_signal_name($signo),
$signo);
}
}
$result = array(
$err,
$this->stdout,
$signal_info.$this->stderr,
);
$this->recordResult($result);
$this->closeProcess();
return true;
}
$elapsed = (microtime(true) - $this->start);
if ($this->terminateTimeout && ($elapsed >= $this->terminateTimeout)) {
if (!$this->didTerminate) {
$this->killedByTimeout = true;
$this->sendTerminateSignal();
return false;
}
}
if ($this->killTimeout && ($elapsed >= $this->killTimeout)) {
$this->killedByTimeout = true;
$this->resolveKill();
return true;
}
}
/**
* @return void
* @task internal
*/
public function __destruct() {
if (!$this->proc) {
return;
}
// NOTE: If we try to proc_close() an open process, we hang indefinitely. To
// avoid this, kill the process explicitly if it's still running.
$status = $this->procGetStatus();
if ($status['running']) {
$this->sendTerminateSignal();
if (!$this->waitForExit(5)) {
$this->resolveKill();
}
} else {
$this->closeProcess();
}
}
/**
* Close and free resources if necessary.
*
* @return void
* @task internal
*/
private function closeProcess() {
foreach ($this->pipes as $pipe) {
if (isset($pipe)) {
@fclose($pipe);
}
}
$this->pipes = array(null, null, null);
if ($this->proc) {
@proc_close($this->proc);
$this->proc = null;
}
$this->stdin = null;
unset($this->windowsStdoutTempFile);
unset($this->windowsStderrTempFile);
}
/**
* Execute `proc_get_status()`, but avoid pitfalls.
*
* @return dict Process status.
* @task internal
*/
private function procGetStatus() {
// After the process exits, we only get one chance to read proc_get_status()
// before it starts returning garbage. Make sure we don't throw away the
// last good read.
if ($this->procStatus) {
if (!$this->procStatus['running']) {
return $this->procStatus;
}
}
// See T13555. This may occur if you call "getPID()" on a future which
// exited immediately without ever creating a valid subprocess.
if (!$this->proc) {
throw new Exception(
pht(
'Attempting to get subprocess status in "ExecFuture" with no '.
'valid subprocess.'));
}
$this->procStatus = proc_get_status($this->proc);
return $this->procStatus;
}
/**
* Try to close stdin, if we're done using it. This keeps us from hanging if
* the process on the other end of the pipe is waiting for EOF.
*
* @return void
* @task internal
*/
private function tryToCloseStdin() {
if (!$this->closePipe) {
// We've been told to keep the pipe open by a call to write(..., true).
return;
}
if ($this->stdin->getByteLength()) {
// We still have bytes to write.
return;
}
list($stdin) = $this->pipes;
if (!$stdin) {
// We've already closed stdin.
return;
}
// There's nothing stopping us from closing stdin, so close it.
@fclose($stdin);
$this->pipes[0] = null;
}
public function getDefaultWait() {
$wait = parent::getDefaultWait();
$next_timeout = $this->getNextTimeout();
if ($next_timeout) {
if (!$this->start) {
$this->start = microtime(true);
}
$elapsed = (microtime(true) - $this->start);
$wait = max(0, min($next_timeout - $elapsed, $wait));
}
return $wait;
}
private function getNextTimeout() {
if ($this->didTerminate) {
return $this->killTimeout;
} else {
return $this->terminateTimeout;
}
}
private function sendTerminateSignal() {
$this->didTerminate = true;
proc_terminate($this->proc);
return $this;
}
private function waitForExit($duration) {
$start = microtime(true);
while (true) {
$status = $this->procGetStatus();
if (!$status['running']) {
return true;
}
$waited = (microtime(true) - $start);
if ($waited > $duration) {
return false;
}
}
}
protected function getServiceProfilerStartParameters() {
return array(
'type' => 'exec',
'command' => phutil_string_cast($this->getCommand()),
);
}
protected function getServiceProfilerResultParameters() {
if ($this->hasResult()) {
$result = $this->getResult();
$err = idx($result, 0);
} else {
$err = null;
}
return array(
'err' => $err,
);
}
}