<?php
/**
* Manages lint execution. When you run 'arc lint' or 'arc diff', Arcanist
* attempts to run lint rules using a lint engine.
*
* Lint engines are high-level strategic classes which do not contain any
* actual linting rules. Linting rules live in `Linter` classes. The lint
* engine builds and configures linters.
*
* Most modern linters can be configured with an `.arclint` file, which is
* managed by the builtin @{class:ArcanistConfigurationDrivenLintEngine}.
* Consult the documentation for more information on these files.
*
* In the majority of cases, you do not need to write a custom lint engine.
* For example, to add new rules for a new language, write a linter instead.
* However, if you have a very advanced or specialized use case, you can write
* a custom lint engine by extending this class; custom lint engines are more
* powerful but much more complex than the builtin engines.
*
* The lint engine is given a list of paths (generally, the paths that you
* modified in your change) and determines which linters to run on them. The
* linters themselves are responsible for actually analyzing file text and
* finding warnings and errors. For example, if the modified paths include some
* JS files and some Python files, you might want to run JSLint on the JS files
* and PyLint on the Python files.
*
* You can also run multiple linters on a single file. For instance, you might
* run one linter on all text files to make sure they don't have trailing
* whitespace, or enforce tab vs space rules, or make sure there are enough
* curse words in them.
*
* You can test an engine like this:
*
* arc lint --engine YourLintEngineClassName --lintall some_file.py
*
* ...which will show you all the lint issues raised in the file.
*
* See @{article@phabricator:Arcanist User Guide: Customizing Lint, Unit Tests
* and Workflows} for more information about configuring lint engines.
*/
abstract class ArcanistLintEngine extends Phobject {
protected $workingCopy;
protected $paths = array();
protected $fileData = array();
protected $charToLine = array();
protected $lineToFirstChar = array();
private $cachedResults;
private $cacheVersion;
private $repositoryVersion;
private $results = array();
private $stopped = array();
private $minimumSeverity = ArcanistLintSeverity::SEVERITY_DISABLED;
private $changedLines = array();
private $configurationManager;
private $linterResources = array();
public function __construct() {}
final public function setConfigurationManager(
ArcanistConfigurationManager $configuration_manager) {
$this->configurationManager = $configuration_manager;
return $this;
}
final public function getConfigurationManager() {
return $this->configurationManager;
}
final public function setWorkingCopy(
ArcanistWorkingCopyIdentity $working_copy) {
$this->workingCopy = $working_copy;
return $this;
}
final public function getWorkingCopy() {
return $this->workingCopy;
}
final public function setPaths($paths) {
$this->paths = $paths;
return $this;
}
public function getPaths() {
return $this->paths;
}
final public function setPathChangedLines($path, $changed) {
if ($changed === null) {
$this->changedLines[$path] = null;
} else {
$this->changedLines[$path] = array_fill_keys($changed, true);
}
return $this;
}
final public function getPathChangedLines($path) {
return idx($this->changedLines, $path);
}
final public function setFileData($data) {
$this->fileData = $data + $this->fileData;
return $this;
}
final public function loadData($path) {
if (!isset($this->fileData[$path])) {
$disk_path = $this->getFilePathOnDisk($path);
$this->fileData[$path] = Filesystem::readFile($disk_path);
}
return $this->fileData[$path];
}
public function pathExists($path) {
$disk_path = $this->getFilePathOnDisk($path);
return Filesystem::pathExists($disk_path);
}
final public function isDirectory($path) {
$disk_path = $this->getFilePathOnDisk($path);
return is_dir($disk_path);
}
final public function isBinaryFile($path) {
try {
$data = $this->loadData($path);
} catch (Exception $ex) {
return false;
}
return ArcanistDiffUtils::isHeuristicBinaryFile($data);
}
final public function isSymbolicLink($path) {
return is_link($this->getFilePathOnDisk($path));
}
final public function getFilePathOnDisk($path) {
return Filesystem::resolvePath(
$path,
$this->getWorkingCopy()->getProjectRoot());
}
final public function setMinimumSeverity($severity) {
$this->minimumSeverity = $severity;
return $this;
}
final public function run() {
$linters = $this->buildLinters();
if (!$linters) {
throw new ArcanistNoEffectException(pht('No linters to run.'));
}
foreach ($linters as $key => $linter) {
$linter->setLinterID($key);
}
$linters = msort($linters, 'getLinterPriority');
foreach ($linters as $linter) {
$linter->setEngine($this);
}
$have_paths = false;
foreach ($linters as $linter) {
if ($linter->getPaths()) {
$have_paths = true;
break;
}
}
if (!$have_paths) {
throw new ArcanistNoEffectException(pht('No paths are lintable.'));
}
$versions = array($this->getCacheVersion());
foreach ($linters as $linter) {
$version = get_class($linter).':'.$linter->getCacheVersion();
$symbols = id(new PhutilSymbolLoader())
->setType('class')
->setName(get_class($linter))
->selectSymbolsWithoutLoading();
$symbol = idx($symbols, 'class$'.get_class($linter));
if ($symbol) {
$version .= ':'.md5_file(
phutil_get_library_root($symbol['library']).'/'.$symbol['where']);
}
$versions[] = $version;
}
$this->cacheVersion = crc32(implode("\n", $versions));
$runnable = $this->getRunnableLinters($linters);
$this->stopped = array();
$exceptions = $this->executeLinters($runnable);
foreach ($runnable as $linter) {
foreach ($linter->getLintMessages() as $message) {
$this->validateLintMessage($linter, $message);
if (!$this->isSeverityEnabled($message->getSeverity())) {
continue;
}
if (!$this->isRelevantMessage($message)) {
continue;
}
$message->setGranularity($linter->getCacheGranularity());
$result = $this->getResultForPath($message->getPath());
$result->addMessage($message);
}
}
if ($this->cachedResults) {
foreach ($this->cachedResults as $path => $messages) {
$messages = idx($messages, $this->cacheVersion, array());
$repository_version = idx($messages, 'repository_version');
unset($messages['stopped']);
unset($messages['repository_version']);
foreach ($messages as $message) {
$use_cache = $this->shouldUseCache(
idx($message, 'granularity'),
$repository_version);
if ($use_cache) {
$this->getResultForPath($path)->addMessage(
ArcanistLintMessage::newFromDictionary($message));
}
}
}
}
foreach ($this->results as $path => $result) {
$disk_path = $this->getFilePathOnDisk($path);
$result->setFilePathOnDisk($disk_path);
if (isset($this->fileData[$path])) {
$result->setData($this->fileData[$path]);
} else if ($disk_path && Filesystem::pathExists($disk_path)) {
// TODO: this may cause us to, e.g., load a large binary when we only
// raised an error about its filename. We could refine this by looking
// through the lint messages and doing this load only if any of them
// have original/replacement text or something like that.
try {
$this->fileData[$path] = Filesystem::readFile($disk_path);
$result->setData($this->fileData[$path]);
} catch (FilesystemException $ex) {
// Ignore this, it's noncritical that we access this data and it
// might be unreadable or a directory or whatever else for plenty
// of legitimate reasons.
}
}
}
if ($exceptions) {
throw new PhutilAggregateException(
pht('Some linters failed:'),
$exceptions);
}
return $this->results;
}
final public function isSeverityEnabled($severity) {
$minimum = $this->minimumSeverity;
return ArcanistLintSeverity::isAtLeastAsSevere($severity, $minimum);
}
private function shouldUseCache(
$cache_granularity,
$repository_version) {
switch ($cache_granularity) {
case ArcanistLinter::GRANULARITY_FILE:
return true;
case ArcanistLinter::GRANULARITY_DIRECTORY:
case ArcanistLinter::GRANULARITY_REPOSITORY:
return ($this->repositoryVersion == $repository_version);
default:
return false;
}
}
/**
* @param dict<string path, dict<string version, list<dict message>>>
* @return this
*/
final public function setCachedResults(array $results) {
$this->cachedResults = $results;
return $this;
}
final public function getResults() {
return $this->results;
}
final public function getStoppedPaths() {
return $this->stopped;
}
abstract public function buildLinters();
final public function setRepositoryVersion($version) {
$this->repositoryVersion = $version;
return $this;
}
private function isRelevantMessage(ArcanistLintMessage $message) {
// When a user runs "arc lint", we default to raising only warnings on
// lines they have changed (errors are still raised anywhere in the
// file). The list of $changed lines may be null, to indicate that the
// path is a directory or a binary file so we should not exclude
// warnings.
if (!$this->changedLines ||
$message->isError() ||
$message->shouldBypassChangedLineFiltering()) {
return true;
}
$locations = $message->getOtherLocations();
$locations[] = $message->toDictionary();
foreach ($locations as $location) {
$path = idx($location, 'path', $message->getPath());
if (!array_key_exists($path, $this->changedLines)) {
if (phutil_is_windows()) {
// We try checking the UNIX path form as well, on Windows. Linters
// store noramlized paths, which use the Windows-style "\" as a
// delimiter; as such, they don't match the UNIX-style paths stored
// in changedLines, which come from the VCS.
$path = str_replace('\\', '/', $path);
if (!array_key_exists($path, $this->changedLines)) {
continue;
}
} else {
continue;
}
}
$changed = $this->getPathChangedLines($path);
if ($changed === null || !$location['line']) {
return true;
}
$last_line = $location['line'];
if (isset($location['original'])) {
$last_line += substr_count($location['original'], "\n");
}
for ($l = $location['line']; $l <= $last_line; $l++) {
if (!empty($changed[$l])) {
return true;
}
}
}
return false;
}
final protected function getResultForPath($path) {
if (empty($this->results[$path])) {
$result = new ArcanistLintResult();
$result->setPath($path);
$result->setCacheVersion($this->cacheVersion);
$this->results[$path] = $result;
}
return $this->results[$path];
}
final public function getLineAndCharFromOffset($path, $offset) {
if (!isset($this->charToLine[$path])) {
$char_to_line = array();
$line_to_first_char = array();
$lines = explode("\n", $this->loadData($path));
$line_number = 0;
$line_start = 0;
foreach ($lines as $line) {
$len = strlen($line) + 1; // Account for "\n".
$line_to_first_char[] = $line_start;
$line_start += $len;
for ($ii = 0; $ii < $len; $ii++) {
$char_to_line[] = $line_number;
}
$line_number++;
}
$this->charToLine[$path] = $char_to_line;
$this->lineToFirstChar[$path] = $line_to_first_char;
}
$line = $this->charToLine[$path][$offset];
$char = $offset - $this->lineToFirstChar[$path][$line];
return array($line, $char);
}
protected function getCacheVersion() {
return 1;
}
/**
* Get a named linter resource shared by another linter.
*
* This mechanism allows linters to share arbitrary resources, like the
* results of computation. If several linters need to perform the same
* expensive computation step, they can use a named resource to synchronize
* construction of the result so it doesn't need to be built multiple
* times.
*
* @param string Resource identifier.
* @param wild Optionally, default value to return if resource does not
* exist.
* @return wild Resource, or default value if not present.
*/
public function getLinterResource($key, $default = null) {
return idx($this->linterResources, $key, $default);
}
/**
* Set a linter resource that other linters can access.
*
* See @{method:getLinterResource} for a description of this mechanism.
*
* @param string Resource identifier.
* @param wild Resource.
* @return this
*/
public function setLinterResource($key, $value) {
$this->linterResources[$key] = $value;
return $this;
}
private function getRunnableLinters(array $linters) {
assert_instances_of($linters, 'ArcanistLinter');
// TODO: The canRun() mechanism is only used by one linter, and just
// silently disables the linter. Almost every other linter handles this
// by throwing `ArcanistMissingLinterException`. Both mechanisms are not
// ideal; linters which can not run should emit a message, get marked as
// "skipped", and allow execution to continue. See T7045.
$runnable = array();
foreach ($linters as $key => $linter) {
if ($linter->canRun()) {
$runnable[$key] = $linter;
}
}
return $runnable;
}
private function executeLinters(array $runnable) {
assert_instances_of($runnable, 'ArcanistLinter');
$all_paths = $this->getPaths();
$path_chunks = array_chunk($all_paths, 32, $preserve_keys = true);
$exception_lists = array();
foreach ($path_chunks as $chunk) {
$exception_lists[] = $this->executeLintersOnChunk($runnable, $chunk);
}
return array_mergev($exception_lists);
}
private function executeLintersOnChunk(array $runnable, array $path_list) {
assert_instances_of($runnable, 'ArcanistLinter');
$path_map = array_fuse($path_list);
$exceptions = array();
$did_lint = array();
foreach ($runnable as $linter) {
$linter_id = $linter->getLinterID();
$paths = $linter->getPaths();
foreach ($paths as $key => $path) {
// If we aren't running this path in the current chunk of paths,
// skip it completely.
if (empty($path_map[$path])) {
unset($paths[$key]);
continue;
}
// Make sure each path has a result generated, even if it is empty
// (i.e., the file has no lint messages).
$result = $this->getResultForPath($path);
// If a linter has stopped all other linters for this path, don't
// actually run the linter.
if (isset($this->stopped[$path])) {
unset($paths[$key]);
continue;
}
// If we have a cached result for this path, don't actually run the
// linter.
if (isset($this->cachedResults[$path][$this->cacheVersion])) {
$cached_result = $this->cachedResults[$path][$this->cacheVersion];
$use_cache = $this->shouldUseCache(
$linter->getCacheGranularity(),
idx($cached_result, 'repository_version'));
if ($use_cache) {
unset($paths[$key]);
if (idx($cached_result, 'stopped') == $linter_id) {
$this->stopped[$path] = $linter_id;
}
}
}
}
$paths = array_values($paths);
if (!$paths) {
continue;
}
try {
$this->executeLinterOnPaths($linter, $paths);
$did_lint[] = array($linter, $paths);
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
foreach ($did_lint as $info) {
list($linter, $paths) = $info;
try {
$this->executeDidLintOnPaths($linter, $paths);
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
return $exceptions;
}
private function beginLintServiceCall(ArcanistLinter $linter, array $paths) {
$profiler = PhutilServiceProfiler::getInstance();
return $profiler->beginServiceCall(
array(
'type' => 'lint',
'linter' => $linter->getInfoName(),
'paths' => $paths,
));
}
private function endLintServiceCall($call_id) {
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($call_id, array());
}
private function executeLinterOnPaths(ArcanistLinter $linter, array $paths) {
$call_id = $this->beginLintServiceCall($linter, $paths);
try {
$linter->willLintPaths($paths);
foreach ($paths as $path) {
$linter->setActivePath($path);
$linter->lintPath($path);
if ($linter->didStopAllLinters()) {
$this->stopped[$path] = $linter->getLinterID();
}
}
} catch (Exception $ex) {
$this->endLintServiceCall($call_id);
throw $ex;
}
$this->endLintServiceCall($call_id);
}
private function executeDidLintOnPaths(ArcanistLinter $linter, array $paths) {
$call_id = $this->beginLintServiceCall($linter, $paths);
try {
$linter->didLintPaths($paths);
} catch (Exception $ex) {
$this->endLintServiceCall($call_id);
throw $ex;
}
$this->endLintServiceCall($call_id);
}
private function validateLintMessage(
ArcanistLinter $linter,
ArcanistLintMessage $message) {
$name = $message->getName();
if (!strlen($name)) {
throw new Exception(
pht(
'Linter "%s" generated a lint message that is invalid because it '.
'does not have a name. Lint messages must have a name.',
get_class($linter)));
}
}
}