File Manager

Editing File: /home/ubuntu/code_review/arcanist/src/moduleutils/PhutilLibraryMapBuilder.php

<?php

/**
 * Build maps of libphutil libraries. libphutil uses the library map to locate
 * and load classes and functions in the library.
 *
 * @task map      Mapping libphutil Libraries
 * @task path     Path Management
 * @task symbol   Symbol Analysis and Caching
 * @task source   Source Management
 */
final class PhutilLibraryMapBuilder extends Phobject {

private $root;
  private $quiet = true;
  private $subprocessLimit = 8;

private $fileSymbolMap;
  private $librarySymbolMap;

const LIBRARY_MAP_VERSION_KEY   = '__library_version__';
  const LIBRARY_MAP_VERSION       = 2;

const SYMBOL_CACHE_VERSION_KEY  = '__symbol_cache_version__';
  const SYMBOL_CACHE_VERSION      = 11;

/* -(  Mapping libphutil Libraries  )---------------------------------------- */

/**
   * Create a new map builder for a library.
   *
   * @param string Path to the library root.
   *
   * @task map
   */
  public function __construct($root) {
    $this->root = $root;
  }

/**
   * Control status output. Use `--quiet` to set this.
   *
   * @param  bool  If true, don't show status output.
   * @return this
   *
   * @task map
   */
  public function setQuiet($quiet) {
    $this->quiet = $quiet;
    return $this;
  }

/**
   * Control subprocess parallelism limit. Use `--limit` to set this.
   *
   * @param  int   Maximum number of subprocesses to run in parallel.
   * @return this
   *
   * @task map
   */
  public function setSubprocessLimit($limit) {
    $this->subprocessLimit = $limit;
    return $this;
  }

/**
   * Get the map of symbols in this library, analyzing the library to build it
   * if necessary.
   *
   * @return map<string, wild> Information about symbols in this library.
   *
   * @task map
   */
  public function buildMap() {
    if ($this->librarySymbolMap === null) {
      $this->analyzeLibrary();
    }
    return $this->librarySymbolMap;
  }

/**
   * Get the map of files in this library, analyzing the library to build it
   * if necessary.
   *
   * Returns a map of file paths to information about symbols used and defined
   * in the file.
   *
   * @return map<string, wild> Information about files in this library.
   *
   * @task map
   */
  public function buildFileSymbolMap() {
    if ($this->fileSymbolMap === null) {
      $this->analyzeLibrary();
    }
    return $this->fileSymbolMap;
  }

/**
   * Build and update the library map.
   *
   * @return void
   *
   * @task map
   */
  public function buildAndWriteMap() {
    $library_map = $this->buildMap();

$this->log(pht('Writing map...'));
    $this->writeLibraryMap($library_map);
  }

/**
   * Write a status message to the user, if not running in quiet mode.
   *
   * @param  string  Message to write.
   * @return this
   *
   * @task map
   */
  private function log($message) {
    if (!$this->quiet) {
      @fwrite(STDERR, "%s\n", $message);
    }
    return $this;
  }

/* -(  Path Management  )---------------------------------------------------- */

/**
   * Get the path to some file in the library.
   *
   * @param  string  A library-relative path. If omitted, returns the library
   *                 root path.
   * @return string  An absolute path.
   *
   * @task path
   */
  private function getPath($path = '') {
    return $this->root.'/'.$path;
  }

/**
   * Get the path to the symbol cache file.
   *
   * @return string Absolute path to symbol cache.
   *
   * @task path
   */
  private function getPathForSymbolCache() {
    return $this->getPath('.phutil_module_cache');
  }

/**
   * Get the path to the map file.
   *
   * @return string Absolute path to the library map.
   *
   * @task path
   */
  private function getPathForLibraryMap() {
    return $this->getPath('__phutil_library_map__.php');
  }

/**
   * Get the path to the library init file.
   *
   * @return string Absolute path to the library init file
   *
   * @task path
   */
  private function getPathForLibraryInit() {
    return $this->getPath('__phutil_library_init__.php');
  }

/* -(  Symbol Analysis and Caching  )---------------------------------------- */

/**
   * Load the library symbol cache, if it exists and is readable and valid.
   *
   * @return dict  Map of content hashes to cache of output from
   *               `extract-symbols.php`.
   *
   * @task symbol
   */
  private function loadSymbolCache() {
    $cache_file = $this->getPathForSymbolCache();

try {
      $cache = Filesystem::readFile($cache_file);
    } catch (Exception $ex) {
      $cache = null;
    }

$symbol_cache = array();
    if ($cache) {
      try {
        $symbol_cache = phutil_json_decode($cache);
      } catch (PhutilJSONParserException $ex) {
        $symbol_cache = array();
      }
    }

$version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY);
    if ($version != self::SYMBOL_CACHE_VERSION) {
      // Throw away caches from a different version of the library.
      $symbol_cache = array();
    }
    unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]);

return $symbol_cache;
  }

/**
   * Write a symbol map to disk cache.
   *
   * @param  dict  Symbol map of relative paths to symbols.
   * @param  dict  Source map (like @{method:loadSourceFileMap}).
   * @return void
   *
   * @task symbol
   */
  private function writeSymbolCache(array $symbol_map, array $source_map) {
    $cache_file = $this->getPathForSymbolCache();

$cache = array(
      self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION,
    );

foreach ($symbol_map as $file => $symbols) {
      $cache[$source_map[$file]] = $symbols;
    }

$json = json_encode($cache);
    try {
      Filesystem::writeFile($cache_file, $json);
    } catch (FilesystemException $ex) {
      $this->log(pht('Unable to save the cache!'));
    }
  }

/**
   * Drop the symbol cache, forcing a clean rebuild.
   *
   * @return this
   *
   * @task symbol
   */
  public function dropSymbolCache() {
    $this->log(pht('Dropping symbol cache...'));
    Filesystem::remove($this->getPathForSymbolCache());
  }

/**
   * Build a future which returns a `extract-symbols.php` analysis of a source
   * file.
   *
   * @param  string  Relative path to the source file to analyze.
   * @return Future  Analysis future.
   *
   * @task symbol
   */
  private function buildSymbolAnalysisFuture($file) {
    $absolute_file = $this->getPath($file);
    return self::newExtractSymbolsFuture(
      array(),
      array($absolute_file));
  }

private static function newExtractSymbolsFuture(array $flags, array $paths) {
    $bin = dirname(__FILE__).'/../../support/lib/extract-symbols.php';

return new ExecFuture(
      'php -f %R -- --ugly %Ls -- %Ls',
      $bin,
      $flags,
      $paths);
  }

public static function newBuiltinMap() {
    $future = self::newExtractSymbolsFuture(
      array('--builtins'),
      array());

list($json) = $future->resolvex();

return phutil_json_decode($json);
  }

/* -(  Source Management  )-------------------------------------------------- */

/**
   * Build a map of all source files in a library to hashes of their content.
   * Returns an array like this:
   *
   *   array(
   *     'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3',
   *     // ...
   *   );
   *
   * @return dict  Map of library-relative paths to content hashes.
   * @task source
   */
  private function loadSourceFileMap() {
    $root = $this->getPath();

$init = $this->getPathForLibraryInit();
    if (!Filesystem::pathExists($init)) {
      throw new Exception(
        pht(
          "Provided path '%s' is not a %s library.",
          $root,
          'phutil'));
    }

$files = id(new FileFinder($root))
      ->withType('f')
      ->withSuffix('php')
      ->excludePath('*/.*')
      ->setGenerateChecksums(true)
      ->find();

$extensions_dir = 'extensions/';
    $extensions_len = strlen($extensions_dir);

$map = array();
    foreach ($files as $file => $hash) {
      $file = Filesystem::readablePath($file, $root);
      $file = ltrim($file, '/');

if (dirname($file) == '.') {
        // We don't permit normal source files at the root level, so just ignore
        // them; they're special library files.
        continue;
      }

// Ignore files in the extensions/ directory.
      if (!strncmp($file, $extensions_dir, $extensions_len)) {
        continue;
      }

// We include also filename in the hash to handle cases when the file is
      // moved without modifying its content.
      $map[$file] = md5($hash.$file);
    }

return $map;
  }

/**
   * Convert the symbol analysis of all the source files in the library into
   * a library map.
   *
   * @param   dict  Symbol analysis of all source files.
   * @return  dict  Library map.
   * @task source
   */
  private function buildLibraryMap(array $symbol_map) {
    $library_map = array(
      'class'     => array(),
      'function'  => array(),
      'xmap'      => array(),
    );

// Detect duplicate symbols within the library.
    foreach ($symbol_map as $file => $info) {
      foreach ($info['have'] as $type => $symbols) {
        foreach ($symbols as $symbol => $declaration) {
          $lib_type = ($type == 'interface') ? 'class' : $type;
          if (!empty($library_map[$lib_type][$symbol])) {
            $prior = $library_map[$lib_type][$symbol];
            throw new Exception(
              pht(
                "Definition of %s '%s' in file '%s' duplicates prior ".
                "definition in file '%s'. You can not declare the ".
                "same symbol twice.",
                $type,
                $symbol,
                $file,
                $prior));
          }
          $library_map[$lib_type][$symbol] = $file;
        }
      }
      $library_map['xmap'] += $info['xmap'];
    }

// Simplify the common case (one parent) to make the file a little easier
    // to deal with.
    foreach ($library_map['xmap'] as $class => $extends) {
      if (count($extends) == 1) {
        $library_map['xmap'][$class] = reset($extends);
      }
    }

// Sort the map so it is relatively stable across changes.
    foreach ($library_map as $key => $symbols) {
      ksort($symbols);
      $library_map[$key] = $symbols;
    }
    ksort($library_map);

return $library_map;
  }

/**
   * Write a finalized library map.
   *
   * @param  dict Library map structure to write.
   * @return void
   *
   * @task source
   */
  private function writeLibraryMap(array $library_map) {
    $map_file = $this->getPathForLibraryMap();
    $version = self::LIBRARY_MAP_VERSION;

$library_map = array(
      self::LIBRARY_MAP_VERSION_KEY => $version,
    ) + $library_map;

$library_map = phutil_var_export($library_map);
    $at = '@';

$source_file = <<<EOPHP
<?php

/**
 * This file is automatically generated. Use 'arc liberate' to rebuild it.
 *
 * {$at}generated
 * {$at}phutil-library-version {$version}
 */
phutil_register_library_map({$library_map});

EOPHP;

Filesystem::writeFile($map_file, $source_file);
  }

/**
   * Analyze the library, generating the file and symbol maps.
   *
   * @return void
   */
  private function analyzeLibrary() {
    // Identify all the ".php" source files in the library.
    $this->log(pht('Finding source files...'));
    $source_map = $this->loadSourceFileMap();
    $this->log(
      pht('Found %s files.', new PhutilNumber(count($source_map))));

// Load the symbol cache with existing parsed symbols. This allows us
    // to remap libraries quickly by analyzing only changed files.
    $this->log(pht('Loading symbol cache...'));
    $symbol_cache = $this->loadSymbolCache();

// If the XHPAST binary is not up-to-date, build it now. Otherwise,
    // `extract-symbols.php` will attempt to build the binary and will fail
    // miserably because it will be trying to build the same file multiple
    // times in parallel.
    if (!PhutilXHPASTBinary::isAvailable()) {
      PhutilXHPASTBinary::build();
    }

// Build out the symbol analysis for all the files in the library. For
    // each file, check if it's in cache. If we miss in the cache, do a fresh
    // analysis.
    $symbol_map = array();
    $futures = array();
    foreach ($source_map as $file => $hash) {
      if (!empty($symbol_cache[$hash])) {
        $symbol_map[$file] = $symbol_cache[$hash];
        continue;
      }
      $futures[$file] = $this->buildSymbolAnalysisFuture($file);
    }
    $this->log(
      pht('Found %s files in cache.', new PhutilNumber(count($symbol_map))));

// Run the analyzer on any files which need analysis.
    if ($futures) {
      $limit = $this->subprocessLimit;

$this->log(
        pht(
          'Analyzing %s file(s) with %s subprocess(es)...',
          phutil_count($futures),
          new PhutilNumber($limit)));

$progress = new PhutilConsoleProgressBar();
      if ($this->quiet) {
        $progress->setQuiet(true);
      }
      $progress->setTotal(count($futures));

$futures = id(new FutureIterator($futures))
        ->limit($limit);
      foreach ($futures as $file => $future) {
        $result = $future->resolveJSON();
        if (empty($result['error'])) {
          $symbol_map[$file] = $result;
        } else {
          $progress->done(false);
          throw new XHPASTSyntaxErrorException(
            $result['line'],
            $file.': '.$result['error']);
        }
        $progress->update(1);
      }
      $progress->done();
    }

$this->fileSymbolMap = $symbol_map;

// We're done building the cache, so write it out immediately. Note that
    // we've only retained entries for files we found, so this implicitly cleans
    // out old cache entries.
    $this->writeSymbolCache($symbol_map, $source_map);

// Our map is up to date, so either show it on stdout or write it to disk.
    $this->log(pht('Building library map...'));

$this->librarySymbolMap = $this->buildLibraryMap($symbol_map);
  }

}

Back to Directory File Manager