Fallbacks to /tmp if the preferred cache folder isn't writable (#4143)

* Fallbacks to /tmp if the preferred cache folder isn't writable

* Adds a preferred-cache-folder option

* Adds an integration test

* Node 4 doesn't expose fs.constants

* Remove the /tmp fallback

* Lints

* Adds a test

* wip

* Lints

Author: arcanis

__tests__/integration.js

@@ -4,6 +4,8 @@
 import execa from 'execa';
 import makeTemp from './_temp.js';
 import * as fs from '../src/util/fs.js';
+import * as misc from '../src/util/misc.js';
+import * as constants from '../src/constants.js';
 
 jasmine.DEFAULT_TIMEOUT_INTERVAL = 90000;
 
@@ -70,3 +72,37 @@ test('--mutex network', async () => {
     execa(command, ['add', 'foo'].concat(args), options),
   ]);
 });
+
+test('cache folder fallback', async () => {
+  const cwd = await makeTemp();
+  const cacheFolder = path.join(cwd, '.cache');
+
+  const command = path.resolve(__dirname, '../bin/yarn');
+  const args = ['--preferred-cache-folder', cacheFolder];
+
+  const options = {cwd};
+
+  function runCacheDir(): Promise<Array<Buffer>> {
+    const {stderr, stdout} = execa(command, ['cache', 'dir'].concat(args), options);
+
+    const stdoutPromise = misc.consumeStream(stdout);
+    const stderrPromise = misc.consumeStream(stderr);
+
+    return Promise.all([stdoutPromise, stderrPromise]);
+  }
+
+  const [stdoutOutput, stderrOutput] = await runCacheDir();
+
+  expect(stdoutOutput.toString().trim()).toEqual(path.join(cacheFolder, `v${constants.CACHE_VERSION}`));
+  expect(stderrOutput.toString()).not.toMatch(/Skipping preferred cache folder/);
+
+  await fs.unlink(cacheFolder);
+  await fs.writeFile(cacheFolder, `not a directory`);
+
+  const [stdoutOutput2, stderrOutput2] = await runCacheDir();
+
+  expect(stdoutOutput2.toString().trim()).toEqual(
+    path.join(constants.PREFERRED_MODULE_CACHE_DIRECTORIES[0], `v${constants.CACHE_VERSION}`),
+  );
+  expect(stderrOutput2.toString()).toMatch(/Skipping preferred cache folder/);
+});

src/cli/commands/cache.js

@@ -55,7 +55,7 @@ export const {run, setFlags, examples} = buildSubCommands('cache', {
   },
 
   dir(config: Config, reporter: Reporter) {
-    reporter.log(config.cacheFolder);
+    reporter.log(config.cacheFolder, {force: true});
   },
 
   async clean(config: Config, reporter: Reporter, flags: Object, args: Array<string>): Promise<void> {

src/cli/index.js

@@ -60,7 +60,8 @@ export function main({
     '--modules-folder <path>',
     'rather than installing modules into the node_modules folder relative to the cwd, output them here',
   );
-  commander.option('--cache-folder <path>', 'specify a custom folder to store the yarn cache');
+  commander.option('--preferred-cache-folder <path>', 'specify a custom folder to store the yarn cache if possible');
+  commander.option('--cache-folder <path>', 'specify a custom folder that must be used to store the yarn cache');
   commander.option('--mutex <type>[:specifier]', 'use a mutex to ensure only one yarn instance is executing');
   commander.option('--emoji [bool]', 'enable emoji in output', process.platform === 'darwin');
   commander.option('-s, --silent', 'skip Yarn console logs, other types of logs (script output) will be printed');
@@ -325,6 +326,7 @@ export function main({
       binLinks: commander.binLinks,
       modulesFolder: commander.modulesFolder,
       globalFolder: commander.globalFolder,
+      preferredCacheFolder: commander.preferredCacheFolder,
       cacheFolder: commander.cacheFolder,
       preferOffline: commander.preferOffline,
       captureHar: commander.har,

src/config.js

@@ -276,9 +276,47 @@ export default class Config {
       networkConcurrency: this.networkConcurrency,
       networkTimeout: this.networkTimeout,
     });
-    this._cacheRootFolder = String(
-      opts.cacheFolder || this.getOption('cache-folder', true) || constants.MODULE_CACHE_DIRECTORY,
-    );
+
+    let cacheRootFolder = opts.cacheFolder || this.getOption('cache-folder', true);
+
+    if (!cacheRootFolder) {
+      let preferredCacheFolders = constants.PREFERRED_MODULE_CACHE_DIRECTORIES;
+      const preferredCacheFolder = opts.preferredCacheFolder || this.getOption('preferred-cache-folder', true);
+
+      if (preferredCacheFolder) {
+        preferredCacheFolders = [preferredCacheFolder].concat(preferredCacheFolders);
+      }
+
+      for (let t = 0; t < preferredCacheFolders.length && !cacheRootFolder; ++t) {
+        const tentativeCacheFolder = String(preferredCacheFolders[t]);
+
+        try {
+          await fs.mkdirp(tentativeCacheFolder);
+
+          const testFile = path.join(tentativeCacheFolder, 'testfile');
+
+          // fs.access is not enough, because the cache folder could actually be a file.
+          await fs.writeFile(testFile, 'content');
+          await fs.readFile(testFile);
+          await fs.unlink(testFile);
+
+          cacheRootFolder = tentativeCacheFolder;
+        } catch (error) {
+          this.reporter.warn(this.reporter.lang('cacheFolderSkipped', tentativeCacheFolder));
+        }
+
+        if (cacheRootFolder && t > 0) {
+          this.reporter.warn(this.reporter.lang('cacheFolderSelected', cacheRootFolder));
+        }
+      }
+    }
+
+    if (!cacheRootFolder) {
+      throw new MessageError(this.reporter.lang('cacheFolderMissing'));
+    } else {
+      this._cacheRootFolder = String(cacheRootFolder);
+    }
+
     this.workspacesEnabled = Boolean(this.getOption('workspaces-experimental'));
 
     this.pruneOfflineMirror = Boolean(this.getOption('yarn-offline-mirror-pruning'));

src/constants.js

@@ -1,5 +1,6 @@
 /* @flow */
 
+const os = require('os');
 const path = require('path');
 const userHome = require('./util/user-home-dir').default;
 
@@ -46,15 +47,21 @@ function getDirectory(category: string): string {
   return path.join(userHome, `.${category}`, 'yarn');
 }
 
-function getCacheDirectory(): string {
+function getPreferredCacheDirectories(): Array<string> {
+  const preferredCacheDirectories = [];
+
   if (process.platform === 'darwin') {
-    return path.join(userHome, 'Library', 'Caches', 'Yarn');
+    preferredCacheDirectories.push(path.join(userHome, 'Library', 'Caches', 'Yarn'));
+  } else {
+    preferredCacheDirectories.push(getDirectory('cache'));
   }
 
-  return getDirectory('cache');
+  preferredCacheDirectories.push(path.join(os.tmpdir(), '.yarn-cache'));
+
+  return preferredCacheDirectories;
 }
 
-export const MODULE_CACHE_DIRECTORY = getCacheDirectory();
+export const PREFERRED_MODULE_CACHE_DIRECTORIES = getPreferredCacheDirectories();
 export const CONFIG_DIRECTORY = getDirectory('config');
 export const LINK_REGISTRY_DIRECTORY = path.join(CONFIG_DIRECTORY, 'link');
 export const GLOBAL_MODULE_DIRECTORY = path.join(CONFIG_DIRECTORY, 'global');
@@ -70,6 +77,7 @@ export const LOCKFILE_FILENAME = 'yarn.lock';
 export const METADATA_FILENAME = '.yarn-metadata.json';
 export const TARBALL_FILENAME = '.yarn-tarball.tgz';
 export const CLEAN_FILENAME = '.yarnclean';
+export const ACCESS_FILENAME = '.yarn-access';
 
 export const DEFAULT_INDENT = '  ';
 export const SINGLE_INSTANCE_PORT = 31997;

src/reporters/base-reporter.js

@@ -195,7 +195,8 @@ export default class BaseReporter {
   success(message: string) {}
 
   // a simple log message
-  log(message: string) {}
+  // TODO: rethink the {force} parameter. In the meantime, please don't use it (cf comments in #4143).
+  log(message: string, {force = false}: {force?: boolean} = {}) {}
 
   // a shell command has been executed
   command(command: string) {}

src/reporters/console/console-reporter.js

@@ -160,13 +160,13 @@ export default class ConsoleReporter extends BaseReporter {
     this.log(this._prependEmoji(msg, '✨'));
   }
 
-  log(msg: string) {
+  log(msg: string, {force = false}: {force?: boolean} = {}) {
     this._lastCategorySize = 0;
-    this._log(msg);
+    this._log(msg, {force});
   }
 
-  _log(msg: string) {
-    if (this.isSilent) {
+  _log(msg: string, {force = false}: {force?: boolean} = {}) {
+    if (this.isSilent && !force) {
       return;
     }
     clearLine(this.stdout);

src/reporters/lang/en.js

@@ -181,6 +181,11 @@ const messages = {
   workspaceNameMandatory: 'Missing name in workspace at $0, ignoring.',
   workspaceNameDuplicate: 'There are more than one workspace with name $0',
 
+  cacheFolderSkipped: 'Skipping preferred cache folder $0 because it is not writable.',
+  cacheFolderMissing:
+    "Yarn hasn't been able to find a cache folder it can use. Please use the explicit --cache-folder option to tell it what location to use, or make one of the preferred locations writable.",
+  cacheFolderSelected: 'Selected the next writable cache folder in the list, will be $0.',
+
   execMissingCommand: 'Missing command name.',
 
   commandNotSpecified: 'No command specified.',

src/util/fs.js

@@ -13,6 +13,15 @@ const globModule = require('glob');
 const os = require('os');
 const path = require('path');
 
+export const constants =
+  typeof fs.constants !== 'undefined'
+    ? fs.constants
+    : {
+        R_OK: fs.R_OK,
+        W_OK: fs.W_OK,
+        X_OK: fs.X_OK,
+      };
+
 export const lockQueue = new BlockingQueue('fs lock');
 
 export const readFileBuffer = promisify(fs.readFile);

src/util/misc.js

@@ -2,6 +2,24 @@
 
 const _camelCase = require('camelcase');
 
+export function consumeStream(stream: Object): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const buffers = [];
+
+    stream.on(`data`, buffer => {
+      buffers.push(buffer);
+    });
+
+    stream.on(`end`, () => {
+      resolve(Buffer.concat(buffers));
+    });
+
+    stream.on(`error`, error => {
+      reject(error);
+    });
+  });
+}
+
 export function sortAlpha(a: string, b: string): number {
   // sort alphabetically in a deterministic way
   const shortLen = Math.min(a.length, b.length);