Add pyenv locator (#14587)

* Add pyenv locator

* Minor tweak

* More minor corrections

* More clean up

* Address comments add more tests

* Fix linting.

* Fix typo

* More review comments

* Remove export on getPythonVersionSpecificity

Author: karthiknadig

src/client/pythonEnvironments/base/info/env.ts

@@ -179,7 +179,7 @@ export function areSameEnv(
  * weighted by most important to least important fields.
  * Wn > Wn-1 + Wn-2 + ... W0
  */
-function getPythonVersionInfoHeuristic(version: PythonVersion): number {
+function getPythonVersionSpecificity(version: PythonVersion): number {
     let infoLevel = 0;
     if (version.major > 0) {
         infoLevel += 20; // W4
@@ -204,6 +204,15 @@ function getPythonVersionInfoHeuristic(version: PythonVersion): number {
     return infoLevel;
 }
 
+/**
+ * Compares two python versions, based on the amount of data each object has. If versionA has
+ * less information then the returned value is negative. If it is same then 0. If versionA has
+ * more information then positive.
+ */
+export function comparePythonVersionSpecificity(versionA: PythonVersion, versionB: PythonVersion): number {
+    return Math.sign(getPythonVersionSpecificity(versionA) - getPythonVersionSpecificity(versionB));
+}
+
 /**
  * Returns a heuristic value on how much information is available in the given executable object.
  * @param {FileInfo} executable executable object to generate heuristic from.
@@ -267,7 +276,7 @@ export function mergeEnvironments(target: PythonEnvInfo, other: PythonEnvInfo):
     const merged = cloneDeep(target);
 
     const version = cloneDeep(
-        getPythonVersionInfoHeuristic(target.version) > getPythonVersionInfoHeuristic(other.version)
+        getPythonVersionSpecificity(target.version) > getPythonVersionSpecificity(other.version)
             ? target.version
             : other.version,
     );

src/client/pythonEnvironments/base/info/pythonVersion.ts

@@ -18,6 +18,14 @@ export function getPythonVersionFromPath(exe:string): PythonVersion {
 
 /**
  * Convert the given string into the corresponding Python version object.
+ * Example:
+ *   3.9.0
+ *   3.9.0a1
+ *   3.9.0b2
+ *   3.9.0rc1
+ *
+ * Does not parse:
+ *   3.9.0.final.0
  */
 export function parseVersion(versionStr: string): PythonVersion {
     const parsed = parseBasicVersionInfo<PythonVersion>(versionStr);
@@ -49,6 +57,47 @@ export function parseVersion(versionStr: string): PythonVersion {
     return version;
 }
 
+/**
+ * Convert the given string into the corresponding Python version object.
+ * Example:
+ *   3.9.0.final.0
+ *   3.9.0.alpha.1
+ *   3.9.0.beta.2
+ *   3.9.0.candidate.1
+ *
+ * Does not parse:
+ *   3.9.0
+ *   3.9.0a1
+ *   3.9.0b2
+ *   3.9.0rc1
+ */
+export function parseVersionInfo(versionInfoStr: string): PythonVersion {
+    const parts = versionInfoStr.split('.');
+    const version = UNKNOWN_PYTHON_VERSION;
+    if (parts.length >= 2) {
+        version.major = parseInt(parts[0], 10);
+        version.minor = parseInt(parts[1], 10);
+    }
+
+    if (parts.length >= 3) {
+        version.micro = parseInt(parts[2], 10);
+    }
+
+    if (parts.length >= 4 && version.release) {
+        const levels = ['alpha', 'beta', 'candidate', 'final'];
+        const level = parts[3].toLowerCase();
+        if (levels.includes(level)) {
+            version.release.level = level as PythonReleaseLevel;
+        }
+    }
+
+    if (parts.length >= 5 && version.release) {
+        version.release.serial = parseInt(parts[4], 10);
+    }
+
+    return version;
+}
+
 /**
  * Checks if all the fields in the version object match.
  * @param {PythonVersion} left

src/client/pythonEnvironments/base/locators/composite/environmentsReducer.ts

@@ -144,10 +144,10 @@ function sortEnvInfoByPriority(...envs: PythonEnvInfo[]): PythonEnvInfo[] {
  * configure the environment, and the fall back for identification.
  * Top level we have the following environment types, since they leave a unique signature
  * in the environment or * use a unique path for the environments they create.
- *  1. Conda
- *  2. Windows Store
- *  3. PipEnv
- *  4. Pyenv
+ *  1. Pyenv (pyenv can also be a conda env or venv, but should be activated as a venv)
+ *  2. Conda
+ *  3. Windows Store
+ *  4. PipEnv
  *  5. Poetry
  *
  * Next level we have the following virtual environment tools. The are here because they
@@ -160,11 +160,11 @@ function sortEnvInfoByPriority(...envs: PythonEnvInfo[]): PythonEnvInfo[] {
  */
 function getPrioritizedEnvironmentKind(): PythonEnvKind[] {
     return [
+        PythonEnvKind.Pyenv,
         PythonEnvKind.CondaBase,
         PythonEnvKind.Conda,
         PythonEnvKind.WindowsStore,
         PythonEnvKind.Pipenv,
-        PythonEnvKind.Pyenv,
         PythonEnvKind.Poetry,
         PythonEnvKind.Venv,
         PythonEnvKind.VirtualEnvWrapper,

src/client/pythonEnvironments/common/commonUtils.ts

@@ -5,27 +5,46 @@ import * as fsapi from 'fs-extra';
 import * as path from 'path';
 import { chain, iterable } from '../../common/utils/async';
 import { getOSType, OSType } from '../../common/utils/platform';
+import { PythonVersion, UNKNOWN_PYTHON_VERSION } from '../base/info';
+import { comparePythonVersionSpecificity } from '../base/info/env';
+import { parseVersion } from '../base/info/pythonVersion';
+import { getPythonVersionFromConda } from '../discovery/locators/services/condaLocator';
+import { getPythonVersionFromVenv } from '../discovery/locators/services/virtualEnvironmentIdentifier';
 import { isPosixPythonBin } from './posixUtils';
 import { isWindowsPythonExe } from './windowsUtils';
 
-export async function* findInterpretersInDir(root:string, recurseLevels?:number): AsyncIterableIterator<string> {
-    const dirContents = (await fsapi.readdir(root)).map((c) => path.join(root, c));
+/**
+ * Searches recursively under the given `root` directory for python interpreters.
+ * @param root : Directory where the search begins.
+ * @param recurseLevels : Number of levels to search for from the root directory.
+ * @param filter : Callback that identifies directories to ignore.
+ */
+export async function* findInterpretersInDir(
+    root: string,
+    recurseLevels?: number,
+    filter?: (x: string) => boolean,
+): AsyncIterableIterator<string> {
     const os = getOSType();
     const checkBin = os === OSType.Windows ? isWindowsPythonExe : isPosixPythonBin;
+    const itemFilter = filter ?? (() => true);
+
+    const dirContents = (await fsapi.readdir(root)).filter(itemFilter);
+
     const generators = dirContents.map((item) => {
         async function* generator() {
-            const stat = await fsapi.lstat(item);
+            const fullPath = path.join(root, item);
+            const stat = await fsapi.lstat(fullPath);
 
             if (stat.isDirectory()) {
                 if (recurseLevels && recurseLevels > 0) {
-                    const subItems = findInterpretersInDir(item, recurseLevels - 1);
+                    const subItems = findInterpretersInDir(fullPath, recurseLevels - 1);
 
                     for await (const subItem of subItems) {
                         yield subItem;
                     }
                 }
-            } else if (checkBin(item)) {
-                yield item;
+            } else if (checkBin(fullPath)) {
+                yield fullPath;
             }
         }
 
@@ -34,3 +53,94 @@ export async function* findInterpretersInDir(root:string, recurseLevels?:number)
 
     yield* iterable(chain(generators));
 }
+
+/**
+ * Looks for files in the same directory which might have version in their name.
+ * @param interpreterPath
+ */
+export async function getPythonVersionFromNearByFiles(interpreterPath:string): Promise<PythonVersion> {
+    const root = path.dirname(interpreterPath);
+    let version = UNKNOWN_PYTHON_VERSION;
+    for await (const interpreter of findInterpretersInDir(root)) {
+        try {
+            const curVersion = parseVersion(path.basename(interpreter));
+            if (comparePythonVersionSpecificity(curVersion, version) > 0) {
+                version = curVersion;
+            }
+        } catch (ex) {
+            // Ignore any parse errors
+        }
+    }
+    return version;
+}
+
+/**
+ * This function does the best effort of finding version of python without running the
+ * python binary.
+ * @param interpreterPath Absolute path to the interpreter.
+ * @param hint Any string that might contain version info.
+ */
+export async function getPythonVersionFromPath(
+    interpreterPath: string | undefined,
+    hint: string | undefined,
+): Promise<PythonVersion> {
+    let versionA;
+    try {
+        versionA = hint ? parseVersion(hint) : UNKNOWN_PYTHON_VERSION;
+    } catch (ex) {
+        versionA = UNKNOWN_PYTHON_VERSION;
+    }
+    const versionB = interpreterPath ? await getPythonVersionFromNearByFiles(interpreterPath) : UNKNOWN_PYTHON_VERSION;
+    const versionC = interpreterPath ? await getPythonVersionFromVenv(interpreterPath) : UNKNOWN_PYTHON_VERSION;
+    const versionD = interpreterPath ? await getPythonVersionFromConda(interpreterPath) : UNKNOWN_PYTHON_VERSION;
+
+    let version = UNKNOWN_PYTHON_VERSION;
+    for (const v of [versionA, versionB, versionC, versionD]) {
+        version = comparePythonVersionSpecificity(version, v) > 0 ? version : v;
+    }
+    return version;
+}
+
+/**
+ * This function looks specifically for 'python' or 'python.exe' binary in the sub folders of a given
+ * environment directory.
+ * @param envDir Absolute path to the environment directory
+ */
+export async function getInterpreterPathFromDir(envDir: string): Promise<string|undefined> {
+    // Ignore any folders or files that not directly python binary related.
+    function filter(str:string):boolean {
+        const lower = str.toLowerCase();
+        return ['bin', 'scripts'].includes(lower) || lower.search('python') >= 0;
+    }
+
+    // Search in the sub-directories for python binary
+    for await (const bin of findInterpretersInDir(envDir, 2, filter)) {
+        const base = path.basename(bin).toLowerCase();
+        if (base === 'python.exe' || base === 'python') {
+            return bin;
+        }
+    }
+    return undefined;
+}
+
+/**
+ * Gets the root environment directory based on the absolute path to the python
+ *  interpreter binary.
+ * @param interpreterPath Absolute path to the python interpreter
+ */
+export function getEnvironmentDirFromPath(interpreterPath:string): string {
+    const skipDirs = ['bin', 'scripts'];
+
+    // env <--- Return this directory if it is not 'bin' or 'scripts'
+    // |__ python  <--- interpreterPath
+    const dir = path.basename(path.dirname(interpreterPath));
+    if (!skipDirs.includes(dir.toLowerCase())) {
+        return path.dirname(interpreterPath);
+    }
+
+    // This is the best next guess.
+    // env <--- Return this directory if it is not 'bin' or 'scripts'
+    // |__ bin or Scripts
+    //     |__ python  <--- interpreterPath
+    return path.dirname(path.dirname(interpreterPath));
+}

src/client/pythonEnvironments/common/environmentIdentifier.ts

@@ -16,10 +16,10 @@ import { EnvironmentType } from '../info';
  * configure the environment, and the fall back for identification.
  * Top level we have the following environment types, since they leave a unique signature
  * in the environment or * use a unique path for the environments they create.
- *  1. Conda
- *  2. Windows Store
- *  3. PipEnv
- *  4. Pyenv
+ *  1. Pyenv (pyenv can also be a conda env or venv, but should be activated as a venv)
+ *  2. Conda
+ *  3. Windows Store
+ *  4. PipEnv
  *  5. Poetry
  *
  * Next level we have the following virtual environment tools. The are here because they
@@ -32,10 +32,10 @@ import { EnvironmentType } from '../info';
  */
 export function getPrioritizedEnvironmentType(): EnvironmentType[] {
     return [
+        EnvironmentType.Pyenv,
         EnvironmentType.Conda,
         EnvironmentType.WindowsStore,
         EnvironmentType.Pipenv,
-        EnvironmentType.Pyenv,
         EnvironmentType.Poetry,
         EnvironmentType.VirtualEnvWrapper,
         EnvironmentType.Venv,

src/client/pythonEnvironments/common/externalDependencies.ts

@@ -5,6 +5,7 @@ import * as fsapi from 'fs-extra';
 import * as path from 'path';
 import { ExecutionResult, IProcessServiceFactory } from '../../common/process/types';
 import { IPersistentStateFactory } from '../../common/types';
+import { chain, iterable } from '../../common/utils/async';
 import { getOSType, OSType } from '../../common/utils/platform';
 import { IServiceContainer } from '../../ioc/types';
 
@@ -73,3 +74,20 @@ export async function resolveSymbolicLink(filepath:string): Promise<string> {
     }
     return filepath;
 }
+
+export async function* getSubDirs(root:string): AsyncIterableIterator<string> {
+    const dirContents = await fsapi.readdir(root);
+    const generators = dirContents.map((item) => {
+        async function* generator() {
+            const stat = await fsapi.lstat(path.join(root, item));
+
+            if (stat.isDirectory()) {
+                yield item;
+            }
+        }
+
+        return generator();
+    });
+
+    yield* iterable(chain(generators));
+}