QMK CLI Environment bootstrapper (#25038)

Co-authored-by: Joel Challis <git@zvecr.com>
Co-authored-by: Pascal Getreuer <getreuer@google.com>

7 files changed, 154 insertions, 31 deletions

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 26905ec134..dc2e4726a5 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -3,6 +3,8 @@
 We list each subcommand here explicitly because all the reliable ways of searching for modules are slow and delay startup.
 """
 import os
+import platform
+import platformdirs
 import shlex
 import sys
 from importlib.util import find_spec
@@ -12,6 +14,28 @@ from subprocess import run
 from milc import cli, __VERSION__
 from milc.questions import yesno
 
+
+def _get_default_distrib_path():
+    if 'windows' in platform.platform().lower():
+        try:
+            result = cli.run(['cygpath', '-w', '/opt/qmk'])
+            if result.returncode == 0:
+                return result.stdout.strip()
+        except Exception:
+            pass
+
+    return platformdirs.user_data_dir('qmk')
+
+
+# Ensure the QMK distribution is on the `$PATH` if present. This must be kept in sync with qmk/qmk_cli.
+QMK_DISTRIB_DIR = Path(os.environ.get('QMK_DISTRIB_DIR', _get_default_distrib_path()))
+if QMK_DISTRIB_DIR.exists():
+    os.environ['PATH'] = str(QMK_DISTRIB_DIR / 'bin') + os.pathsep + os.environ['PATH']
+
+# Prepend any user-defined path prefix
+if 'QMK_PATH_PREFIX' in os.environ:
+    os.environ['PATH'] = os.environ['QMK_PATH_PREFIX'] + os.pathsep + os.environ['PATH']
+
 import_names = {
     # A mapping of package name to importable name
     'pep8-naming': 'pep8ext_naming',
diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py
index 51b0f0c80a..8a13cb0832 100644
--- a/lib/python/qmk/cli/doctor/check.py
+++ b/lib/python/qmk/cli/doctor/check.py
@@ -1,7 +1,6 @@
 """Check for specific programs.
 """
 from enum import Enum
-import re
 import shutil
 from subprocess import DEVNULL, TimeoutExpired
 from tempfile import TemporaryDirectory
@@ -9,6 +8,7 @@ from pathlib import Path
 
 from milc import cli
 from qmk import submodules
+from qmk.commands import find_make
 
 
 class CheckStatus(Enum):
@@ -17,7 +17,13 @@ class CheckStatus(Enum):
     ERROR = 3
 
 
+WHICH_MAKE = Path(find_make()).name
+
 ESSENTIAL_BINARIES = {
+    WHICH_MAKE: {},
+    'git': {},
+    'dos2unix': {},
+    'diff': {},
     'dfu-programmer': {},
     'avrdude': {},
     'dfu-util': {},
@@ -30,14 +36,39 @@ ESSENTIAL_BINARIES = {
 }
 
 
-def _parse_gcc_version(version):
-    m = re.match(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?", version)
+def _check_make_version():
+    last_line = ESSENTIAL_BINARIES[WHICH_MAKE]['output'].split('\n')[0]
+    version_number = last_line.split()[2]
+    cli.log.info('Found %s version %s', WHICH_MAKE, version_number)
 
-    return {
-        'major': int(m.group(1)),
-        'minor': int(m.group(2)) if m.group(2) else 0,
-        'patch': int(m.group(3)) if m.group(3) else 0,
-    }
+    return CheckStatus.OK
+
+
+def _check_git_version():
+    last_line = ESSENTIAL_BINARIES['git']['output'].split('\n')[0]
+    version_number = last_line.split()[2]
+    cli.log.info('Found git version %s', version_number)
+
+    return CheckStatus.OK
+
+
+def _check_dos2unix_version():
+    last_line = ESSENTIAL_BINARIES['dos2unix']['output'].split('\n')[0]
+    version_number = last_line.split()[1]
+    cli.log.info('Found dos2unix version %s', version_number)
+
+    return CheckStatus.OK
+
+
+def _check_diff_version():
+    last_line = ESSENTIAL_BINARIES['diff']['output'].split('\n')[0]
+    if 'Apple diff' in last_line:
+        version_number = last_line
+    else:
+        version_number = last_line.split()[3]
+    cli.log.info('Found diff version %s', version_number)
+
+    return CheckStatus.OK
 
 
 def _check_arm_gcc_version():
@@ -148,16 +179,24 @@ def check_binaries():
     """Iterates through ESSENTIAL_BINARIES and tests them.
     """
     ok = CheckStatus.OK
+    missing_from_path = []
 
     for binary in sorted(ESSENTIAL_BINARIES):
         try:
-            if not is_executable(binary):
+            if not is_in_path(binary):
+                ok = CheckStatus.ERROR
+                missing_from_path.append(binary)
+            elif not is_executable(binary):
                 ok = CheckStatus.ERROR
         except TimeoutExpired:
             cli.log.debug('Timeout checking %s', binary)
             if ok != CheckStatus.ERROR:
                 ok = CheckStatus.WARNING
 
+    if missing_from_path:
+        location_noun = 'its location' if len(missing_from_path) == 1 else 'their locations'
+        cli.log.error('{fg_red}' + ', '.join(missing_from_path) + f' may need to be installed, or {location_noun} added to your path.')
+
     return ok
 
 
@@ -165,6 +204,10 @@ def check_binary_versions():
     """Check the versions of ESSENTIAL_BINARIES
     """
     checks = {
+        WHICH_MAKE: _check_make_version,
+        'git': _check_git_version,
+        'dos2unix': _check_dos2unix_version,
+        'diff': _check_diff_version,
         'arm-none-eabi-gcc': _check_arm_gcc_version,
         'avr-gcc': _check_avr_gcc_version,
         'avrdude': _check_avrdude_version,
@@ -196,15 +239,18 @@ def check_submodules():
     return CheckStatus.OK
 
 
-def is_executable(command):
-    """Returns True if command exists and can be executed.
+def is_in_path(command):
+    """Returns True if command is found in the path.
     """
-    # Make sure the command is in the path.
-    res = shutil.which(command)
-    if res is None:
+    if shutil.which(command) is None:
         cli.log.error("{fg_red}Can't find %s in your path.", command)
         return False
+    return True
+
 
+def is_executable(command):
+    """Returns True if command can be executed.
+    """
     # Make sure the command can be executed
     version_arg = ESSENTIAL_BINARIES[command].get('version_arg', '--version')
     check = cli.run([command, version_arg], combined_output=True, stdin=DEVNULL, timeout=5)
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 391353ebbf..45667e8ce2 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -3,7 +3,6 @@
 Check out the user's QMK environment and make sure it's ready to compile.
 """
 import platform
-from subprocess import DEVNULL
 
 from milc import cli
 from milc.questions import yesno
@@ -16,6 +15,60 @@ from qmk.commands import in_virtualenv
 from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError
 
 
+def distrib_tests():
+    def _load_kvp_file(file):
+        """Load a simple key=value file into a dictionary
+        """
+        vars = {}
+        with open(file, 'r') as f:
+            for line in f:
+                if '=' in line:
+                    key, value = line.split('=', 1)
+                    vars[key.strip()] = value.strip()
+        return vars
+
+    def _parse_toolchain_release_file(file):
+        """Parse the QMK toolchain release info file
+        """
+        try:
+            vars = _load_kvp_file(file)
+            return f'{vars.get("TOOLCHAIN_HOST", "unknown")}:{vars.get("TOOLCHAIN_TARGET", "unknown")}:{vars.get("COMMIT_HASH", "unknown")}'
+        except Exception as e:
+            cli.log.warning('Error reading QMK toolchain release info file: %s', e)
+            return f'Unknown toolchain release info file: {file}'
+
+    def _parse_flashutils_release_file(file):
+        """Parse the QMK flashutils release info file
+        """
+        try:
+            vars = _load_kvp_file(file)
+            return f'{vars.get("FLASHUTILS_HOST", "unknown")}:{vars.get("COMMIT_HASH", "unknown")}'
+        except Exception as e:
+            cli.log.warning('Error reading QMK flashutils release info file: %s', e)
+            return f'Unknown flashutils release info file: {file}'
+
+    try:
+        from qmk.cli import QMK_DISTRIB_DIR
+        if (QMK_DISTRIB_DIR / 'etc').exists():
+            cli.log.info('Found QMK tools distribution directory: {fg_cyan}%s', QMK_DISTRIB_DIR)
+
+            toolchains = [_parse_toolchain_release_file(file) for file in (QMK_DISTRIB_DIR / 'etc').glob('toolchain_release_*')]
+            if len(toolchains) > 0:
+                cli.log.info('Found QMK toolchains: {fg_cyan}%s', ', '.join(toolchains))
+            else:
+                cli.log.warning('No QMK toolchains manifest found.')
+
+            flashutils = [_parse_flashutils_release_file(file) for file in (QMK_DISTRIB_DIR / 'etc').glob('flashutils_release_*')]
+            if len(flashutils) > 0:
+                cli.log.info('Found QMK flashutils: {fg_cyan}%s', ', '.join(flashutils))
+            else:
+                cli.log.warning('No QMK flashutils manifest found.')
+    except ImportError:
+        cli.log.info('QMK tools distribution not found.')
+
+    return CheckStatus.OK
+
+
 def os_tests():
     """Determine our OS and run platform specific tests
     """
@@ -124,10 +177,12 @@ def doctor(cli):
         * [ ] Compile a trivial program with each compiler
     """
     cli.log.info('QMK Doctor is checking your environment.')
+    cli.log.info('Python version: %s', platform.python_version())
     cli.log.info('CLI version: %s', cli.version)
     cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE)
 
     status = os_status = os_tests()
+    distrib_tests()
 
     userspace_tests(None)
 
@@ -141,12 +196,6 @@ def doctor(cli):
 
     # Make sure the basic CLI tools we need are available and can be executed.
     bin_ok = check_binaries()
-
-    if bin_ok == CheckStatus.ERROR:
-        if yesno('Would you like to install dependencies?', default=True):
-            cli.run(['util/qmk_install.sh', '-y'], stdin=DEVNULL, capture_output=False)
-            bin_ok = check_binaries()
-
     if bin_ok == CheckStatus.OK:
         cli.log.info('All dependencies are installed.')
     elif bin_ok == CheckStatus.WARNING:
@@ -163,7 +212,6 @@ def doctor(cli):
 
     # Check out the QMK submodules
     sub_ok = check_submodules()
-
     if sub_ok == CheckStatus.OK:
         cli.log.info('Submodules are up to date.')
     else:
@@ -186,6 +234,7 @@ def doctor(cli):
         cli.log.info('{fg_yellow}QMK is ready to go, but minor problems were found')
         return 1
     else:
-        cli.log.info('{fg_red}Major problems detected, please fix these problems before proceeding.')
-        cli.log.info('{fg_blue}Check out the FAQ (https://docs.qmk.fm/#/faq_build) or join the QMK Discord (https://discord.gg/qmk) for help.')
+        cli.log.info('{fg_red}Major problems detected, please fix these problems before proceeding.{fg_reset}')
+        cli.log.info('{fg_blue}If you\'re missing dependencies, try following the instructions on: https://docs.qmk.fm/newbs_getting_started{fg_reset}')
+        cli.log.info('{fg_blue}Additionally, check out the FAQ (https://docs.qmk.fm/#/faq_build) or join the QMK Discord (https://discord.gg/qmk) for help.{fg_reset}')
         return 2
diff --git a/lib/python/qmk/flashers.py b/lib/python/qmk/flashers.py
index b70b5fb035..6b52f4d35a 100644
--- a/lib/python/qmk/flashers.py
+++ b/lib/python/qmk/flashers.py
@@ -155,10 +155,10 @@ def _flash_atmel_dfu(mcu, file):
 def _flash_hid_bootloader(mcu, details, file):
     cmd = None
     if details == 'halfkay':
-        if shutil.which('teensy-loader-cli'):
-            cmd = 'teensy-loader-cli'
-        elif shutil.which('teensy_loader_cli'):
+        if shutil.which('teensy_loader_cli'):
             cmd = 'teensy_loader_cli'
+        elif shutil.which('teensy-loader-cli'):
+            cmd = 'teensy-loader-cli'
 
     # Use 'hid_bootloader_cli' for QMK HID and as a fallback for HalfKay
     if not cmd:
diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index f63228b2bc..e8aad760de 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -14,7 +14,7 @@ from qmk.json_schema import deep_update, json_load, validate
 from qmk.keyboard import config_h, rules_mk
 from qmk.commands import parse_configurator_json
 from qmk.makefile import parse_rules_mk_file
-from qmk.math import compute
+from qmk.math_ops import compute
 from qmk.util import maybe_exit, truthy
 
 true_values = ['1', 'on', 'yes']
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 254dc62309..e8534492c9 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -175,8 +175,9 @@ def keyboard_completer(prefix, action, parser, parsed_args):
     return list_keyboards()
 
 
+@lru_cache(maxsize=None)
 def list_keyboards():
-    """Returns a list of all keyboards
+    """Returns a list of all keyboards.
     """
     # We avoid pathlib here because this is performance critical code.
     kb_wildcard = os.path.join(base_path, "**", 'keyboard.json')
@@ -184,6 +185,9 @@ def list_keyboards():
 
     found = map(_find_name, paths)
 
+    # Convert to posix paths for consistency
+    found = map(lambda x: str(Path(x).as_posix()), found)
+
     return sorted(set(found))
 
 
diff --git a/lib/python/qmk/math.py b/lib/python/qmk/math_ops.py
index 88dc4a300c..1f14b18f4e 100644
--- a/lib/python/qmk/math.py
+++ b/lib/python/qmk/math_ops.py
@@ -23,8 +23,8 @@ def compute(expr):
 
 
 def _eval(node):
-    if isinstance(node, ast.Num):  # <number>
-        return node.n
+    if isinstance(node, ast.Constant):  # <number>
+        return node.value
     elif isinstance(node, ast.BinOp):  # <left> <operator> <right>
         return operators[type(node.op)](_eval(node.left), _eval(node.right))
     elif isinstance(node, ast.UnaryOp):  # <operator> <operand> e.g., -1