diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 26905ec134..dc2e4726a5 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -3,6 +3,8 @@ We list each subcommand here explicitly because all the reliable ways of searching for modules are slow and delay startup. """ import os +import platform +import platformdirs import shlex import sys from importlib.util import find_spec @@ -12,6 +14,28 @@ from subprocess import run from milc import cli, __VERSION__ from milc.questions import yesno + +def _get_default_distrib_path(): + if 'windows' in platform.platform().lower(): + try: + result = cli.run(['cygpath', '-w', '/opt/qmk']) + if result.returncode == 0: + return result.stdout.strip() + except Exception: + pass + + return platformdirs.user_data_dir('qmk') + + +# Ensure the QMK distribution is on the `$PATH` if present. This must be kept in sync with qmk/qmk_cli. +QMK_DISTRIB_DIR = Path(os.environ.get('QMK_DISTRIB_DIR', _get_default_distrib_path())) +if QMK_DISTRIB_DIR.exists(): + os.environ['PATH'] = str(QMK_DISTRIB_DIR / 'bin') + os.pathsep + os.environ['PATH'] + +# Prepend any user-defined path prefix +if 'QMK_PATH_PREFIX' in os.environ: + os.environ['PATH'] = os.environ['QMK_PATH_PREFIX'] + os.pathsep + os.environ['PATH'] + import_names = { # A mapping of package name to importable name 'pep8-naming': 'pep8ext_naming', diff --git a/lib/python/qmk/cli/ci/validate_aliases.py b/lib/python/qmk/cli/ci/validate_aliases.py index 7f781d4397..4f2fe6c941 100644 --- a/lib/python/qmk/cli/ci/validate_aliases.py +++ b/lib/python/qmk/cli/ci/validate_aliases.py @@ -2,7 +2,7 @@ """ from milc import cli -from qmk.keyboard import resolve_keyboard, keyboard_folder, keyboard_alias_definitions +from qmk.keyboard import keyboard_folder, keyboard_alias_definitions def _safe_keyboard_folder(target): @@ -17,10 +17,6 @@ def _target_keyboard_exists(target): if not target: return False - # If the target directory existed but there was no rules.mk or rules.mk was incorrectly parsed, then we can't build it. - if not resolve_keyboard(target): - return False - # If the target directory exists but it itself has an invalid alias or invalid rules.mk, then we can't build it either. if not _safe_keyboard_folder(target): return False @@ -29,6 +25,21 @@ def _target_keyboard_exists(target): return True +def _alias_not_self(alias): + """Check if alias points to itself, either directly or within a circular reference + """ + aliases = keyboard_alias_definitions() + + found = set() + while alias in aliases: + found.add(alias) + alias = aliases[alias].get('target', alias) + if alias in found: + return False + + return True + + @cli.subcommand('Validates the list of keyboard aliases.', hidden=True) def ci_validate_aliases(cli): aliases = keyboard_alias_definitions() @@ -36,7 +47,11 @@ def ci_validate_aliases(cli): success = True for alias in aliases.keys(): target = aliases[alias].get('target', None) - if not _target_keyboard_exists(target): + if not _alias_not_self(alias): + cli.log.error(f'Keyboard alias {alias} should not point to itself') + success = False + + elif not _target_keyboard_exists(target): cli.log.error(f'Keyboard alias {alias} has a target that doesn\'t exist: {target}') success = False diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py index 51b0f0c80a..8a13cb0832 100644 --- a/lib/python/qmk/cli/doctor/check.py +++ b/lib/python/qmk/cli/doctor/check.py @@ -1,7 +1,6 @@ """Check for specific programs. """ from enum import Enum -import re import shutil from subprocess import DEVNULL, TimeoutExpired from tempfile import TemporaryDirectory @@ -9,6 +8,7 @@ from pathlib import Path from milc import cli from qmk import submodules +from qmk.commands import find_make class CheckStatus(Enum): @@ -17,7 +17,13 @@ class CheckStatus(Enum): ERROR = 3 +WHICH_MAKE = Path(find_make()).name + ESSENTIAL_BINARIES = { + WHICH_MAKE: {}, + 'git': {}, + 'dos2unix': {}, + 'diff': {}, 'dfu-programmer': {}, 'avrdude': {}, 'dfu-util': {}, @@ -30,14 +36,39 @@ ESSENTIAL_BINARIES = { } -def _parse_gcc_version(version): - m = re.match(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?", version) +def _check_make_version(): + last_line = ESSENTIAL_BINARIES[WHICH_MAKE]['output'].split('\n')[0] + version_number = last_line.split()[2] + cli.log.info('Found %s version %s', WHICH_MAKE, version_number) - return { - 'major': int(m.group(1)), - 'minor': int(m.group(2)) if m.group(2) else 0, - 'patch': int(m.group(3)) if m.group(3) else 0, - } + return CheckStatus.OK + + +def _check_git_version(): + last_line = ESSENTIAL_BINARIES['git']['output'].split('\n')[0] + version_number = last_line.split()[2] + cli.log.info('Found git version %s', version_number) + + return CheckStatus.OK + + +def _check_dos2unix_version(): + last_line = ESSENTIAL_BINARIES['dos2unix']['output'].split('\n')[0] + version_number = last_line.split()[1] + cli.log.info('Found dos2unix version %s', version_number) + + return CheckStatus.OK + + +def _check_diff_version(): + last_line = ESSENTIAL_BINARIES['diff']['output'].split('\n')[0] + if 'Apple diff' in last_line: + version_number = last_line + else: + version_number = last_line.split()[3] + cli.log.info('Found diff version %s', version_number) + + return CheckStatus.OK def _check_arm_gcc_version(): @@ -148,16 +179,24 @@ def check_binaries(): """Iterates through ESSENTIAL_BINARIES and tests them. """ ok = CheckStatus.OK + missing_from_path = [] for binary in sorted(ESSENTIAL_BINARIES): try: - if not is_executable(binary): + if not is_in_path(binary): + ok = CheckStatus.ERROR + missing_from_path.append(binary) + elif not is_executable(binary): ok = CheckStatus.ERROR except TimeoutExpired: cli.log.debug('Timeout checking %s', binary) if ok != CheckStatus.ERROR: ok = CheckStatus.WARNING + if missing_from_path: + location_noun = 'its location' if len(missing_from_path) == 1 else 'their locations' + cli.log.error('{fg_red}' + ', '.join(missing_from_path) + f' may need to be installed, or {location_noun} added to your path.') + return ok @@ -165,6 +204,10 @@ def check_binary_versions(): """Check the versions of ESSENTIAL_BINARIES """ checks = { + WHICH_MAKE: _check_make_version, + 'git': _check_git_version, + 'dos2unix': _check_dos2unix_version, + 'diff': _check_diff_version, 'arm-none-eabi-gcc': _check_arm_gcc_version, 'avr-gcc': _check_avr_gcc_version, 'avrdude': _check_avrdude_version, @@ -196,15 +239,18 @@ def check_submodules(): return CheckStatus.OK -def is_executable(command): - """Returns True if command exists and can be executed. +def is_in_path(command): + """Returns True if command is found in the path. """ - # Make sure the command is in the path. - res = shutil.which(command) - if res is None: + if shutil.which(command) is None: cli.log.error("{fg_red}Can't find %s in your path.", command) return False + return True + +def is_executable(command): + """Returns True if command can be executed. + """ # Make sure the command can be executed version_arg = ESSENTIAL_BINARIES[command].get('version_arg', '--version') check = cli.run([command, version_arg], combined_output=True, stdin=DEVNULL, timeout=5) diff --git a/lib/python/qmk/cli/doctor/linux.py b/lib/python/qmk/cli/doctor/linux.py index f0850d4e64..c99cc6baea 100644 --- a/lib/python/qmk/cli/doctor/linux.py +++ b/lib/python/qmk/cli/doctor/linux.py @@ -87,7 +87,7 @@ def check_udev_rules(): line = line.strip() if not line.startswith("#") and len(line): current_rules.add(line) - except PermissionError: + except (PermissionError, FileNotFoundError): cli.log.debug("Failed to read: %s", rule_file) # Check if the desired rules are among the currently present rules diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py index 391353ebbf..45667e8ce2 100755 --- a/lib/python/qmk/cli/doctor/main.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -3,7 +3,6 @@ Check out the user's QMK environment and make sure it's ready to compile. """ import platform -from subprocess import DEVNULL from milc import cli from milc.questions import yesno @@ -16,6 +15,60 @@ from qmk.commands import in_virtualenv from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError +def distrib_tests(): + def _load_kvp_file(file): + """Load a simple key=value file into a dictionary + """ + vars = {} + with open(file, 'r') as f: + for line in f: + if '=' in line: + key, value = line.split('=', 1) + vars[key.strip()] = value.strip() + return vars + + def _parse_toolchain_release_file(file): + """Parse the QMK toolchain release info file + """ + try: + vars = _load_kvp_file(file) + return f'{vars.get("TOOLCHAIN_HOST", "unknown")}:{vars.get("TOOLCHAIN_TARGET", "unknown")}:{vars.get("COMMIT_HASH", "unknown")}' + except Exception as e: + cli.log.warning('Error reading QMK toolchain release info file: %s', e) + return f'Unknown toolchain release info file: {file}' + + def _parse_flashutils_release_file(file): + """Parse the QMK flashutils release info file + """ + try: + vars = _load_kvp_file(file) + return f'{vars.get("FLASHUTILS_HOST", "unknown")}:{vars.get("COMMIT_HASH", "unknown")}' + except Exception as e: + cli.log.warning('Error reading QMK flashutils release info file: %s', e) + return f'Unknown flashutils release info file: {file}' + + try: + from qmk.cli import QMK_DISTRIB_DIR + if (QMK_DISTRIB_DIR / 'etc').exists(): + cli.log.info('Found QMK tools distribution directory: {fg_cyan}%s', QMK_DISTRIB_DIR) + + toolchains = [_parse_toolchain_release_file(file) for file in (QMK_DISTRIB_DIR / 'etc').glob('toolchain_release_*')] + if len(toolchains) > 0: + cli.log.info('Found QMK toolchains: {fg_cyan}%s', ', '.join(toolchains)) + else: + cli.log.warning('No QMK toolchains manifest found.') + + flashutils = [_parse_flashutils_release_file(file) for file in (QMK_DISTRIB_DIR / 'etc').glob('flashutils_release_*')] + if len(flashutils) > 0: + cli.log.info('Found QMK flashutils: {fg_cyan}%s', ', '.join(flashutils)) + else: + cli.log.warning('No QMK flashutils manifest found.') + except ImportError: + cli.log.info('QMK tools distribution not found.') + + return CheckStatus.OK + + def os_tests(): """Determine our OS and run platform specific tests """ @@ -124,10 +177,12 @@ def doctor(cli): * [ ] Compile a trivial program with each compiler """ cli.log.info('QMK Doctor is checking your environment.') + cli.log.info('Python version: %s', platform.python_version()) cli.log.info('CLI version: %s', cli.version) cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE) status = os_status = os_tests() + distrib_tests() userspace_tests(None) @@ -141,12 +196,6 @@ def doctor(cli): # Make sure the basic CLI tools we need are available and can be executed. bin_ok = check_binaries() - - if bin_ok == CheckStatus.ERROR: - if yesno('Would you like to install dependencies?', default=True): - cli.run(['util/qmk_install.sh', '-y'], stdin=DEVNULL, capture_output=False) - bin_ok = check_binaries() - if bin_ok == CheckStatus.OK: cli.log.info('All dependencies are installed.') elif bin_ok == CheckStatus.WARNING: @@ -163,7 +212,6 @@ def doctor(cli): # Check out the QMK submodules sub_ok = check_submodules() - if sub_ok == CheckStatus.OK: cli.log.info('Submodules are up to date.') else: @@ -186,6 +234,7 @@ def doctor(cli): cli.log.info('{fg_yellow}QMK is ready to go, but minor problems were found') return 1 else: - cli.log.info('{fg_red}Major problems detected, please fix these problems before proceeding.') - cli.log.info('{fg_blue}Check out the FAQ (https://docs.qmk.fm/#/faq_build) or join the QMK Discord (https://discord.gg/qmk) for help.') + cli.log.info('{fg_red}Major problems detected, please fix these problems before proceeding.{fg_reset}') + cli.log.info('{fg_blue}If you\'re missing dependencies, try following the instructions on: https://docs.qmk.fm/newbs_getting_started{fg_reset}') + cli.log.info('{fg_blue}Additionally, check out the FAQ (https://docs.qmk.fm/#/faq_build) or join the QMK Discord (https://discord.gg/qmk) for help.{fg_reset}') return 2 diff --git a/lib/python/qmk/cli/generate/autocorrect_data.py b/lib/python/qmk/cli/generate/autocorrect_data.py index 01a29b46fe..4f322adce2 100644 --- a/lib/python/qmk/cli/generate/autocorrect_data.py +++ b/lib/python/qmk/cli/generate/autocorrect_data.py @@ -250,8 +250,8 @@ def to_hex(b: int) -> str: @cli.argument('filename', type=normpath, help='The autocorrection database file') -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') -@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a output file is supplied.') +@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a output file is supplied.') @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") @cli.subcommand('Generate the autocorrection data file from a dictionary file.') @@ -263,7 +263,7 @@ def generate_autocorrect_data(cli): current_keyboard = cli.args.keyboard or cli.config.user.keyboard or cli.config.generate_autocorrect_data.keyboard current_keymap = cli.args.keymap or cli.config.user.keymap or cli.config.generate_autocorrect_data.keymap - if current_keyboard and current_keymap: + if not cli.args.output and current_keyboard and current_keymap: cli.args.output = locate_keymap(current_keyboard, current_keymap).parent / 'autocorrect_data.h' assert all(0 <= b <= 255 for b in data) diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py index d613f7b92c..1ade452f95 100755 --- a/lib/python/qmk/cli/generate/config_h.py +++ b/lib/python/qmk/cli/generate/config_h.py @@ -72,19 +72,6 @@ def generate_matrix_size(kb_info_json, config_h_lines): config_h_lines.append(generate_define('MATRIX_ROWS', kb_info_json['matrix_size']['rows'])) -def generate_matrix_masked(kb_info_json, config_h_lines): - """"Enable matrix mask if required""" - mask_required = False - - if 'matrix_grid' in kb_info_json.get('dip_switch', {}): - mask_required = True - if 'matrix_grid' in kb_info_json.get('split', {}).get('handedness', {}): - mask_required = True - - if mask_required: - config_h_lines.append(generate_define('MATRIX_MASKED')) - - def generate_config_items(kb_info_json, config_h_lines): """Iterate through the info_config map to generate basic config values. """ @@ -138,11 +125,12 @@ def generate_encoder_config(encoder_json, config_h_lines, postfix=''): config_h_lines.append(generate_define(f'ENCODER_A_PINS{postfix}', f'{{ {", ".join(a_pads)} }}')) config_h_lines.append(generate_define(f'ENCODER_B_PINS{postfix}', f'{{ {", ".join(b_pads)} }}')) - if None in resolutions: - cli.log.debug(f"Unable to generate ENCODER_RESOLUTION{postfix} configuration") - elif len(resolutions) == 0: + if len(resolutions) == 0 or all(r is None for r in resolutions): cli.log.debug(f"Skipping ENCODER_RESOLUTION{postfix} configuration") - elif len(set(resolutions)) == 1: + return + + resolutions = [4 if r is None else r for r in resolutions] + if len(set(resolutions)) == 1: config_h_lines.append(generate_define(f'ENCODER_RESOLUTION{postfix}', resolutions[0])) else: config_h_lines.append(generate_define(f'ENCODER_RESOLUTIONS{postfix}', f'{{ {", ".join(map(str,resolutions))} }}')) @@ -202,8 +190,6 @@ def generate_config_h(cli): generate_matrix_size(kb_info_json, config_h_lines) - generate_matrix_masked(kb_info_json, config_h_lines) - if 'matrix_pins' in kb_info_json: config_h_lines.append(matrix_pins(kb_info_json['matrix_pins'])) diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py index d686935fa8..d694202aec 100644 --- a/lib/python/qmk/cli/generate/keycodes.py +++ b/lib/python/qmk/cli/generate/keycodes.py @@ -92,15 +92,15 @@ def _generate_helpers(lines, keycodes): for group, codes in temp.items(): lo = keycodes["keycodes"][f'0x{codes[0]:04X}']['key'] hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key'] - lines.append(f'#define IS_{ _translate_group(group).upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})') + lines.append(f'#define IS_{_translate_group(group).upper()}_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})') lines.append('') lines.append('// Switch statement Helpers') for group, codes in temp.items(): lo = keycodes["keycodes"][f'0x{codes[0]:04X}']['key'] hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key'] - name = f'{ _translate_group(group).upper() }_KEYCODE_RANGE' - lines.append(f'#define { name.ljust(35) } {lo} ... {hi}') + name = f'{_translate_group(group).upper()}_KEYCODE_RANGE' + lines.append(f'#define {name.ljust(35)} {lo} ... {hi}') def _generate_aliases(lines, keycodes): diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py index 358a22fd1d..16084bded1 100755 --- a/lib/python/qmk/cli/generate/rules_mk.py +++ b/lib/python/qmk/cli/generate/rules_mk.py @@ -96,11 +96,10 @@ def generate_rules_mk(cli): rules_mk_lines.append(generate_rule('SPLIT_TRANSPORT', 'custom')) # Set CUSTOM_MATRIX, if needed - if kb_info_json.get('matrix_pins', {}).get('custom'): - if kb_info_json.get('matrix_pins', {}).get('custom_lite'): - rules_mk_lines.append(generate_rule('CUSTOM_MATRIX', 'lite')) - else: - rules_mk_lines.append(generate_rule('CUSTOM_MATRIX', 'yes')) + if kb_info_json.get('matrix_pins', {}).get('custom_lite'): + rules_mk_lines.append(generate_rule('CUSTOM_MATRIX', 'lite')) + elif kb_info_json.get('matrix_pins', {}).get('custom'): + rules_mk_lines.append(generate_rule('CUSTOM_MATRIX', 'yes')) if converter: rules_mk_lines.append(generate_rule('CONVERT_TO', converter)) diff --git a/lib/python/qmk/cli/generate/version_h.py b/lib/python/qmk/cli/generate/version_h.py index fd87df3617..8156e85559 100644 --- a/lib/python/qmk/cli/generate/version_h.py +++ b/lib/python/qmk/cli/generate/version_h.py @@ -8,6 +8,7 @@ from qmk.path import normpath from qmk.commands import dump_lines from qmk.git import git_get_qmk_hash, git_get_version, git_is_dirty from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE +from qmk.util import triplet_to_bcd TIME_FMT = '%Y-%m-%d-%H:%M:%S' @@ -32,12 +33,14 @@ def generate_version_h(cli): git_dirty = False git_version = "NA" git_qmk_hash = "NA" + git_bcd_version = "0x00000000" chibios_version = "NA" chibios_contrib_version = "NA" else: git_dirty = git_is_dirty() git_version = git_get_version() or current_time git_qmk_hash = git_get_qmk_hash() or "Unknown" + git_bcd_version = triplet_to_bcd(git_version) chibios_version = git_get_version("chibios", "os") or current_time chibios_contrib_version = git_get_version("chibios-contrib", "os") or current_time @@ -48,6 +51,7 @@ def generate_version_h(cli): f""" #define QMK_VERSION "{git_version}" #define QMK_BUILDDATE "{current_time}" +#define QMK_VERSION_BCD {git_bcd_version} #define QMK_GIT_HASH "{git_qmk_hash}{'*' if git_dirty else ''}" #define CHIBIOS_VERSION "{chibios_version}" #define CHIBIOS_CONTRIB_VERSION "{chibios_contrib_version}" diff --git a/lib/python/qmk/cli/info.py b/lib/python/qmk/cli/info.py index 5925b57258..26f7f0269d 100755 --- a/lib/python/qmk/cli/info.py +++ b/lib/python/qmk/cli/info.py @@ -102,6 +102,48 @@ def show_matrix(kb_info_json, title_caps=True): print(render_layout(kb_info_json['layouts'][layout_name]['layout'], cli.config.info.ascii, labels)) +def show_leds(kb_info_json, title_caps=True): + """Render LED indices per key, using the keyboard's key layout geometry. + + We build a map from (row, col) -> LED index using rgb_matrix/led_matrix layout, + then label each key with its LED index. Keys without an associated LED are left blank. + """ + # Prefer rgb_matrix, fall back to led_matrix + led_feature = None + for feature in ['rgb_matrix', 'led_matrix']: + if 'layout' in kb_info_json.get(feature, {}): + led_feature = feature + break + + if not led_feature: + cli.echo('{fg_yellow}No rgb_matrix/led_matrix layout found to derive LED indices.{fg_reset}') + return + + # Build mapping from matrix position -> LED indices for faster lookup later + by_matrix = {} + for idx, led in enumerate(kb_info_json[led_feature]['layout']): + if 'matrix' in led: + led_key = tuple(led.get('matrix')) + by_matrix[led_key] = idx + + # For each keyboard layout (e.g., LAYOUT), render keys labeled with LED index (or blank) + for layout_name, layout in kb_info_json['layouts'].items(): + labels = [] + for key in layout['layout']: + led_key = tuple(key.get('matrix')) + label = str(by_matrix[led_key]) if led_key in by_matrix else '' + + labels.append(label) + + # Header + if title_caps: + cli.echo('{fg_blue}LED indices for "%s"{fg_reset}:', layout_name) + else: + cli.echo('{fg_blue}leds_%s{fg_reset}:', layout_name) + + print(render_layout(kb_info_json['layouts'][layout_name]['layout'], cli.config.info.ascii, labels)) + + def print_friendly_output(kb_info_json): """Print the info.json in a friendly text format. """ @@ -169,6 +211,7 @@ def print_parsed_rules_mk(keyboard_name): @cli.argument('-km', '--keymap', help='Keymap to show info for (Optional).') @cli.argument('-l', '--layouts', action='store_true', help='Render the layouts.') @cli.argument('-m', '--matrix', action='store_true', help='Render the layouts with matrix information.') +@cli.argument('-L', '--leds', action='store_true', help='Render the LED layout with LED indices (rgb_matrix/led_matrix).') @cli.argument('-f', '--format', default='friendly', arg_only=True, help='Format to display the data in (friendly, text, json) (Default: friendly).') @cli.argument('--ascii', action='store_true', default=not UNICODE_SUPPORT, help='Render layout box drawings in ASCII only.') @cli.argument('-r', '--rules-mk', action='store_true', help='Render the parsed values of the keyboard\'s rules.mk file.') @@ -227,5 +270,8 @@ def info(cli): if cli.config.info.matrix: show_matrix(kb_info_json, title_caps) + if cli.config.info.leds: + show_leds(kb_info_json, title_caps) + if cli.config.info.keymap: show_keymap(kb_info_json, title_caps) diff --git a/lib/python/qmk/cli/lint.py b/lib/python/qmk/cli/lint.py index 484ddb5bd9..8a128ce6d2 100644 --- a/lib/python/qmk/cli/lint.py +++ b/lib/python/qmk/cli/lint.py @@ -304,6 +304,10 @@ def keyboard_check(kb): # noqa C901 cli.log.error(f'{kb}: The file "{file}" should not exist!') ok = False + if not _get_readme_files(kb): + cli.log.error(f'{kb}: Is missing a readme.md file!') + ok = False + for file in _get_readme_files(kb): if _is_invalid_readme(file): cli.log.error(f'{kb}: The file "{file}" still contains template tokens!') diff --git a/lib/python/qmk/cli/list/keyboards.py b/lib/python/qmk/cli/list/keyboards.py index 405b9210e4..8b6c451673 100644 --- a/lib/python/qmk/cli/list/keyboards.py +++ b/lib/python/qmk/cli/list/keyboards.py @@ -5,10 +5,9 @@ from milc import cli import qmk.keyboard -@cli.argument('--no-resolve-defaults', arg_only=True, action='store_false', help='Ignore any "DEFAULT_FOLDER" within keyboards rules.mk') @cli.subcommand("List the keyboards currently defined within QMK") def list_keyboards(cli): """List the keyboards currently defined within QMK """ - for keyboard_name in qmk.keyboard.list_keyboards(cli.args.no_resolve_defaults): + for keyboard_name in qmk.keyboard.list_keyboards(): print(keyboard_name) diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py index 4c4669d451..e71280f482 100755 --- a/lib/python/qmk/cli/mass_compile.py +++ b/lib/python/qmk/cli/mass_compile.py @@ -16,7 +16,7 @@ from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget from qmk.util import maybe_exit_config -def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env): +def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, print_failures: bool, **env): if len(targets) == 0: return @@ -37,6 +37,30 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, builddir.mkdir(parents=True, exist_ok=True) with open(makefile, "w") as f: + # yapf: disable + f.write( + f"""\ +# This file is auto-generated by qmk mass-compile +# Do not edit this file directly. +all: print_failures +.PHONY: all_targets print_failures +print_failures: all_targets +"""# noqa + ) + if print_failures: + f.write( + f"""\ + @for f in $$(ls .build/failed.log.{os.getpid()}.* 2>/dev/null | sort); do \\ + echo; \\ + echo "======================================================================================"; \\ + echo "Failed build log: $$f"; \\ + echo "------------------------------------------------------"; \\ + cat $$f; \\ + echo "------------------------------------------------------"; \\ + done +"""# noqa + ) + # yapf: enable for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): keyboard_name = target.keyboard keymap_name = target.keymap @@ -58,7 +82,7 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, f.write( f"""\ .PHONY: {target_filename}{target_suffix}_binary -all: {target_filename}{target_suffix}_binary +all_targets: {target_filename}{target_suffix}_binary {target_filename}{target_suffix}_binary: @rm -f "{build_log}" || true @echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}" @@ -98,6 +122,7 @@ all: {target_filename}{target_suffix}_binary @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.") +@cli.argument('-p', '--print-failures', arg_only=True, action='store_true', help="Print failed builds.") @cli.argument( '-f', '--filter', @@ -123,4 +148,4 @@ def mass_compile(cli): else: targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter) - return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env)) + return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.config.mass_compile.parallel, cli.args.print_failures, **build_environment(cli.args.env)) diff --git a/lib/python/qmk/cli/migrate.py b/lib/python/qmk/cli/migrate.py index 0bab5c1949..d0f195d737 100644 --- a/lib/python/qmk/cli/migrate.py +++ b/lib/python/qmk/cli/migrate.py @@ -6,14 +6,14 @@ from dotty_dict import dotty from milc import cli -from qmk.keyboard import keyboard_completer, keyboard_folder, resolve_keyboard +from qmk.keyboard import keyboard_completer, keyboard_folder from qmk.info import info_json, find_info_json from qmk.json_encoders import InfoJSONEncoder from qmk.json_schema import json_load def _candidate_files(keyboard): - kb_dir = Path(resolve_keyboard(keyboard)) + kb_dir = Path(keyboard) cur_dir = Path('keyboards') files = [] diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py index 20a0c0198c..4d19a726a4 100755 --- a/lib/python/qmk/cli/new/keymap.py +++ b/lib/python/qmk/cli/new/keymap.py @@ -9,6 +9,7 @@ from milc import cli from milc.questions import question, choice from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE +from qmk.errors import NoSuchKeyboardError from qmk.path import is_keyboard, keymaps, keymap from qmk.git import git_get_username from qmk.decorators import automagic_keyboard, automagic_keymap @@ -110,13 +111,18 @@ def new_keymap(cli): cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} does not exist! Please choose a valid name.') return False - # generate keymap paths - keymaps_dirs = keymaps(kb_name) - keymap_path_default = keymap(kb_name, 'default') - keymap_path_new = keymaps_dirs[0] / user_name + # validate before any keymap ops + try: + keymaps_dirs = keymaps(kb_name) + keymap_path_new = keymaps_dirs[0] / user_name + except NoSuchKeyboardError: + cli.log.error(f'Keymap folder for {{fg_cyan}}{kb_name}{{fg_reset}} does not exist!') + return False - if not keymap_path_default.exists(): - cli.log.error(f'Default keymap {{fg_cyan}}{keymap_path_default}{{fg_reset}} does not exist!') + keymap_path_default = keymap(kb_name, 'default') + + if not keymap_path_default: + cli.log.error(f'Default keymap for {{fg_cyan}}{kb_name}{{fg_reset}} does not exist!') return False if not validate_keymap_name(user_name): @@ -134,7 +140,7 @@ def new_keymap(cli): _set_converter(keymap_path_new / 'keymap.json', converter) # end message to user - cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}') + cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}{{fg_reset}}.') cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.") # Add to userspace compile if we have userspace available diff --git a/lib/python/qmk/cli/resolve_alias.py b/lib/python/qmk/cli/resolve_alias.py index b9ffb46618..dff2242b28 100644 --- a/lib/python/qmk/cli/resolve_alias.py +++ b/lib/python/qmk/cli/resolve_alias.py @@ -5,7 +5,7 @@ from milc import cli @cli.argument('--allow-unknown', arg_only=True, action='store_true', help="Return original if rule is not a valid keyboard.") @cli.argument('keyboard', arg_only=True, help='The keyboard\'s name') -@cli.subcommand('Resolve DEFAULT_FOLDER and any keyboard_aliases for provided rule') +@cli.subcommand('Resolve any keyboard_aliases for provided rule') def resolve_alias(cli): try: print(keyboard_folder(cli.args.keyboard)) diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py index 0d6f32cd11..eea70efb95 100644 --- a/lib/python/qmk/cli/userspace/add.py +++ b/lib/python/qmk/cli/userspace/add.py @@ -47,6 +47,7 @@ def userspace_add(cli): from qmk.cli.new.keymap import new_keymap cli.config.new_keymap.keyboard = cli.args.keyboard cli.config.new_keymap.keymap = cli.args.keymap + cli.args.skip_converter = True if new_keymap(cli) is not False: userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env) else: diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py index f164ca2ef1..64fa3ed0c9 100644 --- a/lib/python/qmk/cli/userspace/compile.py +++ b/lib/python/qmk/cli/userspace/compile.py @@ -20,6 +20,7 @@ def _extra_arg_setter(target, extra_args): @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.") +@cli.argument('-p', '--print-failures', arg_only=True, action='store_true', help="Print failed builds.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.subcommand('Compiles the build targets specified in userspace `qmk.json`.') def userspace_compile(cli): @@ -42,4 +43,4 @@ def userspace_compile(cli): if len(keyboard_keymap_targets) > 0: build_targets.extend(search_keymap_targets(keyboard_keymap_targets)) - return mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env)) + return mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, cli.args.print_failures, **build_environment(cli.args.env)) diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py index 2b7e29aa7e..7c016e5a2f 100644 --- a/lib/python/qmk/cli/userspace/doctor.py +++ b/lib/python/qmk/cli/userspace/doctor.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: GPL-2.0-or-later from milc import cli -from qmk.constants import QMK_FIRMWARE +from qmk.constants import QMK_FIRMWARE, HAS_QMK_USERSPACE from qmk.cli.doctor.main import userspace_tests @cli.subcommand('Checks userspace configuration.') def userspace_doctor(cli): userspace_tests(QMK_FIRMWARE) + + return 0 if HAS_QMK_USERSPACE else 1 diff --git a/lib/python/qmk/compilation_database.py b/lib/python/qmk/compilation_database.py index 4c88dadbdd..851dc3f157 100755 --- a/lib/python/qmk/compilation_database.py +++ b/lib/python/qmk/compilation_database.py @@ -41,11 +41,10 @@ def cpu_defines(binary: str, compiler_args: str) -> List[str]: if binary.endswith("gcc") or binary.endswith("g++"): invocation = [binary, '-dM', '-E'] if binary.endswith("gcc"): - invocation.extend(['-x', 'c']) + invocation.extend(['-x', 'c', '-std=gnu11']) elif binary.endswith("g++"): - invocation.extend(['-x', 'c++']) - compiler_args = shlex.split(compiler_args) - invocation.extend(compiler_args) + invocation.extend(['-x', 'c++', '-std=gnu++14']) + invocation.extend(shlex.split(compiler_args)) invocation.append('-') result = cli.run(invocation, capture_output=True, check=True, stdin=None, input='\n') define_args = [] @@ -55,7 +54,11 @@ def cpu_defines(binary: str, compiler_args: str) -> List[str]: define_args.append(f'-D{line_args[1]}={line_args[2]}') elif len(line_args) == 2 and line_args[0] == '#define': define_args.append(f'-D{line_args[1]}') - return list(sorted(set(define_args))) + + type_filter = re.compile( + r'^-D__(SIZE|INT|UINT|WINT|WCHAR|BYTE|SHRT|SIG|FLOAT|LONG|CHAR|SCHAR|DBL|FLT|LDBL|PTRDIFF|QQ|DQ|DA|HA|HQ|SA|SQ|TA|TQ|UDA|UDQ|UHA|UHQ|USQ|USA|UTQ|UTA|UQQ|UQA|ACCUM|FRACT|UACCUM|UFRACT|LACCUM|LFRACT|ULACCUM|ULFRACT|LLACCUM|LLFRACT|ULLACCUM|ULLFRACT|SACCUM|SFRACT|USACCUM|USFRACT)' + ) + return list(sorted(set(filter(lambda x: not type_filter.match(x), define_args)))) return [] @@ -92,8 +95,8 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]: for s in system_libs(binary): args += ['-isystem', '%s' % s] args.extend(cpu_defines(binary, ' '.join(shlex.quote(s) for s in compiler_args))) - new_cmd = ' '.join(shlex.quote(s) for s in args) - records.append({"directory": str(QMK_FIRMWARE.resolve()), "command": new_cmd, "file": this_file}) + args[0] = binary + records.append({"arguments": args, "directory": str(QMK_FIRMWARE.resolve()), "file": this_file}) state = 'start' return records diff --git a/lib/python/qmk/flashers.py b/lib/python/qmk/flashers.py index 2cca4941d3..6b52f4d35a 100644 --- a/lib/python/qmk/flashers.py +++ b/lib/python/qmk/flashers.py @@ -96,7 +96,7 @@ def _find_bootloader(): details = 'halfkay' else: details = 'qmk-hid' - elif bl in {'apm32-dfu', 'gd32v-dfu', 'kiibohd', 'stm32-dfu'}: + elif bl in {'apm32-dfu', 'at32-dfu', 'gd32v-dfu', 'kiibohd', 'stm32-dfu'}: details = (vid, pid) else: details = None @@ -153,11 +153,12 @@ def _flash_atmel_dfu(mcu, file): def _flash_hid_bootloader(mcu, details, file): + cmd = None if details == 'halfkay': - if shutil.which('teensy-loader-cli'): - cmd = 'teensy-loader-cli' - elif shutil.which('teensy_loader_cli'): + if shutil.which('teensy_loader_cli'): cmd = 'teensy_loader_cli' + elif shutil.which('teensy-loader-cli'): + cmd = 'teensy-loader-cli' # Use 'hid_bootloader_cli' for QMK HID and as a fallback for HalfKay if not cmd: @@ -176,7 +177,7 @@ def _flash_dfu_util(details, file): # kiibohd elif details[0] == '1c11' and details[1] == 'b007': cli.run(['dfu-util', '-a', '0', '-d', f'{details[0]}:{details[1]}', '-D', file], capture_output=False) - # STM32, APM32, or GD32V DFU + # STM32, APM32, AT32, or GD32V DFU else: cli.run(['dfu-util', '-a', '0', '-d', f'{details[0]}:{details[1]}', '-s', '0x08000000:leave', '-D', file], capture_output=False) @@ -226,7 +227,7 @@ def flasher(mcu, file): return (True, "Please make sure 'teensy_loader_cli' or 'hid_bootloader_cli' is available on your system.") else: return (True, "Specifying the MCU with '-m' is necessary for HalfKay/HID bootloaders!") - elif bl in {'apm32-dfu', 'gd32v-dfu', 'kiibohd', 'stm32-dfu'}: + elif bl in {'apm32-dfu', 'at32-dfu', 'gd32v-dfu', 'kiibohd', 'stm32-dfu'}: _flash_dfu_util(details, file) elif bl == 'wb32-dfu': if _flash_wb32_dfu_updater(file): diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index d95fd3d799..e07fa0ccae 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -5,6 +5,7 @@ import os from pathlib import Path import jsonschema from dotty_dict import dotty +from enum import IntFlag from milc import cli @@ -14,13 +15,22 @@ from qmk.json_schema import deep_update, json_load, validate from qmk.keyboard import config_h, rules_mk from qmk.commands import parse_configurator_json from qmk.makefile import parse_rules_mk_file -from qmk.math import compute +from qmk.math_ops import compute from qmk.util import maybe_exit, truthy true_values = ['1', 'on', 'yes'] false_values = ['0', 'off', 'no'] +class LedFlags(IntFlag): + ALL = 0xFF + NONE = 0x00 + MODIFIER = 0x01 + UNDERGLOW = 0x02 + KEYLIGHT = 0x04 + INDICATOR = 0x08 + + def _keyboard_in_layout_name(keyboard, layout): """Validate that a layout macro does not contain name of keyboard """ @@ -223,12 +233,6 @@ def _validate(keyboard, info_data): def info_json(keyboard, force_layout=None): """Generate the info.json data for a specific keyboard. """ - cur_dir = Path('keyboards') - root_rules_mk = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') - - if 'DEFAULT_FOLDER' in root_rules_mk: - keyboard = root_rules_mk['DEFAULT_FOLDER'] - info_data = { 'keyboard_name': str(keyboard), 'keyboard_folder': str(keyboard), @@ -260,6 +264,7 @@ def info_json(keyboard, force_layout=None): # Ensure that we have various calculated values info_data = _matrix_size(info_data) info_data = _joystick_axis_count(info_data) + info_data = _matrix_masked(info_data) # Merge in data from info_data = _extract_led_config(info_data, str(keyboard)) @@ -307,6 +312,24 @@ def _extract_features(info_data, rules): return info_data +def _extract_matrix_rules(info_data, rules): + """Find all the features enabled in rules.mk. + """ + if rules.get('CUSTOM_MATRIX', 'no') != 'no': + if 'matrix_pins' in info_data and 'custom' in info_data['matrix_pins']: + _log_warning(info_data, 'Custom Matrix is specified in both info.json and rules.mk, the rules.mk values win.') + + if 'matrix_pins' not in info_data: + info_data['matrix_pins'] = {} + + if rules['CUSTOM_MATRIX'] == 'lite': + info_data['matrix_pins']['custom_lite'] = True + else: + info_data['matrix_pins']['custom'] = True + + return info_data + + def _pin_name(pin): """Returns the proper representation for a pin. """ @@ -482,6 +505,9 @@ def _extract_split_serial(info_data, config_c): if 'soft_serial_pin' in split: split['serial'] = split.get('serial', {}) split['serial']['pin'] = split.pop('soft_serial_pin') + if 'soft_serial_speed' in split: + split['serial'] = split.get('serial', {}) + split['serial']['speed'] = split.pop('soft_serial_speed') def _extract_split_transport(info_data, config_c): @@ -554,7 +580,6 @@ def _extract_matrix_info(info_data, config_c): row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip() col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip() direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1] - info_snippet = {} if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c: if 'matrix_size' in info_data: @@ -569,26 +594,20 @@ def _extract_matrix_info(info_data, config_c): if 'matrix_pins' in info_data and 'cols' in info_data['matrix_pins'] and 'rows' in info_data['matrix_pins']: _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.') - info_snippet['cols'] = _extract_pins(col_pins) - info_snippet['rows'] = _extract_pins(row_pins) + if 'matrix_pins' not in info_data: + info_data['matrix_pins'] = {} + + info_data['matrix_pins']['cols'] = _extract_pins(col_pins) + info_data['matrix_pins']['rows'] = _extract_pins(row_pins) if direct_pins: if 'matrix_pins' in info_data and 'direct' in info_data['matrix_pins']: _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.') - info_snippet['direct'] = _extract_direct_matrix(direct_pins) + if 'matrix_pins' not in info_data: + info_data['matrix_pins'] = {} - if config_c.get('CUSTOM_MATRIX', 'no') != 'no': - if 'matrix_pins' in info_data and 'custom' in info_data['matrix_pins']: - _log_warning(info_data, 'Custom Matrix is specified in both info.json and config.h, the config.h values win.') - - info_snippet['custom'] = True - - if config_c['CUSTOM_MATRIX'] == 'lite': - info_snippet['custom_lite'] = True - - if info_snippet: - info_data['matrix_pins'] = info_snippet + info_data['matrix_pins']['direct'] = _extract_direct_matrix(direct_pins) return info_data @@ -757,6 +776,7 @@ def _extract_rules_mk(info_data, rules): # Merge in config values that can't be easily mapped _extract_features(info_data, rules) + _extract_matrix_rules(info_data, rules) return info_data @@ -802,6 +822,25 @@ def _extract_led_config(info_data, keyboard): if info_data[feature].get('layout', None) and not info_data[feature].get('led_count', None): info_data[feature]['led_count'] = len(info_data[feature]['layout']) + if info_data[feature].get('layout', None) and not info_data[feature].get('flag_steps', None): + flags = {LedFlags.ALL, LedFlags.NONE} + default_flags = {LedFlags.MODIFIER | LedFlags.KEYLIGHT, LedFlags.UNDERGLOW} + + # if only a single flag is used, assume only all+none flags + kb_flags = set(x.get('flags', LedFlags.NONE) for x in info_data[feature]['layout']) + if len(kb_flags) > 1: + # check if any part of LED flag is with the defaults + unique_flags = set() + for candidate in default_flags: + if any(candidate & flag for flag in kb_flags): + unique_flags.add(candidate) + + # if we still have a single flag, assume only all+none + if len(unique_flags) > 1: + flags.update(unique_flags) + + info_data[feature]['flag_steps'] = sorted([int(flag) for flag in flags], reverse=True) + return info_data @@ -836,6 +875,25 @@ def _joystick_axis_count(info_data): return info_data +def _matrix_masked(info_data): + """"Add info_data['matrix_pins.masked'] if required""" + mask_required = False + + if 'matrix_grid' in info_data.get('dip_switch', {}): + mask_required = True + if 'matrix_grid' in info_data.get('split', {}).get('handedness', {}): + mask_required = True + + if mask_required: + if 'masked' not in info_data.get('matrix_pins', {}): + if 'matrix_pins' not in info_data: + info_data['matrix_pins'] = {} + + info_data['matrix_pins']['masked'] = True + + return info_data + + def _check_matrix(info_data): """Check the matrix to ensure that row/column count is consistent. """ @@ -1005,11 +1063,6 @@ def find_info_json(keyboard): keyboard_parent = keyboard_path.parent info_jsons = [keyboard_path / 'info.json', keyboard_path / 'keyboard.json'] - # Add DEFAULT_FOLDER before parents, if present - rules = rules_mk(keyboard) - if 'DEFAULT_FOLDER' in rules: - info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json') - # Add in parent folders for least specific for _ in range(5): if keyboard_parent == base_path: diff --git a/lib/python/qmk/json_encoders.py b/lib/python/qmk/json_encoders.py index e8bcf48996..6bad820a76 100755 --- a/lib/python/qmk/json_encoders.py +++ b/lib/python/qmk/json_encoders.py @@ -178,9 +178,9 @@ class KeymapJSONEncoder(QMKJSONEncoder): else: layer[-1].append(f'"{key}"') - layer = [f"{self.indent_str*indent_level}{', '.join(row)}" for row in layer] + layer = [f"{self.indent_str * indent_level}{', '.join(row)}" for row in layer] - return f"{self.indent_str}[\n{newline.join(layer)}\n{self.indent_str*self.indentation_level}]" + return f"{self.indent_str}[\n{newline.join(layer)}\n{self.indent_str * self.indentation_level}]" elif self.primitives_only(obj): return "[" + ", ".join(self.encode(element) for element in obj) + "]" diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index fcf5b5b158..e8534492c9 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -99,8 +99,6 @@ def find_keyboard_from_dir(): keymap_index = len(current_path.parts) - current_path.parts.index('keymaps') - 1 current_path = current_path.parents[keymap_index] - current_path = resolve_keyboard(current_path) - if qmk.path.is_keyboard(current_path): return str(current_path) @@ -121,7 +119,7 @@ def find_readme(keyboard): def keyboard_folder(keyboard): """Returns the actual keyboard folder. - This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard. + This checks aliases to resolve the actual path for a keyboard. """ aliases = keyboard_alias_definitions() @@ -131,8 +129,6 @@ def keyboard_folder(keyboard): if keyboard == last_keyboard: break - keyboard = resolve_keyboard(keyboard) - if not qmk.path.is_keyboard(keyboard): raise ValueError(f'Invalid keyboard: {keyboard}') @@ -158,7 +154,7 @@ def keyboard_aliases(keyboard): def keyboard_folder_or_all(keyboard): """Returns the actual keyboard folder. - This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard. + This checks aliases to resolve the actual path for a keyboard. If the supplied argument is "all", it returns an AllKeyboards object. """ if keyboard == 'all': @@ -179,32 +175,22 @@ def keyboard_completer(prefix, action, parser, parsed_args): return list_keyboards() -def list_keyboards(resolve_defaults=True): - """Returns a list of all keyboards - optionally processing any DEFAULT_FOLDER. +@lru_cache(maxsize=None) +def list_keyboards(): + """Returns a list of all keyboards. """ # We avoid pathlib here because this is performance critical code. - paths = [] - for marker in ['rules.mk', 'keyboard.json']: - kb_wildcard = os.path.join(base_path, "**", marker) - paths += [path for path in glob(kb_wildcard, recursive=True) if os.path.sep + 'keymaps' + os.path.sep not in path] + kb_wildcard = os.path.join(base_path, "**", 'keyboard.json') + paths = [path for path in glob(kb_wildcard, recursive=True) if os.path.sep + 'keymaps' + os.path.sep not in path] found = map(_find_name, paths) - if resolve_defaults: - found = map(resolve_keyboard, found) + + # Convert to posix paths for consistency + found = map(lambda x: str(Path(x).as_posix()), found) return sorted(set(found)) -@lru_cache(maxsize=None) -def resolve_keyboard(keyboard): - cur_dir = Path('keyboards') - rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') - while 'DEFAULT_FOLDER' in rules and keyboard != rules['DEFAULT_FOLDER']: - keyboard = rules['DEFAULT_FOLDER'] - rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') - return keyboard - - def config_h(keyboard): """Parses all the config.h files for a keyboard. @@ -216,7 +202,7 @@ def config_h(keyboard): """ config = {} cur_dir = Path('keyboards') - keyboard = Path(resolve_keyboard(keyboard)) + keyboard = Path(keyboard) for dir in keyboard.parts: cur_dir = cur_dir / dir @@ -235,7 +221,7 @@ def rules_mk(keyboard): a dictionary representing the content of the entire rules.mk tree for a keyboard """ cur_dir = Path('keyboards') - keyboard = Path(resolve_keyboard(keyboard)) + keyboard = Path(keyboard) rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') for i, dir in enumerate(keyboard.parts): diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index b7138aa4a1..0ac04f6f73 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -32,6 +32,7 @@ __INCLUDES__ __KEYMAP_GOES_HERE__ __ENCODER_MAP_GOES_HERE__ +__DIP_SWITCH_MAP_GOES_HERE__ __MACRO_OUTPUT_GOES_HERE__ #ifdef OTHER_KEYMAP_C @@ -66,6 +67,19 @@ def _generate_encodermap_table(keymap_json): return lines +def _generate_dipswitchmap_table(keymap_json): + lines = [ + '#if defined(DIP_SWITCH_ENABLE) && defined(DIP_SWITCH_MAP_ENABLE)', + 'const uint16_t PROGMEM dip_switch_map[NUM_DIP_SWITCHES][NUM_DIP_STATES] = {', + ] + for index, switch in enumerate(keymap_json['dip_switches']): + if index != 0: + lines[-1] = lines[-1] + ',' + lines.append(f' DIP_SWITCH_OFF_ON({_strip_any(switch["off"])}, {_strip_any(switch["on"])})') + lines.extend(['};', '#endif // defined(DIP_SWITCH_ENABLE) && defined(DIP_SWITCH_MAP_ENABLE)']) + return lines + + def _generate_macros_function(keymap_json): macro_txt = [ 'bool process_record_user(uint16_t keycode, keyrecord_t *record) {', @@ -286,6 +300,12 @@ def generate_c(keymap_json): encodermap = '\n'.join(encoder_txt) new_keymap = new_keymap.replace('__ENCODER_MAP_GOES_HERE__', encodermap) + dipswitchmap = '' + if 'dip_switches' in keymap_json and keymap_json['dip_switches'] is not None: + dip_txt = _generate_dipswitchmap_table(keymap_json) + dipswitchmap = '\n'.join(dip_txt) + new_keymap = new_keymap.replace('__DIP_SWITCH_MAP_GOES_HERE__', dipswitchmap) + macros = '' if 'macros' in keymap_json and keymap_json['macros'] is not None: macro_txt = _generate_macros_function(keymap_json) @@ -343,24 +363,21 @@ def locate_keymap(keyboard, keymap, force_layout=None): # Check the keyboard folder first, last match wins keymap_path = '' - search_dirs = [QMK_FIRMWARE] - keyboard_dirs = [keyboard_folder(keyboard)] + search_conf = {QMK_FIRMWARE: [keyboard_folder(keyboard)]} if HAS_QMK_USERSPACE: # When we've got userspace, check there _last_ as we want them to override anything in the main repo. - search_dirs.append(QMK_USERSPACE) # We also want to search for any aliases as QMK's folder structure may have changed, with an alias, but the user # hasn't updated their keymap location yet. - keyboard_dirs.extend(keyboard_aliases(keyboard)) - keyboard_dirs = list(set(keyboard_dirs)) + search_conf[QMK_USERSPACE] = list(set([keyboard_folder(keyboard), *keyboard_aliases(keyboard)])) - for search_dir in search_dirs: + for search_dir, keyboard_dirs in search_conf.items(): for keyboard_dir in keyboard_dirs: checked_dirs = '' - for dir in keyboard_dir.split('/'): + for folder_name in keyboard_dir.split('/'): if checked_dirs: - checked_dirs = '/'.join((checked_dirs, dir)) + checked_dirs = '/'.join((checked_dirs, folder_name)) else: - checked_dirs = dir + checked_dirs = folder_name keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps' diff --git a/lib/python/qmk/math_ops.py b/lib/python/qmk/math_ops.py new file mode 100644 index 0000000000..1f14b18f4e --- /dev/null +++ b/lib/python/qmk/math_ops.py @@ -0,0 +1,33 @@ +"""Parse arbitrary math equations in a safe way. + +Gratefully copied from https://stackoverflow.com/a/9558001 +""" +import ast +import operator as op + +# supported operators +operators = {ast.Add: op.add, ast.Sub: op.sub, ast.Mult: op.mul, ast.Div: op.truediv, ast.Pow: op.pow, ast.BitXor: op.xor, ast.USub: op.neg} + + +def compute(expr): + """Parse a mathematical expression and return the answer. + + >>> compute('2^6') + 4 + >>> compute('2**6') + 64 + >>> compute('1 + 2*3**(4^5) / (6 + -7)') + -5.0 + """ + return _eval(ast.parse(expr, mode='eval').body) + + +def _eval(node): + if isinstance(node, ast.Constant): # + return node.value + elif isinstance(node, ast.BinOp): # + return operators[type(node.op)](_eval(node.left), _eval(node.right)) + elif isinstance(node, ast.UnaryOp): # e.g., -1 + return operators[type(node.op)](_eval(node.operand)) + else: + raise TypeError(node) diff --git a/lib/python/qmk/painter.py b/lib/python/qmk/painter.py index ed0372c163..1a07f7442c 100644 --- a/lib/python/qmk/painter.py +++ b/lib/python/qmk/painter.py @@ -129,7 +129,7 @@ def _render_image_metadata(metadata): px = size["width"] * size["height"] # FIXME: May need need more chars here too - deltas.append(f"// Frame {i:3d}: ({l:3d}, {t:3d}) - ({r:3d}, {b:3d}) >> {delta_px:4d}/{px:4d} pixels ({100*delta_px/px:.2f}%)") + deltas.append(f"// Frame {i:3d}: ({l:3d}, {t:3d}) - ({r:3d}, {b:3d}) >> {delta_px:4d}/{px:4d} pixels ({100 * delta_px / px:.2f}%)") if deltas: lines.append("// Areas on delta frames") diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py index c47ed18362..1739689adf 100644 --- a/lib/python/qmk/path.py +++ b/lib/python/qmk/path.py @@ -21,11 +21,9 @@ def is_keyboard(keyboard_name): if Path(keyboard_name).is_absolute(): return False - keyboard_path = QMK_FIRMWARE / 'keyboards' / keyboard_name - rules_mk = keyboard_path / 'rules.mk' - keyboard_json = keyboard_path / 'keyboard.json' + keyboard_json = QMK_FIRMWARE / 'keyboards' / keyboard_name / 'keyboard.json' - return rules_mk.exists() or keyboard_json.exists() + return keyboard_json.exists() def under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])): diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index dd659fe0f2..2716459989 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -159,6 +159,7 @@ const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { + #ifdef OTHER_KEYMAP_C # include OTHER_KEYMAP_C #endif // OTHER_KEYMAP_C @@ -196,6 +197,7 @@ const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { + #ifdef OTHER_KEYMAP_C # include OTHER_KEYMAP_C #endif // OTHER_KEYMAP_C diff --git a/lib/python/qmk/tests/test_qmk_keymap.py b/lib/python/qmk/tests/test_qmk_keymap.py index 80cc679b00..34360d3b6d 100644 --- a/lib/python/qmk/tests/test_qmk_keymap.py +++ b/lib/python/qmk/tests/test_qmk_keymap.py @@ -27,6 +27,7 @@ const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { + #ifdef OTHER_KEYMAP_C # include OTHER_KEYMAP_C #endif // OTHER_KEYMAP_C diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py index 8f99410e1d..6da684a577 100644 --- a/lib/python/qmk/util.py +++ b/lib/python/qmk/util.py @@ -3,9 +3,12 @@ import contextlib import multiprocessing import sys +import re from milc import cli +TRIPLET_PATTERN = re.compile(r'^(\d+)\.(\d+)\.(\d+)') + maybe_exit_should_exit = True maybe_exit_reraise = False @@ -96,3 +99,10 @@ def parallel_map(*args, **kwargs): # before the results are returned. Returning a list ensures results are # materialised before any worker pool is shut down. return list(map_fn(*args, **kwargs)) + + +def triplet_to_bcd(ver: str): + m = TRIPLET_PATTERN.match(ver) + if not m: + return '0x00000000' + return f'0x{int(m.group(1)):02d}{int(m.group(2)):02d}{int(m.group(3)):04d}' diff --git a/util/drivers.txt b/util/drivers.txt index e8ed7bdb0b..0490bfa4ab 100644 --- a/util/drivers.txt +++ b/util/drivers.txt @@ -5,6 +5,8 @@ # Use Windows Powershell and type [guid]::NewGuid() to generate guids winusb,STM32 Bootloader,0483,DF11,6d98a87f-4ecf-464d-89ed-8c684d857a75 winusb,APM32 Bootloader,314B,0106,9ff3cc31-6772-4a3f-a492-a80d91f7a853 +winusb,WB32 Bootloader,342D,DFA0,89b0fdf0-3d22-4408-8393-32147ba508ce +winusb,GD32V Bootloader,28E9,0189,e1421fd6-f799-4b6c-97e6-39e87d37f858 winusb,STM32duino Bootloader,1EAF,0003,746915ec-99d8-4a90-a722-3c85ba31e4fe libusbk,USBaspLoader,16C0,05DC,e69affdc-0ef0-427c-aefb-4e593c9d2724 winusb,Kiibohd DFU Bootloader,1C11,B007,aa5a3f86-b81e-4416-89ad-0c1ea1ed63af diff --git a/util/env-bootstrap.sh b/util/env-bootstrap.sh new file mode 100755 index 0000000000..07020ea371 --- /dev/null +++ b/util/env-bootstrap.sh @@ -0,0 +1,594 @@ +#!/usr/bin/env sh +# Copyright 2025 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later + +################################################################################ +# This script will install the QMK CLI, toolchains, and flashing utilities. +################################################################################ +# Environment variables: +# CONFIRM: Skip the pre-install delay. (or: --confirm) +# QMK_DISTRIB_DIR: The directory to install the QMK distribution to. (or: --qmk-distrib-dir=...) +# UV_INSTALL_DIR: The directory to install `uv` to. (or: --uv-install-dir=...) +# UV_TOOL_DIR: The directory to install `uv` tools to. (or: --uv-tool-dir=...) +# SKIP_CLEAN: Skip cleaning the distribution directory. (or: --skip-clean) +# SKIP_PACKAGE_MANAGER: Skip installing the necessary packages for the package manager. (or: --skip-package-manager) +# SKIP_UV: Skip installing `uv`. (or: --skip-uv) +# SKIP_QMK_CLI: Skip installing the QMK CLI. (or: --skip-qmk-cli) +# SKIP_QMK_TOOLCHAINS: Skip installing the QMK toolchains. (or: --skip-qmk-toolchains) +# SKIP_QMK_FLASHUTILS: Skip installing the QMK flashing utilities. (or: --skip-qmk-flashutils) +# SKIP_UDEV_RULES: Skip installing the udev rules for Linux. (or: --skip-udev-rules) +# SKIP_WINDOWS_DRIVERS: Skip installing the Windows drivers for the flashing utilities. (or: --skip-windows-drivers) +# +# Arguments above may be negated by prefixing with `--no-` instead (e.g. `--no-skip-clean`). +################################################################################ +# Usage: +# curl -fsSL https://raw.githubusercontent.com/qmk/qmk_firmware/master/util/env-bootstrap.sh | sh +# +# Help: +# curl -fsSL https://raw.githubusercontent.com/qmk/qmk_firmware/master/util/env-bootstrap.sh | sh -s -- --help +# +# An example which skips installing `uv` using environment variables: +# curl -fsSL https://raw.githubusercontent.com/qmk/qmk_firmware/master/util/env-bootstrap.sh | SKIP_UV=1 sh +# +# ...or by using command line arguments: +# curl -fsSL https://raw.githubusercontent.com/qmk/qmk_firmware/master/util/env-bootstrap.sh | sh -s -- --skip-uv +# +# Any other configurable items listed above may be specified in the same way. +################################################################################ + +{ # this ensures the entire script is downloaded # + set -eu + + BOOTSTRAP_TMPDIR="$(mktemp -d /tmp/qmk-bootstrap-failure.XXXXXX)" + trap 'rm -rf "$BOOTSTRAP_TMPDIR" >/dev/null 2>&1 || true' EXIT + FAILURE_FILE="${BOOTSTRAP_TMPDIR}/fail" + + # Work out which `sed` to use + command -v gsed >/dev/null 2>&1 && SED=gsed || SED=sed + + script_args() { + cat <<__EOT__ + --help -- Shows this help text + --confirm -- Skips the delay before installation + --uv-install-dir={path} -- The directory to install \`uv\` into + --uv-tool-dir={path} -- The directory to install \`uv\` tools into + --qmk-distrib-dir={path} -- The directory to install the QMK distribution into + --skip-clean -- Skip cleaning the QMK distribution directory + --skip-package-manager -- Skip installing the necessary packages for the package manager + --skip-uv -- Skip installing \`uv\` + --skip-qmk-cli -- Skip installing the QMK CLI + --skip-qmk-toolchains -- Skip installing the QMK toolchains + --skip-qmk-flashutils -- Skip installing the QMK flashing utilities + --skip-udev-rules -- Skip installing the udev rules for Linux + --skip-windows-drivers -- Skip installing the Windows drivers for the flashing utilities +__EOT__ + # Hidden: + # --wsl-install -- Installs the WSL variant of qmk_flashutils + } + + signal_execution_failure() { + touch "$FAILURE_FILE" >/dev/null 2>&1 || true + } + + exit_if_execution_failed() { + if [ -e "$FAILURE_FILE" ]; then + exit 1 + fi + } + + script_help() { + echo "$(basename ${this_script:-qmk-install.sh}) $(script_args | sort | ${SED} -e 's@^\s*@@g' -e 's@\s\+--.*@@g' -e 's@^@[@' -e 's@$@]@' | tr '\n' ' ')" + echo + echo "Arguments:" + script_args + echo + echo "Switch arguments may be negated by prefixing with '--no-' (e.g. '--no-skip-clean')." + } + + script_parse_args() { + local N + local V + while [ ! -z "${1:-}" ]; do + case "$1" in + --help) + script_help + exit 0 + ;; + --*=*) + N=${1%%=*} + N=${N##--} + N=$(echo $N | tr '-' '_' | tr 'a-z' 'A-Z') + V=${1##*=} + export $N="$V" + ;; + --no-*) + N=${1##--no-} + N=$(echo $N | tr '-' '_' | tr 'a-z' 'A-Z') + unset $N + ;; + --*) + N=${1##--} + N=$(echo $N | tr '-' '_' | tr 'a-z' 'A-Z') + export $N=true + ;; + *) + echo "Unknown argument: '$1'" >&2 + echo + script_help >&2 + exit 1 + ;; + esac + shift + unset N + unset V + done + } + + nsudo() { + if [ "$(fn_os)" = "windows" ]; then + # No need for sudo under QMK MSYS + return + elif [ $(id -u) -ne 0 ]; then + if [ -n "$(command -v sudo 2>/dev/null || true)" ]; then + echo "sudo" + elif [ -n "$(command -v doas 2>/dev/null || true)" ]; then + echo "doas" + else + echo "Please install 'sudo' or 'doas' to continue." >&2 + exit 1 + fi + fi + true + } + + download_url() { + local url=$1 + local filename=${2:-$(basename "$url")} + local quiet='' + if [ -n "$(command -v curl 2>/dev/null || true)" ]; then + [ "$filename" = "-" ] && quiet='-s' || echo "Downloading '$url' => '$filename'" >&2 + curl -LSf $quiet -o "$filename" "$url" + elif [ -n "$(command -v wget 2>/dev/null || true)" ]; then + [ "$filename" = "-" ] && quiet='-q' || echo "Downloading '$url' => '$filename'" >&2 + wget $quiet "-O$filename" "$url" + else + echo "Please install 'curl' to continue." >&2 + exit 1 + fi + } + + github_api_call() { + local url="$1" + local token="${GITHUB_TOKEN:-${GH_TOKEN:-}}" + if [ -n "${token:-}" ]; then + if [ -n "$(command -v curl 2>/dev/null || true)" ]; then + curl -fsSL -H "Authorization: token $token" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/$url" + elif [ -n "$(command -v wget 2>/dev/null || true)" ]; then + wget -q --header="Authorization: token $token" --header="Accept: application/vnd.github.v3+json" "https://api.github.com/$url" -O - + fi + else + download_url "https://api.github.com/$url" - + fi + } + + fn_os() { + local os_name=$(echo ${1:-} | tr 'A-Z' 'a-z') + if [ -z "$os_name" ]; then + os_name=$(uname -s | tr 'A-Z' 'a-z') + fi + case "$os_name" in + *darwin* | *macos* | *apple*) + echo macos + ;; + *windows* | *mingw* | *msys*) + echo windows + ;; + *linux*) + echo linux + ;; + *) + echo unknown + ;; + esac + } + + fn_arch() { + local arch_name=$(echo ${1:-} | tr 'A-Z' 'a-z') + if [ -z "$arch_name" ]; then + arch_name=$(uname -m | tr 'A-Z' 'a-z') + fi + case "$arch_name" in + *arm64* | *aarch64*) + echo ARM64 + ;; + *riscv64*) + echo RV64 + ;; + *x86_64* | *x64*) + echo X64 + ;; + *) + echo unknown + ;; + esac + } + + preinstall_delay() { + [ -z "${CONFIRM:-}" ] || return 0 + echo >&2 + echo "Waiting 10 seconds before proceeding. Press Ctrl+C to cancel installation." >&2 + sleep 10 + } + + get_package_manager_deps() { + case $(fn_os) in + macos) echo "zstd clang-format make hidapi libusb dos2unix git" ;; + windows) echo "base-devel: zstd:p toolchain:p clang:p hidapi:p dos2unix: git: unzip:" ;; + linux) + case $(grep ID /etc/os-release) in + *arch* | *manjaro* | *cachyos*) echo "zstd base-devel clang diffutils wget unzip zip hidapi dos2unix git" ;; + *debian* | *ubuntu*) echo "zstd build-essential clang-format diffutils wget unzip zip libhidapi-hidraw0 dos2unix git" ;; + *fedora*) echo "zstd clang diffutils which gcc git wget unzip zip hidapi dos2unix libusb-devel libusb1-devel libusb-compat-0.1-devel libusb0-devel git epel-release" ;; + *suse*) echo "zstd clang diffutils wget unzip zip libhidapi-hidraw0 dos2unix git libusb-1_0-devel gzip which" ;; + *gentoo*) echo "zstd sys-apps/diffutils wget unzip zip dev-libs/hidapi dos2unix dev-vcs/git dev-libs/libusb app-arch/gzip which" ;; + *) + echo >&2 + echo "Sorry, we don't recognize your distribution." >&2 + echo >&2 + echo "Proceeding with the installation, however you will need to install at least the following tools manually:" >&2 + echo " - make, git, curl, zstd, unzip, [lib]hidapi" >&2 + echo "Other tools may be required depending on your distribution." >&2 + echo >&2 + echo "Alternatively, if you prefer Docker, try using the docker image instead:" >&2 + echo " - https://docs.qmk.fm/#/getting_started_docker" >&2 + ;; + esac + ;; + *) + # We can only really support macOS, Windows, and Linux at this time due to `uv` requirements. + echo >&2 + echo "Sorry, we don't recognize your OS. Try using a compatible OS instead:" >&2 + echo " - https://docs.qmk.fm/newbs_getting_started#set-up-your-environment" >&2 + echo >&2 + echo "If you cannot use a compatible OS, you can try installing the \`qmk\` Python package manually using \`pip\`, most likely requiring a virtual environment:" >&2 + echo " % python3 -m pip install qmk" >&2 + echo >&2 + echo "All other dependencies will need to be installed manually, such as make, git, AVR and ARM toolchains, and associated flashing utilities." >&2 + echo >&2 + echo "**NOTE**: QMK does not provide official support for your environment. Here be dragons, you are on your own." >&2 + signal_execution_failure + ;; + esac + } + + print_package_manager_deps_and_delay() { + get_package_manager_deps | tr ' ' '\n' | sort | xargs -I'{}' echo " - {}" >&2 + exit_if_execution_failed + preinstall_delay || exit 1 + } + + install_package_manager_deps() { + # Install the necessary packages for the package manager + case $(fn_os) in + macos) + if [ -n "$(command -v brew 2>/dev/null || true)" ]; then + echo "It will also install the following system packages using 'brew':" >&2 + print_package_manager_deps_and_delay + + brew update + + local existing="" + local new="" + for dep in $(get_package_manager_deps); do + if brew list --formula | grep -q "^${dep}\$"; then + existing="${existing:-} $dep" + else + new="${new:-} $dep" + fi + done + + if [ -n "${existing:-}" ]; then + brew upgrade $existing + fi + if [ -n "${new:-}" ]; then + brew install $new + fi + else + echo "Please install 'brew' to continue. See https://brew.sh/ for more information." >&2 + exit 1 + fi + ;; + windows) + echo "It will also install the following packages using 'pacman'/'pacboy':" >&2 + print_package_manager_deps_and_delay + $(nsudo) pacman --needed --noconfirm --disable-download-timeout -S pactoys + $(nsudo) pacboy sync --needed --noconfirm --disable-download-timeout $(get_package_manager_deps) + ;; + linux) + case $(grep ID /etc/os-release) in + *arch* | *manjaro* | *cachyos*) + echo "It will also install the following system packages using 'pacman':" >&2 + print_package_manager_deps_and_delay + $(nsudo) pacman --needed --noconfirm -S $(get_package_manager_deps) + ;; + *debian* | *ubuntu*) + echo "It will also install the following system packages using 'apt':" >&2 + print_package_manager_deps_and_delay + $(nsudo) apt-get update + DEBIAN_FRONTEND=noninteractive \ + $(nsudo) apt-get --quiet --yes install $(get_package_manager_deps) + ;; + *fedora*) + echo "It will also install the following system packages using 'dnf':" >&2 + print_package_manager_deps_and_delay + # Some RHEL-likes need EPEL for hidapi + $(nsudo) dnf -y install epel-release 2>/dev/null || true + # RHEL-likes have some naming differences in libusb packages, so manually handle those + $(nsudo) dnf -y install $(get_package_manager_deps | tr ' ' '\n' | grep -v 'epel-release' | grep -v libusb | tr '\n' ' ') + for pkg in $(get_package_manager_deps | tr ' ' '\n' | grep libusb); do + $(nsudo) dnf -y install "$pkg" 2>/dev/null || true + done + ;; + *opensuse* | *suse*) + echo "It will also install development tools as well as the following system packages using 'zypper':" >&2 + print_package_manager_deps_and_delay + $(nsudo) zypper --non-interactive refresh + $(nsudo) zypper --non-interactive install -t pattern devel_basis devel_C_C++ + $(nsudo) zypper --non-interactive install $(get_package_manager_deps) + ;; + *gentoo*) + echo "It will also install the following system packages using 'emerge':" >&2 + print_package_manager_deps_and_delay + $(nsudo) emerge --sync + $(nsudo) emerge --noreplace --ask=n $(get_package_manager_deps | tr ' ' '\n') || signal_execution_failure + exit_if_execution_failed + ;; + *) + print_package_manager_deps_and_delay + echo "Proceeding with the installation, you will need to ensure prerequisites are installed." >&2 + ;; + esac + ;; + *) + print_package_manager_deps_and_delay + ;; + esac + } + + install_uv() { + # Install `uv` (or update as necessary) + download_url https://astral.sh/uv/install.sh - | TMPDIR="$(windows_ish_path "${TMPDIR:-}")" UV_INSTALL_DIR="$(windows_ish_path "${UV_INSTALL_DIR:-}")" sh + } + + setup_paths() { + # Set up the paths for any of the locations `uv` expects + if [ -n "${XDG_BIN_HOME:-}" ]; then + export PATH="$XDG_BIN_HOME:$PATH" + fi + if [ -n "${XDG_DATA_HOME:-}" ]; then + export PATH="$XDG_DATA_HOME/../bin:$PATH" + fi + [ ! -d "$HOME/.local/bin" ] || export PATH="$HOME/.local/bin:$PATH" + + if [ -n "${UV_INSTALL_DIR:-}" ]; then + export PATH="$UV_INSTALL_DIR/bin:$UV_INSTALL_DIR:$PATH" # cater for both "flat" and "hierarchical" installs of `uv` + fi + + if [ -n "${UV_TOOL_BIN_DIR:-}" ]; then + export PATH="$UV_TOOL_BIN_DIR:$PATH" + fi + } + + uv_command() { + if [ "$(fn_os)" = "windows" ]; then + UV_TOOL_DIR="$(windows_ish_path "${UV_TOOL_DIR:-}")" \ + UV_TOOL_BIN_DIR="$(windows_ish_path "${UV_TOOL_BIN_DIR:-}")" \ + uv "$@" + else + uv "$@" + fi + } + + install_qmk_cli() { + # Install the QMK CLI + uv_command tool install --force --with pip --upgrade --python $PYTHON_TARGET_VERSION qmk + + # QMK is installed to... + local qmk_tooldir="$(posix_ish_path "$(uv_command tool dir)/qmk")" + + # Activate the environment + if [ -e "$qmk_tooldir/bin" ]; then + . "$qmk_tooldir/bin/activate" + elif [ -e "$qmk_tooldir/Scripts" ]; then + . "$qmk_tooldir/Scripts/activate" + else + echo "Could not find the QMK environment to activate." >&2 + exit 1 + fi + + # Install the QMK dependencies + uv_command pip install --upgrade -r https://raw.githubusercontent.com/qmk/qmk_firmware/refs/heads/master/requirements.txt + uv_command pip install --upgrade -r https://raw.githubusercontent.com/qmk/qmk_firmware/refs/heads/master/requirements-dev.txt + + # Deactivate the environment + deactivate + } + + install_toolchains() { + # Get the latest toolchain release from https://github.com/qmk/qmk_toolchains + local latest_toolchains_release=$(github_api_call repos/qmk/qmk_toolchains/releases/latest - | grep -oE '"tag_name": "[^"]+' | grep -oE '[^"]+$') + # Download the specific release asset with a matching keyword + local toolchain_url=$(github_api_call repos/qmk/qmk_toolchains/releases/tags/$latest_toolchains_release - | grep -oE '"browser_download_url": "[^"]+"' | grep -oE 'https://[^"]+' | grep $(fn_os)$(fn_arch)) + if [ -z "$toolchain_url" ]; then + echo "No toolchain found for this OS/Arch combination." >&2 + exit 1 + fi + + # Download the toolchain release to the toolchains location + echo "Downloading compiler toolchains..." >&2 + local target_file="$QMK_DISTRIB_DIR/$(basename "$toolchain_url")" + download_url "$toolchain_url" "$target_file" + + # Extract the toolchain + echo "Extracting compiler toolchains to '$QMK_DISTRIB_DIR'..." >&2 + zstdcat "$target_file" | tar xf - -C "$QMK_DISTRIB_DIR" --strip-components=1 + } + + install_flashing_tools() { + local osarchvariant="$(fn_os)$(fn_arch)" + + # Special case for WSL + if [ -n "${WSL_INSTALL:-}" ] || [ -n "${WSL_DISTRO_NAME:-}" ] || [ -f /proc/sys/fs/binfmt_misc/WSLInterop ]; then + osarchvariant="windowsWSL" + fi + + # Get the latest flashing tools release from https://github.com/qmk/qmk_flashutils + local latest_flashutils_release=$(github_api_call repos/qmk/qmk_flashutils/releases/latest - | grep -oE '"tag_name": "[^"]+' | grep -oE '[^"]+$') + # Download the specific release asset with a matching keyword + local flashutils_url=$(github_api_call repos/qmk/qmk_flashutils/releases/tags/$latest_flashutils_release - | grep -oE '"browser_download_url": "[^"]+"' | grep -oE 'https://[^"]+' | grep "$osarchvariant") + if [ -z "$flashutils_url" ]; then + echo "No flashing tools found for this OS/Arch combination." >&2 + exit 1 + fi + + # Download the flashing tools release to the toolchains location + echo "Downloading flashing tools..." >&2 + local target_file="$QMK_DISTRIB_DIR/$(basename "$flashutils_url")" + download_url "$flashutils_url" "$target_file" + + # Extract the flashing tools + echo "Extracting flashing tools to '$QMK_DISTRIB_DIR'..." >&2 + zstdcat "$target_file" | tar xf - -C "$QMK_DISTRIB_DIR/bin" + # Move the release file to etc + mv "$QMK_DISTRIB_DIR/bin/flashutils_release"* "$QMK_DISTRIB_DIR/etc" + } + + install_linux_udev_rules() { + # Download the udev rules to the toolchains location + echo "Downloading QMK udev rules file..." >&2 + local qmk_rules_target_file="$QMK_DISTRIB_DIR/50-qmk.rules" + download_url "https://raw.githubusercontent.com/qmk/qmk_firmware/refs/heads/master/util/udev/50-qmk.rules" "$qmk_rules_target_file" + + # Install the udev rules -- path list is aligned with qmk doctor's linux.py + local udev_rules_paths=" + /usr/lib/udev/rules.d + /usr/local/lib/udev/rules.d + /run/udev/rules.d + /etc/udev/rules.d + " + for udev_rules_dir in $udev_rules_paths; do + if [ -d "$udev_rules_dir" ]; then + echo "Installing udev rules to $udev_rules_dir/50-qmk.rules ..." >&2 + $(nsudo) mv "$qmk_rules_target_file" "$udev_rules_dir" + $(nsudo) chown 0:0 "$udev_rules_dir/50-qmk.rules" + $(nsudo) chmod 644 "$udev_rules_dir/50-qmk.rules" + break + fi + done + + # Reload udev rules + if command -v udevadm >/dev/null 2>&1; then + echo "Reloading udev rules..." >&2 + $(nsudo) udevadm control --reload-rules || true + $(nsudo) udevadm trigger || true + else + echo "udevadm not found, skipping udev rules reload." >&2 + fi + } + + install_windows_drivers() { + # Get the latest driver installer release from https://github.com/qmk/qmk_driver_installer + local latest_driver_installer_release=$(github_api_call repos/qmk/qmk_driver_installer/releases/latest - | grep -oE '"tag_name": "[^"]+' | grep -oE '[^"]+$') + # Download the specific release asset + local driver_installer_url=$(github_api_call repos/qmk/qmk_driver_installer/releases/tags/$latest_driver_installer_release - | grep -oE '"browser_download_url": "[^"]+"' | grep -oE 'https://[^"]+' | grep '\.exe') + if [ -z "$driver_installer_url" ]; then + echo "No driver installer found." >&2 + exit 1 + fi + # Download the driver installer release to the toolchains location + echo "Downloading driver installer..." >&2 + local target_file="$QMK_DISTRIB_DIR/$(basename "$driver_installer_url")" + download_url "$driver_installer_url" "$target_file" + # Download the drivers list + download_url "https://raw.githubusercontent.com/qmk/qmk_firmware/refs/heads/master/util/drivers.txt" "$QMK_DISTRIB_DIR/drivers.txt" + # Execute the driver installer + cd "$QMK_DISTRIB_DIR" + cmd.exe //c "qmk_driver_installer.exe --all --force drivers.txt" + cd - + # Remove the temporary files + rm -f "$QMK_DISTRIB_DIR/qmk_driver_installer.exe" "$QMK_DISTRIB_DIR/drivers.txt" || true + } + + clean_tarballs() { + # Clean up the tarballs + rm -f "$QMK_DISTRIB_DIR"/*.tar.zst || true + } + + windows_ish_path() { + [ -n "$1" ] || return 0 + [ "$(uname -o 2>/dev/null || true)" = "Msys" ] && cygpath -w "$1" || echo "$1" + } + + posix_ish_path() { + [ -n "$1" ] || return 0 + [ "$(uname -o 2>/dev/null || true)" = "Msys" ] && cygpath -u "$1" || echo "$1" + } + + # Set the Python version we want to use with the QMK CLI + export PYTHON_TARGET_VERSION=${PYTHON_TARGET_VERSION:-3.14} + + # Windows/MSYS doesn't like `/tmp` so we need to set a different temporary directory. + # Also set the default `UV_INSTALL_DIR` and `QMK_DISTRIB_DIR` to locations which don't pollute the user's home directory, keeping the installation internal to MSYS. + if [ "$(uname -o 2>/dev/null || true)" = "Msys" ]; then + export TMPDIR="$(posix_ish_path "$TMP")" + export UV_INSTALL_DIR="$(posix_ish_path "${UV_INSTALL_DIR:-/opt/uv}")" + export QMK_DISTRIB_DIR="$(posix_ish_path "${QMK_DISTRIB_DIR:-/opt/qmk}")" + export UV_TOOL_DIR="$(posix_ish_path "${UV_TOOL_DIR:-"$UV_INSTALL_DIR/tools"}")" + export UV_TOOL_BIN_DIR="$(posix_ish_path "$UV_TOOL_DIR/bin")" + fi + + script_parse_args "$@" + + echo "This QMK CLI installation script will install \`uv\`, the QMK CLI, as well as QMK-supplied toolchains and flashing utilities." >&2 + [ -z "${SKIP_PACKAGE_MANAGER:-}" ] || { preinstall_delay || exit 1; } + [ -n "${SKIP_PACKAGE_MANAGER:-}" ] || install_package_manager_deps + [ -n "${SKIP_UV:-}" ] || install_uv + + # Make sure the usual `uv` and other associated directories are on the $PATH + setup_paths + + # Work out where we want to install the distribution and tools now that `uv` is installed + export QMK_DISTRIB_DIR="$(posix_ish_path "${QMK_DISTRIB_DIR:-$(printf 'import platformdirs\nprint(platformdirs.user_data_dir("qmk"))' | uv_command run --quiet --python $PYTHON_TARGET_VERSION --with platformdirs -)}")" + + # Clear out the distrib directory if necessary + if [ -z "${SKIP_CLEAN:-}" ] || [ -z "${SKIP_QMK_TOOLCHAINS:-}" -a -z "${SKIP_QMK_FLASHUTILS:-}" ]; then + if [ -d "$QMK_DISTRIB_DIR" ]; then + echo "Removing old QMK distribution..." >&2 + rm -rf "$QMK_DISTRIB_DIR" + fi + fi + mkdir -p "$QMK_DISTRIB_DIR" + + [ -n "${SKIP_QMK_CLI:-}" ] || install_qmk_cli + [ -n "${SKIP_QMK_TOOLCHAINS:-}" ] || install_toolchains + [ -n "${SKIP_QMK_FLASHUTILS:-}" ] || install_flashing_tools + if [ "$(uname -s 2>/dev/null || true)" = "Linux" ]; then + [ -n "${SKIP_UDEV_RULES:-}" ] || install_linux_udev_rules + fi + if [ "$(uname -o 2>/dev/null || true)" = "Msys" ]; then + [ -n "${SKIP_WINDOWS_DRIVERS:-}" ] || install_windows_drivers + fi + clean_tarballs + + # Notify the user that they may need to restart their shell to get the `qmk` command + echo >&2 + echo "QMK CLI installation complete." >&2 + echo "The QMK CLI has been installed to '$(posix_ish_path "$(dirname "$(command -v qmk)")")'." >&2 + echo "The QMK CLI venv has been created at '$(posix_ish_path "$(uv_command tool dir)/qmk")'." >&2 + echo "Toolchains and flashing utilities have been installed to '$QMK_DISTRIB_DIR'." >&2 + echo >&2 + echo "You may need to restart your shell to gain access to the 'qmk' command." >&2 + echo "Alternatively, add "$(posix_ish_path "$(dirname "$(command -v qmk)")")" to your \$PATH:" >&2 + echo " export PATH=\"$(posix_ish_path "$(dirname "$(command -v qmk)")"):\$PATH\"" >&2 + +} # this ensures the entire script is downloaded # diff --git a/util/qmk_install.sh b/util/qmk_install.sh index 3f49bd255a..0829a917ee 100755 --- a/util/qmk_install.sh +++ b/util/qmk_install.sh @@ -24,7 +24,7 @@ case $(uname -a) in . "$QMK_FIRMWARE_UTIL_DIR/install/linux_shared.sh" case $(grep ID /etc/os-release) in - *arch*|*manjaro*) + *arch*|*manjaro*|*cachyos*) . "$QMK_FIRMWARE_UTIL_DIR/install/arch.sh";; *debian*|*ubuntu*) . "$QMK_FIRMWARE_UTIL_DIR/install/debian.sh";; diff --git a/util/uf2conv.py b/util/uf2conv.py index 67cf92f169..a0507b00b9 100755 --- a/util/uf2conv.py +++ b/util/uf2conv.py @@ -142,9 +142,9 @@ def convert_to_uf2(file_content): return b"".join(outp) class Block: - def __init__(self, addr): + def __init__(self, addr, default_data=0xFF): self.addr = addr - self.bytes = bytearray(256) + self.bytes = bytearray([default_data] * 256) def encode(self, blockno, numblocks): global familyid @@ -210,24 +210,26 @@ def to_str(b): def get_drives(): drives = [] if sys.platform == "win32": - r = subprocess.check_output(["wmic", "PATH", "Win32_LogicalDisk", - "get", "DeviceID,", "VolumeName,", - "FileSystem,", "DriveType"]) - for line in to_str(r).split('\n'): - words = re.split(r'\s+', line) - if len(words) >= 3 and words[1] == "2" and words[2] == "FAT": - drives.append(words[0]) + r = subprocess.check_output([ + "powershell", + "-Command", + '(Get-WmiObject Win32_LogicalDisk -Filter "FileSystem=\'FAT\'").DeviceID' + ]) + drives = [drive.strip() for drive in to_str(r).splitlines()] else: - searchpaths = ["/media"] + searchpaths = ["/mnt", "/media"] if sys.platform == "darwin": searchpaths = ["/Volumes"] elif sys.platform == "linux": - searchpaths += ["/media/" + os.environ["USER"], '/run/media/' + os.environ["USER"]] + searchpaths += ["/media/" + os.environ["USER"], "/run/media/" + os.environ["USER"]] + if "SUDO_USER" in os.environ.keys(): + searchpaths += ["/media/" + os.environ["SUDO_USER"]] + searchpaths += ["/run/media/" + os.environ["SUDO_USER"]] for rootpath in searchpaths: if os.path.isdir(rootpath): for d in os.listdir(rootpath): - if os.path.isdir(rootpath): + if os.path.isdir(os.path.join(rootpath, d)): drives.append(os.path.join(rootpath, d)) diff --git a/util/uf2families.json b/util/uf2families.json index e35bf8d428..cd21783fe3 100644 --- a/util/uf2families.json +++ b/util/uf2families.json @@ -34,6 +34,11 @@ "short_name": "STM32WL", "description": "ST STM32WLxx" }, + { + "id": "0x22e0d6fc", + "short_name": "RTL8710B", + "description": "Realtek AmebaZ RTL8710B" + }, { "id": "0x2abc77ec", "short_name": "LPC55", @@ -49,6 +54,11 @@ "short_name": "GD32F350", "description": "GD32F350" }, + { + "id": "0x3379CFE2", + "short_name": "RTL8720D", + "description": "Realtek AmebaD RTL8720D" + }, { "id": "0x04240bdf", "short_name": "STM32L5", @@ -64,6 +74,11 @@ "short_name": "MIMXRT10XX", "description": "NXP i.MX RT10XX" }, + { + "id": "0x51e903a8", + "short_name": "XR809", + "description": "Xradiotech 809" + }, { "id": "0x53b80f00", "short_name": "STM32F7", @@ -104,11 +119,21 @@ "short_name": "STM32F0", "description": "ST STM32F0xx" }, + { + "id": "0x675a40b0", + "short_name": "BK7231U", + "description": "Beken 7231U/7231T" + }, { "id": "0x68ed2b88", "short_name": "SAMD21", "description": "Microchip (Atmel) SAMD21" }, + { + "id": "0x6a82cc42", + "short_name": "BK7251", + "description": "Beken 7251/7252" + }, { "id": "0x6b846188", "short_name": "STM32F3", @@ -119,6 +144,11 @@ "short_name": "STM32F407", "description": "ST STM32F407" }, + { + "id": "0x4e8f1c5d", + "short_name": "STM32H5", + "description": "ST STM32H5xx" + }, { "id": "0x6db66082", "short_name": "STM32H7", @@ -129,6 +159,11 @@ "short_name": "STM32WB", "description": "ST STM32WBxx" }, + { + "id": "0x7b3ef230", + "short_name": "BK7231N", + "description": "Beken 7231N" + }, { "id": "0x7eab61ed", "short_name": "ESP8266", @@ -144,11 +179,21 @@ "short_name": "STM32F407VG", "description": "ST STM32F407VG" }, + { + "id": "0x9fffd543", + "short_name": "RTL8710A", + "description": "Realtek Ameba1 RTL8710A" + }, { "id": "0xada52840", "short_name": "NRF52840", "description": "Nordic NRF52840" }, + { + "id": "0x820d9a5f", + "short_name": "NRF52820", + "description": "Nordic NRF52820_xxAA" + }, { "id": "0xbfdd4eee", "short_name": "ESP32S2", @@ -194,6 +239,26 @@ "short_name": "ESP32C61", "description": "ESP32-C61" }, + { + "id": "0xb6dd00af", + "short_name": "ESP32H21", + "description": "ESP32-H21" + }, + { + "id": "0x9e0baa8a", + "short_name": "ESP32H4", + "description": "ESP32-H4" + }, + { + "id": "0xde1270b7", + "short_name": "BL602", + "description": "Boufallo 602" + }, + { + "id": "0xe08f7564", + "short_name": "RTL8720C", + "description": "Realtek AmebaZ2 RTL8720C" + }, { "id": "0xe48bff56", "short_name": "RP2040", @@ -293,5 +358,25 @@ "id": "0x7be8976d", "short_name": "RA4M1", "description": "Renesas RA4M1" + }, + { + "id": "0x7410520a", + "short_name": "MAX32690", + "description": "Analog Devices MAX32690" + }, + { + "id": "0xd63f8632", + "short_name": "MAX32650", + "description": "Analog Devices MAX32650/1/2" + }, + { + "id": "0xf0c30d71", + "short_name": "MAX32666", + "description": "Analog Devices MAX32665/6" + }, + { + "id": "0x91d3fd18", + "short_name": "MAX78002", + "description": "Analog Devices MAX78002" } ]