Merge tag '0.18.16' into firmware22

lib/python/qmk/c_parse.py

@@ -24,6 +24,14 @@ def _get_chunks(it, size):
     return iter(lambda: tuple(islice(it, size)), ())
 
 
+def _preprocess_c_file(file):
+    """Load file and strip comments
+    """
+    file_contents = file.read_text(encoding='utf-8')
+    file_contents = comment_remover(file_contents)
+    return file_contents.replace('\\\n', '')
+
+
 def strip_line_comment(string):
     """Removes comments from a single line string.
     """
@@ -58,9 +66,7 @@ def find_layouts(file):
     parsed_layouts = {}
 
     # Search the file for LAYOUT macros and aliases
-    file_contents = file.read_text(encoding='utf-8')
-    file_contents = comment_remover(file_contents)
-    file_contents = file_contents.replace('\\\n', '')
+    file_contents = _preprocess_c_file(file)
 
     for line in file_contents.split('\n'):
         if layout_macro_define_regex.match(line.lstrip()) and '(' in line and 'LAYOUT' in line:
@@ -205,13 +211,23 @@ def _coerce_led_token(_type, value):
         return value_map[value]
 
 
+def _validate_led_config(matrix, matrix_rows, matrix_indexes, position, position_raw, flags):
+    # TODO: Improve crude parsing/validation
+    if len(matrix) != matrix_rows and len(matrix) != (matrix_rows / 2):
+        raise ValueError("Unable to parse g_led_config matrix data")
+    if len(position) != len(flags):
+        raise ValueError("Unable to parse g_led_config position data")
+    if len(matrix_indexes) and (max(matrix_indexes) >= len(flags)):
+        raise ValueError("OOB within g_led_config matrix data")
+    if not all(isinstance(n, int) for n in matrix_indexes):
+        raise ValueError("matrix indexes are not all ints")
+    if (len(position_raw) % 2) != 0:
+        raise ValueError("Malformed g_led_config position data")
+
+
 def _parse_led_config(file, matrix_cols, matrix_rows):
     """Return any 'raw' led/rgb matrix config
     """
-    file_contents = file.read_text(encoding='utf-8')
-    file_contents = comment_remover(file_contents)
-    file_contents = file_contents.replace('\\\n', '')
-
     matrix_raw = []
     position_raw = []
     flags = []
@@ -219,7 +235,7 @@ def _parse_led_config(file, matrix_cols, matrix_rows):
     found_led_config = False
     bracket_count = 0
     section = 0
-    for _type, value in lex(file_contents, CLexer()):
+    for _type, value in lex(_preprocess_c_file(file), CLexer()):
         # Assume g_led_config..stuff..;
         if value == 'g_led_config':
             found_led_config = True
@@ -242,23 +258,21 @@ def _parse_led_config(file, matrix_cols, matrix_rows):
                         position_raw.append(_coerce_led_token(_type, value))
                     if section == 3 and bracket_count == 2:
                         flags.append(_coerce_led_token(_type, value))
+                elif _type in [Token.Comment.Preproc]:
+                    # TODO: Promote to error
+                    return None
 
     # Slightly better intrim format
     matrix = list(_get_chunks(matrix_raw, matrix_cols))
     position = list(_get_chunks(position_raw, 2))
     matrix_indexes = list(filter(lambda x: x is not None, matrix_raw))
 
-    # If we have not found anything - bail
+    # If we have not found anything - bail with no error
     if not section:
         return None
 
-    # TODO: Improve crude parsing/validation
-    if len(matrix) != matrix_rows and len(matrix) != (matrix_rows / 2):
-        raise ValueError("Unable to parse g_led_config matrix data")
-    if len(position) != len(flags):
-        raise ValueError("Unable to parse g_led_config position data")
-    if len(matrix_indexes) and (max(matrix_indexes) >= len(flags)):
-        raise ValueError("OOB within g_led_config matrix data")
+    # Throw any validation errors
+    _validate_led_config(matrix, matrix_rows, matrix_indexes, position, position_raw, flags)
 
     return (matrix, position, flags)
 

lib/python/qmk/cli/__init__.py

@@ -15,6 +15,7 @@ from milc.questions import yesno
 import_names = {
     # A mapping of package name to importable name
     'pep8-naming': 'pep8ext_naming',
+    'pyserial': 'serial',
     'pyusb': 'usb.core',
     'qmk-dotty-dict': 'dotty_dict',
     'pillow': 'PIL'
@@ -59,6 +60,9 @@ subcommands = [
     'qmk.cli.generate.rules_mk',
     'qmk.cli.generate.version_h',
     'qmk.cli.hello',
+    'qmk.cli.import.kbfirmware',
+    'qmk.cli.import.keyboard',
+    'qmk.cli.import.keymap',
     'qmk.cli.info',
     'qmk.cli.json2c',
     'qmk.cli.lint',
@@ -91,7 +95,7 @@ def _install_deps(requirements):
 
     elif not os.access(sys.prefix, os.W_OK):
         # We can't write to sys.prefix, attempt to install locally
-        command.append('--local')
+        command.append('--user')
 
     return _run_cmd(*command, '-r', requirements)
 
@@ -156,6 +160,18 @@ def _broken_module_imports(requirements):
     return False
 
 
+def _yesno(*args):
+    """Wrapper to only prompt if interactive
+    """
+    return sys.stdout.isatty() and yesno(*args)
+
+
+def _eprint(errmsg):
+    """Wrapper to print to stderr
+    """
+    print(errmsg, file=sys.stderr)
+
+
 # Make sure our python is new enough
 #
 # Supported version information
@@ -177,7 +193,7 @@ def _broken_module_imports(requirements):
 # void: 3.9
 
 if sys.version_info[0] != 3 or sys.version_info[1] < 7:
-    print('Error: Your Python is too old! Please upgrade to Python 3.7 or later.')
+    _eprint('Error: Your Python is too old! Please upgrade to Python 3.7 or later.')
     exit(127)
 
 milc_version = __VERSION__.split('.')
@@ -185,7 +201,7 @@ milc_version = __VERSION__.split('.')
 if int(milc_version[0]) < 2 and int(milc_version[1]) < 4:
     requirements = Path('requirements.txt').resolve()
 
-    print(f'Your MILC library is too old! Please upgrade: python3 -m pip install -U -r {str(requirements)}')
+    _eprint(f'Your MILC library is too old! Please upgrade: python3 -m pip install -U -r {str(requirements)}')
     exit(127)
 
 # Make sure we can run binaries in the same directory as our Python interpreter
@@ -195,7 +211,7 @@ if python_dir not in os.environ['PATH'].split(':'):
     os.environ['PATH'] = ":".join((python_dir, os.environ['PATH']))
 
 # Check to make sure we have all our dependencies
-msg_install = f'Please run `{sys.executable} -m pip install -r %s` to install required python dependencies.'
+msg_install = f'\nPlease run `{sys.executable} -m pip install -r %s` to install required python dependencies.'
 args = sys.argv[1:]
 while args and args[0][0] == '-':
     del args[0]
@@ -204,24 +220,20 @@ safe_command = args and args[0] in safe_commands
 
 if not safe_command:
     if _broken_module_imports('requirements.txt'):
-        if yesno('Would you like to install the required Python modules?'):
+        if _yesno('Would you like to install the required Python modules?'):
             _install_deps('requirements.txt')
         else:
-            print()
-            print(msg_install % (str(Path('requirements.txt').resolve()),))
-            print()
+            _eprint(msg_install % (str(Path('requirements.txt').resolve()),))
             exit(1)
 
     if cli.config.user.developer and _broken_module_imports('requirements-dev.txt'):
-        if yesno('Would you like to install the required developer Python modules?'):
+        if _yesno('Would you like to install the required developer Python modules?'):
             _install_deps('requirements-dev.txt')
-        elif yesno('Would you like to disable developer mode?'):
+        elif _yesno('Would you like to disable developer mode?'):
             _run_cmd(sys.argv[0], 'config', 'user.developer=None')
         else:
-            print()
-            print(msg_install % (str(Path('requirements-dev.txt').resolve()),))
-            print('You can also turn off developer mode: qmk config user.developer=None')
-            print()
+            _eprint(msg_install % (str(Path('requirements-dev.txt').resolve()),))
+            _eprint('You can also turn off developer mode: qmk config user.developer=None')
             exit(1)
 
 # Import our subcommands
@@ -231,6 +243,6 @@ for subcommand in subcommands:
 
     except (ImportError, ModuleNotFoundError) as e:
         if safe_command:
-            print(f'Warning: Could not import {subcommand}: {e.__class__.__name__}, {e}')
+            _eprint(f'Warning: Could not import {subcommand}: {e.__class__.__name__}, {e}')
         else:
             raise

lib/python/qmk/cli/compile.py

@@ -32,8 +32,9 @@ def compile(cli):
     If a keyboard and keymap are provided this command will build a firmware based on that.
     """
     if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
-        command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
-        cli.run(command, capture_output=False, stdin=DEVNULL)
+        if cli.config.compile.keyboard and cli.config.compile.keymap:
+            command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
+            cli.run(command, capture_output=False, stdin=DEVNULL)
 
     # Build the environment vars
     envs = {}

lib/python/qmk/cli/doctor/linux.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 from milc import cli
 
-from qmk.constants import QMK_FIRMWARE
+from qmk.constants import QMK_FIRMWARE, BOOTLOADER_VIDS_PIDS
 from .check import CheckStatus
 
 
@@ -26,6 +26,18 @@ def _udev_rule(vid, pid=None, *args):
     return rule
 
 
+def _generate_desired_rules(bootloader_vids_pids):
+    rules = dict()
+    for bl in bootloader_vids_pids.keys():
+        rules[bl] = set()
+        for vid_pid in bootloader_vids_pids[bl]:
+            if bl == 'caterina' or bl == 'md-boot':
+                rules[bl].add(_udev_rule(vid_pid[0], vid_pid[1], 'ENV{ID_MM_DEVICE_IGNORE}="1"'))
+            else:
+                rules[bl].add(_udev_rule(vid_pid[0], vid_pid[1]))
+    return rules
+
+
 def _deprecated_udev_rule(vid, pid=None):
     """ Helper function that return udev rules
 
@@ -47,47 +59,8 @@ def check_udev_rules():
         Path("/run/udev/rules.d/"),
         Path("/etc/udev/rules.d/"),
     ]
-    desired_rules = {
-        'atmel-dfu': {
-            _udev_rule("03eb", "2fef"),  # ATmega16U2
-            _udev_rule("03eb", "2ff0"),  # ATmega32U2
-            _udev_rule("03eb", "2ff3"),  # ATmega16U4
-            _udev_rule("03eb", "2ff4"),  # ATmega32U4
-            _udev_rule("03eb", "2ff9"),  # AT90USB64
-            _udev_rule("03eb", "2ffa"),  # AT90USB162
-            _udev_rule("03eb", "2ffb")  # AT90USB128
-        },
-        'kiibohd': {_udev_rule("1c11", "b007")},
-        'stm32': {
-            _udev_rule("1eaf", "0003"),  # STM32duino
-            _udev_rule("0483", "df11")  # STM32 DFU
-        },
-        'bootloadhid': {_udev_rule("16c0", "05df")},
-        'usbasploader': {_udev_rule("16c0", "05dc")},
-        'massdrop': {_udev_rule("03eb", "6124", 'ENV{ID_MM_DEVICE_IGNORE}="1"')},
-        'caterina': {
-            # Spark Fun Electronics
-            _udev_rule("1b4f", "9203", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # Pro Micro 3V3/8MHz
-            _udev_rule("1b4f", "9205", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # Pro Micro 5V/16MHz
-            _udev_rule("1b4f", "9207", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # LilyPad 3V3/8MHz (and some Pro Micro clones)
-            # Pololu Electronics
-            _udev_rule("1ffb", "0101", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # A-Star 32U4
-            # Arduino SA
-            _udev_rule("2341", "0036", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # Leonardo
-            _udev_rule("2341", "0037", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # Micro
-            # Adafruit Industries LLC
-            _udev_rule("239a", "000c", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # Feather 32U4
-            _udev_rule("239a", "000d", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # ItsyBitsy 32U4 3V3/8MHz
-            _udev_rule("239a", "000e", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # ItsyBitsy 32U4 5V/16MHz
-            # dog hunter AG
-            _udev_rule("2a03", "0036", 'ENV{ID_MM_DEVICE_IGNORE}="1"'),  # Leonardo
-            _udev_rule("2a03", "0037", 'ENV{ID_MM_DEVICE_IGNORE}="1"')  # Micro
-        },
-        'hid-bootloader': {
-            _udev_rule("03eb", "2067"),  # QMK HID
-            _udev_rule("16c0", "0478")  # PJRC halfkay
-        }
-    }
+
+    desired_rules = _generate_desired_rules(BOOTLOADER_VIDS_PIDS)
 
     # These rules are no longer recommended, only use them to check for their presence.
     deprecated_rules = {

lib/python/qmk/cli/doctor/macos.py

@@ -8,6 +8,6 @@ from .check import CheckStatus
 def os_test_macos():
     """Run the Mac specific tests.
     """
-    cli.log.info("Detected {fg_cyan}macOS %s{fg_reset}.", platform.mac_ver()[0])
+    cli.log.info("Detected {fg_cyan}macOS %s (%s){fg_reset}.", platform.mac_ver()[0], 'Apple Silicon' if platform.processor() == 'arm' else 'Intel')
 
     return CheckStatus.OK

lib/python/qmk/cli/doctor/main.py

@@ -11,7 +11,7 @@ from milc.questions import yesno
 from qmk import submodules
 from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM
 from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
-from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_is_dirty, git_get_remotes, git_check_deviation
+from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation
 from qmk.commands import in_virtualenv
 
 
@@ -66,10 +66,32 @@ def git_tests():
                 if git_branch in ['master', 'develop'] and git_deviation:
                     cli.log.warning('{fg_yellow}The local "%s" branch contains commits not found in the upstream branch.', git_branch)
                     status = CheckStatus.WARNING
+                for branch in [git_branch, 'upstream/master', 'upstream/develop']:
+                    cli.log.info('- Latest %s: %s', branch, git_get_last_log_entry(branch))
+                for branch in ['upstream/master', 'upstream/develop']:
+                    cli.log.info('- Common ancestor with %s: %s', branch, git_get_common_ancestor(branch, 'HEAD'))
 
     return status
 
 
+def output_submodule_status():
+    """Prints out information related to the submodule status.
+    """
+    cli.log.info('Submodule status:')
+    sub_status = submodules.status()
+    for s in sub_status.keys():
+        sub_info = sub_status[s]
+        if 'name' in sub_info:
+            sub_name = sub_info['name']
+            sub_shorthash = sub_info['shorthash'] if 'shorthash' in sub_info else ''
+            sub_describe = sub_info['describe'] if 'describe' in sub_info else ''
+            sub_last_log_timestamp = sub_info['last_log_timestamp'] if 'last_log_timestamp' in sub_info else ''
+            if sub_last_log_timestamp != '':
+                cli.log.info(f'- {sub_name}: {sub_last_log_timestamp} -- {sub_describe} ({sub_shorthash})')
+            else:
+                cli.log.error(f'- {sub_name}: <<< missing or unknown >>>')
+
+
 @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.')
 @cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.')
 @cli.subcommand('Basic QMK environment checks')
@@ -129,6 +151,8 @@ def doctor(cli):
         elif sub_ok == CheckStatus.WARNING and status == CheckStatus.OK:
             status = CheckStatus.WARNING
 
+    output_submodule_status()
+
     # Report a summary of our findings to the user
     if status == CheckStatus.OK:
         cli.log.info('{fg_green}QMK is ready to go')

lib/python/qmk/cli/flash.py

@@ -4,6 +4,7 @@ You can compile a keymap already in the repo or using a QMK Configurator export.
 A bootloader must be specified.
 """
 from subprocess import DEVNULL
+import sys
 
 from argcomplete.completers import FilesCompleter
 from milc import cli
@@ -12,6 +13,7 @@ import qmk.path
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json
 from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.flashers import flasher
 
 
 def print_bootloader_help():
@@ -33,12 +35,15 @@ def print_bootloader_help():
     cli.echo('\tdfu-split-right')
     cli.echo('\tdfu-util-split-left')
     cli.echo('\tdfu-util-split-right')
+    cli.echo('\tuf2-split-left')
+    cli.echo('\tuf2-split-right')
     cli.echo('For more info, visit https://docs.qmk.fm/#/flashing')
 
 
-@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export JSON to compile.')
+@cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.')
 @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')
 @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')
+@cli.argument('-m', '--mcu', help='The MCU name. Required for HalfKay, HID, USBAspLoader and ISP flashing.')
 @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
 @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
@@ -51,6 +56,8 @@ def print_bootloader_help():
 def flash(cli):
     """Compile and or flash QMK Firmware or keyboard/layout
 
+    If a binary firmware is supplied, try to flash that.
+
     If a Configurator JSON export is supplied this command will create a new keymap. Keymap and Keyboard arguments
     will be ignored.
 
@@ -58,55 +65,69 @@ def flash(cli):
 
     If bootloader is omitted the make system will use the configured bootloader for that keyboard.
     """
-    if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
-        command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean')
-        cli.run(command, capture_output=False, stdin=DEVNULL)
+    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex']:
+        # Try to flash binary firmware
+        cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n')
+        try:
+            err, msg = flasher(cli.args.mcu, cli.args.filename)
+            if err:
+                cli.log.error(msg)
+                return False
+        except KeyboardInterrupt:
+            cli.log.info('Ctrl-C was pressed, exiting...')
+            sys.exit(0)
+
+    else:
+        if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
+            if cli.config.flash.keyboard and cli.config.flash.keymap:
+                command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean')
+                cli.run(command, capture_output=False, stdin=DEVNULL)
+
+        # Build the environment vars
+        envs = {}
+        for env in cli.args.env:
+            if '=' in env:
+                key, value = env.split('=', 1)
+                envs[key] = value
+            else:
+                cli.log.warning('Invalid environment variable: %s', env)
+
+        # Determine the compile command
+        command = ''
+
+        if cli.args.bootloaders:
+            # Provide usage and list bootloaders
+            cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
+            print_bootloader_help()
+            return False
+
+        if cli.args.filename:
+            # Handle compiling a configurator JSON
+            user_keymap = parse_configurator_json(cli.args.filename)
+            keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+            command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
+
+            cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
 
-    # Build the environment vars
-    envs = {}
-    for env in cli.args.env:
-        if '=' in env:
-            key, value = env.split('=', 1)
-            envs[key] = value
         else:
-            cli.log.warning('Invalid environment variable: %s', env)
+            if cli.config.flash.keyboard and cli.config.flash.keymap:
+                # Generate the make command for a specific keyboard/keymap.
+                command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
 
-    # Determine the compile command
-    command = ''
+            elif not cli.config.flash.keyboard:
+                cli.log.error('Could not determine keyboard!')
+            elif not cli.config.flash.keymap:
+                cli.log.error('Could not determine keymap!')
 
-    if cli.args.bootloaders:
-        # Provide usage and list bootloaders
-        cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
-        print_bootloader_help()
-        return False
+        # Compile the firmware, if we're able to
+        if command:
+            cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
+            if not cli.args.dry_run:
+                cli.echo('\n')
+                compile = cli.run(command, capture_output=False, stdin=DEVNULL)
+                return compile.returncode
 
-    if cli.args.filename:
-        # Handle compiling a configurator JSON
-        user_keymap = parse_configurator_json(cli.args.filename)
-        keymap_path = qmk.path.keymap(user_keymap['keyboard'])
-        command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
-
-        cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
-
-    else:
-        if cli.config.flash.keyboard and cli.config.flash.keymap:
-            # Generate the make command for a specific keyboard/keymap.
-            command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
-
-        elif not cli.config.flash.keyboard:
-            cli.log.error('Could not determine keyboard!')
-        elif not cli.config.flash.keymap:
-            cli.log.error('Could not determine keymap!')
-
-    # Compile the firmware, if we're able to
-    if command:
-        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
-        if not cli.args.dry_run:
-            cli.echo('\n')
-            compile = cli.run(command, capture_output=False, stdin=DEVNULL)
-            return compile.returncode
-
-    else:
-        cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
-        cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
-        return False
+        else:
+            cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')
+            cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]')
+            return False

lib/python/qmk/cli/generate/api.py

@@ -12,29 +12,14 @@ from qmk.json_encoders import InfoJSONEncoder
 from qmk.json_schema import json_load
 from qmk.keyboard import find_readme, list_keyboards
 
-TEMPLATE_PATH = Path('data/templates/api/')
+DATA_PATH = Path('data')
+TEMPLATE_PATH = DATA_PATH / 'templates/api/'
 BUILD_API_PATH = Path('.build/api_data/')
 
 
-@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't write the data to disk.")
-@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on partial name matches the supplied value. May be passed multiple times.")
-@cli.subcommand('Creates a new keymap for the keyboard of your choosing', hidden=False if cli.config.user.developer else True)
-def generate_api(cli):
-    """Generates the QMK API data.
+def _filtered_keyboard_list():
+    """Perform basic filtering of list_keyboards
     """
-    if BUILD_API_PATH.exists():
-        shutil.rmtree(BUILD_API_PATH)
-
-    shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH)
-
-    v1_dir = BUILD_API_PATH / 'v1'
-    keyboard_all_file = v1_dir / 'keyboards.json'  # A massive JSON containing everything
-    keyboard_list_file = v1_dir / 'keyboard_list.json'  # A simple list of keyboard targets
-    keyboard_aliases_file = v1_dir / 'keyboard_aliases.json'  # A list of historical keyboard names and their new name
-    keyboard_metadata_file = v1_dir / 'keyboard_metadata.json'  # All the data configurator/via needs for initialization
-    usb_file = v1_dir / 'usb.json'  # A mapping of USB VID/PID -> keyboard target
-
-    # Filter down when required
     keyboard_list = list_keyboards()
     if cli.args.filter:
         kb_list = []
@@ -42,6 +27,30 @@ def generate_api(cli):
             if any(i in keyboard_name for i in cli.args.filter):
                 kb_list.append(keyboard_name)
         keyboard_list = kb_list
+    return keyboard_list
+
+
+@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't write the data to disk.")
+@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on partial name matches the supplied value. May be passed multiple times.")
+@cli.subcommand('Generate QMK API data', hidden=False if cli.config.user.developer else True)
+def generate_api(cli):
+    """Generates the QMK API data.
+    """
+    v1_dir = BUILD_API_PATH / 'v1'
+    keyboard_all_file = v1_dir / 'keyboards.json'  # A massive JSON containing everything
+    keyboard_list_file = v1_dir / 'keyboard_list.json'  # A simple list of keyboard targets
+    keyboard_aliases_file = v1_dir / 'keyboard_aliases.json'  # A list of historical keyboard names and their new name
+    keyboard_metadata_file = v1_dir / 'keyboard_metadata.json'  # All the data configurator/via needs for initialization
+    usb_file = v1_dir / 'usb.json'  # A mapping of USB VID/PID -> keyboard target
+
+    if BUILD_API_PATH.exists():
+        shutil.rmtree(BUILD_API_PATH)
+
+    shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH)
+    shutil.copytree(DATA_PATH, v1_dir)
+
+    # Filter down when required
+    keyboard_list = _filtered_keyboard_list()
 
     kb_all = {}
     usb_list = {}

lib/python/qmk/cli/generate/config_h.py

@@ -134,6 +134,36 @@ def generate_config_items(kb_info_json, config_h_lines):
             config_h_lines.append(f'#endif // {config_key}')
 
 
+def generate_encoder_config(encoder_json, config_h_lines, postfix=''):
+    """Generate the config.h lines for encoders."""
+    a_pads = []
+    b_pads = []
+    resolutions = []
+    for encoder in encoder_json.get("rotary", []):
+        a_pads.append(encoder["pin_a"])
+        b_pads.append(encoder["pin_b"])
+        resolutions.append(encoder.get("resolution", None))
+
+    config_h_lines.append(f'#ifndef ENCODERS_PAD_A{postfix}')
+    config_h_lines.append(f'#   define ENCODERS_PAD_A{postfix} {{ { ", ".join(a_pads) } }}')
+    config_h_lines.append(f'#endif // ENCODERS_PAD_A{postfix}')
+
+    config_h_lines.append(f'#ifndef ENCODERS_PAD_B{postfix}')
+    config_h_lines.append(f'#   define ENCODERS_PAD_B{postfix} {{ { ", ".join(b_pads) } }}')
+    config_h_lines.append(f'#endif // ENCODERS_PAD_B{postfix}')
+
+    if None in resolutions:
+        cli.log.debug("Unable to generate ENCODER_RESOLUTION configuration")
+    elif len(set(resolutions)) == 1:
+        config_h_lines.append(f'#ifndef ENCODER_RESOLUTION{postfix}')
+        config_h_lines.append(f'#   define ENCODER_RESOLUTION{postfix} { resolutions[0] }')
+        config_h_lines.append(f'#endif // ENCODER_RESOLUTION{postfix}')
+    else:
+        config_h_lines.append(f'#ifndef ENCODER_RESOLUTIONS{postfix}')
+        config_h_lines.append(f'#   define ENCODER_RESOLUTIONS{postfix} {{ { ", ".join(map(str,resolutions)) } }}')
+        config_h_lines.append(f'#endif // ENCODER_RESOLUTIONS{postfix}')
+
+
 def generate_split_config(kb_info_json, config_h_lines):
     """Generate the config.h lines for split boards."""
     if 'primary' in kb_info_json['split']:
@@ -173,6 +203,9 @@ def generate_split_config(kb_info_json, config_h_lines):
     if 'right' in kb_info_json['split'].get('matrix_pins', {}):
         config_h_lines.append(matrix_pins(kb_info_json['split']['matrix_pins']['right'], '_RIGHT'))
 
+    if 'right' in kb_info_json['split'].get('encoder', {}):
+        generate_encoder_config(kb_info_json['split']['encoder']['right'], config_h_lines, '_RIGHT')
+
 
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@@ -198,6 +231,9 @@ def generate_config_h(cli):
     if 'matrix_pins' in kb_info_json:
         config_h_lines.append(matrix_pins(kb_info_json['matrix_pins']))
 
+    if 'encoder' in kb_info_json:
+        generate_encoder_config(kb_info_json['encoder'], config_h_lines)
+
     if 'split' in kb_info_json:
         generate_split_config(kb_info_json, config_h_lines)
 

lib/python/qmk/cli/generate/dfu_header.py

@@ -33,8 +33,8 @@ def generate_dfu_header(cli):
     kb_info_json = dotty(info_json(cli.config.generate_dfu_header.keyboard))
 
     keyboard_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once']
-    keyboard_h_lines.append(f'#define MANUFACTURER {kb_info_json["manufacturer"]}')
-    keyboard_h_lines.append(f'#define PRODUCT {kb_info_json["keyboard_name"]} Bootloader')
+    keyboard_h_lines.append(f'#define MANUFACTURER "{kb_info_json["manufacturer"]}"')
+    keyboard_h_lines.append(f'#define PRODUCT "{kb_info_json["keyboard_name"]} Bootloader"')
 
     # Optional
     if 'qmk_lufa_bootloader.esc_output' in kb_info_json:

lib/python/qmk/cli/generate/info_json.py

@@ -5,7 +5,7 @@ Compile an info.json for a particular keyboard and pretty-print it.
 import json
 
 from argcomplete.completers import FilesCompleter
-from jsonschema import Draft7Validator, RefResolver, validators
+from jsonschema import Draft202012Validator, RefResolver, validators
 from milc import cli
 from pathlib import Path
 
@@ -18,7 +18,7 @@ from qmk.path import is_keyboard, normpath
 
 
 def pruning_validator(validator_class):
-    """Extends Draft7Validator to remove properties that aren't specified in the schema.
+    """Extends Draft202012Validator to remove properties that aren't specified in the schema.
     """
     validate_properties = validator_class.VALIDATORS["properties"]
 
@@ -37,10 +37,10 @@ def strip_info_json(kb_info_json):
     """Remove the API-only properties from the info.json.
     """
     schema_store = compile_schema_store()
-    pruning_draft_7_validator = pruning_validator(Draft7Validator)
+    pruning_draft_validator = pruning_validator(Draft202012Validator)
     schema = schema_store['qmk.keyboard.v1']
     resolver = RefResolver.from_schema(schema_store['qmk.keyboard.v1'], store=schema_store)
-    validator = pruning_draft_7_validator(schema, resolver=resolver).validate
+    validator = pruning_draft_validator(schema, resolver=resolver).validate
 
     return validator(kb_info_json)
 

lib/python/qmk/cli/generate/keyboard_h.py

@@ -41,7 +41,7 @@ def generate_keyboard_h(cli):
     keyboard_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "quantum.h"']
 
     if not has_layout_h:
-        keyboard_h_lines.append('#pragma error("<keyboard>.h is only optional for data driven keyboards - kb.h == bad times")')
+        keyboard_h_lines.append('#error("<keyboard>.h is only optional for data driven keyboards - kb.h == bad times")')
 
     # Show the results
     dump_lines(cli.args.output, keyboard_h_lines, cli.args.quiet)

lib/python/qmk/cli/import/kbfirmware.py

@@ -0,0 +1,25 @@
+from milc import cli
+
+from qmk.importers import import_kbfirmware as _import_kbfirmware
+from qmk.path import FileType
+from qmk.json_schema import json_load
+
+
+@cli.argument('filename', type=FileType('r'), nargs='+', arg_only=True, help='file')
+@cli.subcommand('Import kbfirmware json export')
+def import_kbfirmware(cli):
+    filename = cli.args.filename[0]
+
+    data = json_load(filename)
+
+    cli.log.info(f'{{style_bright}}Importing {filename.name}.{{style_normal}}')
+    cli.echo('')
+
+    cli.log.warn("Support here is basic - Consider using 'qmk new-keyboard' instead")
+
+    kb_name = _import_kbfirmware(data)
+
+    cli.log.info(f'{{fg_green}}Imported a new keyboard named {{fg_cyan}}{kb_name}{{fg_green}}.{{fg_reset}}')
+    cli.log.info(f'To start working on things, `cd` into {{fg_cyan}}keyboards/{kb_name}{{fg_reset}},')
+    cli.log.info('or open the directory in your preferred text editor.')
+    cli.log.info(f"And build with {{fg_yellow}}qmk compile -kb {kb_name} -km default{{fg_reset}}.")

lib/python/qmk/cli/import/keyboard.py

@@ -0,0 +1,23 @@
+from milc import cli
+
+from qmk.importers import import_keyboard as _import_keyboard
+from qmk.path import FileType
+from qmk.json_schema import json_load
+
+
+@cli.argument('filename', type=FileType('r'), nargs='+', arg_only=True, help='file')
+@cli.subcommand('Import data-driven keyboard')
+def import_keyboard(cli):
+    filename = cli.args.filename[0]
+
+    data = json_load(filename)
+
+    cli.log.info(f'{{style_bright}}Importing {filename.name}.{{style_normal}}')
+    cli.echo('')
+
+    kb_name = _import_keyboard(data)
+
+    cli.log.info(f'{{fg_green}}Imported a new keyboard named {{fg_cyan}}{kb_name}{{fg_green}}.{{fg_reset}}')
+    cli.log.info(f'To start working on things, `cd` into {{fg_cyan}}keyboards/{kb_name}{{fg_reset}},')
+    cli.log.info('or open the directory in your preferred text editor.')
+    cli.log.info(f"And build with {{fg_yellow}}qmk compile -kb {kb_name} -km default{{fg_reset}}.")

lib/python/qmk/cli/import/keymap.py

@@ -0,0 +1,23 @@
+from milc import cli
+
+from qmk.importers import import_keymap as _import_keymap
+from qmk.path import FileType
+from qmk.json_schema import json_load
+
+
+@cli.argument('filename', type=FileType('r'), nargs='+', arg_only=True, help='file')
+@cli.subcommand('Import data-driven keymap')
+def import_keymap(cli):
+    filename = cli.args.filename[0]
+
+    data = json_load(filename)
+
+    cli.log.info(f'{{style_bright}}Importing {filename.name}.{{style_normal}}')
+    cli.echo('')
+
+    kb_name, km_name = _import_keymap(data)
+
+    cli.log.info(f'{{fg_green}}Imported a new keymap named {{fg_cyan}}{km_name}{{fg_green}}.{{fg_reset}}')
+    cli.log.info(f'To start working on things, `cd` into {{fg_cyan}}keyboards/{kb_name}/keymaps/{km_name}{{fg_reset}},')
+    cli.log.info('or open the directory in your preferred text editor.')
+    cli.log.info(f"And build with {{fg_yellow}}qmk compile -kb {kb_name} -km {km_name}{{fg_reset}}.")

lib/python/qmk/cli/lint.py

@@ -7,26 +7,43 @@ from milc import cli
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.info import info_json
 from qmk.keyboard import keyboard_completer, list_keyboards
-from qmk.keymap import locate_keymap
+from qmk.keymap import locate_keymap, list_keymaps
 from qmk.path import is_keyboard, keyboard
+from qmk.git import git_get_ignored_files
 
 
-def keymap_check(kb, km):
-    """Perform the keymap level checks.
+def _list_defaultish_keymaps(kb):
+    """Return default like keymaps for a given keyboard
+    """
+    defaultish = ['ansi', 'iso', 'via']
+
+    keymaps = set()
+    for x in list_keymaps(kb):
+        if x in defaultish or x.startswith('default'):
+            keymaps.add(x)
+
+    return keymaps
+
+
+def _handle_json_errors(kb, info):
+    """Convert any json errors into lint errors
     """
     ok = True
-    keymap_path = locate_keymap(kb, km)
-
-    if not keymap_path:
+    # Check for errors in the json
+    if info['parse_errors']:
         ok = False
-        cli.log.error("%s: Can't find %s keymap.", kb, km)
+        cli.log.error(f'{kb}: Errors found when generating info.json.')
 
+    if cli.config.lint.strict and info['parse_warnings']:
+        ok = False
+        cli.log.error(f'{kb}: Warnings found when generating info.json (Strict mode enabled.)')
     return ok
 
 
-def rules_mk_assignment_only(keyboard_path):
+def _rules_mk_assignment_only(kb):
     """Check the keyboard-level rules.mk to ensure it only has assignments.
     """
+    keyboard_path = keyboard(kb)
     current_path = Path()
     errors = []
 
@@ -58,10 +75,58 @@ def rules_mk_assignment_only(keyboard_path):
     return errors
 
 
+def keymap_check(kb, km):
+    """Perform the keymap level checks.
+    """
+    ok = True
+    keymap_path = locate_keymap(kb, km)
+
+    if not keymap_path:
+        ok = False
+        cli.log.error("%s: Can't find %s keymap.", kb, km)
+        return ok
+
+    # Additional checks
+    invalid_files = git_get_ignored_files(keymap_path.parent.as_posix())
+    for file in invalid_files:
+        cli.log.error(f'{kb}/{km}: The file "{file}" should not exist!')
+        ok = False
+
+    return ok
+
+
+def keyboard_check(kb):
+    """Perform the keyboard level checks.
+    """
+    ok = True
+    kb_info = info_json(kb)
+
+    if not _handle_json_errors(kb, kb_info):
+        ok = False
+
+    # Additional checks
+    rules_mk_assignment_errors = _rules_mk_assignment_only(kb)
+    if rules_mk_assignment_errors:
+        ok = False
+        cli.log.error('%s: Non-assignment code found in rules.mk. Move it to post_rules.mk instead.', kb)
+        for assignment_error in rules_mk_assignment_errors:
+            cli.log.error(assignment_error)
+
+    invalid_files = git_get_ignored_files(f'keyboards/{kb}/')
+    for file in invalid_files:
+        if 'keymap' in file:
+            continue
+        cli.log.error(f'{kb}: The file "{file}" should not exist!')
+        ok = False
+
+    return ok
+
+
 @cli.argument('--strict', action='store_true', help='Treat warnings as errors')
 @cli.argument('-kb', '--keyboard', completer=keyboard_completer, help='Comma separated list of keyboards to check')
 @cli.argument('-km', '--keymap', help='The keymap to check')
 @cli.argument('--all-kb', action='store_true', arg_only=True, help='Check all keyboards')
+@cli.argument('--all-km', action='store_true', arg_only=True, help='Check all keymaps')
 @cli.subcommand('Check keyboard and keymap for common mistakes.')
 @automagic_keyboard
 @automagic_keymap
@@ -73,7 +138,7 @@ def lint(cli):
     # Determine our keyboard list
     if cli.args.all_kb:
         if cli.args.keyboard:
-            cli.log.warning('Both --all-kb and --keyboard passed, --all-kb takes presidence.')
+            cli.log.warning('Both --all-kb and --keyboard passed, --all-kb takes precedence.')
 
         keyboard_list = list_keyboards()
     elif not cli.config.lint.keyboard:
@@ -89,38 +154,25 @@ def lint(cli):
             cli.log.error('No such keyboard: %s', kb)
             continue
 
-        # Gather data about the keyboard.
+        # Determine keymaps to also check
+        if cli.args.all_km:
+            keymaps = list_keymaps(kb)
+        elif cli.config.lint.keymap:
+            keymaps = {cli.config.lint.keymap}
+        else:
+            keymaps = _list_defaultish_keymaps(kb)
+            # Ensure that at least a 'default' keymap always exists
+            keymaps.add('default')
+
         ok = True
-        keyboard_path = keyboard(kb)
-        keyboard_info = info_json(kb)
 
-        # Check for errors in the info.json
-        if keyboard_info['parse_errors']:
+        # keyboard level checks
+        if not keyboard_check(kb):
             ok = False
-            cli.log.error('%s: Errors found when generating info.json.', kb)
-
-        if cli.config.lint.strict and keyboard_info['parse_warnings']:
-            ok = False
-            cli.log.error('%s: Warnings found when generating info.json (Strict mode enabled.)', kb)
-
-        # Check the rules.mk file(s)
-        rules_mk_assignment_errors = rules_mk_assignment_only(keyboard_path)
-        if rules_mk_assignment_errors:
-            ok = False
-            cli.log.error('%s: Non-assignment code found in rules.mk. Move it to post_rules.mk instead.', kb)
-            for assignment_error in rules_mk_assignment_errors:
-                cli.log.error(assignment_error)
 
         # Keymap specific checks
-        if cli.config.lint.keymap:
-            if not keymap_check(kb, cli.config.lint.keymap):
-                ok = False
-
-        # Check if all non-data driven macros exist in <keyboard.h>
-        for layout, data in keyboard_info['layouts'].items():
-            # Matrix data should be a list with exactly two integers: [0, 1]
-            if not data['c_macro'] and not all('matrix' in key_data.keys() or len(key_data) == 2 or all(isinstance(n, int) for n in key_data) for key_data in data['layout']):
-                cli.log.error(f'{kb}: "{layout}" has no "matrix" definition in either "info.json" or "<keyboard>.h"!')
+        for keymap in keymaps:
+            if not keymap_check(kb, keymap):
                 ok = False
 
         # Report status

lib/python/qmk/cli/multibuild.py

@@ -28,6 +28,7 @@ def _is_split(keyboard_name):
     return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False
 
 
+@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
 @cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.")
@@ -69,6 +70,7 @@ def multibuild(cli):
 all: {keyboard_safe}_binary
 {keyboard_safe}_binary:
 	@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" || true
+	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{cli.args.keymap}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}"
 	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{cli.args.keymap}" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false {' '.join(cli.args.env)} \\
 		>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" 2>&1 \\
 		|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}"
@@ -76,11 +78,26 @@ all: {keyboard_safe}_binary
 		|| {{ grep '\[WARNINGS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;33m[WARNINGS]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" ; }} \\
 		|| printf "Build %-64s \e[1;32m[OK]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}"
 	@rm -f "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" || true
-
 """# noqa
                 )
                 # yapf: enable
 
+                if cli.args.no_temp:
+                    # yapf: disable
+                    f.write(
+                        f"""\
+	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.elf" 2>/dev/null || true
+	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.map" 2>/dev/null || true
+	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.hex" 2>/dev/null || true
+	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.bin" 2>/dev/null || true
+	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.uf2" 2>/dev/null || true
+	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}" || true
+	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{cli.args.keymap}" || true
+"""# noqa
+                    )
+                    # yapf: enable
+                f.write('\n')
+
     cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
 
     # Check for failures

lib/python/qmk/constants.py

@@ -14,12 +14,13 @@ QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'
 MAX_KEYBOARD_SUBFOLDERS = 5
 
 # Supported processor types
-CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK66FX1M0', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F405', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L432', 'STM32L433', 'STM32L442', 'STM32L443', 'GD32VF103', 'WB32F3G71', 'WB32FQ95'
+CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'MK64FX512', 'MK66FX1M0', 'RP2040', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F405', 'STM32F407', 'STM32F411', 'STM32F446', 'STM32G431', 'STM32G474', 'STM32L412', 'STM32L422', 'STM32L432', 'STM32L433', 'STM32L442', 'STM32L443', 'GD32VF103', 'WB32F3G71', 'WB32FQ95'
 LUFA_PROCESSORS = 'at90usb162', 'atmega16u2', 'atmega32u2', 'atmega16u4', 'atmega32u4', 'at90usb646', 'at90usb647', 'at90usb1286', 'at90usb1287', None
 VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85'
 
 # Bootloaders of the supported processors
 MCU2BOOTLOADER = {
+    "RP2040": "rp2040",
     "MKL26Z64": "halfkay",
     "MK20DX128": "halfkay",
     "MK20DX256": "halfkay",
@@ -58,6 +59,59 @@ MCU2BOOTLOADER = {
     "atmega328": "usbasploader",
 }
 
+# Map of legacy keycodes that can be automatically updated
+LEGACY_KEYCODES = {  # Comment here is to force multiline formatting
+    'RESET': 'QK_BOOT'
+}
+
+# Map VID:PID values to bootloaders
+BOOTLOADER_VIDS_PIDS = {
+    'atmel-dfu': {
+        ("03eb", "2fef"),  # ATmega16U2
+        ("03eb", "2ff0"),  # ATmega32U2
+        ("03eb", "2ff3"),  # ATmega16U4
+        ("03eb", "2ff4"),  # ATmega32U4
+        ("03eb", "2ff9"),  # AT90USB64
+        ("03eb", "2ffa"),  # AT90USB162
+        ("03eb", "2ffb")  # AT90USB128
+    },
+    'kiibohd': {("1c11", "b007")},
+    'stm32-dfu': {
+        ("1eaf", "0003"),  # STM32duino
+        ("0483", "df11")  # STM32 DFU
+    },
+    'apm32-dfu': {("314b", "0106")},
+    'gd32v-dfu': {("28e9", "0189")},
+    'bootloadhid': {("16c0", "05df")},
+    'usbasploader': {("16c0", "05dc")},
+    'usbtinyisp': {("1782", "0c9f")},
+    'md-boot': {("03eb", "6124")},
+    'caterina': {
+        # pid.codes shared PID
+        ("1209", "2302"),  # Keyboardio Atreus 2 Bootloader
+        # Spark Fun Electronics
+        ("1b4f", "9203"),  # Pro Micro 3V3/8MHz
+        ("1b4f", "9205"),  # Pro Micro 5V/16MHz
+        ("1b4f", "9207"),  # LilyPad 3V3/8MHz (and some Pro Micro clones)
+        # Pololu Electronics
+        ("1ffb", "0101"),  # A-Star 32U4
+        # Arduino SA
+        ("2341", "0036"),  # Leonardo
+        ("2341", "0037"),  # Micro
+        # Adafruit Industries LLC
+        ("239a", "000c"),  # Feather 32U4
+        ("239a", "000d"),  # ItsyBitsy 32U4 3V3/8MHz
+        ("239a", "000e"),  # ItsyBitsy 32U4 5V/16MHz
+        # dog hunter AG
+        ("2a03", "0036"),  # Leonardo
+        ("2a03", "0037")  # Micro
+    },
+    'hid-bootloader': {
+        ("03eb", "2067"),  # QMK HID
+        ("16c0", "0478")  # PJRC halfkay
+    }
+}
+
 # Common format strings
 DATE_FORMAT = '%Y-%m-%d'
 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
@@ -67,13 +121,6 @@ TIME_FORMAT = '%H:%M:%S'
 COL_LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijilmnopqrstuvwxyz'
 ROW_LETTERS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop'
 
-# Mapping between info.json and config.h keys
-LED_INDICATORS = {
-    'caps_lock': 'LED_CAPS_LOCK_PIN',
-    'num_lock': 'LED_NUM_LOCK_PIN',
-    'scroll_lock': 'LED_SCROLL_LOCK_PIN',
-}
-
 # Constants that should match their counterparts in make
 BUILD_DIR = environ.get('BUILD_DIR', '.build')
 KEYBOARD_OUTPUT_PREFIX = f'{BUILD_DIR}/obj_'

lib/python/qmk/flashers.py

@@ -0,0 +1,203 @@
+import shutil
+import time
+import os
+import signal
+
+import usb.core
+
+from qmk.constants import BOOTLOADER_VIDS_PIDS
+from milc import cli
+
+# yapf: disable
+_PID_TO_MCU = {
+    '2fef': 'atmega16u2',
+    '2ff0': 'atmega32u2',
+    '2ff3': 'atmega16u4',
+    '2ff4': 'atmega32u4',
+    '2ff9': 'at90usb64',
+    '2ffa': 'at90usb162',
+    '2ffb': 'at90usb128'
+}
+
+AVRDUDE_MCU = {
+    'atmega32a': 'm32',
+    'atmega328p': 'm328p',
+    'atmega328': 'm328',
+}
+# yapf: enable
+
+
+class DelayedKeyboardInterrupt:
+    # Custom interrupt handler to delay the processing of Ctrl-C
+    # https://stackoverflow.com/a/21919644
+    def __enter__(self):
+        self.signal_received = False
+        self.old_handler = signal.signal(signal.SIGINT, self.handler)
+
+    def handler(self, sig, frame):
+        self.signal_received = (sig, frame)
+
+    def __exit__(self, type, value, traceback):
+        signal.signal(signal.SIGINT, self.old_handler)
+        if self.signal_received:
+            self.old_handler(*self.signal_received)
+
+
+# TODO: Make this more generic, so cli/doctor/check.py and flashers.py can share the code
+def _check_dfu_programmer_version():
+    # Return True if version is higher than 0.7.0: supports '--force'
+    check = cli.run(['dfu-programmer', '--version'], combined_output=True, timeout=5)
+    first_line = check.stdout.split('\n')[0]
+    version_number = first_line.split()[1]
+    maj, min_, bug = version_number.split('.')
+    if int(maj) >= 0 and int(min_) >= 7:
+        return True
+    else:
+        return False
+
+
+def _find_bootloader():
+    # To avoid running forever in the background, only look for bootloaders for 10min
+    start_time = time.time()
+    while time.time() - start_time < 600:
+        for bl in BOOTLOADER_VIDS_PIDS:
+            for vid, pid in BOOTLOADER_VIDS_PIDS[bl]:
+                vid_hex = int(f'0x{vid}', 0)
+                pid_hex = int(f'0x{pid}', 0)
+                with DelayedKeyboardInterrupt():
+                    # PyUSB does not like to be interrupted by Ctrl-C
+                    # therefore we catch the interrupt with a custom handler
+                    # and only process it once pyusb finished
+                    dev = usb.core.find(idVendor=vid_hex, idProduct=pid_hex)
+                if dev:
+                    if bl == 'atmel-dfu':
+                        details = _PID_TO_MCU[pid]
+                    elif bl == 'caterina':
+                        details = (vid_hex, pid_hex)
+                    elif bl == 'hid-bootloader':
+                        if vid == '16c0' and pid == '0478':
+                            details = 'halfkay'
+                        else:
+                            details = 'qmk-hid'
+                    elif bl == 'stm32-dfu' or bl == 'apm32-dfu' or bl == 'gd32v-dfu' or bl == 'kiibohd':
+                        details = (vid, pid)
+                    else:
+                        details = None
+                    return (bl, details)
+        time.sleep(0.1)
+    return (None, None)
+
+
+def _find_serial_port(vid, pid):
+    if 'windows' in cli.platform.lower():
+        from serial.tools.list_ports_windows import comports
+        platform = 'windows'
+    else:
+        from serial.tools.list_ports_posix import comports
+        platform = 'posix'
+
+    start_time = time.time()
+    # Caterina times out after 8 seconds
+    while time.time() - start_time < 8:
+        for port in comports():
+            port, desc, hwid = port
+            if f'{vid:04x}:{pid:04x}' in hwid.casefold():
+                if platform == 'windows':
+                    time.sleep(1)
+                    return port
+                else:
+                    start_time = time.time()
+                    # Wait until the port becomes writable before returning
+                    while time.time() - start_time < 8:
+                        if os.access(port, os.W_OK):
+                            return port
+                        else:
+                            time.sleep(0.5)
+                return None
+    return None
+
+
+def _flash_caterina(details, file):
+    port = _find_serial_port(details[0], details[1])
+    if port:
+        cli.run(['avrdude', '-p', 'atmega32u4', '-c', 'avr109', '-U', f'flash:w:{file}:i', '-P', port], capture_output=False)
+        return False
+    else:
+        return True
+
+
+def _flash_atmel_dfu(mcu, file):
+    force = '--force' if _check_dfu_programmer_version() else ''
+    cli.run(['dfu-programmer', mcu, 'erase', force], capture_output=False)
+    cli.run(['dfu-programmer', mcu, 'flash', force, file], capture_output=False)
+    cli.run(['dfu-programmer', mcu, 'reset'], capture_output=False)
+
+
+def _flash_hid_bootloader(mcu, details, file):
+    if details == 'halfkay':
+        if shutil.which('teensy-loader-cli'):
+            cmd = 'teensy-loader-cli'
+        elif shutil.which('teensy_loader_cli'):
+            cmd = 'teensy_loader_cli'
+
+    # Use 'hid_bootloader_cli' for QMK HID and as a fallback for HalfKay
+    if not cmd:
+        if shutil.which('hid_bootloader_cli'):
+            cmd = 'hid_bootloader_cli'
+        else:
+            return True
+
+    cli.run([cmd, f'-mmcu={mcu}', '-w', '-v', file], capture_output=False)
+
+
+def _flash_dfu_util(details, file):
+    # STM32duino
+    if details[0] == '1eaf' and details[1] == '0003':
+        cli.run(['dfu-util', '-a', '2', '-d', f'{details[0]}:{details[1]}', '-R', '-D', file], capture_output=False)
+    # kiibohd
+    elif details[0] == '1c11' and details[1] == 'b007':
+        cli.run(['dfu-util', '-a', '0', '-d', f'{details[0]}:{details[1]}', '-D', file], capture_output=False)
+    # STM32, APM32, or GD32V DFU
+    else:
+        cli.run(['dfu-util', '-a', '0', '-d', f'{details[0]}:{details[1]}', '-s', '0x08000000:leave', '-D', file], capture_output=False)
+
+
+def _flash_isp(mcu, programmer, file):
+    programmer = 'usbasp' if programmer == 'usbasploader' else 'usbtiny'
+    # Check if the provide mcu has an avrdude-specific name, otherwise pass on what the user provided
+    mcu = AVRDUDE_MCU.get(mcu, mcu)
+    cli.run(['avrdude', '-p', mcu, '-c', programmer, '-U', f'flash:w:{file}:i'], capture_output=False)
+
+
+def _flash_mdloader(file):
+    cli.run(['mdloader', '--first', '--download', file, '--restart'], capture_output=False)
+
+
+def flasher(mcu, file):
+    bl, details = _find_bootloader()
+    # Add a small sleep to avoid race conditions
+    time.sleep(1)
+    if bl == 'atmel-dfu':
+        _flash_atmel_dfu(details, file.name)
+    elif bl == 'caterina':
+        if _flash_caterina(details, file.name):
+            return (True, "The Caterina bootloader was found but is not writable. Check 'qmk doctor' output for advice.")
+    elif bl == 'hid-bootloader':
+        if mcu:
+            if _flash_hid_bootloader(mcu, details, file.name):
+                return (True, "Please make sure 'teensy_loader_cli' or 'hid_bootloader_cli' is available on your system.")
+        else:
+            return (True, "Specifying the MCU with '-m' is necessary for HalfKay/HID bootloaders!")
+    elif bl == 'stm32-dfu' or bl == 'apm32-dfu' or bl == 'gd32v-dfu' or bl == 'kiibohd':
+        _flash_dfu_util(details, file.name)
+    elif bl == 'usbasploader' or bl == 'usbtinyisp':
+        if mcu:
+            _flash_isp(mcu, bl, file.name)
+        else:
+            return (True, "Specifying the MCU with '-m' is necessary for ISP flashing!")
+    elif bl == 'md-boot':
+        _flash_mdloader(file.name)
+    else:
+        return (True, "Known bootloader found but flashing not currently supported!")
+
+    return (False, None)

lib/python/qmk/git.py

@@ -62,6 +62,25 @@ def git_get_tag():
         return git_tag.stdout.strip()
 
 
+def git_get_last_log_entry(branch_name):
+    """Retrieves the last log entry for the branch being worked on.
+    """
+    git_lastlog = cli.run(['git', '--no-pager', 'log', '--pretty=format:%ad (%h) -- %s', '--date=iso', '-n1', branch_name])
+
+    if git_lastlog.returncode == 0 and git_lastlog.stdout:
+        return git_lastlog.stdout.strip()
+
+
+def git_get_common_ancestor(branch_a, branch_b):
+    """Retrieves the common ancestor between for the two supplied branches.
+    """
+    git_merge_base = cli.run(['git', 'merge-base', branch_a, branch_b])
+    git_branchpoint_log = cli.run(['git', '--no-pager', 'log', '--pretty=format:%ad (%h) -- %s', '--date=iso', '-n1', git_merge_base.stdout.strip()])
+
+    if git_branchpoint_log.returncode == 0 and git_branchpoint_log.stdout:
+        return git_branchpoint_log.stdout.strip()
+
+
 def git_get_remotes():
     """Returns the current remotes for a repo.
     """
@@ -108,3 +127,12 @@ def git_check_deviation(active_branch):
     cli.run(['git', 'fetch', 'upstream', active_branch])
     deviations = cli.run(['git', '--no-pager', 'log', f'upstream/{active_branch}...{active_branch}'])
     return bool(deviations.returncode)
+
+
+def git_get_ignored_files(check_dir='.'):
+    """Return a list of files that would be captured by the current .gitignore
+    """
+    invalid = cli.run(['git', 'ls-files', '-c', '-o', '-i', '--exclude-from=.gitignore', check_dir])
+    if invalid.returncode != 0:
+        return []
+    return invalid.stdout.strip().splitlines()

lib/python/qmk/importers.py

@@ -0,0 +1,193 @@
+from dotty_dict import dotty
+from datetime import date
+from pathlib import Path
+import json
+
+from qmk.git import git_get_username
+from qmk.json_schema import validate
+from qmk.path import keyboard, keymap
+from qmk.constants import MCU2BOOTLOADER, LEGACY_KEYCODES
+from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder
+from qmk.json_schema import deep_update, json_load
+
+TEMPLATE = Path('data/templates/keyboard/')
+
+
+def replace_placeholders(src, dest, tokens):
+    """Replaces the given placeholders in each template file.
+    """
+    content = src.read_text()
+    for key, value in tokens.items():
+        content = content.replace(f'%{key}%', value)
+
+    dest.write_text(content)
+
+
+def _gen_dummy_keymap(name, info_data):
+    # Pick the first layout macro and just dump in KC_NOs or something?
+    (layout_name, layout_data), *_ = info_data["layouts"].items()
+    layout_length = len(layout_data["layout"])
+
+    keymap_data = {
+        "keyboard": name,
+        "layout": layout_name,
+        "layers": [["KC_NO" for _ in range(0, layout_length)]],
+    }
+
+    return keymap_data
+
+
+def _extract_kbfirmware_layout(kbf_data):
+    layout = []
+    for key in kbf_data['keyboard.keys']:
+        item = {
+            'matrix': [key['row'], key['col']],
+            'x': key['state']['x'],
+            'y': key['state']['y'],
+        }
+        if key['state']['w'] != 1:
+            item['w'] = key['state']['w']
+        if key['state']['h'] != 1:
+            item['h'] = key['state']['h']
+        layout.append(item)
+
+    return layout
+
+
+def _extract_kbfirmware_keymap(kbf_data):
+    keymap_data = {
+        'keyboard': kbf_data['keyboard.settings.name'].lower(),
+        'layout': 'LAYOUT',
+        'layers': [],
+    }
+
+    for i in range(15):
+        layer = []
+        for key in kbf_data['keyboard.keys']:
+            keycode = key['keycodes'][i]['id']
+            keycode = LEGACY_KEYCODES.get(keycode, keycode)
+            if '()' in keycode:
+                fields = key['keycodes'][i]['fields']
+                keycode = f'{keycode.split(")")[0]}{",".join(map(str, fields))})'
+            layer.append(keycode)
+        if set(layer) == {'KC_TRNS'}:
+            break
+        keymap_data['layers'].append(layer)
+
+    return keymap_data
+
+
+def import_keymap(keymap_data):
+    # Validate to ensure we don't have to deal with bad data - handles stdin/file
+    validate(keymap_data, 'qmk.keymap.v1')
+
+    kb_name = keymap_data['keyboard']
+    km_name = keymap_data['keymap']
+
+    km_folder = keymap(kb_name) / km_name
+    keyboard_keymap = km_folder / 'keymap.json'
+
+    # This is the deepest folder in the expected tree
+    keyboard_keymap.parent.mkdir(parents=True, exist_ok=True)
+
+    # Dump out all those lovely files
+    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder))
+
+    return (kb_name, km_name)
+
+
+def import_keyboard(info_data, keymap_data=None):
+    # Validate to ensure we don't have to deal with bad data - handles stdin/file
+    validate(info_data, 'qmk.api.keyboard.v1')
+
+    # And validate some more as everything is optional
+    if not all(key in info_data for key in ['keyboard_name', 'layouts']):
+        raise ValueError('invalid info.json')
+
+    kb_name = info_data['keyboard_name']
+
+    # bail
+    kb_folder = keyboard(kb_name)
+    if kb_folder.exists():
+        raise ValueError(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} already exists! Please choose a different name.')
+
+    if not keymap_data:
+        # TODO: if supports community then grab that instead
+        keymap_data = _gen_dummy_keymap(kb_name, info_data)
+
+    keyboard_info = kb_folder / 'info.json'
+    keyboard_keymap = kb_folder / 'keymaps' / 'default' / 'keymap.json'
+
+    # begin with making the deepest folder in the tree
+    keyboard_keymap.parent.mkdir(parents=True, exist_ok=True)
+
+    user_name = git_get_username()
+    if not user_name:
+        user_name = 'TODO'
+
+    tokens = {  # Comment here is to force multiline formatting
+        'YEAR': str(date.today().year),
+        'KEYBOARD': kb_name,
+        'USER_NAME': user_name,
+        'REAL_NAME': user_name,
+    }
+
+    # Dump out all those lovely files
+    for file in list(TEMPLATE.iterdir()):
+        replace_placeholders(file, kb_folder / file.name, tokens)
+
+    temp = json_load(keyboard_info)
+    deep_update(temp, info_data)
+
+    keyboard_info.write_text(json.dumps(temp, cls=InfoJSONEncoder))
+    keyboard_keymap.write_text(json.dumps(keymap_data, cls=KeymapJSONEncoder))
+
+    return kb_name
+
+
+def import_kbfirmware(kbfirmware_data):
+    kbf_data = dotty(kbfirmware_data)
+
+    diode_direction = ["COL2ROW", "ROW2COL"][kbf_data['keyboard.settings.diodeDirection']]
+    mcu = ["atmega32u2", "atmega32u4", "at90usb1286"][kbf_data['keyboard.controller']]
+    bootloader = MCU2BOOTLOADER.get(mcu, "custom")
+
+    layout = _extract_kbfirmware_layout(kbf_data)
+    keymap_data = _extract_kbfirmware_keymap(kbf_data)
+
+    # convert to d/d info.json
+    info_data = dotty({
+        "keyboard_name": kbf_data['keyboard.settings.name'].lower(),
+        "processor": mcu,
+        "bootloader": bootloader,
+        "diode_direction": diode_direction,
+        "matrix_pins": {
+            "cols": kbf_data['keyboard.pins.col'],
+            "rows": kbf_data['keyboard.pins.row'],
+        },
+        "layouts": {
+            "LAYOUT": {
+                "layout": layout,
+            }
+        }
+    })
+
+    if kbf_data['keyboard.pins.num'] or kbf_data['keyboard.pins.caps'] or kbf_data['keyboard.pins.scroll']:
+        if kbf_data['keyboard.pins.num']:
+            info_data['indicators.num_lock'] = kbf_data['keyboard.pins.num']
+        if kbf_data['keyboard.pins.caps']:
+            info_data['indicators.caps_lock'] = kbf_data['keyboard.pins.caps']
+        if kbf_data['keyboard.pins.scroll']:
+            info_data['indicators.scroll_lock'] = kbf_data['keyboard.pins.scroll']
+
+    if kbf_data['keyboard.pins.rgb']:
+        info_data['rgblight.animations.all'] = True
+        info_data['rgblight.led_count'] = kbf_data['keyboard.settings.rgbNum']
+        info_data['rgblight.pin'] = kbf_data['keyboard.pins.rgb']
+
+    if kbf_data['keyboard.pins.led']:
+        info_data['backlight.levels'] = kbf_data['keyboard.settings.backlightLevels']
+        info_data['backlight.pin'] = kbf_data['keyboard.pins.led']
+
+    # delegate as if it were a regular keyboard import
+    return import_keyboard(info_data.to_dict(), keymap_data)

lib/python/qmk/info.py

@@ -26,13 +26,6 @@ def _valid_community_layout(layout):
     return (Path('layouts/default') / layout).exists()
 
 
-def _remove_newlines_from_labels(layouts):
-    for layout_name, layout_json in layouts.items():
-        for key in layout_json['layout']:
-            if '\n' in key['label']:
-                key['label'] = key['label'].split('\n')[0]
-
-
 def info_json(keyboard):
     """Generate the info.json data for a specific keyboard.
     """
@@ -111,23 +104,13 @@ def info_json(keyboard):
     # Check that the reported matrix size is consistent with the actual matrix size
     _check_matrix(info_data)
 
-    # Remove newline characters from layout labels
-    _remove_newlines_from_labels(layouts)
-
     return info_data
 
 
 def _extract_features(info_data, rules):
     """Find all the features enabled in rules.mk.
     """
-    # Special handling for bootmagic which also supports a "lite" mode.
-    if rules.get('BOOTMAGIC_ENABLE') == 'lite':
-        rules['BOOTMAGIC_LITE_ENABLE'] = 'on'
-        del rules['BOOTMAGIC_ENABLE']
-    if rules.get('BOOTMAGIC_ENABLE') == 'full':
-        rules['BOOTMAGIC_ENABLE'] = 'on'
-
-    # Process the rest of the rules as booleans
+    # Process booleans rules
     for key, value in rules.items():
         if key.endswith('_ENABLE'):
             key = '_'.join(key.split('_')[:-1]).lower()
@@ -228,6 +211,66 @@ def _extract_audio(info_data, config_c):
         info_data['audio'] = {'pins': audio_pins}
 
 
+def _extract_encoders_values(config_c, postfix=''):
+    """Common encoder extraction logic
+    """
+    a_pad = config_c.get(f'ENCODERS_PAD_A{postfix}', '').replace(' ', '')[1:-1]
+    b_pad = config_c.get(f'ENCODERS_PAD_B{postfix}', '').replace(' ', '')[1:-1]
+    resolutions = config_c.get(f'ENCODER_RESOLUTIONS{postfix}', '').replace(' ', '')[1:-1]
+
+    default_resolution = config_c.get('ENCODER_RESOLUTION', None)
+
+    if a_pad and b_pad:
+        a_pad = list(filter(None, a_pad.split(',')))
+        b_pad = list(filter(None, b_pad.split(',')))
+        resolutions = list(filter(None, resolutions.split(',')))
+        if default_resolution:
+            resolutions += [default_resolution] * (len(a_pad) - len(resolutions))
+
+        encoders = []
+        for index in range(len(a_pad)):
+            encoder = {'pin_a': a_pad[index], 'pin_b': b_pad[index]}
+            if index < len(resolutions):
+                encoder['resolution'] = int(resolutions[index])
+            encoders.append(encoder)
+
+        return encoders
+
+
+def _extract_encoders(info_data, config_c):
+    """Populate data about encoder pins
+    """
+    encoders = _extract_encoders_values(config_c)
+    if encoders:
+        if 'encoder' not in info_data:
+            info_data['encoder'] = {}
+
+        if 'rotary' in info_data['encoder']:
+            _log_warning(info_data, 'Encoder config is specified in both config.h and info.json (encoder.rotary) (Value: %s), the config.h value wins.' % info_data['encoder']['rotary'])
+
+        info_data['encoder']['rotary'] = encoders
+
+
+def _extract_split_encoders(info_data, config_c):
+    """Populate data about split encoder pins
+    """
+    encoders = _extract_encoders_values(config_c, '_RIGHT')
+    if encoders:
+        if 'split' not in info_data:
+            info_data['split'] = {}
+
+        if 'encoder' not in info_data['split']:
+            info_data['split']['encoder'] = {}
+
+        if 'right' not in info_data['split']['encoder']:
+            info_data['split']['encoder']['right'] = {}
+
+        if 'rotary' in info_data['split']['encoder']['right']:
+            _log_warning(info_data, 'Encoder config is specified in both config.h and info.json (encoder.rotary) (Value: %s), the config.h value wins.' % info_data['split']['encoder']['right']['rotary'])
+
+        info_data['split']['encoder']['right']['rotary'] = encoders
+
+
 def _extract_secure_unlock(info_data, config_c):
     """Populate data about the secure unlock sequence
     """
@@ -324,12 +367,10 @@ def _extract_split_right_pins(info_data, config_c):
     # Figure out the right half matrix pins
     row_pins = config_c.get('MATRIX_ROW_PINS_RIGHT', '').replace('{', '').replace('}', '').strip()
     col_pins = config_c.get('MATRIX_COL_PINS_RIGHT', '').replace('{', '').replace('}', '').strip()
-    unused_pin_text = config_c.get('UNUSED_PINS_RIGHT')
-    unused_pins = unused_pin_text.replace('{', '').replace('}', '').strip() if isinstance(unused_pin_text, str) else None
     direct_pins = config_c.get('DIRECT_PINS_RIGHT', '').replace(' ', '')[1:-1]
 
-    if row_pins and col_pins:
-        if info_data.get('split', {}).get('matrix_pins', {}).get('right') in info_data:
+    if row_pins or col_pins or direct_pins:
+        if info_data.get('split', {}).get('matrix_pins', {}).get('right', None):
             _log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
 
         if 'split' not in info_data:
@@ -341,37 +382,14 @@ def _extract_split_right_pins(info_data, config_c):
         if 'right' not in info_data['split']['matrix_pins']:
             info_data['split']['matrix_pins']['right'] = {}
 
-        info_data['split']['matrix_pins']['right'] = {
-            'cols': _extract_pins(col_pins),
-            'rows': _extract_pins(row_pins),
-        }
+        if col_pins:
+            info_data['split']['matrix_pins']['right']['cols'] = _extract_pins(col_pins)
 
-    if direct_pins:
-        if info_data.get('split', {}).get('matrix_pins', {}).get('right', {}):
-            _log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.')
+        if row_pins:
+            info_data['split']['matrix_pins']['right']['rows'] = _extract_pins(row_pins)
 
-        if 'split' not in info_data:
-            info_data['split'] = {}
-
-        if 'matrix_pins' not in info_data['split']:
-            info_data['split']['matrix_pins'] = {}
-
-        if 'right' not in info_data['split']['matrix_pins']:
-            info_data['split']['matrix_pins']['right'] = {}
-
-        info_data['split']['matrix_pins']['right']['direct'] = _extract_direct_matrix(direct_pins)
-
-    if unused_pins:
-        if 'split' not in info_data:
-            info_data['split'] = {}
-
-        if 'matrix_pins' not in info_data['split']:
-            info_data['split']['matrix_pins'] = {}
-
-        if 'right' not in info_data['split']['matrix_pins']:
-            info_data['split']['matrix_pins']['right'] = {}
-
-        info_data['split']['matrix_pins']['right']['unused'] = _extract_pins(unused_pins)
+        if direct_pins:
+            info_data['split']['matrix_pins']['right']['direct'] = _extract_direct_matrix(direct_pins)
 
 
 def _extract_matrix_info(info_data, config_c):
@@ -379,8 +397,6 @@ def _extract_matrix_info(info_data, config_c):
     """
     row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip()
     col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip()
-    unused_pin_text = config_c.get('UNUSED_PINS')
-    unused_pins = unused_pin_text.replace('{', '').replace('}', '').strip() if isinstance(unused_pin_text, str) else None
     direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1]
     info_snippet = {}
 
@@ -406,12 +422,6 @@ def _extract_matrix_info(info_data, config_c):
 
         info_snippet['direct'] = _extract_direct_matrix(direct_pins)
 
-    if unused_pins:
-        if 'matrix_pins' not in info_data:
-            info_data['matrix_pins'] = {}
-
-        info_snippet['unused'] = _extract_pins(unused_pins)
-
     if config_c.get('CUSTOM_MATRIX', 'no') != 'no':
         if 'matrix_pins' in info_data and 'custom' in info_data['matrix_pins']:
             _log_warning(info_data, 'Custom Matrix is specified in both info.json and config.h, the config.h values win.')
@@ -440,6 +450,47 @@ def _extract_device_version(info_data):
             info_data['usb']['device_version'] = f'{major}.{minor}.{revision}'
 
 
+def _config_to_json(key_type, config_value):
+    """Convert config value using spec
+    """
+    if key_type.startswith('array'):
+        if '.' in key_type:
+            key_type, array_type = key_type.split('.', 1)
+        else:
+            array_type = None
+
+        config_value = config_value.replace('{', '').replace('}', '').strip()
+
+        if array_type == 'int':
+            return list(map(int, config_value.split(',')))
+        else:
+            return config_value.split(',')
+
+    elif key_type == 'bool':
+        return config_value in true_values
+
+    elif key_type == 'hex':
+        return '0x' + config_value[2:].upper()
+
+    elif key_type == 'list':
+        return config_value.split()
+
+    elif key_type == 'int':
+        return int(config_value)
+
+    elif key_type == 'str':
+        return config_value.strip('"')
+
+    elif key_type == 'bcd_version':
+        major = int(config_value[2:4])
+        minor = int(config_value[4])
+        revision = int(config_value[5])
+
+        return f'{major}.{minor}.{revision}'
+
+    return config_value
+
+
 def _extract_config_h(info_data, config_c):
     """Pull some keyboard information from existing config.h files
     """
@@ -452,47 +503,23 @@ def _extract_config_h(info_data, config_c):
         key_type = info_dict.get('value_type', 'raw')
 
         try:
+            replace_with = info_dict.get('replace_with')
+            if config_key in config_c and info_dict.get('invalid', False):
+                if replace_with:
+                    _log_error(info_data, '%s in config.h is no longer a valid option and should be replaced with %s' % (config_key, replace_with))
+                else:
+                    _log_error(info_data, '%s in config.h is no longer a valid option and should be removed' % config_key)
+            elif config_key in config_c and info_dict.get('deprecated', False):
+                if replace_with:
+                    _log_warning(info_data, '%s in config.h is deprecated in favor of %s and will be removed at a later date' % (config_key, replace_with))
+                else:
+                    _log_warning(info_data, '%s in config.h is deprecated and will be removed at a later date' % config_key)
+
             if config_key in config_c and info_dict.get('to_json', True):
                 if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
                     _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
 
-                if key_type.startswith('array'):
-                    if '.' in key_type:
-                        key_type, array_type = key_type.split('.', 1)
-                    else:
-                        array_type = None
-
-                    config_value = config_c[config_key].replace('{', '').replace('}', '').strip()
-
-                    if array_type == 'int':
-                        dotty_info[info_key] = list(map(int, config_value.split(',')))
-                    else:
-                        dotty_info[info_key] = config_value.split(',')
-
-                elif key_type == 'bool':
-                    dotty_info[info_key] = config_c[config_key] in true_values
-
-                elif key_type == 'hex':
-                    dotty_info[info_key] = '0x' + config_c[config_key][2:].upper()
-
-                elif key_type == 'list':
-                    dotty_info[info_key] = config_c[config_key].split()
-
-                elif key_type == 'int':
-                    dotty_info[info_key] = int(config_c[config_key])
-
-                elif key_type == 'str':
-                    dotty_info[info_key] = config_c[config_key].strip('"')
-
-                elif key_type == 'bcd_version':
-                    major = int(config_c[config_key][2:4])
-                    minor = int(config_c[config_key][4])
-                    revision = int(config_c[config_key][5])
-
-                    dotty_info[info_key] = f'{major}.{minor}.{revision}'
-
-                else:
-                    dotty_info[info_key] = config_c[config_key]
+                dotty_info[info_key] = _config_to_json(key_type, config_c[config_key])
 
         except Exception as e:
             _log_warning(info_data, f'{config_key}->{info_key}: {e}')
@@ -506,6 +533,8 @@ def _extract_config_h(info_data, config_c):
     _extract_split_main(info_data, config_c)
     _extract_split_transport(info_data, config_c)
     _extract_split_right_pins(info_data, config_c)
+    _extract_encoders(info_data, config_c)
+    _extract_split_encoders(info_data, config_c)
     _extract_device_version(info_data)
 
     return info_data
@@ -547,40 +576,23 @@ def _extract_rules_mk(info_data, rules):
         key_type = info_dict.get('value_type', 'raw')
 
         try:
+            replace_with = info_dict.get('replace_with')
+            if rules_key in rules and info_dict.get('invalid', False):
+                if replace_with:
+                    _log_error(info_data, '%s in rules.mk is no longer a valid option and should be replaced with %s' % (rules_key, replace_with))
+                else:
+                    _log_error(info_data, '%s in rules.mk is no longer a valid option and should be removed' % rules_key)
+            elif rules_key in rules and info_dict.get('deprecated', False):
+                if replace_with:
+                    _log_warning(info_data, '%s in rules.mk is deprecated in favor of %s and will be removed at a later date' % (rules_key, replace_with))
+                else:
+                    _log_warning(info_data, '%s in rules.mk is deprecated and will be removed at a later date' % rules_key)
+
             if rules_key in rules and info_dict.get('to_json', True):
                 if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
                     _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
 
-                if key_type.startswith('array'):
-                    if '.' in key_type:
-                        key_type, array_type = key_type.split('.', 1)
-                    else:
-                        array_type = None
-
-                    rules_value = rules[rules_key].replace('{', '').replace('}', '').strip()
-
-                    if array_type == 'int':
-                        dotty_info[info_key] = list(map(int, rules_value.split(',')))
-                    else:
-                        dotty_info[info_key] = rules_value.split(',')
-
-                elif key_type == 'list':
-                    dotty_info[info_key] = rules[rules_key].split()
-
-                elif key_type == 'bool':
-                    dotty_info[info_key] = rules[rules_key] in true_values
-
-                elif key_type == 'hex':
-                    dotty_info[info_key] = '0x' + rules[rules_key][2:].upper()
-
-                elif key_type == 'int':
-                    dotty_info[info_key] = int(rules[rules_key])
-
-                elif key_type == 'str':
-                    dotty_info[info_key] = rules[rules_key].strip('"')
-
-                else:
-                    dotty_info[info_key] = rules[rules_key]
+                dotty_info[info_key] = _config_to_json(key_type, rules[rules_key])
 
         except Exception as e:
             _log_warning(info_data, f'{rules_key}->{info_key}: {e}')
@@ -821,8 +833,11 @@ def merge_info_jsons(keyboard, info_data):
                     for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):
                         existing_key.update(new_key)
             else:
-                layout['c_macro'] = False
-                info_data['layouts'][layout_name] = layout
+                if not all('matrix' in key_data.keys() for key_data in layout['layout']):
+                    _log_error(info_data, f'Layout "{layout_name}" has no "matrix" definition in either "info.json" or "<keyboard>.h"!')
+                else:
+                    layout['c_macro'] = False
+                    info_data['layouts'][layout_name] = layout
 
         # Update info_data with the new data
         if 'layouts' in new_info_data:

lib/python/qmk/json_schema.py

@@ -68,11 +68,7 @@ def create_validator(schema):
     schema_store = compile_schema_store()
     resolver = jsonschema.RefResolver.from_schema(schema_store[schema], store=schema_store)
 
-    # TODO: Remove this after the jsonschema>=4 requirement had time to reach users
-    try:
-        return jsonschema.Draft202012Validator(schema_store[schema], resolver=resolver).validate
-    except AttributeError:
-        return jsonschema.Draft7Validator(schema_store[schema], resolver=resolver).validate
+    return jsonschema.Draft202012Validator(schema_store[schema], resolver=resolver).validate
 
 
 def validate(data, schema):

lib/python/qmk/keyboard.py

@@ -103,7 +103,7 @@ def list_keyboards():
     """
     # We avoid pathlib here because this is performance critical code.
     kb_wildcard = os.path.join(base_path, "**", "rules.mk")
-    paths = [path for path in glob(kb_wildcard, recursive=True) if 'keymaps' not in path]
+    paths = [path for path in glob(kb_wildcard, recursive=True) if os.path.sep + 'keymaps' + os.path.sep not in path]
 
     return sorted(set(map(resolve_keyboard, map(_find_name, paths))))
 

lib/python/qmk/keymap.py

@@ -12,7 +12,7 @@ from pygments.token import Token
 from pygments import lex
 
 import qmk.path
-from qmk.keyboard import find_keyboard_from_dir, rules_mk
+from qmk.keyboard import find_keyboard_from_dir, rules_mk, keyboard_folder
 from qmk.errors import CppError
 
 # The `keymap.c` template to use when a keyboard doesn't have its own
@@ -357,7 +357,7 @@ def locate_keymap(keyboard, keymap):
     checked_dirs = ''
     keymap_path = ''
 
-    for dir in keyboard.split('/'):
+    for dir in keyboard_folder(keyboard).split('/'):
         if checked_dirs:
             checked_dirs = '/'.join((checked_dirs, dir))
         else:

lib/python/qmk/submodules.py

@@ -11,7 +11,11 @@ def status():
         {
             'name': 'submodule_name',
             'status': None/False/True,
-            'githash': '<sha-1 hash for the submodule>
+            'githash': '<sha-1 hash for the submodule>'
+            'shorthash': '<short hash for the submodule>'
+            'describe': '<output of `git describe --tags`>'
+            'last_log_message': 'log message'
+            'last_log_timestamp': 'timestamp'
         }
 
     status is None when the submodule doesn't exist, False when it's out of date, and True when it's current
@@ -36,6 +40,26 @@ def status():
         else:
             raise ValueError('Unknown `git submodule status` sha-1 prefix character: "%s"' % status)
 
+    submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1'])
+    for log_line in submodule_logs.stdout.split('\n'):
+        if not log_line:
+            continue
+
+        r = log_line.split('\x01')
+        submodule = r[0]
+        submodules[submodule]['shorthash'] = r[1] if len(r) > 1 else ''
+        submodules[submodule]['last_log_timestamp'] = r[2] if len(r) > 2 else ''
+        submodules[submodule]['last_log_message'] = r[3] if len(r) > 3 else ''
+
+    submodule_tags = cli.run(['git', 'submodule', '-q', 'foreach', '\'echo $sm_path `git describe --tags`\''])
+    for log_line in submodule_tags.stdout.split('\n'):
+        if not log_line:
+            continue
+
+        r = log_line.split()
+        submodule = r[0]
+        submodules[submodule]['describe'] = r[1] if len(r) > 1 else ''
+
     return submodules
 
 

lib/python/qmk/tests/test_cli_commands.py

@@ -97,13 +97,15 @@ def test_list_keyboards():
 def test_list_keymaps():
     result = check_subcommand('list-keymaps', '-kb', 'handwired/pytest/basic')
     check_returncode(result)
-    assert 'default' and 'default_json' in result.stdout
+    assert 'default' in result.stdout
+    assert 'default_json' in result.stdout
 
 
 def test_list_keymaps_long():
     result = check_subcommand('list-keymaps', '--keyboard', 'handwired/pytest/basic')
     check_returncode(result)
-    assert 'default' and 'default_json' in result.stdout
+    assert 'default' in result.stdout
+    assert 'default_json' in result.stdout
 
 
 def test_list_keymaps_community():
@@ -113,25 +115,24 @@ def test_list_keymaps_community():
 
 
 def test_list_keymaps_kb_only():
-<<<<<<< HEAD
     result = check_subcommand('list-keymaps', '-kb', 'moonlander')
-=======
-    result = check_subcommand('list-keymaps', '-kb', 'contra')
->>>>>>> qmk/master
     check_returncode(result)
-    assert 'default' and 'oyrx' and 'webusb' in result.stdout
+    assert 'default' in result.stdout
+    assert 'oryx' in result.stdout
 
 
 def test_list_keymaps_vendor_kb():
     result = check_subcommand('list-keymaps', '-kb', 'planck/ez')
     check_returncode(result)
-    assert 'default' and 'oryx' and 'webusb' in result.stdout
+    assert 'default' in result.stdout
+    assert 'oryx' in result.stdout
 
 
-# def test_list_keymaps_vendor_kb_rev():
-#     result = check_subcommand('list-keymaps', '-kb', 'kbdfans/kbd67/mkiirgb/v2')
-#     check_returncode(result)
-#     assert 'default' and 'via' in result.stdout
+def test_list_keymaps_vendor_kb_rev():
+    result = check_subcommand('list-keymaps', '-kb', 'kbdfans/kbd67/mkiirgb/v2')
+    check_returncode(result)
+    assert 'default' in result.stdout
+    assert 'oryx' in result.stdout
 
 
 def test_list_keymaps_no_keyboard_found():
@@ -263,7 +264,6 @@ def test_generate_config_h():
     result = check_subcommand('generate-config-h', '-kb', 'handwired/pytest/basic')
     check_returncode(result)
     assert '#   define DEVICE_VER 0x0001' in result.stdout
-    assert '#   define DESCRIPTION "handwired/pytest/basic"' in result.stdout
     assert '#   define DIODE_DIRECTION COL2ROW' in result.stdout
     assert '#   define MANUFACTURER none' in result.stdout
     assert '#   define PRODUCT pytest' in result.stdout