poz/qmk_firmware
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Userspace: add support for adding environment variables during build (#22887)

Browse source
This commit is contained in:
Nick Brassel 2024-08-12 22:34:22 +10:00 committed by GitHub
parent 158aaef78c
commit 380e0c9cad
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 372 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
builddefs/build_keyboard.mk
View file

@ -34,10 +34,16 @@ ifeq ($(strip $(DUMP_CI_METADATA)),yes)
endif
# Force expansion
TARGET := $(TARGET)
override TARGET := $(TARGET)
$(info TARGET=$(TARGET))
ifneq ($(FORCE_LAYOUT),)
TARGET := $(TARGET)_$(FORCE_LAYOUT)
override TARGET := $(TARGET)_$(FORCE_LAYOUT)
$(info TARGET=$(TARGET))
endif
ifneq ($(CONVERT_TO),)
override TARGET := $(TARGET)_$(CONVERT_TO)
$(info TARGET=$(TARGET))
endif
# Object files and generated keymap directory
@ -58,9 +64,6 @@ ifdef SKIP_GIT
VERSION_H_FLAGS += --skip-git
endif
# Generate the board's version.h file.
$(shell $(QMK_BIN) generate-version-h $(VERSION_H_FLAGS) -q -o $(INTERMEDIATE_OUTPUT)/src/version.h)
# Determine which subfolders exist.
KEYBOARD_FOLDER_PATH_1 := $(KEYBOARD)
KEYBOARD_FOLDER_PATH_2 := $(patsubst %/,%,$(dir $(KEYBOARD_FOLDER_PATH_1)))
@ -218,6 +221,9 @@ endif
include $(BUILDDEFS_PATH)/converters.mk
# Generate the board's version.h file.
$(shell $(QMK_BIN) generate-version-h $(VERSION_H_FLAGS) -q -o $(INTERMEDIATE_OUTPUT)/src/version.h)
MCU_ORIG := $(MCU)
include $(wildcard $(PLATFORM_PATH)/*/mcu_selection.mk)

3
builddefs/converters.mk
View file

@ -32,9 +32,6 @@ ifneq ($(CONVERT_TO),)
PLATFORM_KEY = $(shell echo $(CONVERTER) | cut -d "/" -f2)
# force setting as value can be from environment
override TARGET := $(TARGET)_$(CONVERT_TO)
# Configure any defaults
OPT_DEFS += -DCONVERT_TO_$(shell echo $(CONVERT_TO) | tr '[:lower:]' '[:upper:]')
OPT_DEFS += -DCONVERTER_TARGET=\"$(CONVERT_TO)\"

23
data/schemas/definitions.jsonschema
View file

@ -16,12 +16,6 @@
"type": "object",
"additionalProperties": {"type": "boolean"}
},
"build_target": {
"oneOf": [
{"$ref": "#/keyboard_keymap_tuple"},
{"$ref": "#/json_file_path"}
]
},
"filename": {
"type": "string",
"minLength": 1,
@ -53,6 +47,19 @@
{"$ref": "#/keyboard"},
{"$ref": "#/filename"}
],
"minItems": 2,
"maxItems": 2,
"unevaluatedItems": false
},
"keyboard_keymap_env": {
"type": "array",
"prefixItems": [
{"$ref": "#/keyboard"},
{"$ref": "#/filename"},
{"$ref": "#/kvp_object"}
],
"minItems": 3,
"maxItems": 3,
"unevaluatedItems": false
},
"keycode": {
@ -87,6 +94,10 @@
"maxLength": 7,
"pattern": "^[A-Z][A-Zs_0-9]*$"
},
"kvp_object": {
"type": "object",
"additionalProperties": {"type": "string"}
},
"layout_macro": {
"oneOf": [
{

10
data/schemas/user_repo_v1.jsonschema
View file

@ -3,6 +3,14 @@
"$id": "qmk.user_repo.v1",
"title": "User Repository Information",
"type": "object",
"definitions": {
"build_target": {
"oneOf": [
{"$ref": "qmk.definitions.v1#/keyboard_keymap_tuple"},
{"$ref": "qmk.definitions.v1#/json_file_path"}
]
},
},
"required": [
"userspace_version",
"build_targets"
@ -15,7 +23,7 @@
"build_targets": {
"type": "array",
"items": {
"$ref": "qmk.definitions.v1#/build_target"
"$ref": "#/definitions/build_target"
}
}
}

31
data/schemas/user_repo_v1_1.jsonschema Normal file
View file

@ -0,0 +1,31 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema#",
"$id": "qmk.user_repo.v1_1",
"title": "User Repository Information",
"type": "object",
"definitions": {
"build_target": {
"oneOf": [
{"$ref": "qmk.definitions.v1#/keyboard_keymap_tuple"},
{"$ref": "qmk.definitions.v1#/keyboard_keymap_env"},
{"$ref": "qmk.definitions.v1#/json_file_path"}
]
},
},
"required": [
"userspace_version",
"build_targets"
],
"properties": {
"userspace_version": {
"type": "string",
"enum": ["1.1"]
},
"build_targets": {
"type": "array",
"items": {
"$ref": "#/definitions/build_target"
}
}
}
}

107
lib/python/qmk/build_targets.py
View file

@ -1,8 +1,8 @@
# Copyright 2023 Nick Brassel (@tzarc)
# Copyright 2023-2024 Nick Brassel (@tzarc)
# SPDX-License-Identifier: GPL-2.0-or-later
import json
import shutil
from typing import List, Union
from typing import Dict, List, Union
from pathlib import Path
from dotty_dict import dotty, Dotty
from milc import cli
@ -13,6 +13,9 @@ from qmk.info import keymap_json
from qmk.keymap import locate_keymap
from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace
# These must be kept in the order in which they're applied to $(TARGET) in the makefiles in order to ensure consistency.
TARGET_FILENAME_MODIFIERS = ['FORCE_LAYOUT', 'CONVERT_TO']
class BuildTarget:
def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
@ -22,25 +25,25 @@ class BuildTarget:
self._parallel = 1
self._clean = False
self._compiledb = False
self._target = f'{self._keyboard_safe}_{self.keymap}'
self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
self._generated_files_path = self._intermediate_output / 'src'
self._extra_args = {}
self._json = json.to_dict() if isinstance(json, Dotty) else json
def __str__(self):
return f'{self.keyboard}:{self.keymap}'
def __repr__(self):
if len(self._extra_args.items()) > 0:
return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap}, extra_args={json.dumps(self._extra_args, sort_keys=True)})'
return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'
def __lt__(self, __value: object) -> bool:
return self.__repr__() < __value.__repr__()
def __eq__(self, __value: object) -> bool:
if not isinstance(__value, BuildTarget):
return False
return self.__repr__() == __value.__repr__()
def __ne__(self, __value: object) -> bool:
return not self.__eq__(__value)
def __hash__(self) -> int:
return self.__repr__().__hash__()
@ -72,7 +75,34 @@ class BuildTarget:
def dotty(self) -> Dotty:
return dotty(self.json)
def _common_make_args(self, dry_run: bool = False, build_target: str = None):
@property
def extra_args(self) -> Dict[str, str]:
return {k: v for k, v in self._extra_args.items()}
@extra_args.setter
def extra_args(self, ex_args: Dict[str, str]):
if ex_args is not None and isinstance(ex_args, dict):
self._extra_args = {k: v for k, v in ex_args.items()}
def target_name(self, **env_vars) -> str:
# Work out the intended target name
target = f'{self._keyboard_safe}_{self.keymap}'
vars = self._all_vars(**env_vars)
for modifier in TARGET_FILENAME_MODIFIERS:
if modifier in vars:
target += f"_{vars[modifier]}"
return target
def _all_vars(self, **env_vars) -> Dict[str, str]:
vars = {k: v for k, v in env_vars.items()}
for k, v in self._extra_args.items():
vars[k] = v
return vars
def _intermediate_output(self, **env_vars) -> Path:
return Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self.target_name(**env_vars)}')
def _common_make_args(self, dry_run: bool = False, build_target: str = None, **env_vars):
compile_args = [
find_make(),
*get_make_parallel_args(self._parallel),
@ -98,14 +128,17 @@ class BuildTarget:
f'KEYBOARD={self.keyboard}',
f'KEYMAP={self.keymap}',
f'KEYBOARD_FILESAFE={self._keyboard_safe}',
f'TARGET={self._target}',
f'INTERMEDIATE_OUTPUT={self._intermediate_output}',
f'TARGET={self._keyboard_safe}_{self.keymap}', # don't use self.target_name() here, it's rebuilt on the makefile side
f'VERBOSE={verbose}',
f'COLOR={color}',
'SILENT=false',
'QMK_BIN="qmk"',
])
vars = self._all_vars(**env_vars)
for k, v in vars.items():
compile_args.append(f'{k}={v}')
return compile_args
def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
@ -150,6 +183,8 @@ class KeyboardKeymapBuildTarget(BuildTarget):
super().__init__(keyboard=keyboard, keymap=keymap, json=json)
def __repr__(self):
if len(self._extra_args.items()) > 0:
return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, extra_args={self._extra_args})'
return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})'
def _load_json(self):
@ -159,15 +194,13 @@ class KeyboardKeymapBuildTarget(BuildTarget):
pass
def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
for key, value in env_vars.items():
compile_args.append(f'{key}={value}')
compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target, **env_vars)
# Need to override the keymap path if the keymap is a userspace directory.
# This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap
# in an equivalent historical location.
keymap_location = locate_keymap(self.keyboard, self.keymap)
vars = self._all_vars(**env_vars)
keymap_location = locate_keymap(self.keyboard, self.keymap, force_layout=vars.get('FORCE_LAYOUT'))
if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location):
keymap_directory = keymap_location.parent
compile_args.extend([
@ -196,47 +229,51 @@ class JsonKeymapBuildTarget(BuildTarget):
super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json)
self._keymap_json = self._generated_files_path / 'keymap.json'
def __repr__(self):
if len(self._extra_args.items()) > 0:
return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path}, extra_args={self._extra_args})'
return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})'
def _load_json(self):
pass # Already loaded in constructor
def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None:
intermediate_output = self._intermediate_output(**env_vars)
generated_files_path = intermediate_output / 'src'
keymap_json = generated_files_path / 'keymap.json'
if self._clean:
if self._intermediate_output.exists():
shutil.rmtree(self._intermediate_output)
if intermediate_output.exists():
shutil.rmtree(intermediate_output)
# begin with making the deepest folder in the tree
self._generated_files_path.mkdir(exist_ok=True, parents=True)
generated_files_path.mkdir(exist_ok=True, parents=True)
# Compare minified to ensure consistent comparison
new_content = json.dumps(self.json, separators=(',', ':'))
if self._keymap_json.exists():
old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
if keymap_json.exists():
old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':'))
if old_content == new_content:
new_content = None
# Write the keymap.json file if different so timestamps are only updated
# if the content changes -- running `make` won't treat it as modified.
if new_content:
self._keymap_json.write_text(new_content, encoding='utf-8')
keymap_json.write_text(new_content, encoding='utf-8')
def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]:
compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target)
compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target, **env_vars)
intermediate_output = self._intermediate_output(**env_vars)
generated_files_path = intermediate_output / 'src'
keymap_json = generated_files_path / 'keymap.json'
compile_args.extend([
f'MAIN_KEYMAP_PATH_1={self._intermediate_output}',
f'MAIN_KEYMAP_PATH_2={self._intermediate_output}',
f'MAIN_KEYMAP_PATH_3={self._intermediate_output}',
f'MAIN_KEYMAP_PATH_4={self._intermediate_output}',
f'MAIN_KEYMAP_PATH_5={self._intermediate_output}',
f'KEYMAP_JSON={self._keymap_json}',
f'KEYMAP_PATH={self._generated_files_path}',
f'MAIN_KEYMAP_PATH_1={intermediate_output}',
f'MAIN_KEYMAP_PATH_2={intermediate_output}',
f'MAIN_KEYMAP_PATH_3={intermediate_output}',
f'MAIN_KEYMAP_PATH_4={intermediate_output}',
f'MAIN_KEYMAP_PATH_5={intermediate_output}',
f'KEYMAP_JSON={keymap_json}',
f'KEYMAP_PATH={generated_files_path}',
])
for key, value in env_vars.items():
compile_args.append(f'{key}={value}')
return compile_args

7
lib/python/qmk/cli/format/json.py
View file

@ -17,6 +17,13 @@ def _detect_json_format(file, json_data):
"""Detect the format of a json file.
"""
json_encoder = None
try:
validate(json_data, 'qmk.user_repo.v1_1')
json_encoder = UserspaceJSONEncoder
except ValidationError:
pass
if json_encoder is None:
try:
validate(json_data, 'qmk.user_repo.v1')
json_encoder = UserspaceJSONEncoder

24
lib/python/qmk/cli/mass_compile.py
View file

@ -7,6 +7,7 @@ from typing import List
from pathlib import Path
from subprocess import DEVNULL
from milc import cli
import shlex
from qmk.constants import QMK_FIRMWARE
from qmk.commands import find_make, get_make_parallel_args, build_environment
@ -26,7 +27,8 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool,
if dry_run:
cli.log.info('Compilation targets:')
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap}{{fg_reset}}")
extra_args = ' '.join([f"-e {shlex.quote(f'{k}={v}')}" for k, v in target.extra_args.items()])
cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap} {extra_args}{{fg_reset}}")
else:
if clean:
cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
@ -36,18 +38,26 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool,
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
keyboard_name = target.keyboard
keymap_name = target.keymap
keyboard_safe = keyboard_name.replace('/', '_')
target_filename = target.target_name(**env)
target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation
target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
command = target.compile_command(**env)
command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism
keyboard_safe = keyboard_name.replace('/', '_')
extra_args = '_'.join([f"{k}_{v}" for k, v in target.extra_args.items()])
build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
target_suffix = ''
if len(extra_args) > 0:
build_log += f".{extra_args}"
failed_log += f".{extra_args}"
target_suffix = f"_{extra_args}"
# yapf: disable
f.write(
f"""\
all: {keyboard_safe}_{keymap_name}_binary
{keyboard_safe}_{keymap_name}_binary:
.PHONY: {target_filename}{target_suffix}_binary
all: {target_filename}{target_suffix}_binary
{target_filename}{target_suffix}_binary:
@rm -f "{build_log}" || true
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
{' '.join(command)} \\
@ -65,9 +75,9 @@ all: {keyboard_safe}_{keymap_name}_binary
# yapf: disable
f.write(
f"""\
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.elf" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.map" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{keymap_name}" || true
@rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.elf" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.map" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/obj_{target_filename}" || true
"""# noqa
)
# yapf: enable

10
lib/python/qmk/cli/userspace/add.py
View file

@ -1,8 +1,9 @@
# Copyright 2023 Nick Brassel (@tzarc)
# Copyright 2023-2024 Nick Brassel (@tzarc)
# SPDX-License-Identifier: GPL-2.0-or-later
from pathlib import Path
from milc import cli
from qmk.commands import parse_env_vars
from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
from qmk.keymap import keymap_completer, is_keymap_target
@ -12,12 +13,15 @@ from qmk.userspace import UserspaceDefs
@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Extra variables to set during build. May be passed multiple times.")
@cli.subcommand('Adds a build target to userspace `qmk.json`.')
def userspace_add(cli):
if not HAS_QMK_USERSPACE:
cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
return False
build_env = None if len(cli.args.env) == 0 else parse_env_vars(cli.args.env)
userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
if len(cli.args.builds) > 0:
@ -44,8 +48,8 @@ def userspace_add(cli):
cli.config.new_keymap.keyboard = cli.args.keyboard
cli.config.new_keymap.keymap = cli.args.keymap
if new_keymap(cli) is not False:
userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env)
else:
userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env)
return userspace.save()

10
lib/python/qmk/cli/userspace/compile.py
View file

@ -1,4 +1,4 @@
# Copyright 2023 Nick Brassel (@tzarc)
# Copyright 2023-2024 Nick Brassel (@tzarc)
# SPDX-License-Identifier: GPL-2.0-or-later
from pathlib import Path
from milc import cli
@ -12,6 +12,10 @@ from qmk.cli.mass_compile import mass_compile_targets
from qmk.util import maybe_exit_config
def _extra_arg_setter(target, extra_args):
target.extra_args = extra_args
@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@ -33,8 +37,8 @@ def userspace_compile(cli):
if isinstance(e, Path):
build_targets.append(JsonKeymapBuildTarget(e))
elif isinstance(e, dict):
keyboard_keymap_targets.append((e['keyboard'], e['keymap']))
f = e['env'] if 'env' in e else None
keyboard_keymap_targets.append((e['keyboard'], e['keymap'], f))
if len(keyboard_keymap_targets) > 0:
build_targets.extend(search_keymap_targets(keyboard_keymap_targets))

23
lib/python/qmk/cli/userspace/list.py
View file

@ -1,4 +1,4 @@
# Copyright 2023 Nick Brassel (@tzarc)
# Copyright 2023-2024 Nick Brassel (@tzarc)
# SPDX-License-Identifier: GPL-2.0-or-later
from pathlib import Path
from dotty_dict import Dotty
@ -13,6 +13,10 @@ from qmk.search import search_keymap_targets
from qmk.util import maybe_exit_config
def _extra_arg_setter(target, extra_args):
target.extra_args = extra_args
@cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.")
@cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
def userspace_list(cli):
@ -26,11 +30,15 @@ def userspace_list(cli):
if cli.args.expand:
build_targets = []
keyboard_keymap_targets = []
for e in userspace.build_targets:
if isinstance(e, Path):
build_targets.append(e)
elif isinstance(e, dict) or isinstance(e, Dotty):
build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
f = e['env'] if 'env' in e else None
keyboard_keymap_targets.append((e['keyboard'], e['keymap'], f))
if len(keyboard_keymap_targets) > 0:
build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
else:
build_targets = userspace.build_targets
@ -43,12 +51,19 @@ def userspace_list(cli):
# keyboard/keymap dict from userspace
keyboard = e['keyboard']
keymap = e['keymap']
extra_args = e.get('env')
elif isinstance(e, BuildTarget):
# BuildTarget from search_keymap_targets()
keyboard = e.keyboard
keymap = e.keymap
extra_args = e.extra_args
extra_args_str = ''
if extra_args is not None and len(extra_args) > 0:
extra_args_str = ', '.join([f'{{fg_cyan}}{k}={v}{{fg_reset}}' for k, v in extra_args.items()])
extra_args_str = f' ({{fg_cyan}}{extra_args_str}{{fg_reset}})'
if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap):
cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}{extra_args_str}')
else:
cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!')
cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}{extra_args_str} -- not found!')

10
lib/python/qmk/cli/userspace/remove.py
View file

@ -1,8 +1,9 @@
# Copyright 2023 Nick Brassel (@tzarc)
# Copyright 2023-2024 Nick Brassel (@tzarc)
# SPDX-License-Identifier: GPL-2.0-or-later
from pathlib import Path
from milc import cli
from qmk.commands import parse_env_vars
from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
from qmk.keymap import keymap_completer
@ -12,12 +13,15 @@ from qmk.userspace import UserspaceDefs
@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Extra variables to set during build. May be passed multiple times.")
@cli.subcommand('Removes a build target from userspace `qmk.json`.')
def userspace_remove(cli):
if not HAS_QMK_USERSPACE:
cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
return False
build_env = None if len(cli.args.env) == 0 else parse_env_vars(cli.args.env)
userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
if len(cli.args.builds) > 0:
@ -29,9 +33,9 @@ def userspace_remove(cli):
for e in make_like_targets:
s = e.split(':')
userspace.remove_target(keyboard=s[0], keymap=s[1])
userspace.remove_target(keyboard=s[0], keymap=s[1], build_env=build_env)
else:
userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env)
return userspace.save()

7
lib/python/qmk/commands.py
View file

@ -68,7 +68,7 @@ def parse_configurator_json(configurator_file):
return user_keymap
def build_environment(args):
def parse_env_vars(args):
"""Common processing for cli.args.env
"""
envs = {}
@ -78,6 +78,11 @@ def build_environment(args):
envs[key] = value
else:
cli.log.warning('Invalid environment variable: %s', env)
return envs
def build_environment(args):
envs = parse_env_vars(args)
if HAS_QMK_USERSPACE:
envs['QMK_USERSPACE'] = Path(QMK_USERSPACE).resolve()

19
lib/python/qmk/info.py
View file

@ -212,7 +212,7 @@ def _validate(keyboard, info_data):
maybe_exit(1)
def info_json(keyboard):
def info_json(keyboard, force_layout=None):
"""Generate the info.json data for a specific keyboard.
"""
cur_dir = Path('keyboards')
@ -255,6 +255,11 @@ def info_json(keyboard):
# Merge in data from <keyboard.c>
info_data = _extract_led_config(info_data, str(keyboard))
# Force a community layout if requested
community_layouts = info_data.get("community_layouts", [])
if force_layout in community_layouts:
info_data["community_layouts"] = [force_layout]
# Validate
_validate(keyboard, info_data)
@ -988,25 +993,25 @@ def find_info_json(keyboard):
return [info_json for info_json in info_jsons if info_json.exists()]
def keymap_json_config(keyboard, keymap):
def keymap_json_config(keyboard, keymap, force_layout=None):
"""Extract keymap level config
"""
# TODO: resolve keymap.py and info.py circular dependencies
from qmk.keymap import locate_keymap
keymap_folder = locate_keymap(keyboard, keymap).parent
keymap_folder = locate_keymap(keyboard, keymap, force_layout=force_layout).parent
km_info_json = parse_configurator_json(keymap_folder / 'keymap.json')
return km_info_json.get('config', {})
def keymap_json(keyboard, keymap):
def keymap_json(keyboard, keymap, force_layout=None):
"""Generate the info.json data for a specific keymap.
"""
# TODO: resolve keymap.py and info.py circular dependencies
from qmk.keymap import locate_keymap
keymap_folder = locate_keymap(keyboard, keymap).parent
keymap_folder = locate_keymap(keyboard, keymap, force_layout=force_layout).parent
# Files to scan
keymap_config = keymap_folder / 'config.h'
@ -1014,10 +1019,10 @@ def keymap_json(keyboard, keymap):
keymap_file = keymap_folder / 'keymap.json'
# Build the info.json file
kb_info_json = info_json(keyboard)
kb_info_json = info_json(keyboard, force_layout=force_layout)
# Merge in the data from keymap.json
km_info_json = keymap_json_config(keyboard, keymap) if keymap_file.exists() else {}
km_info_json = keymap_json_config(keyboard, keymap, force_layout=force_layout) if keymap_file.exists() else {}
deep_update(kb_info_json, km_info_json)
# Merge in the data from config.h, and rules.mk

4
lib/python/qmk/keymap.py
View file

@ -420,7 +420,7 @@ def write(keymap_json):
return write_file(keymap_file, keymap_content)
def locate_keymap(keyboard, keymap):
def locate_keymap(keyboard, keymap, force_layout=None):
"""Returns the path to a keymap for a specific keyboard.
"""
if not qmk.path.is_keyboard(keyboard):
@ -459,7 +459,7 @@ def locate_keymap(keyboard, keymap):
return keymap_path
# Check community layouts as a fallback
info = info_json(keyboard)
info = info_json(keyboard, force_layout=force_layout)
community_parents = list(Path('layouts').glob('*/'))
if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists():

157
lib/python/qmk/search.py
View file

@ -1,11 +1,13 @@
"""Functions for searching through QMK keyboards and keymaps.
"""
from dataclasses import dataclass
import contextlib
import functools
import fnmatch
import json
import logging
import re
from typing import Callable, List, Optional, Tuple
from typing import Callable, Dict, List, Optional, Tuple, Union
from dotty_dict import dotty, Dotty
from milc import cli
@ -15,7 +17,32 @@ from qmk.keyboard import list_keyboards, keyboard_folder
from qmk.keymap import list_keymaps, locate_keymap
from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget
TargetInfo = Tuple[str, str, dict]
@dataclass
class KeyboardKeymapDesc:
keyboard: str
keymap: str
data: dict = None
extra_args: dict = None
def __hash__(self) -> int:
return self.keyboard.__hash__() ^ self.keymap.__hash__() ^ json.dumps(self.extra_args, sort_keys=True).__hash__()
def __lt__(self, other) -> bool:
return (self.keyboard, self.keymap, json.dumps(self.extra_args, sort_keys=True)) < (other.keyboard, other.keymap, json.dumps(other.extra_args, sort_keys=True))
def load_data(self):
data = keymap_json(self.keyboard, self.keymap)
self.data = data.to_dict() if isinstance(data, Dotty) else data
@property
def dotty(self) -> Dotty:
return dotty(self.data) if self.data is not None else None
def to_build_target(self) -> KeyboardKeymapBuildTarget:
target = KeyboardKeymapBuildTarget(keyboard=self.keyboard, keymap=self.keymap, json=self.data)
target.extra_args = self.extra_args
return target
# by using a class for filters, we dont need to worry about capturing values
@ -36,7 +63,7 @@ class FilterFunction:
value: Optional[str]
func_name: str
apply: Callable[[TargetInfo], bool]
apply: Callable[[KeyboardKeymapDesc], bool]
def __init__(self, key, value):
self.key = key
@ -46,33 +73,29 @@ class FilterFunction:
class Exists(FilterFunction):
func_name = "exists"
def apply(self, target_info: TargetInfo) -> bool:
_kb, _km, info = target_info
return self.key in info
def apply(self, target_info: KeyboardKeymapDesc) -> bool:
return self.key in target_info.data
class Absent(FilterFunction):
func_name = "absent"
def apply(self, target_info: TargetInfo) -> bool:
_kb, _km, info = target_info
return self.key not in info
def apply(self, target_info: KeyboardKeymapDesc) -> bool:
return self.key not in target_info.data
class Length(FilterFunction):
func_name = "length"
def apply(self, target_info: TargetInfo) -> bool:
_kb, _km, info = target_info
return (self.key in info and len(info[self.key]) == int(self.value))
def apply(self, target_info: KeyboardKeymapDesc) -> bool:
return (self.key in target_info.data and len(target_info.data[self.key]) == int(self.value))
class Contains(FilterFunction):
func_name = "contains"
def apply(self, target_info: TargetInfo) -> bool:
_kb, _km, info = target_info
return (self.key in info and self.value in info[self.key])
def apply(self, target_info: KeyboardKeymapDesc) -> bool:
return (self.key in target_info.data and self.value in target_info.data[self.key])
def _get_filter_class(func_name: str, key: str, value: str) -> Optional[FilterFunction]:
@ -109,12 +132,12 @@ def ignore_logging():
_set_log_level(old)
def _all_keymaps(keyboard):
"""Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard.
def _all_keymaps(keyboard) -> List[KeyboardKeymapDesc]:
"""Returns a list of KeyboardKeymapDesc for all keymaps for the given keyboard.
"""
with ignore_logging():
keyboard = keyboard_folder(keyboard)
return [(keyboard, keymap) for keymap in list_keymaps(keyboard)]
return [KeyboardKeymapDesc(keyboard, keymap) for keymap in list_keymaps(keyboard)]
def _keymap_exists(keyboard, keymap):
@ -124,85 +147,91 @@ def _keymap_exists(keyboard, keymap):
return keyboard if locate_keymap(keyboard, keymap) is not None else None
def _load_keymap_info(target: Tuple[str, str]) -> TargetInfo:
"""Returns a tuple of (keyboard, keymap, info.json) for the given keyboard/keymap combination.
def _load_keymap_info(target: KeyboardKeymapDesc) -> KeyboardKeymapDesc:
"""Ensures a KeyboardKeymapDesc has its data loaded.
"""
kb, km = target
with ignore_logging():
return (kb, km, keymap_json(kb, km))
target.load_data() # Ensure we load the data first
return target
def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]:
"""Expand a list of make targets into a list of (keyboard, keymap) tuples.
def expand_make_targets(targets: List[Union[str, Tuple[str, Dict[str, str]]]]) -> List[KeyboardKeymapDesc]:
"""Expand a list of make targets into a list of KeyboardKeymapDesc.
Caters for 'all' in either keyboard or keymap, or both.
"""
split_targets = []
for target in targets:
extra_args = None
if isinstance(target, tuple):
split_target = target[0].split(':')
extra_args = target[1]
else:
split_target = target.split(':')
if len(split_target) != 2:
cli.log.error(f"Invalid build target: {target}")
return []
split_targets.append((split_target[0], split_target[1]))
split_targets.append(KeyboardKeymapDesc(split_target[0], split_target[1], extra_args=extra_args))
return expand_keymap_targets(split_targets)
def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] = None) -> List[Tuple[str, str]]:
"""Expand a keyboard input and keymap input into a list of (keyboard, keymap) tuples.
def _expand_keymap_target(target: KeyboardKeymapDesc, all_keyboards: List[str] = None) -> List[KeyboardKeymapDesc]:
"""Expand a keyboard input and keymap input into a list of KeyboardKeymapDesc.
Caters for 'all' in either keyboard or keymap, or both.
"""
if all_keyboards is None:
all_keyboards = list_keyboards()
if keyboard == 'all':
if keymap == 'all':
if target.keyboard == 'all':
if target.keymap == 'all':
cli.log.info('Retrieving list of all keyboards and keymaps...')
targets = []
for kb in parallel_map(_all_keymaps, all_keyboards):
targets.extend(kb)
for t in targets:
t.extra_args = target.extra_args
return targets
else:
cli.log.info(f'Retrieving list of keyboards with keymap "{keymap}"...')
keyboard_filter = functools.partial(_keymap_exists, keymap=keymap)
return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
cli.log.info(f'Retrieving list of keyboards with keymap "{target.keymap}"...')
keyboard_filter = functools.partial(_keymap_exists, keymap=target.keymap)
return [KeyboardKeymapDesc(kb, target.keymap, extra_args=target.extra_args) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]
else:
if keymap == 'all':
cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...')
return _all_keymaps(keyboard)
if target.keymap == 'all':
cli.log.info(f'Retrieving list of keymaps for keyboard "{target.keyboard}"...')
targets = _all_keymaps(target.keyboard)
for t in targets:
t.extra_args = target.extra_args
return targets
else:
return [(keyboard, keymap)]
return [target]
def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
"""Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.
def expand_keymap_targets(targets: List[KeyboardKeymapDesc]) -> List[KeyboardKeymapDesc]:
"""Expand a list of KeyboardKeymapDesc inclusive of 'all', into a list of explicit KeyboardKeymapDesc.
"""
overall_targets = []
all_keyboards = list_keyboards()
for target in targets:
overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))
overall_targets.extend(_expand_keymap_target(target, all_keyboards))
return list(sorted(set(overall_targets)))
def _construct_build_target_kb_km(e):
return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1])
def _construct_build_target(e: KeyboardKeymapDesc):
return e.to_build_target()
def _construct_build_target_kb_km_json(e):
return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2])
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
"""Filter a list of (keyboard, keymap) tuples based on the supplied filters.
def _filter_keymap_targets(target_list: List[KeyboardKeymapDesc], filters: List[str] = []) -> List[KeyboardKeymapDesc]:
"""Filter a list of KeyboardKeymapDesc based on the supplied filters.
Optionally includes the values of the queried info.json keys.
"""
if len(filters) == 0:
cli.log.info('Preparing target list...')
targets = list(set(parallel_map(_construct_build_target_kb_km, target_list)))
targets = target_list
else:
cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
valid_targets = parallel_map(_load_keymap_info, target_list)
function_re = re.compile(r'^(?P<function>[a-zA-Z]+)\((?P<key>[a-zA-Z0-9_\.]+)(,\s*(?P<value>[^#]+))?\)$')
equals_re = re.compile(r'^(?P<key>[a-zA-Z0-9_\.]+)\s*=\s*(?P<value>[^#]+)$')
@ -220,7 +249,7 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
if filter_class is None:
cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}')
continue
valid_keymaps = filter(filter_class.apply, valid_keymaps)
valid_targets = filter(filter_class.apply, valid_targets)
value_str = f", {{fg_cyan}}{value}{{fg_reset}}" if value is not None else ""
cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}{value_str})...')
@ -234,32 +263,42 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
expr = fnmatch.translate(v)
rule = re.compile(f'^{expr}$', re.IGNORECASE)
def f(e):
lhs = e[2].get(k)
def f(e: KeyboardKeymapDesc):
lhs = e.dotty.get(k)
lhs = str(False if lhs is None else lhs)
return rule.search(lhs) is not None
return f
valid_keymaps = filter(_make_filter(key, value), valid_keymaps)
valid_targets = filter(_make_filter(key, value), valid_targets)
else:
cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
continue
cli.log.info('Preparing target list...')
valid_keymaps = [(e[0], e[1], e[2].to_dict() if isinstance(e[2], Dotty) else e[2]) for e in valid_keymaps] # need to convert dotty_dict back to dict because it doesn't survive parallelisation
targets = list(set(parallel_map(_construct_build_target_kb_km_json, list(valid_keymaps))))
targets = list(sorted(set(valid_targets)))
return targets
def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
def search_keymap_targets(targets: List[Union[Tuple[str, str], Tuple[str, str, Dict[str, str]]]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:
"""Search for build targets matching the supplied criteria.
"""
return _filter_keymap_targets(expand_keymap_targets(targets), filters)
def _make_desc(e):
if len(e) == 3:
return KeyboardKeymapDesc(keyboard=e[0], keymap=e[1], extra_args=e[2])
else:
return KeyboardKeymapDesc(keyboard=e[0], keymap=e[1])
targets = map(_make_desc, targets)
targets = _filter_keymap_targets(expand_keymap_targets(targets), filters)
targets = list(set(parallel_map(_construct_build_target, list(targets))))
return sorted(targets)
def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]:
def search_make_targets(targets: List[Union[str, Tuple[str, Dict[str, str]]]], filters: List[str] = []) -> List[BuildTarget]:
"""Search for build targets matching the supplied criteria.
"""
return _filter_keymap_targets(expand_make_targets(targets), filters)
targets = _filter_keymap_targets(expand_make_targets(targets), filters)
targets = list(set(parallel_map(_construct_build_target, list(targets))))
return sorted(targets)

48
lib/python/qmk/userspace.py
View file

@ -1,4 +1,4 @@
# Copyright 2023 Nick Brassel (@tzarc)
# Copyright 2023-2024 Nick Brassel (@tzarc)
# SPDX-License-Identifier: GPL-2.0-or-later
from os import environ
from pathlib import Path
@ -77,31 +77,43 @@ class UserspaceDefs:
raise exception
# Iterate through each version of the schema, starting with the latest and decreasing to v1
schema_versions = [
('qmk.user_repo.v1_1', self.__load_v1_1), #
('qmk.user_repo.v1', self.__load_v1) #
]
for v in schema_versions:
schema = v[0]
loader = v[1]
try:
validate(json, 'qmk.user_repo.v1')
self.__load_v1(json)
validate(json, schema)
loader(json)
success = True
break
except jsonschema.ValidationError as err:
exception.add('qmk.user_repo.v1', err)
exception.add(schema, err)
if not success:
raise exception
def save(self):
target_json = {
"userspace_version": "1.0", # Needs to match latest version
"userspace_version": "1.1", # Needs to match latest version
"build_targets": []
}
for e in self.build_targets:
if isinstance(e, dict):
target_json['build_targets'].append([e['keyboard'], e['keymap']])
entry = [e['keyboard'], e['keymap']]
if 'env' in e:
entry.append(e['env'])
target_json['build_targets'].append(entry)
elif isinstance(e, Path):
target_json['build_targets'].append(str(e.relative_to(self.path.parent)))
try:
# Ensure what we're writing validates against the latest version of the schema
validate(target_json, 'qmk.user_repo.v1')
validate(target_json, 'qmk.user_repo.v1_1')
except jsonschema.ValidationError as err:
cli.log.error(f'Could not save userspace file: {err}')
return False
@ -114,7 +126,7 @@ class UserspaceDefs:
cli.log.info(f'Saved userspace file to {self.path}.')
return True
def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
def add_target(self, keyboard=None, keymap=None, build_env=None, json_path=None, do_print=True):
if json_path is not None:
# Assume we're adding a json filename/path
json_path = Path(json_path)
@ -128,6 +140,8 @@ class UserspaceDefs:
elif keyboard is not None and keymap is not None:
# Both keyboard/keymap specified
e = {"keyboard": keyboard, "keymap": keymap}
if build_env is not None:
e['env'] = build_env
if e not in self.build_targets:
self.build_targets.append(e)
if do_print:
@ -136,7 +150,7 @@ class UserspaceDefs:
if do_print:
cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.')
def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
def remove_target(self, keyboard=None, keymap=None, build_env=None, json_path=None, do_print=True):
if json_path is not None:
# Assume we're removing a json filename/path
json_path = Path(json_path)
@ -150,6 +164,8 @@ class UserspaceDefs:
elif keyboard is not None and keymap is not None:
# Both keyboard/keymap specified
e = {"keyboard": keyboard, "keymap": keymap}
if build_env is not None:
e['env'] = build_env
if e in self.build_targets:
self.build_targets.remove(e)
if do_print:
@ -160,6 +176,13 @@ class UserspaceDefs:
def __load_v1(self, json):
for e in json['build_targets']:
self.__load_v1_target(e)
def __load_v1_1(self, json):
for e in json['build_targets']:
self.__load_v1_1_target(e)
def __load_v1_target(self, e):
if isinstance(e, list) and len(e) == 2:
self.add_target(keyboard=e[0], keymap=e[1], do_print=False)
if isinstance(e, str):
@ -167,6 +190,13 @@ class UserspaceDefs:
if p.exists() and p.suffix == '.json':
self.add_target(json_path=p, do_print=False)
def __load_v1_1_target(self, e):
# v1.1 adds support for a third item in the build target tuple; kvp's for environment
if isinstance(e, list) and len(e) == 3:
self.add_target(keyboard=e[0], keymap=e[1], build_env=e[2], do_print=False)
else:
self.__load_v1_target(e)
class UserspaceValidationError(Exception):
def __init__(self, *args, **kwargs):