From 8e6c2aea6b890465fe0382f92c2c3d4f1dfe92e6 Mon Sep 17 00:00:00 2001 From: Luca Toniolo <10792599+grandixximo@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:35:32 +0800 Subject: [PATCH 1/2] xhc-hb04: rename dashed ini identifiers to underscores The new ini parser restricts section and tag identifiers to letters, digits and underscore, not starting with a digit. The historic parser had no identifier character restrictions, so the xhc-hb04 pendant files used dashes freely and no longer parse: - the [XHC_HB04_BUTTONS] keys in the sample configs (start-pause, goto-zero, ...) - the [XHC-HB04] section of the pendant's own layout cfg files, which the component parses with the same IniFile class, making it exit at startup Rename the identifiers to use underscores: the INI button keys in the sample configs, the [XHC_HB04] section in the shipped layout cfg files and in the component's hardcoded section name, plus documentation. The HAL pin names are unchanged and keep their dashes (xhc-hb04.button-start-pause). The xhc-hb04.tcl script maps the underscored INI button names back to the dashed pin names, so existing user HAL files that reference the pins keep working. --- .../sim/axis/ja_tests/xyzx_mpg/xhc-hb04.inc | 44 +++++++++---------- configs/sim/axis/xhc-hb04/README | 12 ++--- configs/sim/axis/xhc-hb04/layout1.inc | 18 ++++---- configs/sim/axis/xhc-hb04/layout2.inc | 44 +++++++++---------- .../sim/axis/xhc-hb04/xhc-hb04-layout1.txt | 2 +- .../sim/axis/xhc-hb04/xhc-hb04-layout1_es.txt | 2 +- .../sim/axis/xhc-hb04/xhc-hb04-layout2.txt | 2 +- .../sim/axis/xhc-hb04/xhc-hb04-layout2_es.txt | 2 +- docs/src/man/man1/xhc-hb04.1.adoc | 2 +- lib/hallib/xhc-hb04-layout1.cfg | 2 +- lib/hallib/xhc-hb04-layout2.cfg | 2 +- lib/hallib/xhc-hb04.tcl | 14 ++++-- src/hal/user_comps/xhc-hb04.cc | 4 +- 13 files changed, 79 insertions(+), 71 deletions(-) diff --git a/configs/sim/axis/ja_tests/xyzx_mpg/xhc-hb04.inc b/configs/sim/axis/ja_tests/xyzx_mpg/xhc-hb04.inc index 942882102cc..4313de461ea 100644 --- a/configs/sim/axis/ja_tests/xyzx_mpg/xhc-hb04.inc +++ b/configs/sim/axis/ja_tests/xyzx_mpg/xhc-hb04.inc @@ -28,47 +28,47 @@ mpg_accels = 20 20 20 20 [XHC_HB04_BUTTONS] # use button names according to layout file LIB:xhc-hb04-layout{n}.cfg -# note: "start-pause" is connected for standard behavior +# note: "start_pause" is connected for standard behavior # controlling halui.pause/halui.resume/halui.run # these are examples, edit as required: -goto-zero = halui.mdi-command-00 +goto_zero = halui.mdi-command-00 # synthesized per-axis buttons for goto-zero button: -goto-zero-x = "" -goto-zero-y = "" -goto-zero-z = "" -goto-zero-a = "" +goto_zero_x = "" +goto_zero_y = "" +goto_zero_z = "" +goto_zero_a = "" -start-pause = std_start_pause +start_pause = std_start_pause rewind = halui.program.step # probe-input for simulating a probe: -probe-z = motion.probe-input -macro-3 = halui.mdi-command-03 +probe_z = motion.probe-input +macro_3 = halui.mdi-command-03 half = halui.spindle.0.stop # synthesized per-axis buttons for half button: -half-x = "" -half-y = "" -half-z = "" -half-a = "" +half_x = "" +half_y = "" +half_z = "" +half_a = "" zero = "" # synthesized per-axis buttons for zero button: -zero-x = halui.mdi-command-12 -zero-y = halui.mdi-command-13 -zero-z = halui.mdi-command-14 -zero-a = halui.mdi-command-15 +zero_x = halui.mdi-command-12 +zero_y = halui.mdi-command-13 +zero_z = halui.mdi-command-14 +zero_a = halui.mdi-command-15 -safe-z = halui.mdi-command-10 +safe_z = halui.mdi-command-10 # in this sim, home-all only works first time, hold down till finished: home = halui.home-all -macro-1 = halui.mdi-command-01 -macro-2 = halui.mdi-command-02 +macro_1 = halui.mdi-command-01 +macro_2 = halui.mdi-command-02 spindle = halui.spindle.0.start step = xhc-hb04.stepsize-up mode = "" -macro-6 = halui.mdi-command-06 -macro-7 = halui.mdi-command-07 +macro_6 = halui.mdi-command-06 +macro_7 = halui.mdi-command-07 stop = halui.program.stop reset = halui.estop.activate diff --git a/configs/sim/axis/xhc-hb04/README b/configs/sim/axis/xhc-hb04/README index 6b7aba395c3..c867328afbc 100644 --- a/configs/sim/axis/xhc-hb04/README +++ b/configs/sim/axis/xhc-hb04/README @@ -16,10 +16,10 @@ Sim configurations are provided for two known button configurations: layout1 -- 16 buttons layout2 -- 18 buttons (more common) -The button names are defined in the files LIB:xhc-hb04-layout*.cfg in a stanza labeled XHC-HB04 with definitions as BUTTON=XX:name where XX is a hexadecimal code used by the device. +The button names are defined in the files LIB:xhc-hb04-layout*.cfg in a stanza labeled XHC_HB04 with definitions as BUTTON=XX:name where XX is a hexadecimal code used by the device. Example: - [XHC-HB04] + [XHC_HB04] BUTTON=01:button-stop BUTTON=03:button-rewind ... @@ -111,20 +111,22 @@ sequence: Typically use 1 for mm-based machine, 2 for inch-based machine ---------------------------------- Pendant buttons are connected to hal pins with ini file items. +Ini identifiers cannot contain a dash, so button names use underscores +in the ini file and are mapped to the dashed hal pin names. Examples: [XHC_HB04_BUTTONS] stop = halui.program.stop -goto-zero = halui.mdi-command-00 +goto_zero = halui.mdi-command-00 step = xhc-hb04.stepsize-up rewind = halui.program.step -macro-1 = halui.mdi-command-01 +macro_1 = halui.mdi-command-01 mode = "" (placeholder) With the above items, connections are made as: Signal_name Source pin Destination pin ----------------- ---------------------------- ------------------------ pendant:stop <== xhc-hb04.button-stop ==> halui.program.stop - pendant:goto-zero <== xhc-hb04.button-goto-zero ==> halui.mdi-command-00 + pendant:goto_zero <== xhc-hb04.button-goto-zero ==> halui.mdi-command-00 pendant:step <== xhc-hb04.button-step ==> xhc-hb04.stepsize-up etc ... diff --git a/configs/sim/axis/xhc-hb04/layout1.inc b/configs/sim/axis/xhc-hb04/layout1.inc index 6627b044e88..1117db1242a 100644 --- a/configs/sim/axis/xhc-hb04/layout1.inc +++ b/configs/sim/axis/xhc-hb04/layout1.inc @@ -1,28 +1,28 @@ [XHC_HB04_BUTTONS] # use button names according to layout file LIB:xhc-hb04-layout{n}.cfg -# note: "start-pause" is connected for standard behavior +# note: "start_pause" is connected for standard behavior # controlling halui.pause/halui.resume/halui.run # these are examples, edit as required: stop = halui.program.stop -start-pause = std_start_pause +start_pause = std_start_pause rewind = halui.program.step -safe-z = halui.mdi-command-01 +safe_z = halui.mdi-command-01 # in this sim, home-all only works first time, hold down till finished: home = halui.home-all x2 = halui.mdi-command-02 y2 = halui.mdi-command-03 # probe-input for simulating a probe: -probe-z = motion.probe-input +probe_z = motion.probe-input x0 = halui.mdi-command-04 y0 = halui.mdi-command-05 z0 = halui.mdi-command-06 -goto-zero = halui.mdi-command-00 +goto_zero = halui.mdi-command-00 # synthesized per-axis buttons for goto-zero button: -goto-zero-x = "" -goto-zero-y = "" -goto-zero-z = "" -goto-zero-a = "" +goto_zero_x = "" +goto_zero_y = "" +goto_zero_z = "" +goto_zero_a = "" step = xhc-hb04.stepsize-up mode = "" diff --git a/configs/sim/axis/xhc-hb04/layout2.inc b/configs/sim/axis/xhc-hb04/layout2.inc index 31e51513a28..f31fcb16604 100644 --- a/configs/sim/axis/xhc-hb04/layout2.inc +++ b/configs/sim/axis/xhc-hb04/layout2.inc @@ -1,46 +1,46 @@ [XHC_HB04_BUTTONS] # use button names according to layout file LIB:xhc-hb04-layout{n}.cfg -# note: "start-pause" is connected for standard behavior +# note: "start_pause" is connected for standard behavior # controlling halui.pause/halui.resume/halui.run # these are examples, edit as required: -goto-zero = halui.mdi-command-00 +goto_zero = halui.mdi-command-00 # synthesized per-axis buttons for goto-zero button: -goto-zero-x = "" -goto-zero-y = "" -goto-zero-z = "" -goto-zero-a = "" +goto_zero_x = "" +goto_zero_y = "" +goto_zero_z = "" +goto_zero_a = "" -start-pause = std_start_pause +start_pause = std_start_pause rewind = halui.program.step # probe-input for simulating a probe: -probe-z = motion.probe-input -macro-3 = halui.mdi-command-03 +probe_z = motion.probe-input +macro_3 = halui.mdi-command-03 half = halui.spindle.0.stop # synthesized per-axis buttons for half button: -half-x = "" -half-y = "" -half-z = "" -half-a = "" +half_x = "" +half_y = "" +half_z = "" +half_a = "" zero = "" # synthesized per-axis buttons for zero button: -zero-x = halui.mdi-command-12 -zero-y = halui.mdi-command-13 -zero-z = halui.mdi-command-14 -zero-a = halui.mdi-command-15 +zero_x = halui.mdi-command-12 +zero_y = halui.mdi-command-13 +zero_z = halui.mdi-command-14 +zero_a = halui.mdi-command-15 -safe-z = halui.mdi-command-10 +safe_z = halui.mdi-command-10 # in this sim, home-all only works first time, hold down till finished: home = halui.home-all -macro-1 = halui.mdi-command-01 -macro-2 = halui.mdi-command-02 +macro_1 = halui.mdi-command-01 +macro_2 = halui.mdi-command-02 spindle = halui.spindle.0.start step = xhc-hb04.stepsize-up mode = "" -macro-6 = halui.mdi-command-06 -macro-7 = halui.mdi-command-07 +macro_6 = halui.mdi-command-06 +macro_7 = halui.mdi-command-07 stop = halui.program.stop reset = halui.estop.activate diff --git a/configs/sim/axis/xhc-hb04/xhc-hb04-layout1.txt b/configs/sim/axis/xhc-hb04/xhc-hb04-layout1.txt index 11cd029bb66..a11c4ffc027 100644 --- a/configs/sim/axis/xhc-hb04/xhc-hb04-layout1.txt +++ b/configs/sim/axis/xhc-hb04/xhc-hb04-layout1.txt @@ -25,7 +25,7 @@ xhc-hb04-layout1 Button codes defined in LIB:xhc-hb04-layout1.cfg The Start/Pause button can be connected for standard behavior with: - [XHC_HB04_BUTTON]start-pause = std_start_pause + [XHC_HB04_BUTTONS]start_pause = std_start_pause The STEP button is normally connected with: [XHC_HB04_BUTTON]step = xhc-hb04.stepsize-up diff --git a/configs/sim/axis/xhc-hb04/xhc-hb04-layout1_es.txt b/configs/sim/axis/xhc-hb04/xhc-hb04-layout1_es.txt index 8337ba89b5b..bdf20d7db80 100644 --- a/configs/sim/axis/xhc-hb04/xhc-hb04-layout1_es.txt +++ b/configs/sim/axis/xhc-hb04/xhc-hb04-layout1_es.txt @@ -25,7 +25,7 @@ xhc-hb04-layout1 Button codes defined in LIB:xhc-hb04-layout1.cfg El botón Start/Pause se puede conectar para un comportamiento estándar con: - [XHC_HB04_BUTTON]start-pause = std_start_pause + [XHC_HB04_BUTTONS]start_pause = std_start_pause El botón STEP normalmente está conectado con: [XHC_HB04_BUTTON]step = xhc-hb04.stepsize-up diff --git a/configs/sim/axis/xhc-hb04/xhc-hb04-layout2.txt b/configs/sim/axis/xhc-hb04/xhc-hb04-layout2.txt index 397db18c2f5..b5df8771143 100644 --- a/configs/sim/axis/xhc-hb04/xhc-hb04-layout2.txt +++ b/configs/sim/axis/xhc-hb04/xhc-hb04-layout2.txt @@ -25,7 +25,7 @@ xhc-hb04-layout2 Button codes defined in LIB:xhc-hb04-layout2.cfg The Start/Pause button can be connected for standard behavior with: - [XHC_HB04_BUTTON]start-pause = std_start_pause + [XHC_HB04_BUTTONS]start_pause = std_start_pause The STEP button is normally connected with: [XHC_HB04_BUTTON]step = xhc-hb04.stepsize-up diff --git a/configs/sim/axis/xhc-hb04/xhc-hb04-layout2_es.txt b/configs/sim/axis/xhc-hb04/xhc-hb04-layout2_es.txt index d24421535d4..4c30fdf5389 100644 --- a/configs/sim/axis/xhc-hb04/xhc-hb04-layout2_es.txt +++ b/configs/sim/axis/xhc-hb04/xhc-hb04-layout2_es.txt @@ -25,7 +25,7 @@ xhc-hb04-layout2 Códigos de botón definidos en LIB:xhc-hb04-layout2.cfg El botón Start/Pause se puede conectar para un comportamiento estándar con: - [XHC_HB04_BUTTON]start-pause = std_start_pause + [XHC_HB04_BUTTONS]start_pause = std_start_pause El botón STEP normalmente está conectado con: [XHC_HB04_BUTTON]step = xhc-hb04.stepsize-up diff --git a/docs/src/man/man1/xhc-hb04.1.adoc b/docs/src/man/man1/xhc-hb04.1.adoc index 86cb4815824..c59eb4fec28 100644 --- a/docs/src/man/man1/xhc-hb04.1.adoc +++ b/docs/src/man/man1/xhc-hb04.1.adoc @@ -90,7 +90,7 @@ should use a file suffix of .cfg. .... File format: - [XHC-HB04] + [XHC_HB04] BUTTON=X1:button-thename1 BUTTON=X2:button-thename2 BUTTON=X3:button-thename3 diff --git a/lib/hallib/xhc-hb04-layout1.cfg b/lib/hallib/xhc-hb04-layout1.cfg index 6a774157630..9ff942efb33 100644 --- a/lib/hallib/xhc-hb04-layout1.cfg +++ b/lib/hallib/xhc-hb04-layout1.cfg @@ -1,4 +1,4 @@ -[XHC-HB04] +[XHC_HB04] BUTTON=01:button-stop BUTTON=02:button-start-pause BUTTON=03:button-rewind diff --git a/lib/hallib/xhc-hb04-layout2.cfg b/lib/hallib/xhc-hb04-layout2.cfg index d844658560d..935c6bdda82 100644 --- a/lib/hallib/xhc-hb04-layout2.cfg +++ b/lib/hallib/xhc-hb04-layout2.cfg @@ -1,4 +1,4 @@ -[XHC-HB04] +[XHC_HB04] BUTTON=01:button-goto-zero BUTTON=02:button-start-pause BUTTON=03:button-rewind diff --git a/lib/hallib/xhc-hb04.tcl b/lib/hallib/xhc-hb04.tcl index d9d3c7fdb3e..50ad8fe9b84 100644 --- a/lib/hallib/xhc-hb04.tcl +++ b/lib/hallib/xhc-hb04.tcl @@ -26,13 +26,16 @@ source [file join $::env(HALLIB_DIR) util_lib.tcl] # [XHC_HB04_BUTTONS] # name = pin (connect button to hal pin) # name = "" (no connect button) +# button names use underscores in the INI (start_pause, goto_zero, +# ...) and are mapped to the dashed hal pin names +# (xhc-hb04.button-start-pause, xhc-hb04.button-goto-zero, ...) # special cases: -# start-pause = std_start_pause (for usual behavior) +# start_pause = std_start_pause (for usual behavior) # step = xhc-hb04.stepsize-up (for usual behavior) # (see INI files for more exanples) # Notes: -# 1) the 'start-pause' pin can be set to "std_start_pause" to +# 1) the 'start_pause' INI item can be set to "std_start_pause" to # implement default behavior # 2) the 'step' pin is normally connected to xhc-hb04.stepsize-up # 3) non-root access to the usb device requires an additional @@ -95,8 +98,11 @@ proc connect_pins {} { #puts stderr "$::progname: no pin defined for <$bname>" continue } + # INI identifiers cannot contain a dash, so button names use + # underscores in the INI; the hal pin names keep the dashes + set pin_bname [string map {_ -} $bname] # this pin is can specify std behavior - if { ([string tolower $bname] == "start-pause") + if { ([string tolower $pin_bname] == "start-pause") && ([string tolower $thepin] == "std_start_pause") } { std_start_pause_button @@ -108,7 +114,7 @@ proc connect_pins {} { puts stderr "$::progname: skipping button $bname marked <$thepin>" continue } - set fullbname xhc-hb04.button-$bname + set fullbname xhc-hb04.button-$pin_bname if !$::xhc_hb04_quiet { if ![pin_exists $fullbname] { puts stderr "$::progname: !!! <$fullbname> pin does not exist, continuing" diff --git a/src/hal/user_comps/xhc-hb04.cc b/src/hal/user_comps/xhc-hb04.cc index 884527da2f0..5f8ba75b7fa 100644 --- a/src/hal/user_comps/xhc-hb04.cc +++ b/src/hal/user_comps/xhc-hb04.cc @@ -41,7 +41,7 @@ using namespace linuxcnc; const char *modname = "xhc-hb04"; int hal_comp_id; -const char *section = "XHC-HB04"; +const char *section = "XHC_HB04"; bool simu_mode = true; typedef struct { @@ -713,7 +713,7 @@ static void Usage(char *name) fprintf(stderr, " 5: 1,10,50,100,1000\n"); fprintf(stderr, "\n"); fprintf(stderr, "Configuration file section format:\n"); - fprintf(stderr, "[XHC-HB04]\n"); + fprintf(stderr, "[XHC_HB04]\n"); fprintf(stderr, "BUTTON=XN:button-thenameN\n"); fprintf(stderr, "...\n"); fprintf(stderr, " where XN=hexcode, button-thenameN=nameforbutton\n"); From 22db1772d432d775a377ebc86f702af2d3143301 Mon Sep 17 00:00:00 2001 From: Luca Toniolo <10792599+grandixximo@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:35:51 +0800 Subject: [PATCH 2/2] update_ini: rewrite dashed ini identifiers to underscores Configs using dashed identifiers (old xhc-hb04 pendant setups) can not be parsed at all any more, so the version-triggered update path never gets a chance to run: the launcher exits on the inivalue parse check before reaching the [EMC]VERSION check that invokes update_ini. Add a text pass to update_ini that runs before parsing and independent of [EMC]VERSION: section and tag identifiers containing dashes (but otherwise valid) are rewritten with underscores in the ini file, in its #INCLUDE'd files and in a custom xhc-hb04 layout cfg found via a loose scan of [XHC_HB04_CONFIG]layout. Every modified file keeps a .bak copy and every rename is listed. The pass is available standalone as update_ini --fix-identifiers. The launcher runs this mode when the inivalue parse check fails and retries parsing afterwards, so old pendant configs migrate themselves on first boot. Identifiers that are invalid even without the dash (leading dash or digit) are left alone and still fail with the original error. Add a tests/update_ini/dash-identifiers test covering the ini file, an included file, a custom layout cfg, the .bak copies and the invalid-identifier cases, and extend the inivalue test with plain dash-rejection cases. --- scripts/linuxcnc.in | 8 +- src/emc/ini/update_ini.py | 236 ++++++++++++++++++++- tests/inifile/inivalue/expected | 2 + tests/inifile/inivalue/test.sh | 5 + tests/update_ini/dash-identifiers/expected | 89 ++++++++ tests/update_ini/dash-identifiers/test.sh | 160 ++++++++++++++ 6 files changed, 495 insertions(+), 5 deletions(-) create mode 100644 tests/update_ini/dash-identifiers/expected create mode 100755 tests/update_ini/dash-identifiers/test.sh diff --git a/scripts/linuxcnc.in b/scripts/linuxcnc.in index 3ab645da498..7a8cf94d76b 100644 --- a/scripts/linuxcnc.in +++ b/scripts/linuxcnc.in @@ -418,7 +418,13 @@ fi # Running inivalue without section/variable filter should return the value of # the first variable in the first section of the file. If that fails, there is # a parse error in the file or it is empty. -inivalue "$INIFILE" > /dev/null || { echo "E: The INI-file contains errors that need to be fixed."; exit 1; } +if ! inivalue "$INIFILE" > /dev/null; then + # Older configs may use dashed identifiers (e.g. the xhc-hb04 pendant + # button names), which the ini parser no longer accepts. Try to rewrite + # them with underscores and test again. + update_ini --fix-identifiers "$INIFILE" + inivalue "$INIFILE" > /dev/null || { echo "E: The INI-file contains errors that need to be fixed."; exit 1; } +fi # delete directories from path, save name only INI_NAME="${INIFILE##*/}" diff --git a/src/emc/ini/update_ini.py b/src/emc/ini/update_ini.py index 5ddc9ec2206..c1026f1d104 100755 --- a/src/emc/ini/update_ini.py +++ b/src/emc/ini/update_ini.py @@ -9,8 +9,14 @@ import re import datetime -import tkinter -from tkinter import messagebox +try: + import tkinter + from tkinter import messagebox +except ImportError: + # tkinter is only needed for the -d dialogs. The --fix-identifiers mode + # and the -f conversion must also work on systems without it (e.g. + # headless builds). + tkinter = None def copysection(block): #Just makes a straight copy of blocks that don't need any work @@ -29,14 +35,220 @@ def writeifexists(file, section, src_item, dest_item = "None"): val = ini.find(section, src_item) if val: file.write("%s = %s\n" % (dest_item, val)) +# The ini parser restricts section and tag identifiers to letters, digits +# and underscore, not starting with a digit. Older configs (notably xhc-hb04 +# pendant setups) used dashes in identifiers, which the historic parser +# accepted. The following text pass rewrites such identifiers with +# underscores so the file can be parsed again. It runs before any +# version-based conversion and regardless of the [EMC]VERSION content. +# A .bak copy is kept of every modified file and every rename is listed. + +_ident_sec_re = re.compile(r"^(\s*)\[([^\]]+)\](.*)$") +_ident_tag_re = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_-]*)(\s*=.*)$") +_ident_ok_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +def _ident_fix_name(name): + # Returns the name with dashes replaced by underscores if the result is + # a valid identifier and the original contained a dash, else None. + if '-' not in name: + return None + fixed = name.replace('-', '_') + if _ident_ok_re.match(fixed): + return fixed + return None + +def fix_dashed_identifiers(ini_filename): + # Rewrites dashed section and tag identifiers to underscores in + # ini_filename, its #INCLUDE'd files and a custom xhc-hb04 layout + # cfg found via [XHC_HB04_CONFIG]layout. Returns a list of + # (path, old, new) renames. + # + # Every rewritten file gets a fresh .bak backup of its original + # content. All files are processed in memory first and nothing is + # written if any of the .bak files already exists, since an existing + # .bak may be ancient or unrelated and overwriting it would lose the + # true original. In that case the whole conversion fails. + renames = [] + contents = {} # path => original text, doubles as the processed set + outputs = {} # path => rewritten text + + def process(path): + path = os.path.normpath(path) + if path in contents: + return + try: + with open(path, 'r') as f: + text = f.read() + except (IOError, OSError): + return # unreadable files are reported by the parser later + contents[path] = text + + # Split into physical lines, keeping line endings for the rewrite + phys = [] + endings = [] + for line in text.splitlines(keepends=True): + body = line.rstrip('\r\n') + phys.append(list(body)) + endings.append(line[len(body):]) + + changed = False + i = 0 + while i < len(phys): + # Assemble one logical line, merging continuations like the + # parser does: a line whose right-trimmed content ends in a + # backslash continues on the next physical line. Only the + # identifier of a section or tag is ever modified (a dash is + # replaced by an underscore at the same character position), + # so values, continuations and layout are preserved verbatim. + logical = "" + spans = [] # (physical line index, logical start, length) per part + j = i + while True: + contrib = ''.join(phys[j]).rstrip() + if contrib.endswith('\\'): + spans.append((j, len(logical), len(contrib) - 1)) + logical += contrib[:-1] + j += 1 + if j >= len(phys): + break # continuation on the last line: a parser error, but not ours to fix + else: + spans.append((j, len(logical), len(contrib))) + logical += contrib + j += 1 + break + i = j + + def flip_dash(p): + # Replace the dash at logical position p with an underscore + # at the corresponding physical position. + for phys_idx, log_start, length in spans: + if log_start <= p < log_start + length: + phys[phys_idx][p - log_start] = '_' + return + + def fix_span(start, end, old, new): + nonlocal changed + renames.append((path, old, new)) + for p in range(start, end): + if logical[p] == '-': + flip_dash(p) + changed = True + + if logical.startswith("#INCLUDE") and len(logical) > 8 and logical[8] in ' \t': + inc = os.path.expanduser(logical[8:].strip()) + if not os.path.isabs(inc): + inc = os.path.join(os.path.dirname(path), inc) + process(inc) + continue + stripped = logical.lstrip() + if stripped == '' or stripped[0] in '#;': + continue + m = _ident_sec_re.match(logical) + if m: + raw = m.group(2) + ident = raw.strip() + fixed = _ident_fix_name(ident) + if fixed: + start = m.start(2) + len(raw) - len(raw.lstrip()) + fix_span(start, start + len(ident), "[%s]" % ident, "[%s]" % fixed) + else: + m = _ident_tag_re.match(logical) + if m: + fixed = _ident_fix_name(m.group(2)) + if fixed: + fix_span(m.start(2), m.end(2), m.group(2), fixed) + + if changed: + outputs[path] = ''.join(''.join(body) + ending + for body, ending in zip(phys, endings)) + + process(ini_filename) + + # A custom xhc-hb04 layout cfg is parsed by the same rules. Find it + # with a loose text scan since the ini file may not parse yet. + cfgs = [] + for path, text in contents.items(): + m = re.search(r"(?ms)^\s*\[XHC_HB04_CONFIG\](.*?)(?:^\s*\[|\Z)", text) + if not m: + continue + lm = re.search(r"(?m)^\s*layout\s*=\s*([^;#\s]+)", m.group(1)) + if not lm: + continue + layout = lm.group(1).strip('{}') + if layout in ("1", "2") or layout.startswith("LIB:"): + continue # distribution-provided layout files + cfg = os.path.expanduser(layout) + if not os.path.isabs(cfg): + cfg = os.path.join(os.path.dirname(path), cfg) + if os.path.isfile(cfg): + cfgs.append(cfg) + for cfg in cfgs: + process(cfg) + + # Refuse to overwrite an existing backup. Fail the whole conversion + # before anything is written, so the include cascade stays atomic. + conflicts = [path for path in outputs if os.path.exists(path + ".bak")] + if conflicts: + for path in conflicts: + print("update_ini: %s.bak already exists, will not overwrite it" % path) + print("update_ini: no files were changed, move or remove the existing .bak file(s) and retry") + exit(1) + + # Back up and write file by file. If anything fails midway, restore + # the files already processed from their fresh .bak copies and remove + # those copies again, so the config is never left half-converted and a + # retry is not blocked by leftover .bak files. + written = [] + try: + for path, text in outputs.items(): + shutil.copy(path, path + ".bak") + written.append(path) + with open(path, 'w') as f: + f.write(text) + except OSError as e: + print("update_ini: failed to update %s: %s" % (path, e)) + for path in written: + try: + shutil.copy(path + ".bak", path) + os.remove(path + ".bak") + except OSError as e2: + print("update_ini: could not restore %s: %s" % (path, e2)) + print("update_ini: %s may be inconsistent, its original content is in %s.bak" % (path, path)) + print("update_ini: no files were changed") + exit(1) + + if outputs: + # Verify that the converted cascade parses. The linuxcnc script + # retries the parse itself, but standalone --fix-identifiers use + # should not report success on a file that still fails, e.g. on + # identifiers that can not be mapped to a valid one. + try: + linuxcnc.ini(ini_filename) + except: + print("update_ini: %s still fails to parse after the identifier conversion" % ini_filename) + print("update_ini: the converted files are left in place, the originals are in the .bak files") + exit(1) + + return renames + +def report_identifier_fixes(renames): + for path, old, new in renames: + print("%s: %s => %s" % (path, old, new)) + if renames: + print("update_ini: renamed %d dashed identifier(s) to use underscores (originals saved as .bak)" % len(renames)) + force = 0 dialogs = 0 +ident_only = 0 subs = {} subs2 = {} filename = None for opt in sys.argv[1:]: if opt == '-d': + if tkinter is None: + print("update_ini: the -d option needs tkinter, which is not available") + exit(1) dialogs = 1 r = tkinter.Tk() r.option_add('*Dialog.msg.font', 'Times 12') @@ -44,6 +256,8 @@ def writeifexists(file, section, src_item, dest_item = "None"): elif opt == '-f': force = 1 + elif opt == '--fix-identifiers': + ident_only = 1 elif opt[0] == '-': print ("Unknown command line option to update_ini, exiting") exit() @@ -51,11 +265,13 @@ def writeifexists(file, section, src_item, dest_item = "None"): filename = opt if filename == None: - t = """Usage: update_ini [-d] [-f] filename.ini\n + t = """Usage: update_ini [-d] [-f] [--fix-identifiers] filename.ini\n If the -d flag is used then a dialog box will be displayed describing the purpose of this script, and giving the user the option to change their minds\nIf the -f flag is used then no questions will be -asked and the conversion will proceed blindly""" +asked and the conversion will proceed blindly\nIf --fix-identifiers is +used then only dashed identifiers are rewritten to use underscores and +no version-based conversion is done""" if dialogs: messagebox.showerror('invalid options', str(t)) elif not force: @@ -84,6 +300,18 @@ def writeifexists(file, section, src_item, dest_item = "None"): # We want to work with the base INI file here, not the expanded version if #include is used filename = re.sub(r'\.expanded', '', filename) +# Rewrite dashed identifiers to underscores before anything else, since a +# file containing them can not be parsed at all. This is independent of the +# [EMC]VERSION-based conversion below. +# FIXME: the versioned conversion backs up differently (a .old copy of the +# whole config). A config that needs both migrations ends up with .bak +# files next to a .old directory. Revisit how the two backup schemes fit +# together when the version 1.2 conversion lands (#3704). +report_identifier_fixes(fix_dashed_identifiers(filename)) + +if ident_only: + exit(0) + try: ini = linuxcnc.ini(filename) except: diff --git a/tests/inifile/inivalue/expected b/tests/inifile/inivalue/expected index 257b8d9c6be..9fb61f63387 100644 --- a/tests/inifile/inivalue/expected +++ b/tests/inifile/inivalue/expected @@ -17,6 +17,7 @@ xtest.ini:1: error: Invalid section. No content between '[' and ']' --- test invalid section, invalid identifier xtest.ini:1: error: Invalid section '0SECTION'. Cannot start with a digit xtest.ini:1: error: Invalid section 'xæøåz'. Identifier contains invalid character(s) +xtest.ini:1: error: Invalid section 'SEC-TION'. Identifier contains invalid character(s) --- test duplicate section merge warning xtest.ini:2: warning: Section 'SECTION' already exists. Merging... val @@ -32,6 +33,7 @@ xtest.ini:2: error: Invalid tag 'VAR x'. Expected '=' after tag identifier --- test invalid variable name identifier xtest.ini:2: error: Invalid tag '0VAR'. Tag identifiers cannot start with a digit xtest.ini:2: error: Invalid tag name 'VÅR'. Identifier contains invalid character(s) +xtest.ini:2: error: Invalid tag name 'V-AR'. Identifier contains invalid character(s) xtest.ini:2: error: Invalid tag name 'VAR x'. Identifier contains invalid character(s) --- test invalid variable outside section xtest.ini:1: error: Tag 'VAR' found without prior section definition diff --git a/tests/inifile/inivalue/test.sh b/tests/inifile/inivalue/test.sh index f510cc98e11..591d5d3de45 100755 --- a/tests/inifile/inivalue/test.sh +++ b/tests/inifile/inivalue/test.sh @@ -55,6 +55,8 @@ echo "[0SECTION]" > xtest.ini tst --var=VAR && t "Invalid section" echo "[xæøåz]" > xtest.ini tst --var=VAR && t "Invalid section" +echo "[SEC-TION]" > xtest.ini +tst --var=VAR && t "Invalid section" r "--- test duplicate section merge warning" ( echo "[SECTION]" @@ -89,6 +91,9 @@ tst --var=0VAR && t "Invalid variable name" ( echo "[SECTION]" echo "VÅR=val" ) > xtest.ini tst --var=VAR && t "Invalid variable name" +( echo "[SECTION]" + echo "V-AR=val" ) > xtest.ini +tst --var=VAR && t "Invalid variable name" ( echo "[SECTION]" echo "VAR x=val" ) > xtest.ini tst --var=VAR && t "Invalid variable name" diff --git a/tests/update_ini/dash-identifiers/expected b/tests/update_ini/dash-identifiers/expected new file mode 100644 index 00000000000..fef54c45f5b --- /dev/null +++ b/tests/update_ini/dash-identifiers/expected @@ -0,0 +1,89 @@ +--- update_ini --fix-identifiers output +buttons.inc: start-pause => start_pause +buttons.inc: goto-zero => goto_zero +test.ini: [XHC-HB04-EXTRA] => [XHC_HB04_EXTRA] +test.ini: some-key => some_key +test.ini: multi-key => multi_key +test.ini: split-tag => split_tag +test.ini: [SECTION-SPLIT] => [SECTION_SPLIT] +test.ini: inner-key => inner_key +mylayout.cfg: [XHC-HB04] => [XHC_HB04] +update_ini: renamed 9 dashed identifier(s) to use underscores (originals saved as .bak) +--- converted test.ini +#INCLUDE buttons.inc +[EMC] +VERSION = 1.1 +MACHINE = dash identifier test + +[XHC_HB04_CONFIG] +layout = mylayout.cfg +coords = x y z + +[XHC_HB04_EXTRA] +some_key = 1 +# continuation lines belong to the value and must not be rewritten +note = do not touch start-pause =\ + goto-zero = inside a value +multi_key = first\ + second +# comment ending in a continuation\ +also-not-a-tag = still comment text +# identifiers split by a continuation are rewritten too +split_t\ +ag = 42 +[SECTION_SPL\ +IT] +inner_key = 9 +--- converted buttons.inc +[XHC_HB04_BUTTONS] +start_pause = std_start_pause +goto_zero = halui.mdi-command-00 +step = xhc-hb04.stepsize-up +--- converted mylayout.cfg +[XHC_HB04] +BUTTON=01:button-stop +BUTTON=02:button-start-pause +--- backups kept +buttons.inc.bak +mylayout.cfg.bak +test.ini.bak +--- inivalue after conversion +1 +std_start_pause +halui.mdi-command-00 +--- continued values survive (dashed text inside values untouched) +do not touch start-pause = goto-zero = inside a value +first second +--- split identifiers +42 +9 +--- invalid identifiers untouched +bad.ini unchanged +update_ini: conflict.inc.bak already exists, will not overwrite it +update_ini: no files were changed, move or remove the existing .bak file(s) and retry +exit=1 +--- conflict.ini untouched +#INCLUDE conflict.inc +[XHC-HB04-EXTRA] +some-key = 1 +--- conflict.inc untouched +[XHC_HB04_BUTTONS] +start-pause = std_start_pause +--- converted file that still fails to parse +update_ini: bad2.ini still fails to parse after the identifier conversion +update_ini: the converted files are left in place, the originals are in the .bak files +exit=1 +--- bad2.ini converted, .bak kept +[SEC_TION] +-bad-key = 1 +bad2.ini.bak +--- write failure rolls back +update_ini: failed to update sub/blocked.inc: [Errno 13] Permission denied: 'sub/blocked.inc.bak' +update_ini: no files were changed +exit=1 +--- roll.ini restored +roll.ini unchanged +--- sub/blocked.inc untouched +[XHC_HB04_BUTTONS] +start-pause = std_start_pause +roll.ini.bak removed diff --git a/tests/update_ini/dash-identifiers/test.sh b/tests/update_ini/dash-identifiers/test.sh new file mode 100755 index 00000000000..ea54ea68dc3 --- /dev/null +++ b/tests/update_ini/dash-identifiers/test.sh @@ -0,0 +1,160 @@ +#!/bin/bash + +# Test the dashed-identifier migration of update_ini --fix-identifiers: +# dashed section and tag names in the ini file, in an #INCLUDE'd file and +# in a custom xhc-hb04 layout cfg are rewritten to use underscores, .bak +# copies are kept and invalid identifiers are left alone. + +UPDATE_INI=update_ini +INIVALUE=inivalue + +command -v "$UPDATE_INI" > /dev/null 2>&1 || { echo "*** Missing update_ini executable"; exit 1; } +command -v "$INIVALUE" > /dev/null 2>&1 || { echo "*** Missing inivalue executable"; exit 1; } + +cat > test.ini <<'EOF' +#INCLUDE buttons.inc +[EMC] +VERSION = 1.1 +MACHINE = dash identifier test + +[XHC_HB04_CONFIG] +layout = mylayout.cfg +coords = x y z + +[XHC-HB04-EXTRA] +some-key = 1 +# continuation lines belong to the value and must not be rewritten +note = do not touch start-pause =\ + goto-zero = inside a value +multi-key = first\ + second +# comment ending in a continuation\ +also-not-a-tag = still comment text +# identifiers split by a continuation are rewritten too +split-t\ +ag = 42 +[SECTION-SPL\ +IT] +inner-key = 9 +EOF + +cat > buttons.inc <<'EOF' +[XHC_HB04_BUTTONS] +start-pause = std_start_pause +goto-zero = halui.mdi-command-00 +step = xhc-hb04.stepsize-up +EOF + +cat > mylayout.cfg <<'EOF' +[XHC-HB04] +BUTTON=01:button-stop +BUTTON=02:button-start-pause +EOF + +echo "--- update_ini --fix-identifiers output" +"$UPDATE_INI" --fix-identifiers test.ini + +echo "--- converted test.ini" +cat test.ini +echo "--- converted buttons.inc" +cat buttons.inc +echo "--- converted mylayout.cfg" +cat mylayout.cfg +echo "--- backups kept" +ls -- *.bak + +echo "--- inivalue after conversion" +"$INIVALUE" --sec=XHC_HB04_EXTRA --var=some_key test.ini +"$INIVALUE" --sec=XHC_HB04_BUTTONS --var=start_pause test.ini +"$INIVALUE" --sec=XHC_HB04_BUTTONS --var=goto_zero test.ini +echo "--- continued values survive (dashed text inside values untouched)" +"$INIVALUE" --sec=XHC_HB04_EXTRA --var=note test.ini +"$INIVALUE" --sec=XHC_HB04_EXTRA --var=multi_key test.ini +echo "--- split identifiers" +"$INIVALUE" --sec=XHC_HB04_EXTRA --var=split_tag test.ini +"$INIVALUE" --sec=SECTION_SPLIT --var=inner_key test.ini + +# identifiers that are invalid even without the dash must be left alone +cat > bad.ini <<'EOF' +[SECTION] +-bad-key = 1 +0bad = 2 +EOF +cp bad.ini bad.ini.orig +"$UPDATE_INI" --fix-identifiers bad.ini +echo "--- invalid identifiers untouched" +if diff bad.ini.orig bad.ini; then + echo "bad.ini unchanged" +else + echo "*** bad.ini was modified" +fi + +# an existing .bak blocks the whole conversion, nothing may be written +cat > conflict.ini <<'EOF' +#INCLUDE conflict.inc +[XHC-HB04-EXTRA] +some-key = 1 +EOF +cat > conflict.inc <<'EOF' +[XHC_HB04_BUTTONS] +start-pause = std_start_pause +EOF +echo "ancient backup" > conflict.inc.bak +"$UPDATE_INI" --fix-identifiers conflict.ini +echo "exit=$?" +echo "--- conflict.ini untouched" +cat conflict.ini +echo "--- conflict.inc untouched" +cat conflict.inc + +# a converted file that still does not parse is reported +echo "--- converted file that still fails to parse" +cat > bad2.ini <<'EOF' +[SEC-TION] +-bad-key = 1 +EOF +"$UPDATE_INI" --fix-identifiers bad2.ini 2>/dev/null +echo "exit=$?" +echo "--- bad2.ini converted, .bak kept" +cat bad2.ini +ls -- bad2.ini.bak + +# a write failure midway must roll back the files already written +echo "--- write failure rolls back" +if [ "$(id -u)" = 0 ]; then + echo "skipped: running as root, permissions not enforced" +else + mkdir sub + cat > roll.ini <<'EOF' +#INCLUDE sub/blocked.inc +[XHC-HB04-EXTRA] +some-key = 1 +EOF + cat > sub/blocked.inc <<'EOF' +[XHC_HB04_BUTTONS] +start-pause = std_start_pause +EOF + cp roll.ini roll.ini.orig + chmod a-w sub + "$UPDATE_INI" --fix-identifiers roll.ini + echo "exit=$?" + chmod +w sub + echo "--- roll.ini restored" + if diff roll.ini.orig roll.ini; then + echo "roll.ini unchanged" + else + echo "*** roll.ini was left converted" + fi + echo "--- sub/blocked.inc untouched" + cat sub/blocked.inc + if [ -e roll.ini.bak ]; then + echo "*** roll.ini.bak left behind" + else + echo "roll.ini.bak removed" + fi +fi + +rm -rf test.ini test.ini.bak buttons.inc buttons.inc.bak \ + mylayout.cfg mylayout.cfg.bak bad.ini bad.ini.orig bad2.ini bad2.ini.bak \ + conflict.ini conflict.inc conflict.inc.bak \ + roll.ini roll.ini.orig sub