Use shlex.join. NFC (#24111)

We bumped the minimum python version to 3.8 in #23417. I guess I
overlooked this change at the time.

Author: sbc100

emcc.py

@@ -526,7 +526,7 @@ def run(args):
     args += shlex.split(EMCC_CFLAGS)
 
   if DEBUG:
-    logger.warning(f'invocation: {shared.shlex_join(args)} (in {os.getcwd()})')
+    logger.warning(f'invocation: {shlex.join(args)} (in {os.getcwd()})')
 
   # Strip args[0] (program name)
   args = args[1:]
@@ -596,7 +596,7 @@ def run(args):
         exit_with_error(f'unable to parse output of `{cmd}`:\n{proc.stderr}')
       parts = shlex.split(lines[0].replace('\\', '\\\\'))
       parts = [x for x in parts if x not in ['-c', '-o', '-v', '-emit-llvm'] and input_file not in x and temp_target not in x]
-      print(shared.shlex_join(parts[1:]))
+      print(shlex.join(parts[1:]))
     return 0
 
   if '-dumpmachine' in args or '-print-target-triple' in args or '--print-target-triple' in args:

emcmake.py

@@ -5,6 +5,7 @@
 # found in the LICENSE file.
 
 import os
+import shlex
 import shutil
 import sys
 from tools import shared
@@ -56,7 +57,7 @@ def has_substr(args, substr):
       print('emcmake: no compatible cmake generator found; Please install ninja or mingw32-make, or specify a generator explicitly using -G', file=sys.stderr)
       return 1
 
-  print('configure: ' + shared.shlex_join(args), file=sys.stderr)
+  print(f'configure: {shlex.join(args)}', file=sys.stderr)
   try:
     shared.check_call(args)
     return 0

emconfigure.py

@@ -16,6 +16,7 @@
 tests will work properly.
 """
 
+import shlex
 import sys
 from tools import building
 from tools import shared
@@ -47,7 +48,7 @@ def run():
   # compilation with emcc, but instead do builds natively with Clang. This
   # is a heuristic emulation that may or may not work.
   env['EMMAKEN_JUST_CONFIGURE'] = '1'
-  print('configure: ' + shared.shlex_join(args), file=sys.stderr)
+  print(f'configure: {shlex.join(args)}', file=sys.stderr)
   try:
     shared.check_call(args, env=env)
     return 0

test/common.py

@@ -88,7 +88,7 @@
 EMCMAKE = shared.bat_suffix(path_from_root('emcmake'))
 EMCONFIGURE = shared.bat_suffix(path_from_root('emconfigure'))
 EMRUN = shared.bat_suffix(shared.path_from_root('emrun'))
-WASM_DIS = Path(building.get_binaryen_bin(), 'wasm-dis')
+WASM_DIS = os.path.join(building.get_binaryen_bin(), 'wasm-dis')
 LLVM_OBJDUMP = os.path.expanduser(shared.build_llvm_tool_path(shared.exe_suffix('llvm-objdump')))
 PYTHON = sys.executable
 if not config.NODE_JS_TEST:
@@ -1406,7 +1406,7 @@ def build(self, filename, libraries=None, includes=None, force_c=False, emcc_arg
       output = output_basename + output_suffix
     else:
       output = shared.unsuffixed_basename(filename) + output_suffix
-    cmd = compiler + [filename, '-o', output] + all_emcc_args
+    cmd = compiler + [str(filename), '-o', output] + all_emcc_args
     if libraries:
       cmd += libraries
     if includes:
@@ -1755,7 +1755,7 @@ def run_process(self, cmd, check=True, **args):
       if check and e.returncode != 0:
         print(e.stdout)
         print(e.stderr)
-        self.fail(f'subprocess exited with non-zero return code({e.returncode}): `{shared.shlex_join(cmd)}`')
+        self.fail(f'subprocess exited with non-zero return code({e.returncode}): `{shlex.join(cmd)}`')
 
   def emcc(self, filename, args=[], output_filename=None, **kwargs):  # noqa
     compile_only = '-c' in args or '-sSIDE_MODULE' in args

test/jsrun.py

@@ -6,6 +6,7 @@
 import common
 import logging
 import os
+import shlex
 import subprocess
 import sys
 from subprocess import PIPE, CalledProcessError
@@ -97,7 +98,7 @@ def run_js(filename, engine, args=None,
 
   command = make_command(os.path.abspath(filename), engine, args)
   if common.EMTEST_VERBOSE:
-    print(f"Running: '{shared.shlex_join(command)}'")
+    print(f"Running: '{shlex.join(command)}'")
   try:
     proc = subprocess.run(
         command,

test/test_browser.py

@@ -1399,7 +1399,7 @@ def test_fs_workerfs_package(self):
     create_file('file1.txt', 'first')
     ensure_dir('sub')
     create_file('sub/file2.txt', 'second')
-    self.run_process([FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', Path('sub/file2.txt'), '--separate-metadata', '--js-output=files.js'])
+    self.run_process([FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', 'sub/file2.txt', '--separate-metadata', '--js-output=files.js'])
     self.btest(Path('fs/test_workerfs_package.cpp'), '1', emcc_args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js'])
 
   def test_fs_lz4fs_package(self):
@@ -1415,7 +1415,7 @@ def test_fs_lz4fs_package(self):
     print('emcc-normal')
     self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$ccall')
     self.btest_exit(Path('fs/test_lz4fs.cpp'), 2, emcc_args=['-sLZ4', '--preload-file', 'file1.txt', '--preload-file', 'subdir/file2.txt', '--preload-file', 'file3.txt'])
-    assert os.path.getsize('file1.txt') + os.path.getsize(Path('subdir/file2.txt')) + os.path.getsize('file3.txt') == 3 * 1024 * 128 * 10 + 1
+    assert os.path.getsize('file1.txt') + os.path.getsize('subdir/file2.txt') + os.path.getsize('file3.txt') == 3 * 1024 * 128 * 10 + 1
     assert os.path.getsize('test.data') < (3 * 1024 * 128 * 10) / 2  # over half is gone
     print('    emcc-opts')
     self.btest_exit(Path('fs/test_lz4fs.cpp'), 2, emcc_args=['-sLZ4', '--preload-file', 'file1.txt', '--preload-file', 'subdir/file2.txt', '--preload-file', 'file3.txt', '-O2'])
@@ -5696,7 +5696,7 @@ def test_emrun(self):
         ['--dump_out_directory=foo_bar', '--port', '6942'],
     ]:
       args = args_base + args + [self.in_dir('test_emrun.html'), '--', '1', '2', '--3', 'escaped space', 'with_underscore']
-      print(shared.shlex_join(args))
+      print(shlex.join(args))
       proc = self.run_process(args, check=False)
       self.assertEqual(proc.returncode, 100)
       dump_dir = 'dump_out'

test/test_core.py

@@ -7841,7 +7841,7 @@ def get_dwarf_addr(line, col):
     self.assertLess(get_dwarf_addr(7, 3), get_dwarf_addr(8, 3))
 
     # Get the wat, printing with -g which has binary offsets
-    wat = self.run_process([Path(building.get_binaryen_bin(), 'wasm-opt'),
+    wat = self.run_process([os.path.join(building.get_binaryen_bin(), 'wasm-opt'),
                            wasm_filename, '-g', '--print', '-all'], stdout=PIPE).stdout
 
     # We expect to see a pattern like this in optimized builds (there isn't
@@ -8352,7 +8352,7 @@ def break_wasm(name):
         return False
       create_file('wat.wat', wat)
       shutil.move(name, name + '.orig')
-      self.run_process([Path(building.get_binaryen_bin(), 'wasm-as'), 'wat.wat', '-o', name, '-g', '--all-features'])
+      self.run_process([os.path.join(building.get_binaryen_bin(), 'wasm-as'), 'wat.wat', '-o', name, '-g', '--all-features'])
       return True
 
     def verify_working(args):

test/test_other.py

@@ -11814,7 +11814,7 @@ def print_percent(actual, expected):
         raise
 
     args = [compiler_for(sources[0]), '-o', 'a.html'] + args + sources
-    print(shared.shlex_join(args))
+    print(shlex.join(args))
     self.run_process(args)
 
     # For certain tests, don't just check the output size but check

test/test_sanity.py

@@ -743,18 +743,24 @@ def test_embuilder_with_use_port_syntax(self):
     self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'libSDL2_image-jpg-png.a'))
     self.assertContained('error building port `sdl2_image:formats=invalid` | invalid is not a supported format', self.do([EMBUILDER, 'build', 'sdl2_image:formats=invalid', '--force']))
 
-  def test_embuilder_external_ports(self):
+  def test_embuilder_external_ports_simple(self):
     restore_and_set_up()
     simple_port_path = test_file("other/ports/simple.py")
     # embuilder handles external port target that ends with .py
     self.run_process([EMBUILDER, 'build', f'{simple_port_path}', '--force'])
     self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lib_simple.a'))
+
+  def test_embuilder_external_ports(self):
     # embuilder handles external port target that contains port options
+    restore_and_set_up()
     external_port_path = test_file("other/ports/external.py")
     self.run_process([EMBUILDER, 'build', f'{external_port_path}:value1=12:value2=36', '--force'])
     self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lib_external.a'))
+
+  def test_embuilder_external_ports_options(self):
     # embuilder handles external port target that contains port options (influences library name,
     # like sdl2_image:formats=png)
+    restore_and_set_up()
     external_port_path = test_file("other/ports/external.py")
     self.run_process([EMBUILDER, 'build', f'{external_port_path}:dependency=sdl2', '--force'])
     self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lib_external-sdl2.a'))

tools/building.py

@@ -314,7 +314,7 @@ def get_command_with_possible_response_file(cmd):
   # subprocess can still run into the Command Line Too Long error.
   # Reduce the limit by ~1K for now to be on the safe side, but we might need to
   # adjust this in the future if it turns out not to be enough.
-  if (len(shared.shlex_join(cmd)) <= 7000 and force_response_files != '1') or force_response_files == '0':
+  if (len(shlex.join(cmd)) <= 7000 and force_response_files != '1') or force_response_files == '0':
     return cmd
 
   logger.debug('using response file for %s' % cmd[0])
@@ -473,12 +473,12 @@ def check_closure_compiler(cmd, args, env, allowed_to_fail):
     if isinstance(e, subprocess.CalledProcessError):
       sys.stderr.write(e.stdout)
     sys.stderr.write(str(e) + '\n')
-    exit_with_error('closure compiler (%s) did not execute properly!' % shared.shlex_join(cmd))
+    exit_with_error('closure compiler (%s) did not execute properly!' % shlex.join(cmd))
 
   if 'Version:' not in output:
     if allowed_to_fail:
       return False
-    exit_with_error('unrecognized closure compiler --version output (%s):\n%s' % (shared.shlex_join(cmd), output))
+    exit_with_error('unrecognized closure compiler --version output (%s):\n%s' % (shlex.join(cmd), output))
 
   return True
 
@@ -685,7 +685,7 @@ def move_to_safe_7bit_ascii_filename(filename):
     logger.error(proc.stderr) # print list of errors (possibly long wall of text if input was minified)
 
     # Exit and print final hint to get clearer output
-    msg = f'closure compiler failed (rc: {proc.returncode}): {shared.shlex_join(cmd)}'
+    msg = f'closure compiler failed (rc: {proc.returncode}): {shlex.join(cmd)}'
     if settings.MINIFY_WHITESPACE:
       msg += ' the error message may be clearer with -g1 and EMCC_DEBUG=2 set'
     exit_with_error(msg)

tools/gen_struct_info.py

@@ -61,6 +61,7 @@
 import json
 import argparse
 import tempfile
+import shlex
 import subprocess
 
 __scriptdir__ = os.path.dirname(os.path.abspath(__file__))
@@ -268,7 +269,7 @@ def generate_cmd(js_file_path, src_file_path, cflags):
   # TODO(sbc): Remove this one we remove the test_em_config_env_var test
   cmd += ['-Wno-deprecated']
 
-  show(shared.shlex_join(cmd))
+  show(shlex.join(cmd))
   return cmd
 
 

tools/link.py

@@ -469,9 +469,9 @@ def make_js_executable(script):
     # Using -S (--split-string) here means that arguments to the executable are
     # correctly parsed.  We don't do this by default because old versions of env
     # don't support -S.
-    cmd = '/usr/bin/env -S ' + shared.shlex_join(cmd)
+    cmd = '/usr/bin/env -S ' + shlex.join(cmd)
   else:
-    cmd = shared.shlex_join(cmd)
+    cmd = shlex.join(cmd)
   logger.debug('adding `#!` to JavaScript file: %s' % cmd)
   # add shebang
   with open(script, 'w') as f:

tools/shared.py

@@ -12,6 +12,7 @@
 import os
 import re
 import shutil
+import shlex
 import subprocess
 import signal
 import stat
@@ -95,23 +96,6 @@
 diagnostics.add_warning('closure', enabled=False)
 
 
-# TODO(sbc): Investigate switching to shlex.quote
-def shlex_quote(arg):
-  arg = os.fspath(arg)
-  if ' ' in arg and (not (arg.startswith('"') and arg.endswith('"'))) and (not (arg.startswith("'") and arg.endswith("'"))):
-    return '"' + arg.replace('"', '\\"') + '"'
-
-  return arg
-
-
-# Switch to shlex.join once we can depend on python 3.8:
-# https://docs.python.org/3/library/shlex.html#shlex.join
-def shlex_join(cmd):
-  if type(cmd) is str:
-    return cmd
-  return ' '.join(shlex_quote(x) for x in cmd)
-
-
 def run_process(cmd, check=True, input=None, *args, **kw):
   """Runs a subprocess returning the exit code.
 
@@ -127,7 +111,7 @@ def run_process(cmd, check=True, input=None, *args, **kw):
   kw.setdefault('text', True)
   kw.setdefault('encoding', 'utf-8')
   ret = subprocess.run(cmd, check=check, input=input, *args, **kw)
-  debug_text = '%sexecuted %s' % ('successfully ' if check else '', shlex_join(cmd))
+  debug_text = '%sexecuted %s' % ('successfully ' if check else '', shlex.join(cmd))
   logger.debug(debug_text)
   return ret
 
@@ -218,7 +202,7 @@ def get_finished_process():
       idx = get_finished_process()
       finished_process = processes.pop(idx)
       if finished_process.returncode != 0:
-        exit_with_error('subprocess %d/%d failed (%s)! (cmdline: %s)' % (idx + 1, len(commands), returncode_to_str(finished_process.returncode), shlex_join(commands[idx])))
+        exit_with_error('subprocess %d/%d failed (%s)! (cmdline: %s)' % (idx + 1, len(commands), returncode_to_str(finished_process.returncode), shlex.join(commands[idx])))
       num_completed += 1
 
   if route_stdout_to_temp_files_suffix:
@@ -235,9 +219,9 @@ def check_call(cmd, *args, **kw):
   try:
     return run_process(cmd, *args, **kw)
   except subprocess.CalledProcessError as e:
-    exit_with_error("'%s' failed (%s)", shlex_join(cmd), returncode_to_str(e.returncode))
+    exit_with_error("'%s' failed (%s)", shlex.join(cmd), returncode_to_str(e.returncode))
   except OSError as e:
-    exit_with_error("'%s' failed: %s", shlex_join(cmd), str(e))
+    exit_with_error("'%s' failed: %s", shlex.join(cmd), str(e))
 
 
 def exec_process(cmd):
@@ -638,7 +622,7 @@ def maybe_quote(arg):
     print(' ' + ' '.join([maybe_quote(a) for a in cmd]), file=sys.stderr)
     sys.stderr.flush()
   elif PRINT_SUBPROCS:
-    print(' %s %s' % (maybe_quote(cmd[0]), shlex_join(cmd[1:])), file=sys.stderr)
+    print(' %s %s' % (maybe_quote(cmd[0]), shlex.join(cmd[1:])), file=sys.stderr)
     sys.stderr.flush()