interp.py 49 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367
  1. # interp.py - shell interpreter for pysh.
  2. #
  3. # Copyright 2007 Patrick Mezard
  4. #
  5. # This software may be used and distributed according to the terms
  6. # of the GNU General Public License, incorporated herein by reference.
  7. """Implement the shell interpreter.
  8. Most references are made to "The Open Group Base Specifications Issue 6".
  9. <http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html>
  10. """
  11. # TODO: document the fact input streams must implement fileno() so Popen will work correctly.
  12. # it requires non-stdin stream to be implemented as files. Still to be tested...
  13. # DOC: pathsep is used in PATH instead of ':'. Clearly, there are path syntax issues here.
  14. # TODO: stop command execution upon error.
  15. # TODO: sort out the filename/io_number mess. It should be possible to use filenames only.
  16. # TODO: review subshell implementation
  17. # TODO: test environment cloning for non-special builtins
  18. # TODO: set -x should not rebuild commands from tokens, assignments/redirections are lost
  19. # TODO: unit test for variable assignment
  20. # TODO: test error management wrt error type/utility type
  21. # TODO: test for binary output everywhere
  22. # BUG: debug-parsing does not pass log file to PLY. Maybe a PLY upgrade is necessary.
  23. import base64
  24. import cPickle as pickle
  25. import errno
  26. import glob
  27. import os
  28. import re
  29. import subprocess
  30. import sys
  31. import tempfile
  32. try:
  33. s = set()
  34. del s
  35. except NameError:
  36. from Set import Set as set
  37. import builtin
  38. from sherrors import *
  39. import pyshlex
  40. import pyshyacc
  41. def mappend(func, *args, **kargs):
  42. """Like map but assume func returns a list. Returned lists are merged into
  43. a single one.
  44. """
  45. return reduce(lambda a,b: a+b, map(func, *args, **kargs), [])
  46. class FileWrapper:
  47. """File object wrapper to ease debugging.
  48. Allow mode checking and implement file duplication through a simple
  49. reference counting scheme. Not sure the latter is really useful since
  50. only real file descriptors can be used.
  51. """
  52. def __init__(self, mode, file, close=True):
  53. if mode not in ('r', 'w', 'a'):
  54. raise IOError('invalid mode: %s' % mode)
  55. self._mode = mode
  56. self._close = close
  57. if isinstance(file, FileWrapper):
  58. if file._refcount[0] <= 0:
  59. raise IOError(0, 'Error')
  60. self._refcount = file._refcount
  61. self._refcount[0] += 1
  62. self._file = file._file
  63. else:
  64. self._refcount = [1]
  65. self._file = file
  66. def dup(self):
  67. return FileWrapper(self._mode, self, self._close)
  68. def fileno(self):
  69. """fileno() should be only necessary for input streams."""
  70. return self._file.fileno()
  71. def read(self, size=-1):
  72. if self._mode!='r':
  73. raise IOError(0, 'Error')
  74. return self._file.read(size)
  75. def readlines(self, *args, **kwargs):
  76. return self._file.readlines(*args, **kwargs)
  77. def write(self, s):
  78. if self._mode not in ('w', 'a'):
  79. raise IOError(0, 'Error')
  80. return self._file.write(s)
  81. def flush(self):
  82. self._file.flush()
  83. def close(self):
  84. if not self._refcount:
  85. return
  86. assert self._refcount[0] > 0
  87. self._refcount[0] -= 1
  88. if self._refcount[0] == 0:
  89. self._mode = 'c'
  90. if self._close:
  91. self._file.close()
  92. self._refcount = None
  93. def mode(self):
  94. return self._mode
  95. def __getattr__(self, name):
  96. if name == 'name':
  97. self.name = getattr(self._file, name)
  98. return self.name
  99. else:
  100. raise AttributeError(name)
  101. def __del__(self):
  102. self.close()
  103. def win32_open_devnull(mode):
  104. return open('NUL', mode)
  105. class Redirections:
  106. """Stores open files and their mapping to pseudo-sh file descriptor.
  107. """
  108. # BUG: redirections are not handled correctly: 1>&3 2>&3 3>&4 does
  109. # not make 1 to redirect to 4
  110. def __init__(self, stdin=None, stdout=None, stderr=None):
  111. self._descriptors = {}
  112. if stdin is not None:
  113. self._add_descriptor(0, stdin)
  114. if stdout is not None:
  115. self._add_descriptor(1, stdout)
  116. if stderr is not None:
  117. self._add_descriptor(2, stderr)
  118. def add_here_document(self, interp, name, content, io_number=None):
  119. if io_number is None:
  120. io_number = 0
  121. if name==pyshlex.unquote_wordtree(name):
  122. content = interp.expand_here_document(('TOKEN', content))
  123. # Write document content in a temporary file
  124. tmp = tempfile.TemporaryFile()
  125. try:
  126. tmp.write(content)
  127. tmp.flush()
  128. tmp.seek(0)
  129. self._add_descriptor(io_number, FileWrapper('r', tmp))
  130. except:
  131. tmp.close()
  132. raise
  133. def add(self, interp, op, filename, io_number=None):
  134. if op not in ('<', '>', '>|', '>>', '>&'):
  135. # TODO: add descriptor duplication and here_documents
  136. raise RedirectionError('Unsupported redirection operator "%s"' % op)
  137. if io_number is not None:
  138. io_number = int(io_number)
  139. if (op == '>&' and filename.isdigit()) or filename=='-':
  140. # No expansion for file descriptors, quote them if you want a filename
  141. fullname = filename
  142. else:
  143. if filename.startswith('/'):
  144. # TODO: win32 kludge
  145. if filename=='/dev/null':
  146. fullname = 'NUL'
  147. else:
  148. # TODO: handle absolute pathnames, they are unlikely to exist on the
  149. # current platform (win32 for instance).
  150. raise NotImplementedError()
  151. else:
  152. fullname = interp.expand_redirection(('TOKEN', filename))
  153. if not fullname:
  154. raise RedirectionError('%s: ambiguous redirect' % filename)
  155. # Build absolute path based on PWD
  156. fullname = os.path.join(interp.get_env()['PWD'], fullname)
  157. if op=='<':
  158. return self._add_input_redirection(interp, fullname, io_number)
  159. elif op in ('>', '>|'):
  160. clobber = ('>|'==op)
  161. return self._add_output_redirection(interp, fullname, io_number, clobber)
  162. elif op=='>>':
  163. return self._add_output_appending(interp, fullname, io_number)
  164. elif op=='>&':
  165. return self._dup_output_descriptor(fullname, io_number)
  166. def close(self):
  167. if self._descriptors is not None:
  168. for desc in self._descriptors.itervalues():
  169. desc.flush()
  170. desc.close()
  171. self._descriptors = None
  172. def stdin(self):
  173. return self._descriptors[0]
  174. def stdout(self):
  175. return self._descriptors[1]
  176. def stderr(self):
  177. return self._descriptors[2]
  178. def clone(self):
  179. clone = Redirections()
  180. for desc, fileobj in self._descriptors.iteritems():
  181. clone._descriptors[desc] = fileobj.dup()
  182. return clone
  183. def _add_output_redirection(self, interp, filename, io_number, clobber):
  184. if io_number is None:
  185. # io_number default to standard output
  186. io_number = 1
  187. if not clobber and interp.get_env().has_opt('-C') and os.path.isfile(filename):
  188. # File already exist in no-clobber mode, bail out
  189. raise RedirectionError('File "%s" already exists' % filename)
  190. # Open and register
  191. self._add_file_descriptor(io_number, filename, 'w')
  192. def _add_output_appending(self, interp, filename, io_number):
  193. if io_number is None:
  194. io_number = 1
  195. self._add_file_descriptor(io_number, filename, 'a')
  196. def _add_input_redirection(self, interp, filename, io_number):
  197. if io_number is None:
  198. io_number = 0
  199. self._add_file_descriptor(io_number, filename, 'r')
  200. def _add_file_descriptor(self, io_number, filename, mode):
  201. try:
  202. if filename.startswith('/'):
  203. if filename=='/dev/null':
  204. f = win32_open_devnull(mode+'b')
  205. else:
  206. # TODO: handle absolute pathnames, they are unlikely to exist on the
  207. # current platform (win32 for instance).
  208. raise NotImplementedError('cannot open absolute path %s' % repr(filename))
  209. else:
  210. f = file(filename, mode+'b')
  211. except IOError as e:
  212. raise RedirectionError(str(e))
  213. wrapper = None
  214. try:
  215. wrapper = FileWrapper(mode, f)
  216. f = None
  217. self._add_descriptor(io_number, wrapper)
  218. except:
  219. if f: f.close()
  220. if wrapper: wrapper.close()
  221. raise
  222. def _dup_output_descriptor(self, source_fd, dest_fd):
  223. if source_fd is None:
  224. source_fd = 1
  225. self._dup_file_descriptor(source_fd, dest_fd, 'w')
  226. def _dup_file_descriptor(self, source_fd, dest_fd, mode):
  227. source_fd = int(source_fd)
  228. if source_fd not in self._descriptors:
  229. raise RedirectionError('"%s" is not a valid file descriptor' % str(source_fd))
  230. source = self._descriptors[source_fd]
  231. if source.mode()!=mode:
  232. raise RedirectionError('Descriptor %s cannot be duplicated in mode "%s"' % (str(source), mode))
  233. if dest_fd=='-':
  234. # Close the source descriptor
  235. del self._descriptors[source_fd]
  236. source.close()
  237. else:
  238. dest_fd = int(dest_fd)
  239. if dest_fd not in self._descriptors:
  240. raise RedirectionError('Cannot replace file descriptor %s' % str(dest_fd))
  241. dest = self._descriptors[dest_fd]
  242. if dest.mode()!=mode:
  243. raise RedirectionError('Descriptor %s cannot be cannot be redirected in mode "%s"' % (str(dest), mode))
  244. self._descriptors[dest_fd] = source.dup()
  245. dest.close()
  246. def _add_descriptor(self, io_number, file):
  247. io_number = int(io_number)
  248. if io_number in self._descriptors:
  249. # Close the current descriptor
  250. d = self._descriptors[io_number]
  251. del self._descriptors[io_number]
  252. d.close()
  253. self._descriptors[io_number] = file
  254. def __str__(self):
  255. names = [('%d=%r' % (k, getattr(v, 'name', None))) for k,v
  256. in self._descriptors.iteritems()]
  257. names = ','.join(names)
  258. return 'Redirections(%s)' % names
  259. def __del__(self):
  260. self.close()
  261. def cygwin_to_windows_path(path):
  262. """Turn /cygdrive/c/foo into c:/foo, or return path if it
  263. is not a cygwin path.
  264. """
  265. if not path.startswith('/cygdrive/'):
  266. return path
  267. path = path[len('/cygdrive/'):]
  268. path = path[:1] + ':' + path[1:]
  269. return path
  270. def win32_to_unix_path(path):
  271. if path is not None:
  272. path = path.replace('\\', '/')
  273. return path
  274. _RE_SHEBANG = re.compile(r'^\#!\s?([^\s]+)(?:\s([^\s]+))?')
  275. _SHEBANG_CMDS = {
  276. '/usr/bin/env': 'env',
  277. '/bin/sh': 'pysh',
  278. 'python': 'python',
  279. }
  280. def resolve_shebang(path, ignoreshell=False):
  281. """Return a list of arguments as shebang interpreter call or an empty list
  282. if path does not refer to an executable script.
  283. See <http://www.opengroup.org/austin/docs/austin_51r2.txt>.
  284. ignoreshell - set to True to ignore sh shebangs. Return an empty list instead.
  285. """
  286. try:
  287. f = file(path)
  288. try:
  289. # At most 80 characters in the first line
  290. header = f.read(80).splitlines()[0]
  291. finally:
  292. f.close()
  293. m = _RE_SHEBANG.search(header)
  294. if not m:
  295. return []
  296. cmd, arg = m.group(1,2)
  297. if os.path.isfile(cmd):
  298. # Keep this one, the hg script for instance contains a weird windows
  299. # shebang referencing the current python install.
  300. cmdfile = os.path.basename(cmd).lower()
  301. if cmdfile == 'python.exe':
  302. cmd = 'python'
  303. pass
  304. elif cmd not in _SHEBANG_CMDS:
  305. raise CommandNotFound('Unknown interpreter "%s" referenced in '\
  306. 'shebang' % header)
  307. cmd = _SHEBANG_CMDS.get(cmd)
  308. if cmd is None or (ignoreshell and cmd == 'pysh'):
  309. return []
  310. if arg is None:
  311. return [cmd, win32_to_unix_path(path)]
  312. return [cmd, arg, win32_to_unix_path(path)]
  313. except IOError as e:
  314. if e.errno!=errno.ENOENT and \
  315. (e.errno!=errno.EPERM and not os.path.isdir(path)): # Opening a directory raises EPERM
  316. raise
  317. return []
  318. def win32_find_in_path(name, path):
  319. if isinstance(path, str):
  320. path = path.split(os.pathsep)
  321. exts = os.environ.get('PATHEXT', '').lower().split(os.pathsep)
  322. for p in path:
  323. p_name = os.path.join(p, name)
  324. prefix = resolve_shebang(p_name)
  325. if prefix:
  326. return prefix
  327. for ext in exts:
  328. p_name_ext = p_name + ext
  329. if os.path.exists(p_name_ext):
  330. return [win32_to_unix_path(p_name_ext)]
  331. return []
  332. class Traps(dict):
  333. def __setitem__(self, key, value):
  334. if key not in ('EXIT',):
  335. raise NotImplementedError()
  336. super(Traps, self).__setitem__(key, value)
  337. # IFS white spaces character class
  338. _IFS_WHITESPACES = (' ', '\t', '\n')
  339. class Environment:
  340. """Environment holds environment variables, export table, function
  341. definitions and whatever is defined in 2.12 "Shell Execution Environment",
  342. redirection excepted.
  343. """
  344. def __init__(self, pwd):
  345. self._opt = set() #Shell options
  346. self._functions = {}
  347. self._env = {'?': '0', '#': '0'}
  348. self._exported = set([
  349. 'HOME', 'IFS', 'PATH'
  350. ])
  351. # Set environment vars with side-effects
  352. self._ifs_ws = None # Set of IFS whitespace characters
  353. self._ifs_re = None # Regular expression used to split between words using IFS classes
  354. self['IFS'] = ''.join(_IFS_WHITESPACES) #Default environment values
  355. self['PWD'] = pwd
  356. self.traps = Traps()
  357. def clone(self, subshell=False):
  358. env = Environment(self['PWD'])
  359. env._opt = set(self._opt)
  360. for k,v in self.get_variables().iteritems():
  361. if k in self._exported:
  362. env.export(k,v)
  363. elif subshell:
  364. env[k] = v
  365. if subshell:
  366. env._functions = dict(self._functions)
  367. return env
  368. def __getitem__(self, key):
  369. if key in ('@', '*', '-', '$'):
  370. raise NotImplementedError('%s is not implemented' % repr(key))
  371. return self._env[key]
  372. def get(self, key, defval=None):
  373. try:
  374. return self[key]
  375. except KeyError:
  376. return defval
  377. def __setitem__(self, key, value):
  378. if key=='IFS':
  379. # Update the whitespace/non-whitespace classes
  380. self._update_ifs(value)
  381. elif key=='PWD':
  382. pwd = os.path.abspath(value)
  383. if not os.path.isdir(pwd):
  384. raise VarAssignmentError('Invalid directory %s' % value)
  385. value = pwd
  386. elif key in ('?', '!'):
  387. value = str(int(value))
  388. self._env[key] = value
  389. def __delitem__(self, key):
  390. if key in ('IFS', 'PWD', '?'):
  391. raise VarAssignmentError('%s cannot be unset' % key)
  392. del self._env[key]
  393. def __contains__(self, item):
  394. return item in self._env
  395. def set_positional_args(self, args):
  396. """Set the content of 'args' as positional argument from 1 to len(args).
  397. Return previous argument as a list of strings.
  398. """
  399. # Save and remove previous arguments
  400. prevargs = []
  401. for i in range(int(self._env['#'])):
  402. i = str(i+1)
  403. prevargs.append(self._env[i])
  404. del self._env[i]
  405. self._env['#'] = '0'
  406. #Set new ones
  407. for i,arg in enumerate(args):
  408. self._env[str(i+1)] = str(arg)
  409. self._env['#'] = str(len(args))
  410. return prevargs
  411. def get_positional_args(self):
  412. return [self._env[str(i+1)] for i in range(int(self._env['#']))]
  413. def get_variables(self):
  414. return dict(self._env)
  415. def export(self, key, value=None):
  416. if value is not None:
  417. self[key] = value
  418. self._exported.add(key)
  419. def get_exported(self):
  420. return [(k,self._env.get(k)) for k in self._exported]
  421. def split_fields(self, word):
  422. if not self._ifs_ws or not word:
  423. return [word]
  424. return re.split(self._ifs_re, word)
  425. def _update_ifs(self, value):
  426. """Update the split_fields related variables when IFS character set is
  427. changed.
  428. """
  429. # TODO: handle NULL IFS
  430. # Separate characters in whitespace and non-whitespace
  431. chars = set(value)
  432. ws = [c for c in chars if c in _IFS_WHITESPACES]
  433. nws = [c for c in chars if c not in _IFS_WHITESPACES]
  434. # Keep whitespaces in a string for left and right stripping
  435. self._ifs_ws = ''.join(ws)
  436. # Build a regexp to split fields
  437. trailing = '[' + ''.join([re.escape(c) for c in ws]) + ']'
  438. if nws:
  439. # First, the single non-whitespace occurence.
  440. nws = '[' + ''.join([re.escape(c) for c in nws]) + ']'
  441. nws = '(?:' + trailing + '*' + nws + trailing + '*' + '|' + trailing + '+)'
  442. else:
  443. # Then mix all parts with quantifiers
  444. nws = trailing + '+'
  445. self._ifs_re = re.compile(nws)
  446. def has_opt(self, opt, val=None):
  447. return (opt, val) in self._opt
  448. def set_opt(self, opt, val=None):
  449. self._opt.add((opt, val))
  450. def find_in_path(self, name, pwd=False):
  451. path = self._env.get('PATH', '').split(os.pathsep)
  452. if pwd:
  453. path[:0] = [self['PWD']]
  454. if os.name == 'nt':
  455. return win32_find_in_path(name, self._env.get('PATH', ''))
  456. else:
  457. raise NotImplementedError()
  458. def define_function(self, name, body):
  459. if not is_name(name):
  460. raise ShellSyntaxError('%s is not a valid function name' % repr(name))
  461. self._functions[name] = body
  462. def remove_function(self, name):
  463. del self._functions[name]
  464. def is_function(self, name):
  465. return name in self._functions
  466. def get_function(self, name):
  467. return self._functions.get(name)
  468. name_charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
  469. name_charset = dict(zip(name_charset,name_charset))
  470. def match_name(s):
  471. """Return the length in characters of the longest prefix made of name
  472. allowed characters in s.
  473. """
  474. for i,c in enumerate(s):
  475. if c not in name_charset:
  476. return s[:i]
  477. return s
  478. def is_name(s):
  479. return len([c for c in s if c not in name_charset])<=0
  480. def is_special_param(c):
  481. return len(c)==1 and c in ('@','*','#','?','-','$','!','0')
  482. def utility_not_implemented(name, *args, **kwargs):
  483. raise NotImplementedError('%s utility is not implemented' % name)
  484. class Utility:
  485. """Define utilities properties:
  486. func -- utility callable. See builtin module for utility samples.
  487. is_special -- see XCU 2.8.
  488. """
  489. def __init__(self, func, is_special=0):
  490. self.func = func
  491. self.is_special = bool(is_special)
  492. def encodeargs(args):
  493. def encodearg(s):
  494. lines = base64.encodestring(s)
  495. lines = [l.splitlines()[0] for l in lines]
  496. return ''.join(lines)
  497. s = pickle.dumps(args)
  498. return encodearg(s)
  499. def decodeargs(s):
  500. s = base64.decodestring(s)
  501. return pickle.loads(s)
  502. class GlobError(Exception):
  503. pass
  504. class Options:
  505. def __init__(self):
  506. # True if Mercurial operates with binary streams
  507. self.hgbinary = True
  508. class Interpreter:
  509. # Implementation is very basic: the execute() method just makes a DFS on the
  510. # AST and execute nodes one by one. Nodes are tuple (name,obj) where name
  511. # is a string identifier and obj the AST element returned by the parser.
  512. #
  513. # Handler are named after the node identifiers.
  514. # TODO: check node names and remove the switch in execute with some
  515. # dynamic getattr() call to find node handlers.
  516. """Shell interpreter.
  517. The following debugging flags can be passed:
  518. debug-parsing - enable PLY debugging.
  519. debug-tree - print the generated AST.
  520. debug-cmd - trace command execution before word expansion, plus exit status.
  521. debug-utility - trace utility execution.
  522. """
  523. # List supported commands.
  524. COMMANDS = {
  525. 'cat': Utility(builtin.utility_cat,),
  526. 'cd': Utility(builtin.utility_cd,),
  527. ':': Utility(builtin.utility_colon,),
  528. 'echo': Utility(builtin.utility_echo),
  529. 'env': Utility(builtin.utility_env),
  530. 'exit': Utility(builtin.utility_exit),
  531. 'export': Utility(builtin.builtin_export, is_special=1),
  532. 'egrep': Utility(builtin.utility_egrep),
  533. 'fgrep': Utility(builtin.utility_fgrep),
  534. 'gunzip': Utility(builtin.utility_gunzip),
  535. 'kill': Utility(builtin.utility_kill),
  536. 'mkdir': Utility(builtin.utility_mkdir),
  537. 'netstat': Utility(builtin.utility_netstat),
  538. 'printf': Utility(builtin.utility_printf),
  539. 'pwd': Utility(builtin.utility_pwd),
  540. 'return': Utility(builtin.builtin_return, is_special=1),
  541. 'sed': Utility(builtin.utility_sed,),
  542. 'set': Utility(builtin.builtin_set,),
  543. 'shift': Utility(builtin.builtin_shift,),
  544. 'sleep': Utility(builtin.utility_sleep,),
  545. 'sort': Utility(builtin.utility_sort,),
  546. 'trap': Utility(builtin.builtin_trap, is_special=1),
  547. 'true': Utility(builtin.utility_true),
  548. 'unset': Utility(builtin.builtin_unset, is_special=1),
  549. 'wait': Utility(builtin.builtin_wait, is_special=1),
  550. }
  551. def __init__(self, pwd, debugflags = [], env=None, redirs=None, stdin=None,
  552. stdout=None, stderr=None, opts=Options()):
  553. self._env = env
  554. if self._env is None:
  555. self._env = Environment(pwd)
  556. self._children = {}
  557. self._redirs = redirs
  558. self._close_redirs = False
  559. if self._redirs is None:
  560. if stdin is None:
  561. stdin = sys.stdin
  562. if stdout is None:
  563. stdout = sys.stdout
  564. if stderr is None:
  565. stderr = sys.stderr
  566. stdin = FileWrapper('r', stdin, False)
  567. stdout = FileWrapper('w', stdout, False)
  568. stderr = FileWrapper('w', stderr, False)
  569. self._redirs = Redirections(stdin, stdout, stderr)
  570. self._close_redirs = True
  571. self._debugflags = list(debugflags)
  572. self._logfile = sys.stderr
  573. self._options = opts
  574. def close(self):
  575. """Must be called when the interpreter is no longer used."""
  576. script = self._env.traps.get('EXIT')
  577. if script:
  578. try:
  579. self.execute_script(script=script)
  580. except:
  581. pass
  582. if self._redirs is not None and self._close_redirs:
  583. self._redirs.close()
  584. self._redirs = None
  585. def log(self, s):
  586. self._logfile.write(s)
  587. self._logfile.flush()
  588. def __getitem__(self, key):
  589. return self._env[key]
  590. def __setitem__(self, key, value):
  591. self._env[key] = value
  592. def options(self):
  593. return self._options
  594. def redirect(self, redirs, ios):
  595. def add_redir(io):
  596. if isinstance(io, pyshyacc.IORedirect):
  597. redirs.add(self, io.op, io.filename, io.io_number)
  598. else:
  599. redirs.add_here_document(self, io.name, io.content, io.io_number)
  600. map(add_redir, ios)
  601. return redirs
  602. def execute_script(self, script=None, ast=None, sourced=False,
  603. scriptpath=None):
  604. """If script is not None, parse the input. Otherwise takes the supplied
  605. AST. Then execute the AST.
  606. Return the script exit status.
  607. """
  608. try:
  609. if scriptpath is not None:
  610. self._env['0'] = os.path.abspath(scriptpath)
  611. if script is not None:
  612. debug_parsing = ('debug-parsing' in self._debugflags)
  613. cmds, script = pyshyacc.parse(script, True, debug_parsing)
  614. if 'debug-tree' in self._debugflags:
  615. pyshyacc.print_commands(cmds, self._logfile)
  616. self._logfile.flush()
  617. else:
  618. cmds, script = ast, ''
  619. status = 0
  620. for cmd in cmds:
  621. try:
  622. status = self.execute(cmd)
  623. except ExitSignal as e:
  624. if sourced:
  625. raise
  626. status = int(e.args[0])
  627. return status
  628. except ShellError:
  629. self._env['?'] = 1
  630. raise
  631. if 'debug-utility' in self._debugflags or 'debug-cmd' in self._debugflags:
  632. self.log('returncode ' + str(status)+ '\n')
  633. return status
  634. except CommandNotFound as e:
  635. print >>self._redirs.stderr, str(e)
  636. self._redirs.stderr.flush()
  637. # Command not found by non-interactive shell
  638. # return 127
  639. raise
  640. except RedirectionError as e:
  641. # TODO: should be handled depending on the utility status
  642. print >>self._redirs.stderr, str(e)
  643. self._redirs.stderr.flush()
  644. # Command not found by non-interactive shell
  645. # return 127
  646. raise
  647. def dotcommand(self, env, args):
  648. if len(args) < 1:
  649. raise ShellError('. expects at least one argument')
  650. path = args[0]
  651. if '/' not in path:
  652. found = env.find_in_path(args[0], True)
  653. if found:
  654. path = found[0]
  655. script = file(path).read()
  656. return self.execute_script(script=script, sourced=True)
  657. def execute(self, token, redirs=None):
  658. """Execute and AST subtree with supplied redirections overriding default
  659. interpreter ones.
  660. Return the exit status.
  661. """
  662. if not token:
  663. return 0
  664. if redirs is None:
  665. redirs = self._redirs
  666. if isinstance(token, list):
  667. # Commands sequence
  668. res = 0
  669. for t in token:
  670. res = self.execute(t, redirs)
  671. return res
  672. type, value = token
  673. status = 0
  674. if type=='simple_command':
  675. redirs_copy = redirs.clone()
  676. try:
  677. # TODO: define and handle command return values
  678. # TODO: implement set -e
  679. status = self._execute_simple_command(value, redirs_copy)
  680. finally:
  681. redirs_copy.close()
  682. elif type=='pipeline':
  683. status = self._execute_pipeline(value, redirs)
  684. elif type=='and_or':
  685. status = self._execute_and_or(value, redirs)
  686. elif type=='for_clause':
  687. status = self._execute_for_clause(value, redirs)
  688. elif type=='while_clause':
  689. status = self._execute_while_clause(value, redirs)
  690. elif type=='function_definition':
  691. status = self._execute_function_definition(value, redirs)
  692. elif type=='brace_group':
  693. status = self._execute_brace_group(value, redirs)
  694. elif type=='if_clause':
  695. status = self._execute_if_clause(value, redirs)
  696. elif type=='subshell':
  697. status = self.subshell(ast=value.cmds, redirs=redirs)
  698. elif type=='async':
  699. status = self._asynclist(value)
  700. elif type=='redirect_list':
  701. redirs_copy = self.redirect(redirs.clone(), value.redirs)
  702. try:
  703. status = self.execute(value.cmd, redirs_copy)
  704. finally:
  705. redirs_copy.close()
  706. else:
  707. raise NotImplementedError('Unsupported token type ' + type)
  708. if status < 0:
  709. status = 255
  710. return status
  711. def _execute_if_clause(self, if_clause, redirs):
  712. cond_status = self.execute(if_clause.cond, redirs)
  713. if cond_status==0:
  714. return self.execute(if_clause.if_cmds, redirs)
  715. else:
  716. return self.execute(if_clause.else_cmds, redirs)
  717. def _execute_brace_group(self, group, redirs):
  718. status = 0
  719. for cmd in group.cmds:
  720. status = self.execute(cmd, redirs)
  721. return status
  722. def _execute_function_definition(self, fundef, redirs):
  723. self._env.define_function(fundef.name, fundef.body)
  724. return 0
  725. def _execute_while_clause(self, while_clause, redirs):
  726. status = 0
  727. while 1:
  728. cond_status = 0
  729. for cond in while_clause.condition:
  730. cond_status = self.execute(cond, redirs)
  731. if cond_status:
  732. break
  733. for cmd in while_clause.cmds:
  734. status = self.execute(cmd, redirs)
  735. return status
  736. def _execute_for_clause(self, for_clause, redirs):
  737. if not is_name(for_clause.name):
  738. raise ShellSyntaxError('%s is not a valid name' % repr(for_clause.name))
  739. items = mappend(self.expand_token, for_clause.items)
  740. status = 0
  741. for item in items:
  742. self._env[for_clause.name] = item
  743. for cmd in for_clause.cmds:
  744. status = self.execute(cmd, redirs)
  745. return status
  746. def _execute_and_or(self, or_and, redirs):
  747. res = self.execute(or_and.left, redirs)
  748. if (or_and.op=='&&' and res==0) or (or_and.op!='&&' and res!=0):
  749. res = self.execute(or_and.right, redirs)
  750. return res
  751. def _execute_pipeline(self, pipeline, redirs):
  752. if len(pipeline.commands)==1:
  753. status = self.execute(pipeline.commands[0], redirs)
  754. else:
  755. # Execute all commands one after the other
  756. status = 0
  757. inpath, outpath = None, None
  758. try:
  759. # Commands inputs and outputs cannot really be plugged as done
  760. # by a real shell. Run commands sequentially and chain their
  761. # input/output throught temporary files.
  762. tmpfd, inpath = tempfile.mkstemp()
  763. os.close(tmpfd)
  764. tmpfd, outpath = tempfile.mkstemp()
  765. os.close(tmpfd)
  766. inpath = win32_to_unix_path(inpath)
  767. outpath = win32_to_unix_path(outpath)
  768. for i, cmd in enumerate(pipeline.commands):
  769. call_redirs = redirs.clone()
  770. try:
  771. if i!=0:
  772. call_redirs.add(self, '<', inpath)
  773. if i!=len(pipeline.commands)-1:
  774. call_redirs.add(self, '>', outpath)
  775. status = self.execute(cmd, call_redirs)
  776. # Chain inputs/outputs
  777. inpath, outpath = outpath, inpath
  778. finally:
  779. call_redirs.close()
  780. finally:
  781. if inpath: os.remove(inpath)
  782. if outpath: os.remove(outpath)
  783. if pipeline.reverse_status:
  784. status = int(not status)
  785. self._env['?'] = status
  786. return status
  787. def _execute_function(self, name, args, interp, env, stdin, stdout, stderr, *others):
  788. assert interp is self
  789. func = env.get_function(name)
  790. #Set positional parameters
  791. prevargs = None
  792. try:
  793. prevargs = env.set_positional_args(args)
  794. try:
  795. redirs = Redirections(stdin.dup(), stdout.dup(), stderr.dup())
  796. try:
  797. status = self.execute(func, redirs)
  798. finally:
  799. redirs.close()
  800. except ReturnSignal as e:
  801. status = int(e.args[0])
  802. env['?'] = status
  803. return status
  804. finally:
  805. #Reset positional parameters
  806. if prevargs is not None:
  807. env.set_positional_args(prevargs)
  808. def _execute_simple_command(self, token, redirs):
  809. """Can raise ReturnSignal when return builtin is called, ExitSignal when
  810. exit is called, and other shell exceptions upon builtin failures.
  811. """
  812. debug_command = 'debug-cmd' in self._debugflags
  813. if debug_command:
  814. self.log('word' + repr(token.words) + '\n')
  815. self.log('assigns' + repr(token.assigns) + '\n')
  816. self.log('redirs' + repr(token.redirs) + '\n')
  817. is_special = None
  818. env = self._env
  819. try:
  820. # Word expansion
  821. args = []
  822. for word in token.words:
  823. args += self.expand_token(word)
  824. if is_special is None and args:
  825. is_special = env.is_function(args[0]) or \
  826. (args[0] in self.COMMANDS and self.COMMANDS[args[0]].is_special)
  827. if debug_command:
  828. self.log('_execute_simple_command' + str(args) + '\n')
  829. if not args:
  830. # Redirections happen is a subshell
  831. redirs = redirs.clone()
  832. elif not is_special:
  833. env = self._env.clone()
  834. # Redirections
  835. self.redirect(redirs, token.redirs)
  836. # Variables assignments
  837. res = 0
  838. for type,(k,v) in token.assigns:
  839. status, expanded = self.expand_variable((k,v))
  840. if status is not None:
  841. res = status
  842. if args:
  843. env.export(k, expanded)
  844. else:
  845. env[k] = expanded
  846. if args and args[0] in ('.', 'source'):
  847. res = self.dotcommand(env, args[1:])
  848. elif args:
  849. if args[0] in self.COMMANDS:
  850. command = self.COMMANDS[args[0]]
  851. elif env.is_function(args[0]):
  852. command = Utility(self._execute_function, is_special=True)
  853. else:
  854. if not '/' in args[0].replace('\\', '/'):
  855. cmd = env.find_in_path(args[0])
  856. if not cmd:
  857. # TODO: test error code on unknown command => 127
  858. raise CommandNotFound('Unknown command: "%s"' % args[0])
  859. else:
  860. # Handle commands like '/cygdrive/c/foo.bat'
  861. cmd = cygwin_to_windows_path(args[0])
  862. if not os.path.exists(cmd):
  863. raise CommandNotFound('%s: No such file or directory' % args[0])
  864. shebang = resolve_shebang(cmd)
  865. if shebang:
  866. cmd = shebang
  867. else:
  868. cmd = [cmd]
  869. args[0:1] = cmd
  870. command = Utility(builtin.run_command)
  871. # Command execution
  872. if 'debug-cmd' in self._debugflags:
  873. self.log('redirections ' + str(redirs) + '\n')
  874. res = command.func(args[0], args[1:], self, env,
  875. redirs.stdin(), redirs.stdout(),
  876. redirs.stderr(), self._debugflags)
  877. if self._env.has_opt('-x'):
  878. # Trace command execution in shell environment
  879. # BUG: would be hard to reproduce a real shell behaviour since
  880. # the AST is not annotated with source lines/tokens.
  881. self._redirs.stdout().write(' '.join(args))
  882. except ReturnSignal:
  883. raise
  884. except ShellError as e:
  885. if is_special or isinstance(e, (ExitSignal,
  886. ShellSyntaxError, ExpansionError)):
  887. raise e
  888. self._redirs.stderr().write(str(e)+'\n')
  889. return 1
  890. return res
  891. def expand_token(self, word):
  892. """Expand a word as specified in [2.6 Word Expansions]. Return the list
  893. of expanded words.
  894. """
  895. status, wtrees = self._expand_word(word)
  896. return map(pyshlex.wordtree_as_string, wtrees)
  897. def expand_variable(self, word):
  898. """Return a status code (or None if no command expansion occurred)
  899. and a single word.
  900. """
  901. status, wtrees = self._expand_word(word, pathname=False, split=False)
  902. words = map(pyshlex.wordtree_as_string, wtrees)
  903. assert len(words)==1
  904. return status, words[0]
  905. def expand_here_document(self, word):
  906. """Return the expanded document as a single word. The here document is
  907. assumed to be unquoted.
  908. """
  909. status, wtrees = self._expand_word(word, pathname=False,
  910. split=False, here_document=True)
  911. words = map(pyshlex.wordtree_as_string, wtrees)
  912. assert len(words)==1
  913. return words[0]
  914. def expand_redirection(self, word):
  915. """Return a single word."""
  916. return self.expand_variable(word)[1]
  917. def get_env(self):
  918. return self._env
  919. def _expand_word(self, token, pathname=True, split=True, here_document=False):
  920. wtree = pyshlex.make_wordtree(token[1], here_document=here_document)
  921. # TODO: implement tilde expansion
  922. def expand(wtree):
  923. """Return a pseudo wordtree: the tree or its subelements can be empty
  924. lists when no value result from the expansion.
  925. """
  926. status = None
  927. for part in wtree:
  928. if not isinstance(part, list):
  929. continue
  930. if part[0]in ("'", '\\'):
  931. continue
  932. elif part[0] in ('`', '$('):
  933. status, result = self._expand_command(part)
  934. part[:] = result
  935. elif part[0] in ('$', '${'):
  936. part[:] = self._expand_parameter(part, wtree[0]=='"', split)
  937. elif part[0] in ('', '"'):
  938. status, result = expand(part)
  939. part[:] = result
  940. else:
  941. raise NotImplementedError('%s expansion is not implemented'
  942. % part[0])
  943. # [] is returned when an expansion result in no-field,
  944. # like an empty $@
  945. wtree = [p for p in wtree if p != []]
  946. if len(wtree) < 3:
  947. return status, []
  948. return status, wtree
  949. status, wtree = expand(wtree)
  950. if len(wtree) == 0:
  951. return status, wtree
  952. wtree = pyshlex.normalize_wordtree(wtree)
  953. if split:
  954. wtrees = self._split_fields(wtree)
  955. else:
  956. wtrees = [wtree]
  957. if pathname:
  958. wtrees = mappend(self._expand_pathname, wtrees)
  959. wtrees = map(self._remove_quotes, wtrees)
  960. return status, wtrees
  961. def _expand_command(self, wtree):
  962. # BUG: there is something to do with backslashes and quoted
  963. # characters here
  964. command = pyshlex.wordtree_as_string(wtree[1:-1])
  965. status, output = self.subshell_output(command)
  966. return status, ['', output, '']
  967. def _expand_parameter(self, wtree, quoted=False, split=False):
  968. """Return a valid wtree or an empty list when no parameter results."""
  969. # Get the parameter name
  970. # TODO: implement weird expansion rules with ':'
  971. name = pyshlex.wordtree_as_string(wtree[1:-1])
  972. if not is_name(name) and not is_special_param(name):
  973. raise ExpansionError('Bad substitution "%s"' % name)
  974. # TODO: implement special parameters
  975. if name in ('@', '*'):
  976. args = self._env.get_positional_args()
  977. if len(args) == 0:
  978. return []
  979. if len(args)<2:
  980. return ['', ''.join(args), '']
  981. sep = self._env.get('IFS', '')[:1]
  982. if split and quoted and name=='@':
  983. # Introduce a new token to tell the caller that these parameters
  984. # cause a split as specified in 2.5.2
  985. return ['@'] + args + ['']
  986. else:
  987. return ['', sep.join(args), '']
  988. return ['', self._env.get(name, ''), '']
  989. def _split_fields(self, wtree):
  990. def is_empty(split):
  991. return split==['', '', '']
  992. def split_positional(quoted):
  993. # Return a list of wtree split according positional parameters rules.
  994. # All remaining '@' groups are removed.
  995. assert quoted[0]=='"'
  996. splits = [[]]
  997. for part in quoted:
  998. if not isinstance(part, list) or part[0]!='@':
  999. splits[-1].append(part)
  1000. else:
  1001. # Empty or single argument list were dealt with already
  1002. assert len(part)>3
  1003. # First argument must join with the beginning part of the original word
  1004. splits[-1].append(part[1])
  1005. # Create double-quotes expressions for every argument after the first
  1006. for arg in part[2:-1]:
  1007. splits[-1].append('"')
  1008. splits.append(['"', arg])
  1009. return splits
  1010. # At this point, all expansions but pathnames have occured. Only quoted
  1011. # and positional sequences remain. Thus, all candidates for field splitting
  1012. # are in the tree root, or are positional splits ('@') and lie in root
  1013. # children.
  1014. if not wtree or wtree[0] not in ('', '"'):
  1015. # The whole token is quoted or empty, nothing to split
  1016. return [wtree]
  1017. if wtree[0]=='"':
  1018. wtree = ['', wtree, '']
  1019. result = [['', '']]
  1020. for part in wtree[1:-1]:
  1021. if isinstance(part, list):
  1022. if part[0]=='"':
  1023. splits = split_positional(part)
  1024. if len(splits)<=1:
  1025. result[-1] += [part, '']
  1026. else:
  1027. # Terminate the current split
  1028. result[-1] += [splits[0], '']
  1029. result += splits[1:-1]
  1030. # Create a new split
  1031. result += [['', splits[-1], '']]
  1032. else:
  1033. result[-1] += [part, '']
  1034. else:
  1035. splits = self._env.split_fields(part)
  1036. if len(splits)<=1:
  1037. # No split
  1038. result[-1][-1] += part
  1039. else:
  1040. # Terminate the current resulting part and create a new one
  1041. result[-1][-1] += splits[0]
  1042. result[-1].append('')
  1043. result += [['', r, ''] for r in splits[1:-1]]
  1044. result += [['', splits[-1]]]
  1045. result[-1].append('')
  1046. # Leading and trailing empty groups come from leading/trailing blanks
  1047. if result and is_empty(result[-1]):
  1048. result[-1:] = []
  1049. if result and is_empty(result[0]):
  1050. result[:1] = []
  1051. return result
  1052. def _expand_pathname(self, wtree):
  1053. """See [2.6.6 Pathname Expansion]."""
  1054. if self._env.has_opt('-f'):
  1055. return [wtree]
  1056. # All expansions have been performed, only quoted sequences should remain
  1057. # in the tree. Generate the pattern by folding the tree, escaping special
  1058. # characters when appear quoted
  1059. special_chars = '*?[]'
  1060. def make_pattern(wtree):
  1061. subpattern = []
  1062. for part in wtree[1:-1]:
  1063. if isinstance(part, list):
  1064. part = make_pattern(part)
  1065. elif wtree[0]!='':
  1066. for c in part:
  1067. # Meta-characters cannot be quoted
  1068. if c in special_chars:
  1069. raise GlobError()
  1070. subpattern.append(part)
  1071. return ''.join(subpattern)
  1072. def pwd_glob(pattern):
  1073. cwd = os.getcwd()
  1074. os.chdir(self._env['PWD'])
  1075. try:
  1076. return glob.glob(pattern)
  1077. finally:
  1078. os.chdir(cwd)
  1079. #TODO: check working directory issues here wrt relative patterns
  1080. try:
  1081. pattern = make_pattern(wtree)
  1082. paths = pwd_glob(pattern)
  1083. except GlobError:
  1084. # BUG: Meta-characters were found in quoted sequences. The should
  1085. # have been used literally but this is unsupported in current glob module.
  1086. # Instead we consider the whole tree must be used literally and
  1087. # therefore there is no point in globbing. This is wrong when meta
  1088. # characters are mixed with quoted meta in the same pattern like:
  1089. # < foo*"py*" >
  1090. paths = []
  1091. if not paths:
  1092. return [wtree]
  1093. return [['', path, ''] for path in paths]
  1094. def _remove_quotes(self, wtree):
  1095. """See [2.6.7 Quote Removal]."""
  1096. def unquote(wtree):
  1097. unquoted = []
  1098. for part in wtree[1:-1]:
  1099. if isinstance(part, list):
  1100. part = unquote(part)
  1101. unquoted.append(part)
  1102. return ''.join(unquoted)
  1103. return ['', unquote(wtree), '']
  1104. def subshell(self, script=None, ast=None, redirs=None):
  1105. """Execute the script or AST in a subshell, with inherited redirections
  1106. if redirs is not None.
  1107. """
  1108. if redirs:
  1109. sub_redirs = redirs
  1110. else:
  1111. sub_redirs = redirs.clone()
  1112. subshell = None
  1113. try:
  1114. subshell = Interpreter(None, self._debugflags, self._env.clone(True),
  1115. sub_redirs, opts=self._options)
  1116. return subshell.execute_script(script, ast)
  1117. finally:
  1118. if not redirs: sub_redirs.close()
  1119. if subshell: subshell.close()
  1120. def subshell_output(self, script):
  1121. """Execute the script in a subshell and return the captured output."""
  1122. # Create temporary file to capture subshell output
  1123. tmpfd, tmppath = tempfile.mkstemp()
  1124. try:
  1125. tmpfile = os.fdopen(tmpfd, 'wb')
  1126. stdout = FileWrapper('w', tmpfile)
  1127. redirs = Redirections(self._redirs.stdin().dup(),
  1128. stdout,
  1129. self._redirs.stderr().dup())
  1130. try:
  1131. status = self.subshell(script=script, redirs=redirs)
  1132. finally:
  1133. redirs.close()
  1134. redirs = None
  1135. # Extract subshell standard output
  1136. tmpfile = open(tmppath, 'rb')
  1137. try:
  1138. output = tmpfile.read()
  1139. return status, output.rstrip('\n')
  1140. finally:
  1141. tmpfile.close()
  1142. finally:
  1143. os.remove(tmppath)
  1144. def _asynclist(self, cmd):
  1145. args = (self._env.get_variables(), cmd)
  1146. arg = encodeargs(args)
  1147. assert len(args) < 30*1024
  1148. cmd = ['pysh.bat', '--ast', '-c', arg]
  1149. p = subprocess.Popen(cmd, cwd=self._env['PWD'])
  1150. self._children[p.pid] = p
  1151. self._env['!'] = p.pid
  1152. return 0
  1153. def wait(self, pids=None):
  1154. if not pids:
  1155. pids = self._children.keys()
  1156. status = 127
  1157. for pid in pids:
  1158. if pid not in self._children:
  1159. continue
  1160. p = self._children.pop(pid)
  1161. status = p.wait()
  1162. return status