spdxcheck.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. #!/usr/bin/env python3
  2. # SPDX-License-Identifier: GPL-2.0
  3. # Copyright Thomas Gleixner <tglx@linutronix.de>
  4. from argparse import ArgumentParser
  5. from ply import lex, yacc
  6. import locale
  7. import traceback
  8. import sys
  9. import git
  10. import re
  11. import os
  12. class ParserException(Exception):
  13. def __init__(self, tok, txt):
  14. self.tok = tok
  15. self.txt = txt
  16. class SPDXException(Exception):
  17. def __init__(self, el, txt):
  18. self.el = el
  19. self.txt = txt
  20. class SPDXdata(object):
  21. def __init__(self):
  22. self.license_files = 0
  23. self.exception_files = 0
  24. self.licenses = [ ]
  25. self.exceptions = { }
  26. # Read the spdx data from the LICENSES directory
  27. def read_spdxdata(repo):
  28. # The subdirectories of LICENSES in the kernel source
  29. # Note: exceptions needs to be parsed as last directory.
  30. license_dirs = [ "preferred", "dual", "deprecated", "exceptions" ]
  31. lictree = repo.head.commit.tree['LICENSES']
  32. spdx = SPDXdata()
  33. for d in license_dirs:
  34. for el in lictree[d].traverse():
  35. if not os.path.isfile(el.path):
  36. continue
  37. exception = None
  38. for l in open(el.path, encoding="utf-8").readlines():
  39. if l.startswith('Valid-License-Identifier:'):
  40. lid = l.split(':')[1].strip().upper()
  41. if lid in spdx.licenses:
  42. raise SPDXException(el, 'Duplicate License Identifier: %s' %lid)
  43. else:
  44. spdx.licenses.append(lid)
  45. elif l.startswith('SPDX-Exception-Identifier:'):
  46. exception = l.split(':')[1].strip().upper()
  47. spdx.exceptions[exception] = []
  48. elif l.startswith('SPDX-Licenses:'):
  49. for lic in l.split(':')[1].upper().strip().replace(' ', '').replace('\t', '').split(','):
  50. if not lic in spdx.licenses:
  51. raise SPDXException(None, 'Exception %s missing license %s' %(exception, lic))
  52. spdx.exceptions[exception].append(lic)
  53. elif l.startswith("License-Text:"):
  54. if exception:
  55. if not len(spdx.exceptions[exception]):
  56. raise SPDXException(el, 'Exception %s is missing SPDX-Licenses' %exception)
  57. spdx.exception_files += 1
  58. else:
  59. spdx.license_files += 1
  60. break
  61. return spdx
  62. class id_parser(object):
  63. reserved = [ 'AND', 'OR', 'WITH' ]
  64. tokens = [ 'LPAR', 'RPAR', 'ID', 'EXC' ] + reserved
  65. precedence = ( ('nonassoc', 'AND', 'OR'), )
  66. t_ignore = ' \t'
  67. def __init__(self, spdx):
  68. self.spdx = spdx
  69. self.lasttok = None
  70. self.lastid = None
  71. self.lexer = lex.lex(module = self, reflags = re.UNICODE)
  72. # Initialize the parser. No debug file and no parser rules stored on disk
  73. # The rules are small enough to be generated on the fly
  74. self.parser = yacc.yacc(module = self, write_tables = False, debug = False)
  75. self.lines_checked = 0
  76. self.checked = 0
  77. self.spdx_valid = 0
  78. self.spdx_errors = 0
  79. self.curline = 0
  80. self.deepest = 0
  81. # Validate License and Exception IDs
  82. def validate(self, tok):
  83. id = tok.value.upper()
  84. if tok.type == 'ID':
  85. if not id in self.spdx.licenses:
  86. raise ParserException(tok, 'Invalid License ID')
  87. self.lastid = id
  88. elif tok.type == 'EXC':
  89. if id not in self.spdx.exceptions:
  90. raise ParserException(tok, 'Invalid Exception ID')
  91. if self.lastid not in self.spdx.exceptions[id]:
  92. raise ParserException(tok, 'Exception not valid for license %s' %self.lastid)
  93. self.lastid = None
  94. elif tok.type != 'WITH':
  95. self.lastid = None
  96. # Lexer functions
  97. def t_RPAR(self, tok):
  98. r'\)'
  99. self.lasttok = tok.type
  100. return tok
  101. def t_LPAR(self, tok):
  102. r'\('
  103. self.lasttok = tok.type
  104. return tok
  105. def t_ID(self, tok):
  106. r'[A-Za-z.0-9\-+]+'
  107. if self.lasttok == 'EXC':
  108. print(tok)
  109. raise ParserException(tok, 'Missing parentheses')
  110. tok.value = tok.value.strip()
  111. val = tok.value.upper()
  112. if val in self.reserved:
  113. tok.type = val
  114. elif self.lasttok == 'WITH':
  115. tok.type = 'EXC'
  116. self.lasttok = tok.type
  117. self.validate(tok)
  118. return tok
  119. def t_error(self, tok):
  120. raise ParserException(tok, 'Invalid token')
  121. def p_expr(self, p):
  122. '''expr : ID
  123. | ID WITH EXC
  124. | expr AND expr
  125. | expr OR expr
  126. | LPAR expr RPAR'''
  127. pass
  128. def p_error(self, p):
  129. if not p:
  130. raise ParserException(None, 'Unfinished license expression')
  131. else:
  132. raise ParserException(p, 'Syntax error')
  133. def parse(self, expr):
  134. self.lasttok = None
  135. self.lastid = None
  136. self.parser.parse(expr, lexer = self.lexer)
  137. def parse_lines(self, fd, maxlines, fname):
  138. self.checked += 1
  139. self.curline = 0
  140. try:
  141. for line in fd:
  142. line = line.decode(locale.getpreferredencoding(False), errors='ignore')
  143. self.curline += 1
  144. if self.curline > maxlines:
  145. break
  146. self.lines_checked += 1
  147. if line.find("SPDX-License-Identifier:") < 0:
  148. continue
  149. expr = line.split(':')[1].strip()
  150. # Remove trailing comment closure
  151. if line.strip().endswith('*/'):
  152. expr = expr.rstrip('*/').strip()
  153. # Remove trailing xml comment closure
  154. if line.strip().endswith('-->'):
  155. expr = expr.rstrip('-->').strip()
  156. # Special case for SH magic boot code files
  157. if line.startswith('LIST \"'):
  158. expr = expr.rstrip('\"').strip()
  159. self.parse(expr)
  160. self.spdx_valid += 1
  161. #
  162. # Should we check for more SPDX ids in the same file and
  163. # complain if there are any?
  164. #
  165. break
  166. except ParserException as pe:
  167. if pe.tok:
  168. col = line.find(expr) + pe.tok.lexpos
  169. tok = pe.tok.value
  170. sys.stdout.write('%s: %d:%d %s: %s\n' %(fname, self.curline, col, pe.txt, tok))
  171. else:
  172. sys.stdout.write('%s: %d:0 %s\n' %(fname, self.curline, col, pe.txt))
  173. self.spdx_errors += 1
  174. def scan_git_tree(tree):
  175. for el in tree.traverse():
  176. # Exclude stuff which would make pointless noise
  177. # FIXME: Put this somewhere more sensible
  178. if el.path.startswith("LICENSES"):
  179. continue
  180. if el.path.find("license-rules.rst") >= 0:
  181. continue
  182. if not os.path.isfile(el.path):
  183. continue
  184. with open(el.path, 'rb') as fd:
  185. parser.parse_lines(fd, args.maxlines, el.path)
  186. def scan_git_subtree(tree, path):
  187. for p in path.strip('/').split('/'):
  188. tree = tree[p]
  189. scan_git_tree(tree)
  190. if __name__ == '__main__':
  191. ap = ArgumentParser(description='SPDX expression checker')
  192. ap.add_argument('path', nargs='*', help='Check path or file. If not given full git tree scan. For stdin use "-"')
  193. ap.add_argument('-m', '--maxlines', type=int, default=15,
  194. help='Maximum number of lines to scan in a file. Default 15')
  195. ap.add_argument('-v', '--verbose', action='store_true', help='Verbose statistics output')
  196. args = ap.parse_args()
  197. # Sanity check path arguments
  198. if '-' in args.path and len(args.path) > 1:
  199. sys.stderr.write('stdin input "-" must be the only path argument\n')
  200. sys.exit(1)
  201. try:
  202. # Use git to get the valid license expressions
  203. repo = git.Repo(os.getcwd())
  204. assert not repo.bare
  205. # Initialize SPDX data
  206. spdx = read_spdxdata(repo)
  207. # Initialize the parser
  208. parser = id_parser(spdx)
  209. except SPDXException as se:
  210. if se.el:
  211. sys.stderr.write('%s: %s\n' %(se.el.path, se.txt))
  212. else:
  213. sys.stderr.write('%s\n' %se.txt)
  214. sys.exit(1)
  215. except Exception as ex:
  216. sys.stderr.write('FAIL: %s\n' %ex)
  217. sys.stderr.write('%s\n' %traceback.format_exc())
  218. sys.exit(1)
  219. try:
  220. if len(args.path) and args.path[0] == '-':
  221. stdin = os.fdopen(sys.stdin.fileno(), 'rb')
  222. parser.parse_lines(stdin, args.maxlines, '-')
  223. else:
  224. if args.path:
  225. for p in args.path:
  226. if os.path.isfile(p):
  227. parser.parse_lines(open(p, 'rb'), args.maxlines, p)
  228. elif os.path.isdir(p):
  229. scan_git_subtree(repo.head.reference.commit.tree, p)
  230. else:
  231. sys.stderr.write('path %s does not exist\n' %p)
  232. sys.exit(1)
  233. else:
  234. # Full git tree scan
  235. scan_git_tree(repo.head.commit.tree)
  236. if args.verbose:
  237. sys.stderr.write('\n')
  238. sys.stderr.write('License files: %12d\n' %spdx.license_files)
  239. sys.stderr.write('Exception files: %12d\n' %spdx.exception_files)
  240. sys.stderr.write('License IDs %12d\n' %len(spdx.licenses))
  241. sys.stderr.write('Exception IDs %12d\n' %len(spdx.exceptions))
  242. sys.stderr.write('\n')
  243. sys.stderr.write('Files checked: %12d\n' %parser.checked)
  244. sys.stderr.write('Lines checked: %12d\n' %parser.lines_checked)
  245. sys.stderr.write('Files with SPDX: %12d\n' %parser.spdx_valid)
  246. sys.stderr.write('Files with errors: %12d\n' %parser.spdx_errors)
  247. sys.exit(0)
  248. except Exception as ex:
  249. sys.stderr.write('FAIL: %s\n' %ex)
  250. sys.stderr.write('%s\n' %traceback.format_exc())
  251. sys.exit(1)