CheckUnicodeSourceFiles.py 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175
  1. ## @file
  2. # Unit tests for AutoGen.UniClassObject
  3. #
  4. # Copyright (c) 2015, Intel Corporation. All rights reserved.<BR>
  5. #
  6. # SPDX-License-Identifier: BSD-2-Clause-Patent
  7. #
  8. ##
  9. # Import Modules
  10. #
  11. import os
  12. import unittest
  13. import codecs
  14. import TestTools
  15. from Common.Misc import PathClass
  16. import AutoGen.UniClassObject as BtUni
  17. from Common import EdkLogger
  18. EdkLogger.InitializeForUnitTest()
  19. class Tests(TestTools.BaseToolsTest):
  20. SampleData = u'''
  21. #langdef en-US "English"
  22. #string STR_A #language en-US "STR_A for en-US"
  23. '''
  24. def EncodeToFile(self, encoding, string=None):
  25. if string is None:
  26. string = self.SampleData
  27. if encoding is not None:
  28. data = codecs.encode(string, encoding)
  29. else:
  30. data = string
  31. path = 'input.uni'
  32. self.WriteTmpFile(path, data)
  33. return PathClass(self.GetTmpFilePath(path))
  34. def ErrorFailure(self, error, encoding, shouldPass):
  35. msg = error + ' should '
  36. if shouldPass:
  37. msg += 'not '
  38. msg += 'be generated for '
  39. msg += '%s data in a .uni file' % encoding
  40. self.fail(msg)
  41. def UnicodeErrorFailure(self, encoding, shouldPass):
  42. self.ErrorFailure('UnicodeError', encoding, shouldPass)
  43. def EdkErrorFailure(self, encoding, shouldPass):
  44. self.ErrorFailure('EdkLogger.FatalError', encoding, shouldPass)
  45. def CheckFile(self, encoding, shouldPass, string=None):
  46. path = self.EncodeToFile(encoding, string)
  47. try:
  48. BtUni.UniFileClassObject([path])
  49. if shouldPass:
  50. return
  51. except UnicodeError:
  52. if not shouldPass:
  53. return
  54. else:
  55. self.UnicodeErrorFailure(encoding, shouldPass)
  56. except EdkLogger.FatalError:
  57. if not shouldPass:
  58. return
  59. else:
  60. self.EdkErrorFailure(encoding, shouldPass)
  61. except Exception:
  62. pass
  63. self.EdkErrorFailure(encoding, shouldPass)
  64. def testUtf16InUniFile(self):
  65. self.CheckFile('utf_16', shouldPass=True)
  66. def testSupplementaryPlaneUnicodeCharInUtf16File(self):
  67. #
  68. # Supplementary Plane characters can exist in UTF-16 files,
  69. # but they are not valid UCS-2 characters.
  70. #
  71. # This test makes sure that BaseTools rejects these characters
  72. # if seen in a .uni file.
  73. #
  74. data = u'''
  75. #langdef en-US "English"
  76. #string STR_A #language en-US "CodePoint (\U00010300) > 0xFFFF"
  77. '''
  78. self.CheckFile('utf_16', shouldPass=False, string=data)
  79. def testSurrogatePairUnicodeCharInUtf16File(self):
  80. #
  81. # Surrogate Pair code points are used in UTF-16 files to
  82. # encode the Supplementary Plane characters. But, a Surrogate
  83. # Pair code point which is not followed by another Surrogate
  84. # Pair code point might be interpreted as a single code point
  85. # with the Surrogate Pair code point.
  86. #
  87. # This test makes sure that BaseTools rejects these characters
  88. # if seen in a .uni file.
  89. #
  90. data = codecs.BOM_UTF16_LE + b'//\x01\xd8 '
  91. self.CheckFile(encoding=None, shouldPass=False, string=data)
  92. def testValidUtf8File(self):
  93. self.CheckFile(encoding='utf_8', shouldPass=True)
  94. def testValidUtf8FileWithBom(self):
  95. #
  96. # Same test as testValidUtf8File, but add the UTF-8 BOM
  97. #
  98. data = codecs.BOM_UTF8 + codecs.encode(self.SampleData, 'utf_8')
  99. self.CheckFile(encoding=None, shouldPass=True, string=data)
  100. def test32bitUnicodeCharInUtf8File(self):
  101. data = u'''
  102. #langdef en-US "English"
  103. #string STR_A #language en-US "CodePoint (\U00010300) > 0xFFFF"
  104. '''
  105. self.CheckFile('utf_16', shouldPass=False, string=data)
  106. def test32bitUnicodeCharInUtf8File(self):
  107. data = u'''
  108. #langdef en-US "English"
  109. #string STR_A #language en-US "CodePoint (\U00010300) > 0xFFFF"
  110. '''
  111. self.CheckFile('utf_8', shouldPass=False, string=data)
  112. def test32bitUnicodeCharInUtf8Comment(self):
  113. data = u'''
  114. // Even in comments, we reject non-UCS-2 chars: \U00010300
  115. #langdef en-US "English"
  116. #string STR_A #language en-US "A"
  117. '''
  118. self.CheckFile('utf_8', shouldPass=False, string=data)
  119. def testSurrogatePairUnicodeCharInUtf8File(self):
  120. #
  121. # Surrogate Pair code points are used in UTF-16 files to
  122. # encode the Supplementary Plane characters. In UTF-8, it is
  123. # trivial to encode these code points, but they are not valid
  124. # code points for characters, since they are reserved for the
  125. # UTF-16 Surrogate Pairs.
  126. #
  127. # This test makes sure that BaseTools rejects these characters
  128. # if seen in a .uni file.
  129. #
  130. data = b'\xed\xa0\x81'
  131. self.CheckFile(encoding=None, shouldPass=False, string=data)
  132. def testSurrogatePairUnicodeCharInUtf8FileWithBom(self):
  133. #
  134. # Same test as testSurrogatePairUnicodeCharInUtf8File, but add
  135. # the UTF-8 BOM
  136. #
  137. data = codecs.BOM_UTF8 + b'\xed\xa0\x81'
  138. self.CheckFile(encoding=None, shouldPass=False, string=data)
  139. TheTestSuite = TestTools.MakeTheTestSuite(locals())
  140. if __name__ == '__main__':
  141. allTests = TheTestSuite()
  142. unittest.TextTestRunner().run(allTests)