UniClassObject.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690
  1. ## @file
  2. # This file is used to collect all defined strings in multiple uni files
  3. #
  4. #
  5. # Copyright (c) 2014 Hewlett-Packard Development Company, L.P.<BR>
  6. #
  7. # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
  8. # This program and the accompanying materials
  9. # are licensed and made available under the terms and conditions of the BSD License
  10. # which accompanies this distribution. The full text of the license may be found at
  11. # http://opensource.org/licenses/bsd-license.php
  12. #
  13. # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  15. ##
  16. # Import Modules
  17. #
  18. from __future__ import print_function
  19. import Common.LongFilePathOs as os, codecs, re
  20. import distutils.util
  21. import Common.EdkLogger as EdkLogger
  22. from io import BytesIO
  23. from Common.BuildToolError import *
  24. from Common.StringUtils import GetLineNo
  25. from Common.Misc import PathClass
  26. from Common.LongFilePathSupport import LongFilePath
  27. from Common.GlobalData import *
  28. ##
  29. # Static definitions
  30. #
  31. UNICODE_WIDE_CHAR = u'\\wide'
  32. UNICODE_NARROW_CHAR = u'\\narrow'
  33. UNICODE_NON_BREAKING_CHAR = u'\\nbr'
  34. UNICODE_UNICODE_CR = '\r'
  35. UNICODE_UNICODE_LF = '\n'
  36. NARROW_CHAR = u'\uFFF0'
  37. WIDE_CHAR = u'\uFFF1'
  38. NON_BREAKING_CHAR = u'\uFFF2'
  39. CR = u'\u000D'
  40. LF = u'\u000A'
  41. NULL = u'\u0000'
  42. TAB = u'\t'
  43. BACK_SLASH_PLACEHOLDER = u'\u0006'
  44. gIncludePattern = re.compile("^#include +[\"<]+([^\"< >]+)[>\"]+$", re.MULTILINE | re.UNICODE)
  45. ## Convert a unicode string to a Hex list
  46. #
  47. # Convert a unicode string to a Hex list
  48. # UniToHexList('ABC') is ['0x41', '0x00', '0x42', '0x00', '0x43', '0x00']
  49. #
  50. # @param Uni: The python unicode string
  51. #
  52. # @retval List: The formatted hex list
  53. #
  54. def UniToHexList(Uni):
  55. List = []
  56. for Item in Uni:
  57. Temp = '%04X' % ord(Item)
  58. List.append('0x' + Temp[2:4])
  59. List.append('0x' + Temp[0:2])
  60. return List
  61. LangConvTable = {'eng':'en', 'fra':'fr', \
  62. 'aar':'aa', 'abk':'ab', 'ave':'ae', 'afr':'af', 'aka':'ak', 'amh':'am', \
  63. 'arg':'an', 'ara':'ar', 'asm':'as', 'ava':'av', 'aym':'ay', 'aze':'az', \
  64. 'bak':'ba', 'bel':'be', 'bul':'bg', 'bih':'bh', 'bis':'bi', 'bam':'bm', \
  65. 'ben':'bn', 'bod':'bo', 'bre':'br', 'bos':'bs', 'cat':'ca', 'che':'ce', \
  66. 'cha':'ch', 'cos':'co', 'cre':'cr', 'ces':'cs', 'chu':'cu', 'chv':'cv', \
  67. 'cym':'cy', 'dan':'da', 'deu':'de', 'div':'dv', 'dzo':'dz', 'ewe':'ee', \
  68. 'ell':'el', 'epo':'eo', 'spa':'es', 'est':'et', 'eus':'eu', 'fas':'fa', \
  69. 'ful':'ff', 'fin':'fi', 'fij':'fj', 'fao':'fo', 'fry':'fy', 'gle':'ga', \
  70. 'gla':'gd', 'glg':'gl', 'grn':'gn', 'guj':'gu', 'glv':'gv', 'hau':'ha', \
  71. 'heb':'he', 'hin':'hi', 'hmo':'ho', 'hrv':'hr', 'hat':'ht', 'hun':'hu', \
  72. 'hye':'hy', 'her':'hz', 'ina':'ia', 'ind':'id', 'ile':'ie', 'ibo':'ig', \
  73. 'iii':'ii', 'ipk':'ik', 'ido':'io', 'isl':'is', 'ita':'it', 'iku':'iu', \
  74. 'jpn':'ja', 'jav':'jv', 'kat':'ka', 'kon':'kg', 'kik':'ki', 'kua':'kj', \
  75. 'kaz':'kk', 'kal':'kl', 'khm':'km', 'kan':'kn', 'kor':'ko', 'kau':'kr', \
  76. 'kas':'ks', 'kur':'ku', 'kom':'kv', 'cor':'kw', 'kir':'ky', 'lat':'la', \
  77. 'ltz':'lb', 'lug':'lg', 'lim':'li', 'lin':'ln', 'lao':'lo', 'lit':'lt', \
  78. 'lub':'lu', 'lav':'lv', 'mlg':'mg', 'mah':'mh', 'mri':'mi', 'mkd':'mk', \
  79. 'mal':'ml', 'mon':'mn', 'mar':'mr', 'msa':'ms', 'mlt':'mt', 'mya':'my', \
  80. 'nau':'na', 'nob':'nb', 'nde':'nd', 'nep':'ne', 'ndo':'ng', 'nld':'nl', \
  81. 'nno':'nn', 'nor':'no', 'nbl':'nr', 'nav':'nv', 'nya':'ny', 'oci':'oc', \
  82. 'oji':'oj', 'orm':'om', 'ori':'or', 'oss':'os', 'pan':'pa', 'pli':'pi', \
  83. 'pol':'pl', 'pus':'ps', 'por':'pt', 'que':'qu', 'roh':'rm', 'run':'rn', \
  84. 'ron':'ro', 'rus':'ru', 'kin':'rw', 'san':'sa', 'srd':'sc', 'snd':'sd', \
  85. 'sme':'se', 'sag':'sg', 'sin':'si', 'slk':'sk', 'slv':'sl', 'smo':'sm', \
  86. 'sna':'sn', 'som':'so', 'sqi':'sq', 'srp':'sr', 'ssw':'ss', 'sot':'st', \
  87. 'sun':'su', 'swe':'sv', 'swa':'sw', 'tam':'ta', 'tel':'te', 'tgk':'tg', \
  88. 'tha':'th', 'tir':'ti', 'tuk':'tk', 'tgl':'tl', 'tsn':'tn', 'ton':'to', \
  89. 'tur':'tr', 'tso':'ts', 'tat':'tt', 'twi':'tw', 'tah':'ty', 'uig':'ug', \
  90. 'ukr':'uk', 'urd':'ur', 'uzb':'uz', 'ven':'ve', 'vie':'vi', 'vol':'vo', \
  91. 'wln':'wa', 'wol':'wo', 'xho':'xh', 'yid':'yi', 'yor':'yo', 'zha':'za', \
  92. 'zho':'zh', 'zul':'zu'}
  93. ## GetLanguageCode
  94. #
  95. # Check the language code read from .UNI file and convert ISO 639-2 codes to RFC 4646 codes if appropriate
  96. # ISO 639-2 language codes supported in compatibility mode
  97. # RFC 4646 language codes supported in native mode
  98. #
  99. # @param LangName: Language codes read from .UNI file
  100. #
  101. # @retval LangName: Valid language code in RFC 4646 format or None
  102. #
  103. def GetLanguageCode(LangName, IsCompatibleMode, File):
  104. length = len(LangName)
  105. if IsCompatibleMode:
  106. if length == 3 and LangName.isalpha():
  107. TempLangName = LangConvTable.get(LangName.lower())
  108. if TempLangName is not None:
  109. return TempLangName
  110. return LangName
  111. else:
  112. EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid ISO 639-2 language code : %s" % LangName, File)
  113. if (LangName[0] == 'X' or LangName[0] == 'x') and LangName[1] == '-':
  114. return LangName
  115. if length == 2:
  116. if LangName.isalpha():
  117. return LangName
  118. elif length == 3:
  119. if LangName.isalpha() and LangConvTable.get(LangName.lower()) is None:
  120. return LangName
  121. elif length == 5:
  122. if LangName[0:2].isalpha() and LangName[2] == '-':
  123. return LangName
  124. elif length >= 6:
  125. if LangName[0:2].isalpha() and LangName[2] == '-':
  126. return LangName
  127. if LangName[0:3].isalpha() and LangConvTable.get(LangName.lower()) is None and LangName[3] == '-':
  128. return LangName
  129. EdkLogger.error("Unicode File Parser", FORMAT_INVALID, "Invalid RFC 4646 language code : %s" % LangName, File)
  130. ## Ucs2Codec
  131. #
  132. # This is only a partial codec implementation. It only supports
  133. # encoding, and is primarily used to check that all the characters are
  134. # valid for UCS-2.
  135. #
  136. class Ucs2Codec(codecs.Codec):
  137. def __init__(self):
  138. self.__utf16 = codecs.lookup('utf-16')
  139. def encode(self, input, errors='strict'):
  140. for Char in input:
  141. CodePoint = ord(Char)
  142. if CodePoint >= 0xd800 and CodePoint <= 0xdfff:
  143. raise ValueError("Code Point is in range reserved for " +
  144. "UTF-16 surrogate pairs")
  145. elif CodePoint > 0xffff:
  146. raise ValueError("Code Point too large to encode in UCS-2")
  147. return self.__utf16.encode(input)
  148. TheUcs2Codec = Ucs2Codec()
  149. def Ucs2Search(name):
  150. if name == 'ucs-2':
  151. return codecs.CodecInfo(
  152. name=name,
  153. encode=TheUcs2Codec.encode,
  154. decode=TheUcs2Codec.decode)
  155. else:
  156. return None
  157. codecs.register(Ucs2Search)
  158. ## StringDefClassObject
  159. #
  160. # A structure for language definition
  161. #
  162. class StringDefClassObject(object):
  163. def __init__(self, Name = None, Value = None, Referenced = False, Token = None, UseOtherLangDef = ''):
  164. self.StringName = ''
  165. self.StringNameByteList = []
  166. self.StringValue = ''
  167. self.StringValueByteList = ''
  168. self.Token = 0
  169. self.Referenced = Referenced
  170. self.UseOtherLangDef = UseOtherLangDef
  171. self.Length = 0
  172. if Name is not None:
  173. self.StringName = Name
  174. self.StringNameByteList = UniToHexList(Name)
  175. if Value is not None:
  176. self.StringValue = Value + u'\x00' # Add a NULL at string tail
  177. self.StringValueByteList = UniToHexList(self.StringValue)
  178. self.Length = len(self.StringValueByteList)
  179. if Token is not None:
  180. self.Token = Token
  181. def __str__(self):
  182. return repr(self.StringName) + ' ' + \
  183. repr(self.Token) + ' ' + \
  184. repr(self.Referenced) + ' ' + \
  185. repr(self.StringValue) + ' ' + \
  186. repr(self.UseOtherLangDef)
  187. def UpdateValue(self, Value = None):
  188. if Value is not None:
  189. self.StringValue = Value + u'\x00' # Add a NULL at string tail
  190. self.StringValueByteList = UniToHexList(self.StringValue)
  191. self.Length = len(self.StringValueByteList)
  192. def StripComments(Line):
  193. Comment = u'//'
  194. CommentPos = Line.find(Comment)
  195. while CommentPos >= 0:
  196. # if there are non matched quotes before the comment header
  197. # then we are in the middle of a string
  198. # but we need to ignore the escaped quotes and backslashes.
  199. if ((Line.count(u'"', 0, CommentPos) - Line.count(u'\\"', 0, CommentPos)) & 1) == 1:
  200. CommentPos = Line.find (Comment, CommentPos + 1)
  201. else:
  202. return Line[:CommentPos].strip()
  203. return Line.strip()
  204. ## UniFileClassObject
  205. #
  206. # A structure for .uni file definition
  207. #
  208. class UniFileClassObject(object):
  209. def __init__(self, FileList = [], IsCompatibleMode = False, IncludePathList = []):
  210. self.FileList = FileList
  211. self.Token = 2
  212. self.LanguageDef = [] #[ [u'LanguageIdentifier', u'PrintableName'], ... ]
  213. self.OrderedStringList = {} #{ u'LanguageIdentifier' : [StringDefClassObject] }
  214. self.OrderedStringDict = {} #{ u'LanguageIdentifier' : {StringName:(IndexInList)} }
  215. self.OrderedStringListByToken = {} #{ u'LanguageIdentifier' : {Token: StringDefClassObject} }
  216. self.IsCompatibleMode = IsCompatibleMode
  217. self.IncludePathList = IncludePathList
  218. if len(self.FileList) > 0:
  219. self.LoadUniFiles(FileList)
  220. #
  221. # Get Language definition
  222. #
  223. def GetLangDef(self, File, Line):
  224. Lang = distutils.util.split_quoted((Line.split(u"//")[0]))
  225. if len(Lang) != 3:
  226. try:
  227. FileIn = UniFileClassObject.OpenUniFile(LongFilePath(File.Path))
  228. except UnicodeError as X:
  229. EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File);
  230. except:
  231. EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File);
  232. LineNo = GetLineNo(FileIn, Line, False)
  233. EdkLogger.error("Unicode File Parser", PARSER_ERROR, "Wrong language definition",
  234. ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, File=File, Line=LineNo)
  235. else:
  236. LangName = GetLanguageCode(Lang[1], self.IsCompatibleMode, self.File)
  237. LangPrintName = Lang[2]
  238. IsLangInDef = False
  239. for Item in self.LanguageDef:
  240. if Item[0] == LangName:
  241. IsLangInDef = True
  242. break;
  243. if not IsLangInDef:
  244. self.LanguageDef.append([LangName, LangPrintName])
  245. #
  246. # Add language string
  247. #
  248. self.AddStringToList(u'$LANGUAGE_NAME', LangName, LangName, 0, True, Index=0)
  249. self.AddStringToList(u'$PRINTABLE_LANGUAGE_NAME', LangName, LangPrintName, 1, True, Index=1)
  250. if not IsLangInDef:
  251. #
  252. # The found STRING tokens will be added into new language string list
  253. # so that the unique STRING identifier is reserved for all languages in the package list.
  254. #
  255. FirstLangName = self.LanguageDef[0][0]
  256. if LangName != FirstLangName:
  257. for Index in range (2, len (self.OrderedStringList[FirstLangName])):
  258. Item = self.OrderedStringList[FirstLangName][Index]
  259. if Item.UseOtherLangDef != '':
  260. OtherLang = Item.UseOtherLangDef
  261. else:
  262. OtherLang = FirstLangName
  263. self.OrderedStringList[LangName].append (StringDefClassObject(Item.StringName, '', Item.Referenced, Item.Token, OtherLang))
  264. self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1
  265. return True
  266. @staticmethod
  267. def OpenUniFile(FileName):
  268. #
  269. # Read file
  270. #
  271. try:
  272. UniFile = open(FileName, mode='rb')
  273. FileIn = UniFile.read()
  274. UniFile.close()
  275. except:
  276. EdkLogger.Error("build", FILE_OPEN_FAILURE, ExtraData=File)
  277. #
  278. # Detect Byte Order Mark at beginning of file. Default to UTF-8
  279. #
  280. Encoding = 'utf-8'
  281. if (FileIn.startswith(codecs.BOM_UTF16_BE) or
  282. FileIn.startswith(codecs.BOM_UTF16_LE)):
  283. Encoding = 'utf-16'
  284. UniFileClassObject.VerifyUcs2Data(FileIn, FileName, Encoding)
  285. UniFile = BytesIO(FileIn)
  286. Info = codecs.lookup(Encoding)
  287. (Reader, Writer) = (Info.streamreader, Info.streamwriter)
  288. return codecs.StreamReaderWriter(UniFile, Reader, Writer)
  289. @staticmethod
  290. def VerifyUcs2Data(FileIn, FileName, Encoding):
  291. Ucs2Info = codecs.lookup('ucs-2')
  292. #
  293. # Convert to unicode
  294. #
  295. try:
  296. FileDecoded = codecs.decode(FileIn, Encoding)
  297. Ucs2Info.encode(FileDecoded)
  298. except:
  299. UniFile = BytesIO(FileIn)
  300. Info = codecs.lookup(Encoding)
  301. (Reader, Writer) = (Info.streamreader, Info.streamwriter)
  302. File = codecs.StreamReaderWriter(UniFile, Reader, Writer)
  303. LineNumber = 0
  304. ErrMsg = lambda Encoding, LineNumber: \
  305. '%s contains invalid %s characters on line %d.' % \
  306. (FileName, Encoding, LineNumber)
  307. while True:
  308. LineNumber = LineNumber + 1
  309. try:
  310. Line = File.readline()
  311. if Line == '':
  312. EdkLogger.error('Unicode File Parser', PARSER_ERROR,
  313. ErrMsg(Encoding, LineNumber))
  314. Ucs2Info.encode(Line)
  315. except:
  316. EdkLogger.error('Unicode File Parser', PARSER_ERROR,
  317. ErrMsg('UCS-2', LineNumber))
  318. #
  319. # Get String name and value
  320. #
  321. def GetStringObject(self, Item):
  322. Language = ''
  323. Value = ''
  324. Name = Item.split()[1]
  325. # Check the string name
  326. if Name != '':
  327. MatchString = gIdentifierPattern.match(Name)
  328. if MatchString is None:
  329. EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))
  330. LanguageList = Item.split(u'#language ')
  331. for IndexI in range(len(LanguageList)):
  332. if IndexI == 0:
  333. continue
  334. else:
  335. Language = LanguageList[IndexI].split()[0]
  336. Value = LanguageList[IndexI][LanguageList[IndexI].find(u'\"') + len(u'\"') : LanguageList[IndexI].rfind(u'\"')] #.replace(u'\r\n', u'')
  337. Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File)
  338. self.AddStringToList(Name, Language, Value)
  339. #
  340. # Get include file list and load them
  341. #
  342. def GetIncludeFile(self, Item, Dir):
  343. FileName = Item[Item.find(u'#include ') + len(u'#include ') :Item.find(u' ', len(u'#include '))][1:-1]
  344. self.LoadUniFile(FileName)
  345. #
  346. # Pre-process before parse .uni file
  347. #
  348. def PreProcess(self, File):
  349. if not os.path.exists(File.Path) or not os.path.isfile(File.Path):
  350. EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, ExtraData=File.Path)
  351. try:
  352. FileIn = UniFileClassObject.OpenUniFile(LongFilePath(File.Path))
  353. except UnicodeError as X:
  354. EdkLogger.error("build", FILE_READ_FAILURE, "File read failure: %s" % str(X), ExtraData=File.Path);
  355. except:
  356. EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File.Path);
  357. Lines = []
  358. #
  359. # Use unique identifier
  360. #
  361. for Line in FileIn:
  362. Line = Line.strip()
  363. Line = Line.replace(u'\\\\', BACK_SLASH_PLACEHOLDER)
  364. Line = StripComments(Line)
  365. #
  366. # Ignore empty line
  367. #
  368. if len(Line) == 0:
  369. continue
  370. Line = Line.replace(u'/langdef', u'#langdef')
  371. Line = Line.replace(u'/string', u'#string')
  372. Line = Line.replace(u'/language', u'#language')
  373. Line = Line.replace(u'/include', u'#include')
  374. Line = Line.replace(UNICODE_WIDE_CHAR, WIDE_CHAR)
  375. Line = Line.replace(UNICODE_NARROW_CHAR, NARROW_CHAR)
  376. Line = Line.replace(UNICODE_NON_BREAKING_CHAR, NON_BREAKING_CHAR)
  377. Line = Line.replace(u'\\r\\n', CR + LF)
  378. Line = Line.replace(u'\\n', CR + LF)
  379. Line = Line.replace(u'\\r', CR)
  380. Line = Line.replace(u'\\t', u' ')
  381. Line = Line.replace(u'\t', u' ')
  382. Line = Line.replace(u'\\"', u'"')
  383. Line = Line.replace(u"\\'", u"'")
  384. Line = Line.replace(BACK_SLASH_PLACEHOLDER, u'\\')
  385. StartPos = Line.find(u'\\x')
  386. while (StartPos != -1):
  387. EndPos = Line.find(u'\\', StartPos + 1, StartPos + 7)
  388. if EndPos != -1 and EndPos - StartPos == 6 :
  389. if g4HexChar.match(Line[StartPos + 2 : EndPos], re.UNICODE):
  390. EndStr = Line[EndPos: ]
  391. UniStr = Line[StartPos + 2: EndPos]
  392. if EndStr.startswith(u'\\x') and len(EndStr) >= 7:
  393. if EndStr[6] == u'\\' and g4HexChar.match(EndStr[2 : 6], re.UNICODE):
  394. Line = Line[0 : StartPos] + UniStr + EndStr
  395. else:
  396. Line = Line[0 : StartPos] + UniStr + EndStr[1:]
  397. StartPos = Line.find(u'\\x', StartPos + 1)
  398. IncList = gIncludePattern.findall(Line)
  399. if len(IncList) == 1:
  400. for Dir in [File.Dir] + self.IncludePathList:
  401. IncFile = PathClass(str(IncList[0]), Dir)
  402. if os.path.isfile(IncFile.Path):
  403. Lines.extend(self.PreProcess(IncFile))
  404. break
  405. else:
  406. EdkLogger.error("Unicode File Parser", FILE_NOT_FOUND, Message="Cannot find include file", ExtraData=str(IncList[0]))
  407. continue
  408. Lines.append(Line)
  409. return Lines
  410. #
  411. # Load a .uni file
  412. #
  413. def LoadUniFile(self, File = None):
  414. if File is None:
  415. EdkLogger.error("Unicode File Parser", PARSER_ERROR, 'No unicode file is given')
  416. self.File = File
  417. #
  418. # Process special char in file
  419. #
  420. Lines = self.PreProcess(File)
  421. #
  422. # Get Unicode Information
  423. #
  424. for IndexI in range(len(Lines)):
  425. Line = Lines[IndexI]
  426. if (IndexI + 1) < len(Lines):
  427. SecondLine = Lines[IndexI + 1]
  428. if (IndexI + 2) < len(Lines):
  429. ThirdLine = Lines[IndexI + 2]
  430. #
  431. # Get Language def information
  432. #
  433. if Line.find(u'#langdef ') >= 0:
  434. self.GetLangDef(File, Line)
  435. continue
  436. Name = ''
  437. Language = ''
  438. Value = ''
  439. #
  440. # Get string def information format 1 as below
  441. #
  442. # #string MY_STRING_1
  443. # #language eng
  444. # My first English string line 1
  445. # My first English string line 2
  446. # #string MY_STRING_1
  447. # #language spa
  448. # Mi segunda secuencia 1
  449. # Mi segunda secuencia 2
  450. #
  451. if Line.find(u'#string ') >= 0 and Line.find(u'#language ') < 0 and \
  452. SecondLine.find(u'#string ') < 0 and SecondLine.find(u'#language ') >= 0 and \
  453. ThirdLine.find(u'#string ') < 0 and ThirdLine.find(u'#language ') < 0:
  454. Name = Line[Line.find(u'#string ') + len(u'#string ') : ].strip(' ')
  455. Language = SecondLine[SecondLine.find(u'#language ') + len(u'#language ') : ].strip(' ')
  456. for IndexJ in range(IndexI + 2, len(Lines)):
  457. if Lines[IndexJ].find(u'#string ') < 0 and Lines[IndexJ].find(u'#language ') < 0:
  458. Value = Value + Lines[IndexJ]
  459. else:
  460. IndexI = IndexJ
  461. break
  462. # Value = Value.replace(u'\r\n', u'')
  463. Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File)
  464. # Check the string name
  465. if not self.IsCompatibleMode and Name != '':
  466. MatchString = gIdentifierPattern.match(Name)
  467. if MatchString is None:
  468. EdkLogger.error('Unicode File Parser', FORMAT_INVALID, 'The string token name %s defined in UNI file %s contains the invalid character.' % (Name, self.File))
  469. self.AddStringToList(Name, Language, Value)
  470. continue
  471. #
  472. # Get string def information format 2 as below
  473. #
  474. # #string MY_STRING_1 #language eng "My first English string line 1"
  475. # "My first English string line 2"
  476. # #language spa "Mi segunda secuencia 1"
  477. # "Mi segunda secuencia 2"
  478. # #string MY_STRING_2 #language eng "My first English string line 1"
  479. # "My first English string line 2"
  480. # #string MY_STRING_2 #language spa "Mi segunda secuencia 1"
  481. # "Mi segunda secuencia 2"
  482. #
  483. if Line.find(u'#string ') >= 0 and Line.find(u'#language ') >= 0:
  484. StringItem = Line
  485. for IndexJ in range(IndexI + 1, len(Lines)):
  486. if Lines[IndexJ].find(u'#string ') >= 0 and Lines[IndexJ].find(u'#language ') >= 0:
  487. IndexI = IndexJ
  488. break
  489. elif Lines[IndexJ].find(u'#string ') < 0 and Lines[IndexJ].find(u'#language ') >= 0:
  490. StringItem = StringItem + Lines[IndexJ]
  491. elif Lines[IndexJ].count(u'\"') >= 2:
  492. StringItem = StringItem[ : StringItem.rfind(u'\"')] + Lines[IndexJ][Lines[IndexJ].find(u'\"') + len(u'\"') : ]
  493. self.GetStringObject(StringItem)
  494. continue
  495. #
  496. # Load multiple .uni files
  497. #
  498. def LoadUniFiles(self, FileList):
  499. if len(FileList) > 0:
  500. for File in FileList:
  501. self.LoadUniFile(File)
  502. #
  503. # Add a string to list
  504. #
  505. def AddStringToList(self, Name, Language, Value, Token = None, Referenced = False, UseOtherLangDef = '', Index = -1):
  506. for LangNameItem in self.LanguageDef:
  507. if Language == LangNameItem[0]:
  508. break
  509. else:
  510. EdkLogger.error('Unicode File Parser', FORMAT_NOT_SUPPORTED, "The language '%s' for %s is not defined in Unicode file %s." \
  511. % (Language, Name, self.File))
  512. if Language not in self.OrderedStringList:
  513. self.OrderedStringList[Language] = []
  514. self.OrderedStringDict[Language] = {}
  515. IsAdded = True
  516. if Name in self.OrderedStringDict[Language]:
  517. IsAdded = False
  518. if Value is not None:
  519. ItemIndexInList = self.OrderedStringDict[Language][Name]
  520. Item = self.OrderedStringList[Language][ItemIndexInList]
  521. Item.UpdateValue(Value)
  522. Item.UseOtherLangDef = ''
  523. if IsAdded:
  524. Token = len(self.OrderedStringList[Language])
  525. if Index == -1:
  526. self.OrderedStringList[Language].append(StringDefClassObject(Name, Value, Referenced, Token, UseOtherLangDef))
  527. self.OrderedStringDict[Language][Name] = Token
  528. for LangName in self.LanguageDef:
  529. #
  530. # New STRING token will be added into all language string lists.
  531. # so that the unique STRING identifier is reserved for all languages in the package list.
  532. #
  533. if LangName[0] != Language:
  534. if UseOtherLangDef != '':
  535. OtherLangDef = UseOtherLangDef
  536. else:
  537. OtherLangDef = Language
  538. self.OrderedStringList[LangName[0]].append(StringDefClassObject(Name, '', Referenced, Token, OtherLangDef))
  539. self.OrderedStringDict[LangName[0]][Name] = len(self.OrderedStringList[LangName[0]]) - 1
  540. else:
  541. self.OrderedStringList[Language].insert(Index, StringDefClassObject(Name, Value, Referenced, Token, UseOtherLangDef))
  542. self.OrderedStringDict[Language][Name] = Index
  543. #
  544. # Set the string as referenced
  545. #
  546. def SetStringReferenced(self, Name):
  547. #
  548. # String stoken are added in the same order in all language string lists.
  549. # So, only update the status of string stoken in first language string list.
  550. #
  551. Lang = self.LanguageDef[0][0]
  552. if Name in self.OrderedStringDict[Lang]:
  553. ItemIndexInList = self.OrderedStringDict[Lang][Name]
  554. Item = self.OrderedStringList[Lang][ItemIndexInList]
  555. Item.Referenced = True
  556. #
  557. # Search the string in language definition by Name
  558. #
  559. def FindStringValue(self, Name, Lang):
  560. if Name in self.OrderedStringDict[Lang]:
  561. ItemIndexInList = self.OrderedStringDict[Lang][Name]
  562. return self.OrderedStringList[Lang][ItemIndexInList]
  563. return None
  564. #
  565. # Search the string in language definition by Token
  566. #
  567. def FindByToken(self, Token, Lang):
  568. for Item in self.OrderedStringList[Lang]:
  569. if Item.Token == Token:
  570. return Item
  571. return None
  572. #
  573. # Re-order strings and re-generate tokens
  574. #
  575. def ReToken(self):
  576. #
  577. # Retoken all language strings according to the status of string stoken in the first language string.
  578. #
  579. FirstLangName = self.LanguageDef[0][0]
  580. # Convert the OrderedStringList to be OrderedStringListByToken in order to faciliate future search by token
  581. for LangNameItem in self.LanguageDef:
  582. self.OrderedStringListByToken[LangNameItem[0]] = {}
  583. #
  584. # Use small token for all referred string stoken.
  585. #
  586. RefToken = 0
  587. for Index in range (0, len (self.OrderedStringList[FirstLangName])):
  588. FirstLangItem = self.OrderedStringList[FirstLangName][Index]
  589. if FirstLangItem.Referenced == True:
  590. for LangNameItem in self.LanguageDef:
  591. LangName = LangNameItem[0]
  592. OtherLangItem = self.OrderedStringList[LangName][Index]
  593. OtherLangItem.Referenced = True
  594. OtherLangItem.Token = RefToken
  595. self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem
  596. RefToken = RefToken + 1
  597. #
  598. # Use big token for all unreferred string stoken.
  599. #
  600. UnRefToken = 0
  601. for Index in range (0, len (self.OrderedStringList[FirstLangName])):
  602. FirstLangItem = self.OrderedStringList[FirstLangName][Index]
  603. if FirstLangItem.Referenced == False:
  604. for LangNameItem in self.LanguageDef:
  605. LangName = LangNameItem[0]
  606. OtherLangItem = self.OrderedStringList[LangName][Index]
  607. OtherLangItem.Token = RefToken + UnRefToken
  608. self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem
  609. UnRefToken = UnRefToken + 1
  610. #
  611. # Show the instance itself
  612. #
  613. def ShowMe(self):
  614. print(self.LanguageDef)
  615. #print self.OrderedStringList
  616. for Item in self.OrderedStringList:
  617. print(Item)
  618. for Member in self.OrderedStringList[Item]:
  619. print(str(Member))
  620. # This acts like the main() function for the script, unless it is 'import'ed into another
  621. # script.
  622. if __name__ == '__main__':
  623. EdkLogger.Initialize()
  624. EdkLogger.SetLevel(EdkLogger.DEBUG_0)
  625. a = UniFileClassObject([PathClass("C:\\Edk\\Strings.uni"), PathClass("C:\\Edk\\Strings2.uni")])
  626. a.ReToken()
  627. a.ShowMe()