EDK2 doxygen online documents - Firmware Encoding Index 1
EDK2 doxygen online documents - Firmware Encoding Index

BaseTools/Source/Python/UPT/Library/CommentParsing.py

Go to the documentation of this file.
00001 ## @file
00002 # This file is used to define comment parsing interface
00003 #
00004 # Copyright (c) 2011, Intel Corporation. All rights reserved.<BR>
00005 #
00006 # This program and the accompanying materials are licensed and made available 
00007 # under the terms and conditions of the BSD License which accompanies this 
00008 # distribution. The full text of the license may be found at 
00009 # http://opensource.org/licenses/bsd-license.php
00010 #
00011 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
00012 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
00013 #
00014 
00015 '''
00016 CommentParsing
00017 '''
00018 
00019 ##
00020 # Import Modules
00021 #
00022 import re
00023 
00024 from Library.String import GetSplitValueList
00025 from Library.String import CleanString2
00026 from Library.DataType import HEADER_COMMENT_NOT_STARTED
00027 from Library.DataType import TAB_COMMENT_SPLIT
00028 from Library.DataType import HEADER_COMMENT_LICENSE
00029 from Library.DataType import HEADER_COMMENT_ABSTRACT
00030 from Library.DataType import HEADER_COMMENT_COPYRIGHT
00031 from Library.DataType import HEADER_COMMENT_DESCRIPTION
00032 from Library.DataType import TAB_SPACE_SPLIT
00033 from Library.DataType import TAB_COMMA_SPLIT
00034 from Library.DataType import SUP_MODULE_LIST
00035 from Object.POM.CommonObject import TextObject
00036 from Object.POM.CommonObject import PcdErrorObject
00037 import Logger.Log as Logger
00038 from Logger.ToolError import FORMAT_INVALID
00039 from Logger.ToolError import FORMAT_NOT_SUPPORTED
00040 from Logger import StringTable as ST
00041 
00042 ## ParseHeaderCommentSection
00043 #
00044 # Parse Header comment section lines, extract Abstract, Description, Copyright
00045 # , License lines
00046 #
00047 # @param CommentList:   List of (Comment, LineNumber)
00048 # @param FileName:      FileName of the comment
00049 #
00050 def ParseHeaderCommentSection(CommentList, FileName = None):
00051     Abstract = ''
00052     Description = ''
00053     Copyright = ''
00054     License = ''
00055     EndOfLine = "\n"
00056     STR_HEADER_COMMENT_START = "@file"
00057     HeaderCommentStage = HEADER_COMMENT_NOT_STARTED
00058     
00059     #
00060     # first find the last copyright line
00061     #
00062     Last = 0
00063     for Index in xrange(len(CommentList)-1, 0, -1):
00064         Line = CommentList[Index][0]
00065         if _IsCopyrightLine(Line):
00066             Last = Index
00067             break
00068     
00069     for Item in CommentList:
00070         Line = Item[0]
00071         LineNo = Item[1]
00072         
00073         if not Line.startswith(TAB_COMMENT_SPLIT) and Line:
00074             Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1])
00075         Comment = CleanString2(Line)[1]
00076         Comment = Comment.strip()
00077         #
00078         # if there are blank lines between License or Description, keep them as they would be 
00079         # indication of different block; or in the position that Abstract should be, also keep it
00080         # as it indicates that no abstract
00081         #
00082         if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \
00083                                                       HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:
00084             continue
00085         
00086         if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:
00087             if Comment.startswith(STR_HEADER_COMMENT_START):
00088                 HeaderCommentStage = HEADER_COMMENT_ABSTRACT
00089             else:
00090                 License += Comment + EndOfLine
00091         else:
00092             if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:
00093                 #
00094                 # in case there is no abstract and description
00095                 #
00096                 if not Comment:
00097                     Abstract = ''
00098                     HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
00099                 elif _IsCopyrightLine(Comment):
00100                     Result, ErrMsg = _ValidateCopyright(Comment)
00101                     ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
00102                     Copyright += Comment + EndOfLine
00103                     HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
00104                 else:                    
00105                     Abstract += Comment + EndOfLine
00106                     HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
00107             elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:
00108                 #
00109                 # in case there is no description
00110                 #                
00111                 if _IsCopyrightLine(Comment):
00112                     Result, ErrMsg = _ValidateCopyright(Comment)
00113                     ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
00114                     Copyright += Comment + EndOfLine
00115                     HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
00116                 else:
00117                     Description += Comment + EndOfLine                
00118             elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:
00119                 if _IsCopyrightLine(Comment):
00120                     Result, ErrMsg = _ValidateCopyright(Comment)
00121                     ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
00122                     Copyright += Comment + EndOfLine
00123                 else:
00124                     #
00125                     # Contents after copyright line are license, those non-copyright lines in between
00126                     # copyright line will be discarded 
00127                     #
00128                     if LineNo > Last:
00129                         if License:
00130                             License += EndOfLine
00131                         License += Comment + EndOfLine
00132                         HeaderCommentStage = HEADER_COMMENT_LICENSE                
00133             else:
00134                 if not Comment and not License:
00135                     continue
00136                 License += Comment + EndOfLine
00137     
00138     if not Copyright:
00139         Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_COPYRIGHT_MISSING, \
00140                      FileName)
00141 
00142     if not License:
00143         Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_LICENSE_MISSING, FileName)
00144                      
00145     return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()
00146 
00147 ## _IsCopyrightLine
00148 # check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright" 
00149 # followed by zero or more white space characters followed by a "(" character 
00150 #
00151 # @param LineContent:  the line need to be checked
00152 # @return: True if current line is copyright line, False else
00153 #
00154 def _IsCopyrightLine (LineContent):
00155     LineContent = LineContent.upper()
00156     Result = False
00157     
00158     ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)
00159     if ReIsCopyrightRe.search(LineContent):
00160         Result = True
00161         
00162     return Result
00163 
00164 ## ParseGenericComment
00165 #
00166 # @param GenericComment: Generic comment list, element of 
00167 #                        (CommentLine, LineNum)
00168 # @param ContainerFile:  Input value for filename of Dec file
00169 # 
00170 def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None):
00171     if ContainerFile:
00172         pass
00173     HelpTxt = None         
00174     HelpStr = '' 
00175         
00176     for Item in GenericComment:
00177         CommentLine = Item[0]
00178         Comment = CleanString2(CommentLine)[1]
00179         if SkipTag is not None and Comment.startswith(SkipTag):
00180             Comment = Comment.replace(SkipTag, '', 1)
00181         HelpStr += Comment + '\n'
00182         
00183     if HelpStr:
00184         HelpTxt = TextObject()
00185         if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n':
00186             HelpStr = HelpStr[:-1]
00187         HelpTxt.SetString(HelpStr)
00188 
00189     return HelpTxt
00190 
00191     
00192 ## ParseDecPcdGenericComment
00193 #
00194 # @param GenericComment: Generic comment list, element of (CommentLine, 
00195 #                         LineNum)
00196 # @param ContainerFile:  Input value for filename of Dec file
00197 # 
00198 def ParseDecPcdGenericComment (GenericComment, ContainerFile):       
00199     HelpStr = '' 
00200     PcdErr = None
00201         
00202     for (CommentLine, LineNum) in GenericComment:
00203         Comment = CleanString2(CommentLine)[1]
00204         if Comment.startswith("@ValidRange"):
00205             if PcdErr:
00206                 Logger.Error('Parser', 
00207                              FORMAT_NOT_SUPPORTED,
00208                              ST.WRN_MULTI_PCD_RANGES,
00209                              File = ContainerFile, 
00210                              Line = LineNum)
00211             ValidRange = Comment.replace("@ValidRange", "", 1)
00212             if _CheckRangeExpression(ValidRange):
00213                 PcdErr = PcdErrorObject()
00214                 PcdErr.SetValidValueRange(ValidRange)
00215         elif Comment.startswith("@ValidList"):
00216             if PcdErr:
00217                 Logger.Error('Parser', 
00218                              FORMAT_NOT_SUPPORTED,
00219                              ST.WRN_MULTI_PCD_RANGES,
00220                              File = ContainerFile, 
00221                              Line = LineNum)
00222             ValidValue = Comment.replace("@ValidList", "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT)
00223             PcdErr = PcdErrorObject()
00224             PcdErr.SetValidValue(ValidValue)
00225         elif Comment.startswith("@Expression"):
00226             if PcdErr:
00227                 Logger.Error('Parser', 
00228                              FORMAT_NOT_SUPPORTED,
00229                              ST.WRN_MULTI_PCD_RANGES,
00230                              File = ContainerFile, 
00231                              Line = LineNum)
00232             Expression = Comment.replace("@Expression", "", 1)
00233             if _CheckRangeExpression(Expression):
00234                 PcdErr = PcdErrorObject()
00235                 PcdErr.SetExpression(Expression)
00236         else:
00237             HelpStr += Comment + '\n'
00238     
00239     #
00240     # remove the last EOL if the comment is of format 'FOO\n'
00241     #
00242     if HelpStr.endswith('\n'):
00243         if HelpStr != '\n' and not HelpStr.endswith('\n\n'):
00244             HelpStr = HelpStr[:-1]
00245 
00246     return HelpStr, PcdErr
00247 
00248 ## ParseDecPcdTailComment
00249 #
00250 # @param TailCommentList:    Tail comment list of Pcd, item of format (Comment, LineNum)
00251 # @param ContainerFile:      Input value for filename of Dec file
00252 # @retVal SupModuleList:  The supported module type list detected
00253 # @retVal HelpStr:  The generic help text string detected
00254 #
00255 def ParseDecPcdTailComment (TailCommentList, ContainerFile):
00256     assert(len(TailCommentList) == 1)
00257     TailComment = TailCommentList[0][0]
00258     LineNum = TailCommentList[0][1]
00259 
00260     Comment = TailComment.lstrip(" #")
00261     
00262     ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL)
00263     
00264     #
00265     # get first word and compare with SUP_MODULE_LIST
00266     #
00267     MatchObject = ReFindFirstWordRe.match(Comment)
00268     if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST):
00269         return None, Comment
00270 
00271     #
00272     # parse line, it must have supported module type specified
00273     #
00274     if Comment.find(TAB_COMMENT_SPLIT) == -1:
00275         Comment += TAB_COMMENT_SPLIT    
00276     SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1)
00277     SupModuleList = []
00278     for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT):
00279         if not Mod:
00280             continue
00281         elif Mod not in SUP_MODULE_LIST:
00282             Logger.Error("UPT",
00283                          FORMAT_INVALID,
00284                          ST.WRN_INVALID_MODULE_TYPE%Mod, 
00285                          ContainerFile, 
00286                          LineNum)
00287         else:
00288             SupModuleList.append(Mod)
00289 
00290     return SupModuleList, HelpStr
00291 
00292 
00293 ## _CheckRangeExpression
00294 #
00295 # @param Expression:    Pcd range expression
00296 #          
00297 def _CheckRangeExpression(Expression):
00298     #
00299     # check grammar for Pcd range expression is not required yet
00300     #
00301     if Expression:
00302         pass
00303     return True
00304 
00305 ## ValidateCopyright
00306 #
00307 #
00308 #
00309 def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg):
00310     if not Result:
00311         Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg) 
00312 
00313 ## _ValidateCopyright
00314 #
00315 # @param Line:    Line that contains copyright information, # stripped
00316 # 
00317 # @retval Result: True if line is conformed to Spec format, False else
00318 # @retval ErrMsg: the detailed error description
00319 #  
00320 def _ValidateCopyright(Line):
00321     if Line:
00322         pass
00323     Result = True
00324     ErrMsg = ''
00325     
00326     return Result, ErrMsg
00327 
00328 def GenerateTokenList (Comment):
00329     #
00330     # Tokenize Comment using '#' and ' ' as token seperators
00331     #
00332     RelplacedComment = None    
00333     while Comment != RelplacedComment:
00334         RelplacedComment = Comment
00335         Comment = Comment.replace('##', '#').replace('  ', ' ').replace(' ', '#').strip('# ')
00336     return Comment.split('#')
00337 
00338 
00339 #
00340 # Comment       - Comment to parse
00341 # TypeTokens    - A dictionary of type token synonyms
00342 # RemoveTokens  - A list of tokens to remove from help text
00343 # ParseVariable - True for parsing [Guids].  Otherwise False
00344 #
00345 def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable):
00346     #
00347     # Initialize return values
00348     #
00349     Usage = None
00350     Type = None
00351     String = None
00352     HelpText = None
00353     
00354     Comment = Comment[0]
00355     
00356     NumTokens = 2  
00357     if ParseVariable:
00358         # 
00359         # Remove white space around first instance of ':' from Comment if 'Variable' 
00360         # is in front of ':' and Variable is the 1st or 2nd token in Comment.
00361         #
00362         List = Comment.split(':', 1)    
00363         if len(List) > 1:
00364             SubList = GenerateTokenList (List[0].strip())
00365             if len(SubList) in [1, 2] and SubList[-1] == 'Variable':
00366                 if List[1].strip().find('L"') == 0:      
00367                     Comment = List[0].strip() + ':' + List[1].strip()
00368         
00369         # 
00370         # Remove first instance of L"<VariableName> from Comment and put into String
00371         # if and only if L"<VariableName>" is the 1st token, the 2nd token.  Or 
00372         # L"<VariableName>" is the third token immediately following 'Variable:'.
00373         #
00374         End = -1
00375         Start = Comment.find('Variable:L"')
00376         if Start >= 0:
00377             String = Comment[Start + 9:]
00378             End = String[2:].find('"')
00379         else:
00380             Start = Comment.find('L"')
00381             if Start >= 0:
00382                 String = Comment[Start:]
00383                 End = String[2:].find('"')
00384         if End >= 0:
00385             SubList = GenerateTokenList (Comment[:Start])
00386             if len(SubList) < 2: 
00387                 Comment = Comment[:Start] + String[End + 3:]
00388                 String = String[:End + 3]
00389                 Type = 'Variable'
00390                 NumTokens = 1  
00391     
00392     #
00393     # Initialze HelpText to Comment.  
00394     # Content will be remove from HelpText as matching tokens are found
00395     #  
00396     HelpText = Comment
00397     
00398     #
00399     # Tokenize Comment using '#' and ' ' as token seperators
00400     #
00401     List = GenerateTokenList (Comment)
00402     
00403     #
00404     # Search first two tokens for Usage and Type and remove any matching tokens 
00405     # from HelpText
00406     #
00407     for Token in List[0:NumTokens]:
00408         if Usage == None and Token in UsageTokens:
00409             Usage = UsageTokens[Token]
00410             HelpText = HelpText.replace(Token, '')
00411     if Usage != None or not ParseVariable:
00412         for Token in List[0:NumTokens]:
00413             if Type == None and Token in TypeTokens:
00414                 Type = TypeTokens[Token]
00415                 HelpText = HelpText.replace(Token, '')
00416             if Usage != None:    
00417                 for Token in List[0:NumTokens]:
00418                     if Token in RemoveTokens:
00419                         HelpText = HelpText.replace(Token, '')
00420     
00421     #
00422     # If no Usage token is present and set Usage to UNDEFINED
00423     #  
00424     if Usage == None:
00425         Usage = 'UNDEFINED'
00426     
00427     #
00428     # If no Type token is present and set Type to UNDEFINED
00429     #  
00430     if Type == None:
00431         Type = 'UNDEFINED'
00432     
00433     #
00434     # If Type is not 'Variable:', then set String to None
00435     #  
00436     if Type != 'Variable':
00437         String = None  
00438     
00439     #
00440     # Strip ' ' and '#' from the beginning of HelpText
00441     # If HelpText is an empty string after all parsing is 
00442     # complete then set HelpText to None
00443     #  
00444     HelpText = HelpText.lstrip('# ')
00445     if HelpText == '':
00446         HelpText = None
00447       
00448     #
00449     # Return parsing results
00450     #  
00451     return Usage, Type, String, HelpText  
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Defines