BaseTools: Fix flexible PCD single quote and double quote bugs

1.The " and ' inside the string, must use escape character format
  (\", \')
2.'string' and L'string' format in --pcd, it must be double quoted
  first.

  Some examples that to match --pcd format and DSC format
  --pcd                             DSC format
  L"ABC"                            L"ABC"
  "AB\\\"C"                         "AB\"C"
  "AB\\\'C"                         "AB\'C"
  L"\'AB\\\"C\'"                    L'AB\"C'
  "\'AB\\\'C\'"                     'AB\'C'
  H"{0, L\"AB\\\"B\", \'ab\\\"c\'}" {0, L"AB\"B", 'ab\"c'}

Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Yunhua Feng <yunhuax.feng@intel.com>
Reviewed-by: Yonghong Zhu <yonghong.zhu@intel.com>

5 files changed, 123 insertions, 65 deletions

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index afe6f2f99c..4b924d21e0 100644
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1555,13 +1555,19 @@ class TopLevelMakefile(BuildFile):
             for index, option in enumerate(GlobalData.gCommand):

                 if "--pcd" == option and GlobalData.gCommand[index+1]:

                     pcdName, pcdValue = GlobalData.gCommand[index+1].split('=')

-                    if pcdValue.startswith('H'):

-                        pcdValue = 'H' + '"' + pcdValue[1:] + '"'

-                        ExtraOption += " --pcd " + pcdName + '=' + pcdValue

-                    elif pcdValue.startswith("L'"):

-                        ExtraOption += "--pcd " + pcdName + '=' + pcdValue

-                    elif pcdValue.startswith('L'):

-                        pcdValue = 'L' + '"' + pcdValue[1:] + '"'

+                    for Item in GlobalData.BuildOptionPcd:

+                        if '.'.join(Item[0:2]) == pcdName:

+                            pcdValue = Item[2]

+                            if pcdValue.startswith('L') or pcdValue.startswith('"'):

+                                pcdValue, Size = ParseFieldValue(pcdValue)

+                                NewVal = '{'

+                                for S in range(Size):

+                                    NewVal = NewVal + '0x%02X' % ((pcdValue >> S * 8) & 0xff)

+                                    NewVal += ','

+                                pcdValue =  NewVal[:-1] + '}'

+                            break

+                    if pcdValue.startswith('{'):

+                        pcdValue = 'H' + '"' + pcdValue + '"'

                         ExtraOption += " --pcd " + pcdName + '=' + pcdValue

                     else:

                         ExtraOption += " --pcd " + GlobalData.gCommand[index+1]

diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py
index edb0a60de6..f1516d5c7b 100644
--- a/BaseTools/Source/Python/Common/Expression.py
+++ b/BaseTools/Source/Python/Common/Expression.py
@@ -45,15 +45,28 @@ ERR_IN_OPERAND          = 'Macro after IN operator can only be: $(FAMILY), $(ARC
 #  For example: abc"de\"f"ghi"jkl"mn will be: ['abc', '"de\"f"', 'ghi', '"jkl"', 'mn']

 #

 def SplitString(String):

-    # There might be escaped quote: "abc\"def\\\"ghi"

-    Str = String.replace('\\\\', '//').replace('\\\"', '\\\'')

+    # There might be escaped quote: "abc\"def\\\"ghi", 'abc\'def\\\'ghi'

+    Str = String

     RetList = []

-    InQuote = False

+    InSingleQuote = False

+    InDoubleQuote = False

     Item = ''

     for i, ch in enumerate(Str):

-        if ch == '"':

-            InQuote = not InQuote

-            if not InQuote:

+        if ch == '"' and not InSingleQuote:

+            if Str[i - 1] != '\\':

+                InDoubleQuote = not InDoubleQuote

+            if not InDoubleQuote:

+                Item += String[i]

+                RetList.append(Item)

+                Item = ''

+                continue

+            if Item:

+                RetList.append(Item)

+                Item = ''

+        elif ch == "'" and not InDoubleQuote:

+            if Str[i - 1] != '\\':

+                InSingleQuote = not InSingleQuote

+            if not InSingleQuote:

                 Item += String[i]

                 RetList.append(Item)

                 Item = ''

@@ -62,7 +75,7 @@ def SplitString(String):
                 RetList.append(Item)

                 Item = ''

         Item += String[i]

-    if InQuote:

+    if InSingleQuote or InDoubleQuote:

         raise BadExpression(ERR_STRING_TOKEN % Item)

     if Item:

         RetList.append(Item)

@@ -483,6 +496,8 @@ class ValueExpression(object):
             Flag = 0

             for Index in range(len(self._Token)):

                 if self._Token[Index] in ['"']:

+                    if self._Token[Index - 1] == '\\':

+                        continue

                     Flag += 1

             if Flag == 2 and self._Token.endswith('"'):

                 return True

@@ -490,6 +505,8 @@ class ValueExpression(object):
             Flag = 0

             for Index in range(len(self._Token)):

                 if self._Token[Index] in ["'"]:

+                    if self._Token[Index - 1] == '\\':

+                        continue

                     Flag += 1

             if Flag == 2 and self._Token.endswith("'"):

                 return True

@@ -537,16 +554,25 @@ class ValueExpression(object):
         self._Idx += 1

 

         # Replace escape \\\", \"

-        Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'')

-        for Ch in Expr:

-            self._Idx += 1

-            if Ch == '"' or Ch == "'":

-                break

-        self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]

-        if self._Token.startswith('"') and not self._Token.endswith('"'):

-            raise BadExpression(ERR_STRING_TOKEN % self._Token)

-        if self._Token.startswith("'") and not self._Token.endswith("'"):

-            raise BadExpression(ERR_STRING_TOKEN % self._Token)

+        if self._Expr[Idx] == '"':

+            Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'')

+            for Ch in Expr:

+                self._Idx += 1

+                if Ch == '"':

+                    break

+            self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]

+            if not self._Token.endswith('"'):

+                raise BadExpression(ERR_STRING_TOKEN % self._Token)

+        #Replace escape \\\', \'

+        elif self._Expr[Idx] == "'":

+            Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace("\\\'", "\\\"")

+            for Ch in Expr:

+                self._Idx += 1

+                if Ch == "'":

+                    break

+            self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]

+            if not self._Token.endswith("'"):

+                raise BadExpression(ERR_STRING_TOKEN % self._Token)

         self._Token = self._Token[1:-1]

         return self._Token

 

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 1461d00669..a7e7797d04 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -1443,21 +1443,26 @@ def ParseConsoleLog(Filename):
 

 def AnalyzePcdExpression(Setting):

     Setting = Setting.strip()

-    # There might be escaped quote in a string: \", \\\"

-    Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')

+    # There might be escaped quote in a string: \", \\\" , \', \\\'

+    Data = Setting

     # There might be '|' in string and in ( ... | ... ), replace it with '-'

     NewStr = ''

-    InStr = False

+    InSingleQuoteStr = False

+    InDoubleQuoteStr = False

     Pair = 0

-    for ch in Data:

-        if ch == '"':

-            InStr = not InStr

-        elif ch == '(' and not InStr:

+    for Index, ch in enumerate(Data):

+        if ch == '"' and not InSingleQuoteStr:

+            if Data[Index - 1] != '\\':

+                InDoubleQuoteStr = not InDoubleQuoteStr

+        elif ch == "'" and not InDoubleQuoteStr:

+            if Data[Index - 1] != '\\':

+                InSingleQuoteStr = not InSingleQuoteStr

+        elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):

             Pair += 1

-        elif ch == ')' and not InStr:

+        elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):

             Pair -= 1

 

-        if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:

+        if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:

             NewStr += '-'

         else:

             NewStr += ch

@@ -1549,7 +1554,7 @@ def ParseFieldValue (Value):
         return Value, 16

     if Value.startswith('L"') and Value.endswith('"'):

         # Unicode String

-        List = list(Value[2:-1])

+        List = list(eval(Value[1:]))  # translate escape character

         List.reverse()

         Value = 0

         for Char in List:

@@ -1557,7 +1562,7 @@ def ParseFieldValue (Value):
         return Value, (len(List) + 1) * 2

     if Value.startswith('"') and Value.endswith('"'):

         # ASCII String

-        List = list(Value[1:-1])

+        List = list(eval(Value))  # translate escape character

         List.reverse()

         Value = 0

         for Char in List:

@@ -1565,7 +1570,7 @@ def ParseFieldValue (Value):
         return Value, len(List) + 1

     if Value.startswith("L'") and Value.endswith("'"):

         # Unicode Character Constant

-        List = list(Value[2:-1])

+        List = list(eval(Value[1:])) # translate escape character

         if len(List) == 0:

             raise BadExpression('Length %s is %s' % (Value, len(List)))

         List.reverse()

@@ -1575,7 +1580,7 @@ def ParseFieldValue (Value):
         return Value, len(List) * 2

     if Value.startswith("'") and Value.endswith("'"):

         # Character constant

-        List = list(Value[1:-1])

+        List = list(eval(Value)) # translate escape character

         if len(List) == 0:

             raise BadExpression('Length %s is %s' % (Value, len(List)))

         List.reverse()

diff --git a/BaseTools/Source/Python/Common/String.py b/BaseTools/Source/Python/Common/String.py
index 4a8c03e88e..5e50beff5c 100644
--- a/BaseTools/Source/Python/Common/String.py
+++ b/BaseTools/Source/Python/Common/String.py
@@ -45,26 +45,32 @@ def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1):
     ValueList = []

     Last = 0

     Escaped = False

-    InString = False

+    InSingleQuoteString = False

+    InDoubleQuoteString = False

     InParenthesis = 0

     for Index in range(0, len(String)):

         Char = String[Index]

 

         if not Escaped:

             # Found a splitter not in a string, split it

-            if not InString and InParenthesis == 0 and Char == SplitTag:

+            if (not InSingleQuoteString or not InDoubleQuoteString) and InParenthesis == 0 and Char == SplitTag:

                 ValueList.append(String[Last:Index].strip())

                 Last = Index + 1

                 if MaxSplit > 0 and len(ValueList) >= MaxSplit:

                     break

 

-            if Char == '\\' and InString:

+            if Char == '\\' and (InSingleQuoteString or InDoubleQuoteString):

                 Escaped = True

-            elif Char == '"':

-                if not InString:

-                    InString = True

+            elif Char == '"' and not InSingleQuoteString:

+                if not InDoubleQuoteString:

+                    InDoubleQuoteString = True

                 else:

-                    InString = False

+                    InDoubleQuoteString = False

+            elif Char == "'" and not InDoubleQuoteString:

+                if not InSingleQuoteString:

+                    InSingleQuoteString = True

+                else:

+                    InSingleQuoteString = False

             elif Char == '(':

                 InParenthesis = InParenthesis + 1

             elif Char == ')':

@@ -345,14 +351,17 @@ def CleanString(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyle
     #

     # remove comments, but we should escape comment character in string

     #

-    InString = False

+    InDoubleQuoteString = False

+    InSingleQuoteString = False

     CommentInString = False

     for Index in range(0, len(Line)):

-        if Line[Index] == '"':

-            InString = not InString

-        elif Line[Index] == CommentCharacter and InString :

+        if Line[Index] == '"' and not InSingleQuoteString:

+            InDoubleQuoteString = not InDoubleQuoteString

+        elif Line[Index] == "'" and not InDoubleQuoteString:

+            InSingleQuoteString = not InSingleQuoteString

+        elif Line[Index] == CommentCharacter and (InSingleQuoteString or InDoubleQuoteString):

             CommentInString = True

-        elif Line[Index] == CommentCharacter and not InString :

+        elif Line[Index] == CommentCharacter and not (InSingleQuoteString or InDoubleQuoteString):

             Line = Line[0: Index]

             break

 

@@ -402,15 +411,18 @@ def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyl
     #

     # separate comments and statements, but we should escape comment character in string

     #

-    InString = False

+    InDoubleQuoteString = False

+    InSingleQuoteString = False

     CommentInString = False

     Comment = ''

     for Index in range(0, len(Line)):

-        if Line[Index] == '"':

-            InString = not InString

-        elif Line[Index] == CommentCharacter and InString:

+        if Line[Index] == '"' and not InSingleQuoteString:

+            InDoubleQuoteString = not InDoubleQuoteString

+        elif Line[Index] == "'" and not InDoubleQuoteString:

+            InSingleQuoteString = not InSingleQuoteString

+        elif Line[Index] == CommentCharacter and (InDoubleQuoteString or InSingleQuoteString):

             CommentInString = True

-        elif Line[Index] == CommentCharacter and not InString:

+        elif Line[Index] == CommentCharacter and not (InDoubleQuoteString or InSingleQuoteString):

             Comment = Line[Index:].strip()

             Line = Line[0:Index].strip()

             break

diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index f90da8e501..ea8d1847f7 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -991,6 +991,8 @@ class DscBuildData(PlatformBuildClassObject):
                     NewValue = self.GetFieldValueFromComm(pcdvalue, TokenSpaceGuidCName, TokenCName, FieldName)

                     GlobalData.BuildOptionPcd[i] = (TokenSpaceGuidCName, TokenCName, FieldName,NewValue,("build command options",1))

                 else:

+                    # Replace \' to ', \\\' to \'

+                    pcdvalue = pcdvalue.replace("\\\\\\'", '\\\\\\"').replace('\\\'', '\'').replace('\\\\\\"', "\\'")

                     for key in self.DecPcds:

                         PcdItem = self.DecPcds[key]

                         if HasTokenSpace:

@@ -1002,7 +1004,7 @@ class DscBuildData(PlatformBuildClassObject):
                                     except BadExpression, Value:

                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %

                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

-                                    if PcdDatumType == "VOID*":

+                                    if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:

                                         pcdvalue = 'H' + pcdvalue

                                 elif pcdvalue.startswith("L'"):

                                     try:

@@ -1010,7 +1012,7 @@ class DscBuildData(PlatformBuildClassObject):
                                     except BadExpression, Value:

                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %

                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

-                                    if pcdvalue.startswith('{'):

+                                    if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:

                                         pcdvalue = 'H' + pcdvalue

                                 elif pcdvalue.startswith("'"):

                                     try:

@@ -1018,7 +1020,7 @@ class DscBuildData(PlatformBuildClassObject):
                                     except BadExpression, Value:

                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %

                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

-                                    if pcdvalue.startswith('{'):

+                                    if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:

                                         pcdvalue = 'H' + pcdvalue

                                 elif pcdvalue.startswith('L'):

                                     pcdvalue = 'L"' + pcdvalue[1:] + '"'

@@ -1031,8 +1033,12 @@ class DscBuildData(PlatformBuildClassObject):
                                     try:

                                         pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)

                                     except BadExpression, Value:

-                                        EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %

-                                                        (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

+                                        try:

+                                            pcdvalue = '"' + pcdvalue + '"'

+                                            pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)

+                                        except BadExpression, Value:

+                                            EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %

+                                                            (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

                                 NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue)

                                 FoundFlag = True

                         else:

@@ -1048,7 +1054,7 @@ class DscBuildData(PlatformBuildClassObject):
                                             except BadExpression, Value:

                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' %

                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

-                                            if PcdDatumType == "VOID*":

+                                            if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64,'BOOLEAN']:

                                                 pcdvalue = 'H' + pcdvalue

                                         elif pcdvalue.startswith("L'"):

                                             try:

@@ -1057,7 +1063,7 @@ class DscBuildData(PlatformBuildClassObject):
                                             except BadExpression, Value:

                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %

                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

-                                            if pcdvalue.startswith('{'):

+                                            if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:

                                                 pcdvalue = 'H' + pcdvalue

                                         elif pcdvalue.startswith("'"):

                                             try:

@@ -1066,7 +1072,7 @@ class DscBuildData(PlatformBuildClassObject):
                                             except BadExpression, Value:

                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %

                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

-                                            if pcdvalue.startswith('{'):

+                                            if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:

                                                 pcdvalue = 'H' + pcdvalue

                                         elif pcdvalue.startswith('L'):

                                             pcdvalue = 'L"' + pcdvalue[1:] + '"'

@@ -1080,9 +1086,12 @@ class DscBuildData(PlatformBuildClassObject):
                                             try:

                                                 pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)

                                             except BadExpression, Value:

-                                                EdkLogger.error('Parser', FORMAT_INVALID,

-                                                                'PCD [%s.%s] Value "%s",  %s' %

-                                                                (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

+                                                try:

+                                                    pcdvalue = '"' + pcdvalue + '"'

+                                                    pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)

+                                                except BadExpression, Value:

+                                                    EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %

+                                                                    (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))

                                         NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue)

                                         FoundFlag = True

                                     else: