BaseTools: Fix flexible PCD single quote and double quote bugs

1.The " and ' inside the string, must use escape character format
  (\", \')
2.'string' and L'string' format in --pcd, it must be double quoted
  first.

  Some examples that to match --pcd format and DSC format
  --pcd                             DSC format
  L"ABC"                            L"ABC"
  "AB\\\"C"                         "AB\"C"
  "AB\\\'C"                         "AB\'C"
  L"\'AB\\\"C\'"                    L'AB\"C'
  "\'AB\\\'C\'"                     'AB\'C'
  H"{0, L\"AB\\\"B\", \'ab\\\"c\'}" {0, L"AB\"B", 'ab\"c'}

Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Yunhua Feng <yunhuax.feng@intel.com>
Reviewed-by: Yonghong Zhu <yonghong.zhu@intel.com>

3 files changed, 90 insertions, 47 deletions

diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py
index edb0a60de6..f1516d5c7b 100644
--- a/BaseTools/Source/Python/Common/Expression.py
+++ b/BaseTools/Source/Python/Common/Expression.py
@@ -45,15 +45,28 @@ ERR_IN_OPERAND          = 'Macro after IN operator can only be: $(FAMILY), $(ARC
 #  For example: abc"de\"f"ghi"jkl"mn will be: ['abc', '"de\"f"', 'ghi', '"jkl"', 'mn']

 #

 def SplitString(String):

-    # There might be escaped quote: "abc\"def\\\"ghi"

-    Str = String.replace('\\\\', '//').replace('\\\"', '\\\'')

+    # There might be escaped quote: "abc\"def\\\"ghi", 'abc\'def\\\'ghi'

+    Str = String

     RetList = []

-    InQuote = False

+    InSingleQuote = False

+    InDoubleQuote = False

     Item = ''

     for i, ch in enumerate(Str):

-        if ch == '"':

-            InQuote = not InQuote

-            if not InQuote:

+        if ch == '"' and not InSingleQuote:

+            if Str[i - 1] != '\\':

+                InDoubleQuote = not InDoubleQuote

+            if not InDoubleQuote:

+                Item += String[i]

+                RetList.append(Item)

+                Item = ''

+                continue

+            if Item:

+                RetList.append(Item)

+                Item = ''

+        elif ch == "'" and not InDoubleQuote:

+            if Str[i - 1] != '\\':

+                InSingleQuote = not InSingleQuote

+            if not InSingleQuote:

                 Item += String[i]

                 RetList.append(Item)

                 Item = ''

@@ -62,7 +75,7 @@ def SplitString(String):
                 RetList.append(Item)

                 Item = ''

         Item += String[i]

-    if InQuote:

+    if InSingleQuote or InDoubleQuote:

         raise BadExpression(ERR_STRING_TOKEN % Item)

     if Item:

         RetList.append(Item)

@@ -483,6 +496,8 @@ class ValueExpression(object):
             Flag = 0

             for Index in range(len(self._Token)):

                 if self._Token[Index] in ['"']:

+                    if self._Token[Index - 1] == '\\':

+                        continue

                     Flag += 1

             if Flag == 2 and self._Token.endswith('"'):

                 return True

@@ -490,6 +505,8 @@ class ValueExpression(object):
             Flag = 0

             for Index in range(len(self._Token)):

                 if self._Token[Index] in ["'"]:

+                    if self._Token[Index - 1] == '\\':

+                        continue

                     Flag += 1

             if Flag == 2 and self._Token.endswith("'"):

                 return True

@@ -537,16 +554,25 @@ class ValueExpression(object):
         self._Idx += 1

 

         # Replace escape \\\", \"

-        Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'')

-        for Ch in Expr:

-            self._Idx += 1

-            if Ch == '"' or Ch == "'":

-                break

-        self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]

-        if self._Token.startswith('"') and not self._Token.endswith('"'):

-            raise BadExpression(ERR_STRING_TOKEN % self._Token)

-        if self._Token.startswith("'") and not self._Token.endswith("'"):

-            raise BadExpression(ERR_STRING_TOKEN % self._Token)

+        if self._Expr[Idx] == '"':

+            Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'')

+            for Ch in Expr:

+                self._Idx += 1

+                if Ch == '"':

+                    break

+            self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]

+            if not self._Token.endswith('"'):

+                raise BadExpression(ERR_STRING_TOKEN % self._Token)

+        #Replace escape \\\', \'

+        elif self._Expr[Idx] == "'":

+            Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace("\\\'", "\\\"")

+            for Ch in Expr:

+                self._Idx += 1

+                if Ch == "'":

+                    break

+            self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]

+            if not self._Token.endswith("'"):

+                raise BadExpression(ERR_STRING_TOKEN % self._Token)

         self._Token = self._Token[1:-1]

         return self._Token

 

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 1461d00669..a7e7797d04 100644
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -1443,21 +1443,26 @@ def ParseConsoleLog(Filename):
 

 def AnalyzePcdExpression(Setting):

     Setting = Setting.strip()

-    # There might be escaped quote in a string: \", \\\"

-    Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')

+    # There might be escaped quote in a string: \", \\\" , \', \\\'

+    Data = Setting

     # There might be '|' in string and in ( ... | ... ), replace it with '-'

     NewStr = ''

-    InStr = False

+    InSingleQuoteStr = False

+    InDoubleQuoteStr = False

     Pair = 0

-    for ch in Data:

-        if ch == '"':

-            InStr = not InStr

-        elif ch == '(' and not InStr:

+    for Index, ch in enumerate(Data):

+        if ch == '"' and not InSingleQuoteStr:

+            if Data[Index - 1] != '\\':

+                InDoubleQuoteStr = not InDoubleQuoteStr

+        elif ch == "'" and not InDoubleQuoteStr:

+            if Data[Index - 1] != '\\':

+                InSingleQuoteStr = not InSingleQuoteStr

+        elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):

             Pair += 1

-        elif ch == ')' and not InStr:

+        elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):

             Pair -= 1

 

-        if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:

+        if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:

             NewStr += '-'

         else:

             NewStr += ch

@@ -1549,7 +1554,7 @@ def ParseFieldValue (Value):
         return Value, 16

     if Value.startswith('L"') and Value.endswith('"'):

         # Unicode String

-        List = list(Value[2:-1])

+        List = list(eval(Value[1:]))  # translate escape character

         List.reverse()

         Value = 0

         for Char in List:

@@ -1557,7 +1562,7 @@ def ParseFieldValue (Value):
         return Value, (len(List) + 1) * 2

     if Value.startswith('"') and Value.endswith('"'):

         # ASCII String

-        List = list(Value[1:-1])

+        List = list(eval(Value))  # translate escape character

         List.reverse()

         Value = 0

         for Char in List:

@@ -1565,7 +1570,7 @@ def ParseFieldValue (Value):
         return Value, len(List) + 1

     if Value.startswith("L'") and Value.endswith("'"):

         # Unicode Character Constant

-        List = list(Value[2:-1])

+        List = list(eval(Value[1:])) # translate escape character

         if len(List) == 0:

             raise BadExpression('Length %s is %s' % (Value, len(List)))

         List.reverse()

@@ -1575,7 +1580,7 @@ def ParseFieldValue (Value):
         return Value, len(List) * 2

     if Value.startswith("'") and Value.endswith("'"):

         # Character constant

-        List = list(Value[1:-1])

+        List = list(eval(Value)) # translate escape character

         if len(List) == 0:

             raise BadExpression('Length %s is %s' % (Value, len(List)))

         List.reverse()

diff --git a/BaseTools/Source/Python/Common/String.py b/BaseTools/Source/Python/Common/String.py
index 4a8c03e88e..5e50beff5c 100644
--- a/BaseTools/Source/Python/Common/String.py
+++ b/BaseTools/Source/Python/Common/String.py
@@ -45,26 +45,32 @@ def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1):
     ValueList = []

     Last = 0

     Escaped = False

-    InString = False

+    InSingleQuoteString = False

+    InDoubleQuoteString = False

     InParenthesis = 0

     for Index in range(0, len(String)):

         Char = String[Index]

 

         if not Escaped:

             # Found a splitter not in a string, split it

-            if not InString and InParenthesis == 0 and Char == SplitTag:

+            if (not InSingleQuoteString or not InDoubleQuoteString) and InParenthesis == 0 and Char == SplitTag:

                 ValueList.append(String[Last:Index].strip())

                 Last = Index + 1

                 if MaxSplit > 0 and len(ValueList) >= MaxSplit:

                     break

 

-            if Char == '\\' and InString:

+            if Char == '\\' and (InSingleQuoteString or InDoubleQuoteString):

                 Escaped = True

-            elif Char == '"':

-                if not InString:

-                    InString = True

+            elif Char == '"' and not InSingleQuoteString:

+                if not InDoubleQuoteString:

+                    InDoubleQuoteString = True

                 else:

-                    InString = False

+                    InDoubleQuoteString = False

+            elif Char == "'" and not InDoubleQuoteString:

+                if not InSingleQuoteString:

+                    InSingleQuoteString = True

+                else:

+                    InSingleQuoteString = False

             elif Char == '(':

                 InParenthesis = InParenthesis + 1

             elif Char == ')':

@@ -345,14 +351,17 @@ def CleanString(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyle
     #

     # remove comments, but we should escape comment character in string

     #

-    InString = False

+    InDoubleQuoteString = False

+    InSingleQuoteString = False

     CommentInString = False

     for Index in range(0, len(Line)):

-        if Line[Index] == '"':

-            InString = not InString

-        elif Line[Index] == CommentCharacter and InString :

+        if Line[Index] == '"' and not InSingleQuoteString:

+            InDoubleQuoteString = not InDoubleQuoteString

+        elif Line[Index] == "'" and not InDoubleQuoteString:

+            InSingleQuoteString = not InSingleQuoteString

+        elif Line[Index] == CommentCharacter and (InSingleQuoteString or InDoubleQuoteString):

             CommentInString = True

-        elif Line[Index] == CommentCharacter and not InString :

+        elif Line[Index] == CommentCharacter and not (InSingleQuoteString or InDoubleQuoteString):

             Line = Line[0: Index]

             break

 

@@ -402,15 +411,18 @@ def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyl
     #

     # separate comments and statements, but we should escape comment character in string

     #

-    InString = False

+    InDoubleQuoteString = False

+    InSingleQuoteString = False

     CommentInString = False

     Comment = ''

     for Index in range(0, len(Line)):

-        if Line[Index] == '"':

-            InString = not InString

-        elif Line[Index] == CommentCharacter and InString:

+        if Line[Index] == '"' and not InSingleQuoteString:

+            InDoubleQuoteString = not InDoubleQuoteString

+        elif Line[Index] == "'" and not InDoubleQuoteString:

+            InSingleQuoteString = not InSingleQuoteString

+        elif Line[Index] == CommentCharacter and (InDoubleQuoteString or InSingleQuoteString):

             CommentInString = True

-        elif Line[Index] == CommentCharacter and not InString:

+        elif Line[Index] == CommentCharacter and not (InDoubleQuoteString or InSingleQuoteString):

             Comment = Line[Index:].strip()

             Line = Line[0:Index].strip()

             break