BaseTools: Handle the bytes and str difference

Deal with bytes and str is different, remove the unicode()
Using utcfromtimestamp instead of fromtimestamp.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Yunhua Feng <yunhuax.feng@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>

7 files changed, 43 insertions, 55 deletions

diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index 2815091de8..1e5f86937d 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -661,7 +661,7 @@ class WorkspaceAutoGen(AutoGen):
             for files in AllWorkSpaceMetaFiles:

                 if files.endswith('.dec'):

                     continue

-                f = open(files, 'r')

+                f = open(files, 'rb')

                 Content = f.read()

                 f.close()

                 m.update(Content)

@@ -690,7 +690,7 @@ class WorkspaceAutoGen(AutoGen):
         HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash')

         m = hashlib.md5()

         # Get .dec file's hash value

-        f = open(Pkg.MetaFile.Path, 'r')

+        f = open(Pkg.MetaFile.Path, 'rb')

         Content = f.read()

         f.close()

         m.update(Content)

@@ -700,7 +700,7 @@ class WorkspaceAutoGen(AutoGen):
                 for Root, Dirs, Files in os.walk(str(inc)):

                     for File in sorted(Files):

                         File_Path = os.path.join(Root, File)

-                        f = open(File_Path, 'r')

+                        f = open(File_Path, 'rb')

                         Content = f.read()

                         f.close()

                         m.update(Content)

@@ -1602,7 +1602,7 @@ class PlatformAutoGen(AutoGen):
         for pcd in self._DynamicPcdList:

             if len(pcd.SkuInfoList) == 1:

                 for (SkuName, SkuId) in allskuset:

-                    if type(SkuId) in (str, unicode) and eval(SkuId) == 0 or SkuId == 0:

+                    if isinstance(SkuId, str) and eval(SkuId) == 0 or SkuId == 0:

                         continue

                     pcd.SkuInfoList[SkuName] = copy.deepcopy(pcd.SkuInfoList[TAB_DEFAULT])

                     pcd.SkuInfoList[SkuName].SkuId = SkuId

@@ -3207,7 +3207,7 @@ class ModuleAutoGen(AutoGen):
             AutoFile = PathClass(gAutoGenStringFileName % {"module_name":self.Name}, self.DebugDir)

             RetVal[AutoFile] = str(StringH)

             self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE)

-        if UniStringBinBuffer is not None and UniStringBinBuffer.getvalue() != "":

+        if UniStringBinBuffer is not None and UniStringBinBuffer.getvalue() != b"":

             AutoFile = PathClass(gAutoGenStringFormFileName % {"module_name":self.Name}, self.OutputDir)

             RetVal[AutoFile] = UniStringBinBuffer.getvalue()

             AutoFile.IsBinary = True

@@ -3218,7 +3218,7 @@ class ModuleAutoGen(AutoGen):
             AutoFile = PathClass(gAutoGenImageDefFileName % {"module_name":self.Name}, self.DebugDir)

             RetVal[AutoFile] = str(StringIdf)

             self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE)

-        if IdfGenBinBuffer is not None and IdfGenBinBuffer.getvalue() != "":

+        if IdfGenBinBuffer is not None and IdfGenBinBuffer.getvalue() != b"":

             AutoFile = PathClass(gAutoGenIdfFileName % {"module_name":self.Name}, self.OutputDir)

             RetVal[AutoFile] = IdfGenBinBuffer.getvalue()

             AutoFile.IsBinary = True

@@ -3449,7 +3449,7 @@ class ModuleAutoGen(AutoGen):
             EdkLogger.error("build", FILE_OPEN_FAILURE, "File open failed for %s" % UniVfrOffsetFileName, None)

 

         # Use a instance of BytesIO to cache data

-        fStringIO = BytesIO('')

+        fStringIO = BytesIO()

 

         for Item in VfrUniOffsetList:

             if (Item[0].find("Strings") != -1):

@@ -3459,8 +3459,7 @@ class ModuleAutoGen(AutoGen):
                 # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } }

                 #

                 UniGuid = [0xe0, 0xc5, 0x13, 0x89, 0xf6, 0x33, 0x86, 0x4d, 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66]

-                UniGuid = [chr(ItemGuid) for ItemGuid in UniGuid]

-                fStringIO.write(''.join(UniGuid))

+                fStringIO.write(bytes(UniGuid))

                 UniValue = pack ('Q', int (Item[1], 16))

                 fStringIO.write (UniValue)

             else:

@@ -3470,8 +3469,7 @@ class ModuleAutoGen(AutoGen):
                 # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } };

                 #

                 VfrGuid = [0xb4, 0x7c, 0xbc, 0xd0, 0x47, 0x6a, 0x5f, 0x49, 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2]

-                VfrGuid = [chr(ItemGuid) for ItemGuid in VfrGuid]

-                fStringIO.write(''.join(VfrGuid))

+                fStringIO.write(bytes(VfrGuid))

                 VfrValue = pack ('Q', int (Item[1], 16))

                 fStringIO.write (VfrValue)

         #

@@ -4019,29 +4017,29 @@ class ModuleAutoGen(AutoGen):
             GlobalData.gModuleHash[self.Arch] = {}

         m = hashlib.md5()

         # Add Platform level hash

-        m.update(GlobalData.gPlatformHash)

+        m.update(GlobalData.gPlatformHash.encode('utf-8'))

         # Add Package level hash

         if self.DependentPackageList:

             for Pkg in sorted(self.DependentPackageList, key=lambda x: x.PackageName):

                 if Pkg.PackageName in GlobalData.gPackageHash[self.Arch]:

-                    m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName])

+                    m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName].encode('utf-8'))

 

         # Add Library hash

         if self.LibraryAutoGenList:

             for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.Name):

                 if Lib.Name not in GlobalData.gModuleHash[self.Arch]:

                     Lib.GenModuleHash()

-                m.update(GlobalData.gModuleHash[self.Arch][Lib.Name])

+                m.update(GlobalData.gModuleHash[self.Arch][Lib.Name].encode('utf-8'))

 

         # Add Module self

-        f = open(str(self.MetaFile), 'r')

+        f = open(str(self.MetaFile), 'rb')

         Content = f.read()

         f.close()

         m.update(Content)

         # Add Module's source files

         if self.SourceFileList:

             for File in sorted(self.SourceFileList, key=lambda x: str(x)):

-                f = open(str(File), 'r')

+                f = open(str(File), 'rb')

                 Content = f.read()

                 f.close()

                 m.update(Content)

diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py
index c189269d4d..28539d8239 100644
--- a/BaseTools/Source/Python/AutoGen/GenC.py
+++ b/BaseTools/Source/Python/AutoGen/GenC.py
@@ -1795,7 +1795,7 @@ def CreateIdfFileCode(Info, AutoGenC, StringH, IdfGenCFlag, IdfGenBinBuffer):
                                 TempBuffer += Buffer

                             elif File.Ext.upper() == '.JPG':

                                 ImageType, = struct.unpack('4s', Buffer[6:10])

-                                if ImageType != 'JFIF':

+                                if ImageType != b'JFIF':

                                     EdkLogger.error("build", FILE_TYPE_MISMATCH, "The file %s is not a standard JPG file." % File.Path)

                                 TempBuffer = pack('B', EFI_HII_IIBT_IMAGE_JPEG)

                                 TempBuffer += pack('I', len(Buffer))

@@ -1895,7 +1895,7 @@ def CreateIdfFileCode(Info, AutoGenC, StringH, IdfGenCFlag, IdfGenBinBuffer):
 

 def BmpImageDecoder(File, Buffer, PaletteIndex, TransParent):

     ImageType, = struct.unpack('2s', Buffer[0:2])

-    if ImageType!= 'BM': # BMP file type is 'BM'

+    if ImageType!= b'BM': # BMP file type is 'BM'

         EdkLogger.error("build", FILE_TYPE_MISMATCH, "The file %s is not a standard BMP file." % File.Path)

     BMP_IMAGE_HEADER = collections.namedtuple('BMP_IMAGE_HEADER', ['bfSize', 'bfReserved1', 'bfReserved2', 'bfOffBits', 'biSize', 'biWidth', 'biHeight', 'biPlanes', 'biBitCount', 'biCompression', 'biSizeImage', 'biXPelsPerMeter', 'biYPelsPerMeter', 'biClrUsed', 'biClrImportant'])

     BMP_IMAGE_HEADER_STRUCT = struct.Struct('IHHIIIIHHIIIIII')

@@ -1967,7 +1967,7 @@ def BmpImageDecoder(File, Buffer, PaletteIndex, TransParent):
         for Index in range(0, len(PaletteBuffer)):

             if Index % 4 == 3:

                 continue

-            PaletteTemp += PaletteBuffer[Index]

+            PaletteTemp += bytes([PaletteBuffer[Index]])

         PaletteBuffer = PaletteTemp[1:]

     return ImageBuffer, PaletteBuffer

 

diff --git a/BaseTools/Source/Python/AutoGen/GenPcdDb.py b/BaseTools/Source/Python/AutoGen/GenPcdDb.py
index f99d9f5d18..26017c6544 100644
--- a/BaseTools/Source/Python/AutoGen/GenPcdDb.py
+++ b/BaseTools/Source/Python/AutoGen/GenPcdDb.py
@@ -291,7 +291,7 @@ class DbItemList:
 

         PackStr = PACK_CODE_BY_SIZE[self.ItemSize]

 

-        Buffer = ''

+        Buffer = bytearray()

         for Datas in self.RawDataList:

             if type(Datas) in (list, tuple):

                 for Data in Datas:

@@ -316,7 +316,7 @@ class DbExMapTblItemList (DbItemList):
         DbItemList.__init__(self, ItemSize, DataList, RawDataList)

 

     def PackData(self):

-        Buffer = ''

+        Buffer = bytearray()

         PackStr = "=LHH"

         for Datas in self.RawDataList:

             Buffer += pack(PackStr,

@@ -365,7 +365,7 @@ class DbComItemList (DbItemList):
     def PackData(self):

         PackStr = PACK_CODE_BY_SIZE[self.ItemSize]

 

-        Buffer = ''

+        Buffer = bytearray()

         for DataList in self.RawDataList:

             for Data in DataList:

                 if type(Data) in (list, tuple):

@@ -386,7 +386,7 @@ class DbVariableTableItemList (DbComItemList):
 

     def PackData(self):

         PackStr = "=LLHHLHH"

-        Buffer = ''

+        Buffer = bytearray()

         for DataList in self.RawDataList:

             for Data in DataList:

                 Buffer += pack(PackStr,

@@ -447,7 +447,7 @@ class DbSkuHeadTableItemList (DbItemList):
 

     def PackData(self):

         PackStr = "=LL"

-        Buffer = ''

+        Buffer = bytearray()

         for Data in self.RawDataList:

             Buffer += pack(PackStr,

                            GetIntegerValue(Data[0]),

@@ -469,7 +469,7 @@ class DbSizeTableItemList (DbItemList):
         return length * self.ItemSize

     def PackData(self):

         PackStr = "=H"

-        Buffer = ''

+        Buffer = bytearray()

         for Data in self.RawDataList:

             Buffer += pack(PackStr,

                            GetIntegerValue(Data[0]))

@@ -849,7 +849,7 @@ def BuildExDataBase(Dict):
     Index = 0

     for Item in DbItemTotal:

         Index +=1

-        b = Item.PackData()

+        b = bytes(Item.PackData())

         Buffer += b

         if Index == InitTableNum:

             if len(Buffer) % 8:

@@ -917,9 +917,9 @@ def CreatePcdDataBase(PcdDBData):
     totallenbuff = pack("=L", totallen)

     newbuffer = databasebuff[:32]

     for i in range(4):

-        newbuffer += totallenbuff[i]

+        newbuffer += bytes([totallenbuff[i]])

     for i in range(36, totallen):

-        newbuffer += databasebuff[i]

+        newbuffer += bytes([databasebuff[i]])

 

     return newbuffer

 

@@ -962,7 +962,7 @@ def NewCreatePcdDatabasePhaseSpecificAutoGen(Platform, Phase):
             AdditionalAutoGenH, AdditionalAutoGenC, PcdDbBuffer, VarCheckTab = CreatePcdDatabasePhaseSpecificAutoGen (Platform, DynamicPcdSet_Sku[(skuname, skuid)], Phase)

             final_data = ()

             for item in PcdDbBuffer:

-                final_data += unpack("B", item)

+                final_data += unpack("B", bytes([item]))

             PcdDBData[(skuname, skuid)] = (PcdDbBuffer, final_data)

             PcdDriverAutoGenData[(skuname, skuid)] = (AdditionalAutoGenH, AdditionalAutoGenC)

             VarCheckTableData[(skuname, skuid)] = VarCheckTab

@@ -975,7 +975,7 @@ def NewCreatePcdDatabasePhaseSpecificAutoGen(Platform, Phase):
         AdditionalAutoGenH, AdditionalAutoGenC, PcdDbBuffer, VarCheckTab = CreatePcdDatabasePhaseSpecificAutoGen (Platform, {}, Phase)

         final_data = ()

         for item in PcdDbBuffer:

-            final_data += unpack("B", item)

+            final_data += unpack("B", bytes([item]))

         PcdDBData[(TAB_DEFAULT, "0")] = (PcdDbBuffer, final_data)

 

     return AdditionalAutoGenH, AdditionalAutoGenC, CreatePcdDataBase(PcdDBData)

diff --git a/BaseTools/Source/Python/AutoGen/GenVar.py b/BaseTools/Source/Python/AutoGen/GenVar.py
index f7806b1dd3..d06fbe7e17 100644
--- a/BaseTools/Source/Python/AutoGen/GenVar.py
+++ b/BaseTools/Source/Python/AutoGen/GenVar.py
@@ -66,7 +66,7 @@ class VariableMgr(object):
                 data = value_list[0]

                 value_list = []

                 for data_byte in pack(data_flag, int(data, 16) if data.upper().startswith('0X') else int(data)):

-                    value_list.append(hex(unpack("B", data_byte)[0]))

+                    value_list.append(hex(unpack("B", bytes([data_byte]))[0]))

             newvalue[int(item.var_offset, 16) if item.var_offset.upper().startswith("0X") else int(item.var_offset)] = value_list

         try:

             newvaluestr = "{" + ",".join(VariableMgr.assemble_variable(newvalue)) +"}"

@@ -87,7 +87,7 @@ class VariableMgr(object):
                 data = value_list[0]

                 value_list = []

                 for data_byte in pack(data_flag, int(data, 16) if data.upper().startswith('0X') else int(data)):

-                    value_list.append(hex(unpack("B", data_byte)[0]))

+                    value_list.append(hex(unpack("B", bytes([data_byte]))[0]))

             newvalue[int(item.var_offset, 16) if item.var_offset.upper().startswith("0X") else int(item.var_offset)] = (value_list,item.pcdname,item.PcdDscLine)

         for offset in newvalue:

             value_list,itemPcdname,itemPcdDscLine = newvalue[offset]

@@ -161,7 +161,7 @@ class VariableMgr(object):
 

             default_data_array = ()

             for item in default_data_buffer:

-                default_data_array += unpack("B", item)

+                default_data_array += unpack("B", bytes([item]))

 

             var_data[(DataType.TAB_DEFAULT, DataType.TAB_DEFAULT_STORES_DEFAULT)][index] = (default_data_buffer, sku_var_info[(DataType.TAB_DEFAULT, DataType.TAB_DEFAULT_STORES_DEFAULT)])

 

@@ -179,7 +179,7 @@ class VariableMgr(object):
 

                 others_data_array = ()

                 for item in others_data_buffer:

-                    others_data_array += unpack("B", item)

+                    others_data_array += unpack("B", bytes([item]))

 

                 data_delta = VariableMgr.calculate_delta(default_data_array, others_data_array)

 

@@ -195,7 +195,7 @@ class VariableMgr(object):
             return []

 

         pcds_default_data = var_data.get((DataType.TAB_DEFAULT, DataType.TAB_DEFAULT_STORES_DEFAULT), {})

-        NvStoreDataBuffer = ""

+        NvStoreDataBuffer = bytearray()

         var_data_offset = collections.OrderedDict()

         offset = NvStorageHeaderSize

         for default_data, default_info in pcds_default_data.values():

@@ -222,7 +222,7 @@ class VariableMgr(object):
 

         nv_default_part = VariableMgr.AlignData(VariableMgr.PACK_DEFAULT_DATA(0, 0, VariableMgr.unpack_data(variable_storage_header_buffer+NvStoreDataBuffer)), 8)

 

-        data_delta_structure_buffer = ""

+        data_delta_structure_buffer = bytearray()

         for skuname, defaultstore in var_data:

             if (skuname, defaultstore) == (DataType.TAB_DEFAULT, DataType.TAB_DEFAULT_STORES_DEFAULT):

                 continue

@@ -254,7 +254,7 @@ class VariableMgr(object):
     def unpack_data(data):

         final_data = ()

         for item in data:

-            final_data += unpack("B", item)

+            final_data += unpack("B", bytes([item]))

         return final_data

 

     @staticmethod

@@ -322,7 +322,7 @@ class VariableMgr(object):
 

     @staticmethod

     def PACK_VARIABLES_DATA(var_value,data_type, tail = None):

-        Buffer = ""

+        Buffer = bytearray()

         data_len = 0

         if data_type == DataType.TAB_VOID:

             for value_char in var_value.strip("{").strip("}").split(","):

@@ -352,7 +352,7 @@ class VariableMgr(object):
 

     @staticmethod

     def PACK_DEFAULT_DATA(defaultstoragename, skuid, var_value):

-        Buffer = ""

+        Buffer = bytearray()

         Buffer += pack("=L", 4+8+8)

         Buffer += pack("=Q", int(skuid))

         Buffer += pack("=Q", int(defaultstoragename))

@@ -377,7 +377,7 @@ class VariableMgr(object):
     def PACK_DELTA_DATA(self, skuname, defaultstoragename, delta_list):

         skuid = self.GetSkuId(skuname)

         defaultstorageid = self.GetDefaultStoreId(defaultstoragename)

-        Buffer = ""

+        Buffer = bytearray()

         Buffer += pack("=L", 4+8+8)

         Buffer += pack("=Q", int(skuid))

         Buffer += pack("=Q", int(defaultstorageid))

@@ -400,7 +400,7 @@ class VariableMgr(object):
 

     @staticmethod

     def PACK_VARIABLE_NAME(var_name):

-        Buffer = ""

+        Buffer = bytearray()

         for name_char in var_name.strip("{").strip("}").split(","):

             Buffer += pack("=B", int(name_char, 16))

 

diff --git a/BaseTools/Source/Python/AutoGen/StrGather.py b/BaseTools/Source/Python/AutoGen/StrGather.py
index 3f384db1df..05e670ac06 100644
--- a/BaseTools/Source/Python/AutoGen/StrGather.py
+++ b/BaseTools/Source/Python/AutoGen/StrGather.py
@@ -122,6 +122,8 @@ def DecToHexList(Dec, Digit = 8):
 # @retval:       A list for formatted hex string

 #

 def AscToHexList(Ascii):

+    if isinstance(Ascii, bytes):

+        return ['0x{0:02X}'.format(Item) for Item in Ascii]

     return ['0x{0:02X}'.format(ord(Item)) for Item in Ascii]

 

 ## Create content of .h file

diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py
index b435ffa569..1c86be4429 100644
--- a/BaseTools/Source/Python/AutoGen/UniClassObject.py
+++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py
@@ -45,18 +45,6 @@ BACK_SLASH_PLACEHOLDER = u'\u0006'
 

 gIncludePattern = re.compile("^#include +[\"<]+([^\"< >]+)[>\"]+$", re.MULTILINE | re.UNICODE)

 

-## Convert a python unicode string to a normal string

-#

-# Convert a python unicode string to a normal string

-# UniToStr(u'I am a string') is 'I am a string'

-#

-# @param Uni:  The python unicode string

-#

-# @retval:     The formatted normal string

-#

-def UniToStr(Uni):

-    return repr(Uni)[2:-1]

-

 ## Convert a unicode string to a Hex list

 #

 # Convert a unicode string to a Hex list

@@ -438,7 +426,7 @@ class UniFileClassObject(object):
                 if EndPos != -1 and EndPos - StartPos == 6 :

                     if g4HexChar.match(Line[StartPos + 2 : EndPos], re.UNICODE):

                         EndStr = Line[EndPos: ]

-                        UniStr = ('\u' + (Line[StartPos + 2 : EndPos])).decode('unicode_escape')

+                        UniStr = Line[StartPos + 2: EndPos]

                         if EndStr.startswith(u'\\x') and len(EndStr) >= 7:

                             if EndStr[6] == u'\\' and g4HexChar.match(EndStr[2 : 6], re.UNICODE):

                                 Line = Line[0 : StartPos] + UniStr + EndStr

diff --git a/BaseTools/Source/Python/AutoGen/ValidCheckingInfoObject.py b/BaseTools/Source/Python/AutoGen/ValidCheckingInfoObject.py
index 379d404c76..77518fc1cc 100644
--- a/BaseTools/Source/Python/AutoGen/ValidCheckingInfoObject.py
+++ b/BaseTools/Source/Python/AutoGen/ValidCheckingInfoObject.py
@@ -41,7 +41,7 @@ class VAR_CHECK_PCD_VARIABLE_TAB_CONTAINER(object):
             os.mkdir(dest)

         BinFileName = "PcdVarCheck.bin"

         BinFilePath = os.path.join(dest, BinFileName)

-        Buffer = ''

+        Buffer = bytearray()

         index = 0

         for var_check_tab in self.var_check_info:

             index += 1