Revert BaseTools: PYTHON3 migration

This reverts commit 6693f359b3c213513c5096a06c6f67244a44dc52..
678f85131238622e576705117e299d81cff755c9.

Python3 migration is the fundamental change. It requires every developer
to install Python3. Before this migration, the well communication and wide
verification must be done. But now, most people is not aware of this change,
and not try it. So, Python3 migration is reverted and be moved to edk2-staging
Python3 branch for the edk2 user evaluation.

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Liming Gao <liming.gao@intel.com>

2 files changed, 67 insertions, 89 deletions

diff --git a/BaseTools/Source/Python/build/BuildReport.py b/BaseTools/Source/Python/build/BuildReport.py
index 30c5b10bcc..4fe29f124d 100644
--- a/BaseTools/Source/Python/build/BuildReport.py
+++ b/BaseTools/Source/Python/build/BuildReport.py
@@ -28,7 +28,7 @@ import hashlib
 import subprocess

 import threading

 from datetime import datetime

-from io import StringIO

+from io import BytesIO

 from Common import EdkLogger

 from Common.Misc import SaveFileOnChange

 from Common.Misc import GuidStructureByteArrayToGuidString

@@ -79,7 +79,7 @@ gGlueLibEntryPoint = re.compile(r"__EDKII_GLUE_MODULE_ENTRY_POINT__\s*=\s*(\w+)"
 gLineMaxLength = 120

 

 ## Tags for end of line in report

-gEndOfLine = "\n"

+gEndOfLine = "\r\n"

 

 ## Tags for section start, end and separator

 gSectionStart = ">" + "=" * (gLineMaxLength - 2) + "<"

@@ -600,7 +600,7 @@ class ModuleReport(object):
             # Collect all module used PCD set: module INF referenced directly or indirectly.

             # It also saves module INF default values of them in case they exist.

             #

-            for Pcd in list(M.ModulePcdList) + list(M.LibraryPcdList):

+            for Pcd in M.ModulePcdList + M.LibraryPcdList:

                 self.ModulePcdSet.setdefault((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, Pcd.Type), (Pcd.InfDefaultValue, Pcd.DefaultValue))

 

         self.LibraryReport = None

@@ -634,14 +634,14 @@ class ModuleReport(object):
         FwReportFileName = os.path.join(self._BuildDir, "DEBUG", self.ModuleName + ".txt")

         if os.path.isfile(FwReportFileName):

             try:

-                FileContents = open(FwReportFileName, 'r').read()

+                FileContents = open(FwReportFileName).read()

                 Match = gModuleSizePattern.search(FileContents)

                 if Match:

                     self.Size = int(Match.group(1))

 

                 Match = gTimeStampPattern.search(FileContents)

                 if Match:

-                    self.BuildTimeStamp = datetime.utcfromtimestamp(int(Match.group(1)))

+                    self.BuildTimeStamp = datetime.fromtimestamp(int(Match.group(1)))

             except IOError:

                 EdkLogger.warn(None, "Fail to read report file", FwReportFileName)

 

@@ -726,8 +726,8 @@ def ReadMessage(From, To, ExitFlag):
         # read one line a time

         Line = From.readline()

         # empty string means "end"

-        if Line is not None and Line != b"":

-            To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))

+        if Line is not None and Line != "":

+            To(Line.rstrip())

         else:

             break

         if ExitFlag.isSet():

@@ -853,7 +853,7 @@ class PcdReport(object):
                 #

                 # Collect module override PCDs

                 #

-                for ModulePcd in list(Module.M.ModulePcdList) + list(Module.M.LibraryPcdList):

+                for ModulePcd in Module.M.ModulePcdList + Module.M.LibraryPcdList:

                     TokenCName = ModulePcd.TokenCName

                     TokenSpaceGuid = ModulePcd.TokenSpaceGuidCName

                     ModuleDefault = ModulePcd.DefaultValue

@@ -1031,10 +1031,7 @@ class PcdReport(object):
 

 

                 if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES:

-                    try:

-                        PcdValueNumber = int(PcdValue.strip(), 0)

-                    except:

-                        PcdValueNumber = int(PcdValue.lstrip('0'))

+                    PcdValueNumber = int(PcdValue.strip(), 0)

                     if DecDefaultValue is None:

                         DecMatch = True

                     else:

@@ -1050,10 +1047,7 @@ class PcdReport(object):
                     if DscDefaultValue is None:

                         DscMatch = True

                     else:

-                        try:

-                            DscDefaultValueNumber = int(DscDefaultValue.strip(), 0)

-                        except:

-                            DscDefaultValueNumber = int(DscDefaultValue.lstrip('0'))

+                        DscDefaultValueNumber = int(DscDefaultValue.strip(), 0)

                         DscMatch = (DscDefaultValueNumber == PcdValueNumber)

                 else:

                     if DecDefaultValue is None:

@@ -1085,48 +1079,37 @@ class PcdReport(object):
                         DecMatch = False

                     elif Pcd.SkuOverrideValues:

                         DscOverride = False

-                        if Pcd.DefaultFromDSC:

-                            DscOverride = True

+                        if not Pcd.SkuInfoList:

+                            OverrideValues = Pcd.SkuOverrideValues

+                            if OverrideValues:

+                                Keys = OverrideValues.keys()

+                                Data = OverrideValues[Keys[0]]

+                                Struct = Data.values()[0]

+                                DscOverride = self.ParseStruct(Struct)

                         else:

-                            DictLen = 0

-                            for item in Pcd.SkuOverrideValues:

-                                DictLen += len(Pcd.SkuOverrideValues[item])

-                            if not DictLen:

-                                DscOverride = False

-                            else:

-                                if not Pcd.SkuInfoList:

-                                    OverrideValues = Pcd.SkuOverrideValues

-                                    if OverrideValues:

-                                        Keys = list(OverrideValues.keys())

-                                        Data = OverrideValues[Keys[0]]

-                                        Struct = list(Data.values())

-                                        DscOverride = self.ParseStruct(Struct[0])

-                                else:

-                                    SkuList = sorted(Pcd.SkuInfoList.keys())

-                                    for Sku in SkuList:

-                                        SkuInfo = Pcd.SkuInfoList[Sku]

-                                        if TypeName in ('DYNHII', 'DEXHII'):

-                                            if SkuInfo.DefaultStoreDict:

-                                                DefaultStoreList = sorted(SkuInfo.DefaultStoreDict.keys())

-                                                for DefaultStore in DefaultStoreList:

-                                                    OverrideValues = Pcd.SkuOverrideValues[Sku]

-                                                    DscOverride = self.ParseStruct(OverrideValues[DefaultStore])

-                                                    if DscOverride:

-                                                        break

-                                        else:

+                            SkuList = sorted(Pcd.SkuInfoList.keys())

+                            for Sku in SkuList:

+                                SkuInfo = Pcd.SkuInfoList[Sku]

+                                if TypeName in ('DYNHII', 'DEXHII'):

+                                    if SkuInfo.DefaultStoreDict:

+                                        DefaultStoreList = sorted(SkuInfo.DefaultStoreDict.keys())

+                                        for DefaultStore in DefaultStoreList:

                                             OverrideValues = Pcd.SkuOverrideValues[Sku]

-                                            if OverrideValues:

-                                                Keys = list(OverrideValues.keys())

-                                                OverrideFieldStruct = self.OverrideFieldValue(Pcd, OverrideValues[Keys[0]])

-                                                DscOverride = self.ParseStruct(OverrideFieldStruct)

-                                        if DscOverride:

-                                            break

+                                            DscOverride = self.ParseStruct(OverrideValues[DefaultStore])

+                                            if DscOverride:

+                                                break

+                                else:

+                                    OverrideValues = Pcd.SkuOverrideValues[Sku]

+                                    if OverrideValues:

+                                        Keys = OverrideValues.keys()

+                                        OverrideFieldStruct = self.OverrideFieldValue(Pcd, OverrideValues[Keys[0]])

+                                        DscOverride = self.ParseStruct(OverrideFieldStruct)

+                                if DscOverride:

+                                    break

                         if DscOverride:

                             DscDefaultValue = True

                             DscMatch = True

                             DecMatch = False

-                        else:

-                            DecMatch = True

                     else:

                         DscDefaultValue = True

                         DscMatch = True

@@ -1169,10 +1152,7 @@ class PcdReport(object):
                         for ModulePath in ModuleOverride:

                             ModuleDefault = ModuleOverride[ModulePath]

                             if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES:

-                                try:

-                                    ModulePcdDefaultValueNumber = int(ModuleDefault.strip(), 0)

-                                except:

-                                    ModulePcdDefaultValueNumber = int(ModuleDefault.lstrip('0'))

+                                ModulePcdDefaultValueNumber = int(ModuleDefault.strip(), 0)

                                 Match = (ModulePcdDefaultValueNumber == PcdValueNumber)

                                 if Pcd.DatumType == 'BOOLEAN':

                                     ModuleDefault = str(ModulePcdDefaultValueNumber)

@@ -1251,10 +1231,7 @@ class PcdReport(object):
                     if Value.startswith(('0x', '0X')):

                         Value = '{} ({:d})'.format(Value, int(Value, 0))

                     else:

-                        try:

-                            Value = "0x{:X} ({})".format(int(Value, 0), Value)

-                        except:

-                            Value = "0x{:X} ({})".format(int(Value.lstrip('0')), Value)

+                        Value = "0x{:X} ({})".format(int(Value, 0), Value)

                 FileWrite(File, '    %*s = %s' % (self.MaxLen + 19, 'DEC DEFAULT', Value))

             if IsStructure:

                 self.PrintStructureInfo(File, Pcd.DefaultValues)

@@ -1279,12 +1256,11 @@ class PcdReport(object):
             if IsStructure:

                 OverrideValues = Pcd.SkuOverrideValues

                 if OverrideValues:

-                    Keys = list(OverrideValues.keys())

+                    Keys = OverrideValues.keys()

                     Data = OverrideValues[Keys[0]]

-                    Struct = list(Data.values())

-                    if Struct:

-                        OverrideFieldStruct = self.OverrideFieldValue(Pcd, Struct[0])

-                        self.PrintStructureInfo(File, OverrideFieldStruct)

+                    Struct = Data.values()[0]

+                    OverrideFieldStruct = self.OverrideFieldValue(Pcd, Struct)

+                    self.PrintStructureInfo(File, OverrideFieldStruct)

             self.PrintPcdDefault(File, Pcd, IsStructure, DscMatch, DscDefaultValue, InfMatch, InfDefaultValue, DecMatch, DecDefaultValue)

         else:

             FirstPrint = True

@@ -1406,7 +1382,7 @@ class PcdReport(object):
                     if IsStructure:

                         OverrideValues = Pcd.SkuOverrideValues[Sku]

                         if OverrideValues:

-                            Keys = list(OverrideValues.keys())

+                            Keys = OverrideValues.keys()

                             OverrideFieldStruct = self.OverrideFieldValue(Pcd, OverrideValues[Keys[0]])

                             self.PrintStructureInfo(File, OverrideFieldStruct)

                     self.PrintPcdDefault(File, Pcd, IsStructure, DscMatch, DscDefaultValue, InfMatch, InfDefaultValue, DecMatch, DecDefaultValue)

@@ -1638,7 +1614,7 @@ class PredictionReport(object):
         TempFile.close()

 

         try:

-            from Eot.EotMain import Eot

+            from Eot.Eot import Eot

 

             #

             # Invoke EOT tool and echo its runtime performance

@@ -2270,7 +2246,7 @@ class BuildReport(object):
     def GenerateReport(self, BuildDuration, AutoGenTime, MakeTime, GenFdsTime):

         if self.ReportFile:

             try:

-                File = StringIO('')

+                File = BytesIO('')

                 for (Wa, MaList) in self.ReportList:

                     PlatformReport(Wa, MaList, self.ReportType).GenerateReport(File, BuildDuration, AutoGenTime, MakeTime, GenFdsTime, self.ReportType)

                 Content = FileLinesSplit(File.getvalue(), gLineMaxLength)

diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 1a21395699..d74082fc26 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -16,15 +16,17 @@
 ##

 # Import Modules

 #

+from __future__ import print_function

 import Common.LongFilePathOs as os

 import re

-from io import StringIO

+from io import BytesIO

 import sys

 import glob

 import time

 import platform

 import traceback

 import encodings.ascii

+import itertools

 import multiprocessing

 

 from struct import *

@@ -242,8 +244,8 @@ def ReadMessage(From, To, ExitFlag):
         # read one line a time

         Line = From.readline()

         # empty string means "end"

-        if Line is not None and Line != b"":

-            To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))

+        if Line is not None and Line != "":

+            To(Line.rstrip())

         else:

             break

         if ExitFlag.isSet():

@@ -503,7 +505,7 @@ class BuildTask:
 

                 # get all pending tasks

                 BuildTask._PendingQueueLock.acquire()

-                BuildObjectList = list(BuildTask._PendingQueue.keys())

+                BuildObjectList = BuildTask._PendingQueue.keys()

                 #

                 # check if their dependency is resolved, and if true, move them

                 # into ready queue

@@ -724,7 +726,7 @@ class PeImageInfo():
         self.OutputDir        = OutputDir

         self.DebugDir         = DebugDir

         self.Image            = ImageClass

-        self.Image.Size       = (self.Image.Size // 0x1000 + 1) * 0x1000

+        self.Image.Size       = (self.Image.Size / 0x1000 + 1) * 0x1000

 

 ## The class implementing the EDK2 build process

 #

@@ -1171,9 +1173,9 @@ class Build():
                 f = open(PrebuildEnvFile)

                 envs = f.readlines()

                 f.close()

-                envs = map(lambda l: l.split('=', 1), envs)

-                envs = filter(lambda l: len(l) == 2, envs)

-                envs = map(lambda l: [i.strip() for i in l], envs)

+                envs = itertools.imap(lambda l: l.split('=', 1), envs)

+                envs = itertools.ifilter(lambda l: len(l) == 2, envs)

+                envs = itertools.imap(lambda l: [i.strip() for i in l], envs)

                 os.environ.update(dict(envs))

             EdkLogger.info("\n- Prebuild Done -\n")

 

@@ -1597,7 +1599,7 @@ class Build():
                         RtModuleList[Module.MetaFile] = ImageInfo

                         #IPF runtime driver needs to be at 2 page alignment.

                         if IsIpfPlatform and ImageInfo.Image.Size % 0x2000 != 0:

-                            ImageInfo.Image.Size = (ImageInfo.Image.Size // 0x2000 + 1) * 0x2000

+                            ImageInfo.Image.Size = (ImageInfo.Image.Size / 0x2000 + 1) * 0x2000

                         RtSize += ImageInfo.Image.Size

                     elif Module.ModuleType in [SUP_MODULE_SMM_CORE, SUP_MODULE_DXE_SMM_DRIVER, SUP_MODULE_MM_STANDALONE, SUP_MODULE_MM_CORE_STANDALONE]:

                         SmmModuleList[Module.MetaFile] = ImageInfo

@@ -1666,21 +1668,21 @@ class Build():
             for PcdInfo in PcdTable:

                 ReturnValue = 0

                 if PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE:

-                    ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize // 0x1000))

+                    ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize / 0x1000))

                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE:

-                    ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize // 0x1000))

+                    ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize / 0x1000))

                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE:

-                    ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize // 0x1000))

+                    ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize / 0x1000))

                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE and len (SmmModuleList) > 0:

-                    ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize // 0x1000))

+                    ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize / 0x1000))

                 if ReturnValue != 0:

                     EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)

 

-        MapBuffer.write('PEI_CODE_PAGE_NUMBER      = 0x%x\n' % (PeiSize // 0x1000))

-        MapBuffer.write('BOOT_CODE_PAGE_NUMBER     = 0x%x\n' % (BtSize // 0x1000))

-        MapBuffer.write('RUNTIME_CODE_PAGE_NUMBER  = 0x%x\n' % (RtSize // 0x1000))

+        MapBuffer.write('PEI_CODE_PAGE_NUMBER      = 0x%x\n' % (PeiSize / 0x1000))

+        MapBuffer.write('BOOT_CODE_PAGE_NUMBER     = 0x%x\n' % (BtSize / 0x1000))

+        MapBuffer.write('RUNTIME_CODE_PAGE_NUMBER  = 0x%x\n' % (RtSize / 0x1000))

         if len (SmmModuleList) > 0:

-            MapBuffer.write('SMM_CODE_PAGE_NUMBER      = 0x%x\n' % (SmmSize // 0x1000))

+            MapBuffer.write('SMM_CODE_PAGE_NUMBER      = 0x%x\n' % (SmmSize / 0x1000))

 

         PeiBaseAddr = TopMemoryAddress - RtSize - BtSize

         BtBaseAddr  = TopMemoryAddress - RtSize

@@ -1780,7 +1782,7 @@ class Build():
                             if not Ma.IsLibrary:

                                 ModuleList[Ma.Guid.upper()] = Ma

 

-                    MapBuffer = StringIO('')

+                    MapBuffer = BytesIO('')

                     if self.LoadFixAddress != 0:

                         #

                         # Rebase module to the preferred memory address before GenFds

@@ -1938,7 +1940,7 @@ class Build():
                             if not Ma.IsLibrary:

                                 ModuleList[Ma.Guid.upper()] = Ma

 

-                    MapBuffer = StringIO('')

+                    MapBuffer = BytesIO('')

                     if self.LoadFixAddress != 0:

                         #

                         # Rebase module to the preferred memory address before GenFds

@@ -2125,7 +2127,7 @@ class Build():
                     #

                     # Rebase module to the preferred memory address before GenFds

                     #

-                    MapBuffer = StringIO('')

+                    MapBuffer = BytesIO('')

                     if self.LoadFixAddress != 0:

                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)

 

@@ -2177,7 +2179,7 @@ class Build():
 

                     # Look through the tool definitions for GUIDed tools

                     guidAttribs = []

-                    for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.items():

+                    for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.iteritems():

                         if attrib.upper().endswith('_GUID'):

                             split = attrib.split('_')

                             thisPrefix = '_'.join(split[0:3]) + '_'