Check In tool source code based on Build tool project revision r1655.

git-svn-id: https://edk2.svn.sourceforge.net/svnroot/edk2/trunk/edk2@8964 6f19259b-4bc3-4df7-8a09-765794883524

2 files changed, 621 insertions, 0 deletions

diff --git a/BaseTools/gcc/README.txt b/BaseTools/gcc/README.txt
new file mode 100644
index 0000000000..ea7dd64576
--- /dev/null
+++ b/BaseTools/gcc/README.txt
@@ -0,0 +1,71 @@
+You may run these scripts to build a UEFI/PI gcc cross compiler.

+

+Cross compilers built with these scripts are tested on

+Linux, OS X and Cygwin.

+

+Please note that you may need to modify your edk2 tree's Conf/tools_def.txt

+file to point to the location where you installed the cross compiler.

+

+=== tianoCross-gcc-4.1 ===

+

+This script will build an x86 (ia32) cross compiler.

+

+The results of this script are very similar to the 'mingw' cross compiler

+which is commonly available on linux and OS X.  But, since the cross

+compiler produced by this script is tested, it is the only 'supported' way

+to build UEFI/PI images.

+

+To use this script, you will need:

+

+ * A recent version (3.0 or later should be fine) of gcc that is able to produce

+   executables for the machine that you want to run this compiler on (the host

+   machine).

+ * wget or curl

+ * tar

+ * bzip

+ * gzip

+ * bash

+ * As well as (possibly) others tools and development packages

+

+=== x86_64-mingw-gcc-build.py ==

+

+This script will build an x86_64 (x64/Intel 64/amd64) cross compiler.

+

+To use this script, you will need:

+

+ * A recent version (3.0 or later should be fine) of gcc that is able to

+   produce executables for the machine that you want to run this compiler

+   on (the host machine).

+ * Python 2.5

+ * texinfo

+ * bison

+ * flex

+ * libmpfr

+ * libgmp

+ * As well as (possibly) others tools and development packages

+

+=== Ubuntu Notes ===

+

+On Ubuntu, the following command should install all the necessary build

+packages to utilize the x86_64-mingw-gcc-build.py script:

+

+  sudo apt-get install build-essential texinfo bison flex libgmp3-dev libmpfr-dev

+

+=== CYGWIN Notes ===

+

+You should setup cygwin to use binmode on all mounts. When you initially

+install cygwin it gives you the choice of Unix file mode (recommended) or DOS

+file mode. Unix mode will cause all the cygwin directories to be mounted in

+binmode, while DOS will mount the dirs in textmode. Here is an example of a

+cygwin install where the dirs are (properly) mounted in binmode.

+

+C:\cygwin\bin on /usr/bin type user (binmode)

+C:\cygwin\lib on /usr/lib type user (binmode)

+c:\workspace on /workspace type system (binmode)

+C:\cygwin on / type user (binmode)

+

+If you use textmode, it is likely that the build will fail in a way that is

+hard to debug.

+

+Cygwin is pretty slow, so it is not recommended for large builds.

+

diff --git a/BaseTools/gcc/mingw-gcc-build.py b/BaseTools/gcc/mingw-gcc-build.py
new file mode 100644
index 0000000000..73eb3de0df
--- /dev/null
+++ b/BaseTools/gcc/mingw-gcc-build.py
@@ -0,0 +1,550 @@
+#!/usr/bin/env python
+
+#
+# Automation of instructions from:
+#   http://mingw-w64.svn.sourceforge.net/viewvc/mingw-w64/trunk/mingw-w64-doc/
+#     howto-build/mingw-w64-howto-build.txt?revision=216&view=markup
+#
+
+from optparse import OptionParser
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+import urllib
+import urlparse
+try:
+    from hashlib import md5
+except Exception:
+    from md5 import md5
+
+#
+# Version and Copyright
+#
+VersionNumber = "0.01"
+__version__ = "%prog Version " + VersionNumber
+__copyright__ = "Copyright (c) 2008, Intel Corporation.  All rights reserved."
+
+class Config:
+    """class Config
+
+    Stores the configuration options for the rest of the script.
+
+    Handles the command line options, and allows the code within
+    the script to easily interact with the 'config' requested by
+    the user.
+    """
+
+    def __init__(self):
+        self.base_dir = os.getcwd()
+        (self.options, self.args) = self.CheckOptions()
+        self.__init_dirs__()
+
+    def CheckOptions(self):
+        Parser = \
+            OptionParser(
+                description=__copyright__,
+                version=__version__,
+                prog="mingw-gcc-build",
+                usage="%prog [options] [target]"
+                )
+        Parser.add_option(
+            "--arch",
+            action = "store", type = "string",
+            default = '',
+            dest = "arch",
+            help = "Processor architecture to build gcc for."
+            )
+        Parser.add_option(
+            "--src-dir",
+            action = "store", type = "string", dest = "src_dir",
+            default = os.path.join(self.base_dir, 'src'),
+            help = "Directory to download/extract binutils/gcc sources"
+            )
+        Parser.add_option(
+            "--build-dir",
+            action = "store", type = "string", dest = "build_dir",
+            default = os.path.join(self.base_dir, 'build'),
+            help = "Directory to download/extract binutils/gcc sources"
+            )
+        Parser.add_option(
+            "--prefix",
+            action = "store", type = "string", dest = "prefix",
+            default = os.path.join(self.base_dir, 'install'),
+            help = "Prefix to install binutils/gcc into"
+            )
+        Parser.add_option(
+            "--symlinks",
+            action = "store", type = "string", dest = "symlinks",
+            default = os.path.join(self.base_dir, 'symlinks'),
+            help = "Directory to create binutils/gcc symbolic links into."
+            )
+        Parser.add_option(
+            "-v", "--verbose",
+            action="store_true",
+            type=None, help="Print verbose messages"
+            )
+
+        (Opt, Args) = Parser.parse_args()
+
+        self.arch = Opt.arch.lower()
+        allowedArchs = ('ia32', 'x64', 'ipf')
+        if self.arch not in allowedArchs:
+            Parser.error(
+                'Please use --arch to specify one of: %s' %
+                    ', '.join(allowedArchs)
+                )
+        self.target_arch = {'ia32': 'i686', 'x64': 'x86_64', 'ipf': 'ia64'}[self.arch]
+        self.target_sys = {'ia32': 'pc', 'x64': 'pc', 'ipf': 'pc'}[self.arch]
+        self.target_bin = {'ia32': 'mingw32', 'x64': 'mingw32', 'ipf': 'elf'}[self.arch]
+        self.target_combo = '-'.join((self.target_arch, self.target_sys, self.target_bin))
+
+        return (Opt, Args)
+
+    def __init_dirs__(self):
+        self.src_dir = os.path.realpath(os.path.expanduser(self.options.src_dir))
+        self.build_dir = os.path.realpath(os.path.expanduser(self.options.build_dir))
+        self.prefix = os.path.realpath(os.path.expanduser(self.options.prefix))
+        self.symlinks = os.path.realpath(os.path.expanduser(self.options.symlinks))
+
+    def IsConfigOk(self):
+                
+        print "Current directory:"
+        print "   ", self.base_dir
+        print "Sources download/extraction:", self.Relative(self.src_dir)
+        print "Build directory            :", self.Relative(self.build_dir)
+        print "Prefix (install) directory :", self.Relative(self.prefix)
+        print "Create symlinks directory  :", self.Relative(self.symlinks)
+        print
+        answer = raw_input("Is this configuration ok? (default = no): ")
+        if (answer.lower() not in ('y', 'yes')):
+            print
+            print "Please try using --help and then change the configuration."
+            return False
+
+        if self.arch.lower() == 'ipf':
+            print
+            print 'Please note that the IPF compiler built by this script has'
+            print 'not yet been validated!'
+            print
+            answer = raw_input("Are you sure you want to build it? (default = no): ")
+            if (answer.lower() not in ('y', 'yes')):
+                print
+                print "Please try using --help and then change the configuration."
+                return False
+
+        print
+        return True
+
+    def Relative(self, path):
+        if path.startswith(self.base_dir):
+            return '.' + path[len(self.base_dir):]
+        return path
+
+    def MakeDirs(self):
+        for path in (self.src_dir, self.build_dir,self.prefix, self.symlinks):
+            if not os.path.exists(path):
+                os.makedirs(path)
+
+class SourceFiles:
+    """class SourceFiles
+
+    Handles the downloading of source files used by the script.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.source_files = self.source_files[config.arch]
+
+    source_files_common = {
+        'binutils': {
+            'url': 'http://www.kernel.org/pub/linux/devel/binutils/' + \
+                   'binutils-$version.tar.bz2',
+            'version': '2.18.50.0.5',
+            'md5': 'daee18dbbf0a6ccfc186141bee18bf62',
+            },
+        }
+
+    source_files_x64 = {
+        'gcc': {
+            'url': 'http://gcc-ca.internet.bs/releases/' + \
+                   'gcc-$version/gcc-$version.tar.bz2',
+            'version': '4.3.0',
+            'md5': '197ed8468b38db1d3481c3111691d85b',
+            },
+        'mingw_hdr': {
+            'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
+                   'mingw-w64/mingw-w64-snapshot-$version.tar.bz2',
+            'extract-dir': os.path.join('trunk', 'mingw-w64-headers'),
+            'version': '20080310',
+            'md5': '235b2d15c2411f7d213c0c0977b2162f',
+            },
+        }
+
+    source_files_ia32 = {
+        'gcc': {
+            'url': 'http://superb-east.dl.sourceforge.net/sourceforge/' + \
+                   'mingw/gcc-$version-mingw-$minor_version-src.tar.gz',
+            'version': '4.3.0',
+            'minor_version': 'alpha-20080403',
+            'extract-dir': 'gcc-$version',
+            'md5': '27961d80e304f4ef32c980833c6e8e44',
+            'configure-params': ('--with-gnu-as', '--with-gnu-ld', '--with-newlib',
+                                 '--verbose', '--disable-libssp', '--disable-nls',
+                                 '--enable-languages=c,c++'
+                                )
+            },
+        'mingw_hdr': {
+            'url': 'http://superb-west.dl.sourceforge.net/sourceforge/' + \
+                   'mingw/mingw-runtime-$version-src.tar.gz',
+            'extract-dir': 'mingw-runtime-$version',
+            'version': '3.14',
+            'md5': '7d049a8331efcfe34600c0cda6934ac6',
+            },
+        }
+
+    source_files_ipf = source_files_x64.copy()
+    source_files_ipf['gcc']['configure-params'] = (
+        '--with-gnu-as', '--with-gnu-ld', '--with-newlib',
+        '--verbose', '--disable-libssp', '--disable-nls',
+        '--enable-languages=c,c++'
+        )
+
+    source_files = {
+        'ia32': [source_files_common, source_files_ia32],
+        'x64': [source_files_common, source_files_x64],
+        'ipf': [source_files_common, source_files_ipf],
+        }
+
+    for arch in source_files:
+        mergedSourceFiles = {}
+        for source_files_dict in source_files[arch]:
+            mergedSourceFiles.update(source_files_dict)
+        for downloadItem in mergedSourceFiles:
+            fdata = mergedSourceFiles[downloadItem]
+            fdata['filename'] = fdata['url'].split('/')[-1]
+            if 'extract-dir' not in fdata:
+                for ext in ('.tar.gz', '.tar.bz2', '.zip'):
+                    if fdata['filename'].endswith(ext):
+                        fdata['extract-dir'] = fdata['filename'][:-len(ext)]
+                        break
+            replaceables = ('extract-dir', 'filename', 'url')
+            for replaceItem in fdata:
+                if replaceItem in replaceables: continue
+                if type(fdata[replaceItem]) != str: continue
+                for replaceable in replaceables:
+                    if type(fdata[replaceable]) != str: continue
+                    if replaceable in fdata:
+                        fdata[replaceable] = \
+                            fdata[replaceable].replace(
+                                '$' + replaceItem,
+                                fdata[replaceItem]
+                                )
+        source_files[arch] = mergedSourceFiles
+    #print 'source_files:', source_files
+
+    def GetAll(self):
+
+        def progress(received, blockSize, fileSize):
+            if fileSize < 0: return
+            wDots = (100 * received * blockSize) / fileSize / 10
+            if wDots > self.dots:
+                for i in range(wDots - self.dots):
+                    print '.',
+                    sys.stdout.flush()
+                    self.dots += 1
+
+        maxRetries = 3
+        for (fname, fdata) in self.source_files.items():
+            for retries in range(maxRetries):
+                try:
+                    self.dots = 0
+                    local_file = os.path.join(self.config.src_dir, fdata['filename'])
+                    url = fdata['url']
+                    print 'Downloading %s:' % fname,
+                    if retries > 0:
+                        print '(retry)',
+                    sys.stdout.flush()
+
+                    completed = False
+                    if os.path.exists(local_file):
+                        md5_pass = self.checkHash(fdata)
+                        if md5_pass:
+                            print '[md5 match]',
+                        else:
+                            print '[md5 mismatch]',
+                        sys.stdout.flush()
+                        completed = md5_pass
+
+                    if not completed:
+                        urllib.urlretrieve(url, local_file, progress)
+
+                    #
+                    # BUGBUG: Suggest proxy to user if download fails.
+                    #
+                    # export http_proxy=http://proxyservername.mycompany.com:911
+                    # export ftp_proxy=http://proxyservername.mycompany.com:911
+
+                    if not completed and os.path.exists(local_file):
+                        md5_pass = self.checkHash(fdata)
+                        if md5_pass:
+                            print '[md5 match]',
+                        else:
+                            print '[md5 mismatch]',
+                        sys.stdout.flush()
+                        completed = md5_pass
+
+                    if completed:
+                        print '[done]'
+                        break
+                    else:
+                        print '[failed]'
+                        raise Exception()
+                
+                except KeyboardInterrupt:
+                    print '[KeyboardInterrupt]'
+                    return False
+
+                except:
+                    pass
+
+            if not completed: return False
+
+        return True
+
+    def checkHash(self, fdata):
+        local_file = os.path.join(self.config.src_dir, fdata['filename'])
+        expect_md5 = fdata['md5']
+        data = open(local_file).read()
+        md5sum = md5()
+        md5sum.update(data)
+        return md5sum.hexdigest().lower() == expect_md5.lower()
+
+    def GetModules(self):
+        return self.source_files.keys()
+
+    def GetFilenameOf(self, module):
+        return self.source_files[module]['filename']
+
+    def GetMd5Of(self, module):
+        return self.source_files[module]['md5']
+
+    def GetExtractDirOf(self, module):
+        return self.source_files[module]['extract-dir']
+
+    def GetAdditionalParameters(self, module, step):
+        key = step + '-params'
+        if key in self.source_files[module]:
+            return self.source_files[module][key]
+        else:
+            return tuple()
+
+class Extracter:
+    """class Extracter
+
+    Handles the extraction of the source files from their downloaded
+    archive files.
+    """
+
+    def __init__(self, source_files, config):
+        self.source_files = source_files
+        self.config = config
+
+    def Extract(self, module):
+        src = self.config.src_dir
+        extractDst = os.path.join(src, self.config.arch)
+        local_file = os.path.join(src, self.source_files.GetFilenameOf(module))
+        moduleMd5 = self.source_files.GetMd5Of(module)
+        extracted = os.path.join(extractDst, os.path.split(local_file)[1] + '.extracted')
+        if not os.path.exists(extractDst):
+            os.mkdir(extractDst)
+
+        extractedMd5 = None
+        if os.path.exists(extracted):
+            extractedMd5 = open(extracted).read()
+
+        if extractedMd5 != moduleMd5:
+            print 'Extracting %s:' % self.config.Relative(local_file)
+            tar = tarfile.open(local_file)
+            tar.extractall(extractDst)
+            open(extracted, 'w').write(moduleMd5)
+        else:
+            pass
+            #print 'Previously extracted', self.config.Relative(local_file)
+
+    def ExtractAll(self):
+        for module in self.source_files.GetModules():
+            self.Extract(module)
+
+class Builder:
+    """class Builder
+
+    Builds and installs the GCC tool suite.
+    """
+
+    def __init__(self, source_files, config):
+        self.source_files = source_files
+        self.config = config
+
+    def Build(self):
+        self.BuildModule('binutils')
+        self.CopyIncludeDirectory()
+        self.BuildModule('gcc')
+        self.MakeSymLinks()
+
+    def IsBuildStepComplete(self, step):
+        return \
+            os.path.exists(
+                os.path.join(
+                    self.config.build_dir, self.config.arch, step + '.completed'
+                    )
+                )
+
+    def MarkBuildStepComplete(self, step):
+        open(
+            os.path.join(
+                self.config.build_dir, self.config.arch, step + '.completed'
+                ),
+            "w"
+            ).close()
+
+    def CopyIncludeDirectory(self):
+        linkdst = os.path.join(self.config.prefix, 'mingw')
+        src = os.path.join(
+            self.config.src_dir,
+            self.config.arch,
+            self.source_files.GetExtractDirOf('mingw_hdr'),
+            'include'
+            )
+        dst_parent = os.path.join(self.config.prefix, self.config.target_combo)
+        dst = os.path.join(dst_parent, 'include')
+        if not os.path.exists(dst):
+            if not os.path.exists(dst_parent):
+                os.makedirs(dst_parent)
+            print 'Copying headers to', self.config.Relative(dst)
+            shutil.copytree(src, dst, True)
+        if not os.path.lexists(linkdst):
+            print 'Making symlink at', self.config.Relative(linkdst)
+            os.symlink(self.config.target_combo, linkdst)
+
+    def BuildModule(self, module):
+        base_dir = os.getcwd()
+        build_dir = os.path.join(self.config.build_dir, self.config.arch, module)
+        module_dir = self.source_files.GetExtractDirOf(module)
+        module_dir = os.path.realpath(os.path.join('src', self.config.arch, module_dir))
+        configure = os.path.join(module_dir, 'configure')
+        prefix = self.config.prefix
+        if not os.path.exists(build_dir):
+            os.makedirs(build_dir)
+        os.chdir(build_dir)
+
+        cmd = (
+            configure,
+            '--target=%s' % self.config.target_combo,
+            '--prefix=' + prefix,
+            '--with-sysroot=' + prefix,
+            '--disable-werror',
+            )
+        if os.path.exists('/opt/local/include/gmp.h'):
+            cmd += ('--with-gmp=/opt/local',)
+        if module == 'gcc': cmd += ('--oldincludedir=/opt/local/include',)
+        cmd += self.source_files.GetAdditionalParameters(module, 'configure')
+        self.RunCommand(cmd, module, 'config', skipable=True)
+
+        cmd = ('make',)
+        if module == 'gcc':
+            cmd += ('all-gcc',)
+        self.RunCommand(cmd, module, 'build')
+
+        cmd = ('make',)
+        if module == 'gcc':
+            cmd += ('install-gcc',)
+        else:
+            cmd += ('install',)
+        self.RunCommand(cmd, module, 'install')
+
+        os.chdir(base_dir)
+
+        print '%s module is now built and installed' % module
+
+    def RunCommand(self, cmd, module, stage, skipable=False):
+        if skipable:
+            if self.IsBuildStepComplete('%s.%s' % (module, stage)):
+                return
+
+        popen = lambda cmd: \
+            subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT
+                )
+
+        print '%s [%s] ...' % (module, stage),
+        sys.stdout.flush()
+        p = popen(cmd)
+        output = p.stdout.read()
+        p.wait()
+        if p.returncode != 0:
+            print '[failed!]'
+            logFile = os.path.join(self.config.build_dir, 'log.txt')
+            f = open(logFile, "w")
+            f.write(output)
+            f.close()
+            raise Exception, 'Failed to %s %s\n' % (stage, module) + \
+                'See output log at %s' % self.config.Relative(logFile)
+        else:
+            print '[done]'
+
+        if skipable:
+            self.MarkBuildStepComplete('%s.%s' % (module, stage))
+
+    def MakeSymLinks(self):
+        links_dir = os.path.join(self.config.symlinks, self.config.arch)
+        if not os.path.exists(links_dir):
+            os.makedirs(links_dir)
+        startPrinted = False
+        for link in ('ar', 'ld', 'gcc'):
+            src = os.path.join(
+                self.config.prefix, 'bin', self.config.target_combo + '-' + link
+                )
+            linkdst = os.path.join(links_dir, link)
+            if not os.path.lexists(linkdst):
+                if not startPrinted:
+                    print 'Making symlinks in %s:' % self.config.Relative(links_dir),
+                    startPrinted = True
+                print link,
+                os.symlink(src, linkdst)
+
+        if startPrinted:
+            print '[done]'
+
+class App:
+    """class App
+
+    The main body of the application.
+    """
+
+    def __init__(self):
+        config = Config()
+
+        if not config.IsConfigOk():
+            return
+
+        config.MakeDirs()
+
+        sources = SourceFiles(config)
+        result = sources.GetAll()
+        if result:
+            print 'All files have been downloaded & verified'
+        else:
+            print 'An error occured while downloading a file'
+            return
+
+        Extracter(sources, config).ExtractAll()
+
+        Builder(sources, config).Build()
+
+App()
+