BaseTools: Add Brotli algorithm tool

- Add Brotli algorithm tool support

Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Bell Song <binx.song@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>

10 files changed, 318 insertions, 20 deletions

diff --git a/BaseTools/BinWrappers/PosixLike/Brotli b/BaseTools/BinWrappers/PosixLike/Brotli
new file mode 100644
index 0000000000..a244ecc095
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/Brotli
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash

+

+full_cmd=${BASH_SOURCE:-$0} # see http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is not a good choice here

+dir=$(dirname "$full_cmd")

+cmd=${full_cmd##*/}

+

+if [ -n "$WORKSPACE" ] && [ -e "$WORKSPACE/Conf/BaseToolsCBinaries" ]

+then

+  exec "$WORKSPACE/Conf/BaseToolsCBinaries/$cmd"

+elif [ -n "$WORKSPACE" ] && [ -e "$EDK_TOOLS_PATH/Source/C" ]

+then

+  if [ ! -e "$EDK_TOOLS_PATH/Source/C/bin/$cmd" ]

+  then

+    echo "BaseTools C Tool binary was not found ($cmd)"

+    echo "You may need to run:"

+    echo "  make -C $EDK_TOOLS_PATH/Source/C"

+  else

+    exec "$EDK_TOOLS_PATH/Source/C/bin/$cmd" "$@"

+  fi

+elif [ -e "$dir/../../Source/C/bin/$cmd" ]

+then

+  exec "$dir/../../Source/C/bin/$cmd" "$@"

+else

+  echo "Unable to find the real '$cmd' to run"

+  echo "This message was printed by"

+  echo "  $0"

+  exit 127

+fi

+

diff --git a/BaseTools/BinWrappers/PosixLike/BrotliCompress b/BaseTools/BinWrappers/PosixLike/BrotliCompress
new file mode 100644
index 0000000000..a571ff6892
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/BrotliCompress
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash

+#

+# This script will exec Brotli tool.

+#

+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>

+# This program and the accompanying materials

+# are licensed and made available under the terms and conditions of the BSD License

+# which accompanies this distribution.  The full text of the license may be found at

+# http://opensource.org/licenses/bsd-license.php

+#

+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,

+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.

+#

+LVL="--quality 9"

+

+while [ $# != 0 ];do

+  case $1 in

+    -d)

+      ARGS+="--decompress "

+    ;;

+    -e)

+    ;;

+    -g)

+      ARGS+="--gap $2 "

+      shift

+    ;;

+    -l)

+      LVL="--quality $2 "

+      shift

+    ;;

+    -o)

+      ARGS+="--output $2 "

+      shift

+    ;;

+    *)

+      ARGS+="--input $1 "

+  esac

+

+shift

+done

+

+exec Brotli $ARGS $LVL

diff --git a/BaseTools/Conf/tools_def.template b/BaseTools/Conf/tools_def.template
index ab4f936e7a..0aabdeb2d9 100755
--- a/BaseTools/Conf/tools_def.template
+++ b/BaseTools/Conf/tools_def.template
@@ -7707,6 +7707,12 @@ RELEASE_RVCTCYGWIN_ARM_CC_FLAGS  = "$(CCPATH_FLAG)" $(ARCHCC_FLAGS) $(PLATFORM_F
 *_*_*_RSA2048SHA256SIGN_GUID = A7717414-C616-4977-9420-844712A735BF

 

 ##################

+# BrotliCompress tool definitions

+##################

+*_*_*_BROTLI_PATH        = BrotliCompress

+*_*_*_BROTLI_GUID        = 3D532050-5CDA-4FD0-879E-0F7F630D5AFB

+

+##################

 # LzmaCompress tool definitions

 ##################

 *_*_*_LZMA_PATH          = LzmaCompress

diff --git a/BaseTools/Source/C/BrotliCompress/BrotliCompress.bat b/BaseTools/Source/C/BrotliCompress/BrotliCompress.bat
new file mode 100644
index 0000000000..1bca076feb
--- /dev/null
+++ b/BaseTools/Source/C/BrotliCompress/BrotliCompress.bat
@@ -0,0 +1,48 @@
+@echo off

+@setlocal

+

+set LVL=--quality 9

+

+:Begin

+if "%1"=="" goto End

+

+if "%1"=="-d" (

+  set ARGS=%ARGS% --decompress

+  shift

+  goto Begin

+)

+

+if "%1"=="-e" (

+  shift

+  goto Begin

+)

+

+if "%1"=="-g" (

+  set ARGS=%ARGS% --gap %2

+  shift

+  shift

+  goto Begin

+)

+

+if "%1"=="-l" (

+  set LVL=--quality %2

+  shift

+  shift

+  goto Begin

+)

+

+if "%1"=="-o" (

+  set ARGS=%ARGS% --output %2

+  set INTMP=%2

+  shift

+  shift

+  goto Begin

+)

+

+set ARGS=%ARGS% --input %1

+shift

+goto Begin

+

+:End

+Brotli %ARGS% %LVL%

+@echo on
\ No newline at end of file
diff --git a/BaseTools/Source/C/BrotliCompress/GNUmakefile b/BaseTools/Source/C/BrotliCompress/GNUmakefile
new file mode 100644
index 0000000000..368edbd3c4
--- /dev/null
+++ b/BaseTools/Source/C/BrotliCompress/GNUmakefile
@@ -0,0 +1,43 @@
+## @file

+# GNU/Linux makefile for 'Brotli' module build.

+#

+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>

+# This program and the accompanying materials

+# are licensed and made available under the terms and conditions of the BSD License

+# which accompanies this distribution.  The full text of the license may be found at

+# http://opensource.org/licenses/bsd-license.php

+#

+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,

+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.

+#

+ARCH ?= IA32

+MAKEROOT ?= ..

+

+APPNAME = Brotli

+

+OBJECTS = \

+  tools/bro.o \

+  common/dictionary.o \

+  dec/bit_reader.o \

+  dec/decode.o \

+  dec/huffman.o \

+  dec/state.o \

+  enc/backward_references.o \

+  enc/bit_cost.o \

+  enc/block_splitter.o \

+  enc/brotli_bit_stream.o \

+  enc/cluster.o \

+  enc/compress_fragment.o \

+  enc/compress_fragment_two_pass.o \

+  enc/encode.o \

+  enc/entropy_encode.o \

+  enc/histogram.o \

+  enc/literal_cost.o \

+  enc/memory.o \

+  enc/metablock.o \

+  enc/static_dict.o \

+  enc/utf8_util.o

+

+include $(MAKEROOT)/Makefiles/app.makefile

+

+LIBS += -lm

diff --git a/BaseTools/Source/C/BrotliCompress/Makefile b/BaseTools/Source/C/BrotliCompress/Makefile
new file mode 100644
index 0000000000..9d45ea555d
--- /dev/null
+++ b/BaseTools/Source/C/BrotliCompress/Makefile
@@ -0,0 +1,60 @@
+## @file

+# Windows makefile for 'Brotli' module build.

+#

+# Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>

+# This program and the accompanying materials

+# are licensed and made available under the terms and conditions of the BSD License

+# which accompanies this distribution.  The full text of the license may be found at

+# http://opensource.org/licenses/bsd-license.php

+#

+# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,

+# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.

+#

+!INCLUDE ..\Makefiles\ms.common

+

+CFLAGS = $(CFLAGS) /W2

+

+APPNAME = Brotli

+

+#LIBS = $(LIB_PATH)\Common.lib

+

+COMMON_OBJ = common\dictionary.obj

+DEC_OBJ = \

+  dec\bit_reader.obj \

+  dec\decode.obj \

+  dec\huffman.obj \

+  dec\state.obj

+ENC_OBJ = \

+  enc\backward_references.obj \

+  enc\bit_cost.obj \

+  enc\block_splitter.obj \

+  enc\brotli_bit_stream.obj \

+  enc\cluster.obj \

+  enc\compress_fragment.obj \

+  enc\compress_fragment_two_pass.obj \

+  enc\encode.obj \

+  enc\entropy_encode.obj \

+  enc\histogram.obj \

+  enc\literal_cost.obj \

+  enc\memory.obj \

+  enc\metablock.obj \

+  enc\static_dict.obj \

+  enc\utf8_util.obj

+

+OBJECTS = \

+  tools\bro.obj \

+  $(COMMON_OBJ) \

+  $(DEC_OBJ) \

+  $(ENC_OBJ)

+

+!INCLUDE ..\Makefiles\ms.app

+

+all: $(BIN_PATH)\BrotliCompress.bat

+

+$(BIN_PATH)\BrotliCompress.bat: BrotliCompress.bat

+  copy BrotliCompress.bat $(BIN_PATH)\BrotliCompress.bat /Y

+

+cleanall: localCleanall

+

+localCleanall:

+  del /f /q $(BIN_PATH)\BrotliCompress.bat > nul

diff --git a/BaseTools/Source/C/BrotliCompress/ReadMe.txt b/BaseTools/Source/C/BrotliCompress/ReadMe.txt
new file mode 100644
index 0000000000..c19c0a162f
--- /dev/null
+++ b/BaseTools/Source/C/BrotliCompress/ReadMe.txt
@@ -0,0 +1,2 @@
+It is based on the Brotli v0.5.2.

+Brotli was released on the website https://github.com/google/brotli.

diff --git a/BaseTools/Source/C/BrotliCompress/tools/bro.c b/BaseTools/Source/C/BrotliCompress/tools/bro.c
index 57e053aa29..2fa9f05e7f 100644
--- a/BaseTools/Source/C/BrotliCompress/tools/bro.c
+++ b/BaseTools/Source/C/BrotliCompress/tools/bro.c
@@ -73,6 +73,7 @@ static void ParseArgv(int argc, char **argv,
                       char **dictionary_path,

                       int *force,

                       int *quality,

+                      int *gapmem,

                       int *decompress,

                       int *repeat,

                       int *verbose,

@@ -160,6 +161,13 @@ static void ParseArgv(int argc, char **argv,
         }

         ++k;

         continue;

+      } else if (!strcmp("--gap", argv[k]) ||

+                  !strcmp("-g", argv[k])) {

+        if (!ParseQuality(argv[k + 1], gapmem)) {

+          goto error;

+        }

+        ++k;

+        continue;

       }

     }

     goto error;

@@ -167,7 +175,7 @@ static void ParseArgv(int argc, char **argv,
   return;

 error:

   fprintf(stderr,

-          "Usage: %s [--force] [--quality n] [--decompress]"

+          "Usage: %s [--force] [--quality n] [--gap n] [--decompress]"

           " [--input filename] [--output filename] [--repeat iters]"

           " [--verbose] [--window n] [--custom-dictionary filename]\n",

           argv[0]);

@@ -225,9 +233,20 @@ static int64_t FileSize(const char *path) {
   return retval;

 }

 

+/* Brotli specified memory allocate function */

+static void *BrAlloc (void *memsize, size_t size) {

+  *(int64_t *)memsize += size;

+  return malloc(size);

+}

+

+/* Brotli specified memory free function */

+static void BrFree (void *memsize, void *ptr) {

+  free(ptr);

+}

+

 /* Result ownersip is passed to caller.

    |*dictionary_size| is set to resulting buffer size. */

-static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {

+static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size, void *memsize) {

   static const int kMaxDictionarySize = (1 << 24) - 16;

   FILE *f = fopen(path, "rb");

   int64_t file_size_64;

@@ -252,7 +271,7 @@ static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
   }

   *dictionary_size = (size_t)file_size_64;

 

-  buffer = (uint8_t*)malloc(*dictionary_size);

+  buffer = (uint8_t *)BrAlloc(memsize, *dictionary_size);

   if (!buffer) {

     fprintf(stderr, "could not read dictionary: out of memory\n");

     exit(1);

@@ -268,7 +287,7 @@ static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
 

 static const size_t kFileBufferSize = 65536;

 

-static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {

+static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path, void *memsize) {

   /* Dictionary should be kept during first rounds of decompression. */

   uint8_t* dictionary = NULL;

   uint8_t* input;

@@ -279,18 +298,18 @@ static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
   size_t available_out = kFileBufferSize;

   uint8_t* next_out;

   BrotliResult result = BROTLI_RESULT_ERROR;

-  BrotliState* s = BrotliCreateState(NULL, NULL, NULL);

+  BrotliState* s = BrotliCreateState(BrAlloc, BrFree, memsize);

   if (!s) {

     fprintf(stderr, "out of memory\n");

     return 0;

   }

   if (dictionary_path != NULL) {

     size_t dictionary_size = 0;

-    dictionary = ReadDictionary(dictionary_path, &dictionary_size);

+    dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);

     BrotliSetCustomDictionary(dictionary_size, dictionary, s);

   }

-  input = (uint8_t*)malloc(kFileBufferSize);

-  output = (uint8_t*)malloc(kFileBufferSize);

+  input = (uint8_t*)BrAlloc(memsize, kFileBufferSize);

+  output = (uint8_t*)BrAlloc(memsize, kFileBufferSize);

   if (!input || !output) {

     fprintf(stderr, "out of memory\n");

     goto end;

@@ -331,17 +350,17 @@ static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
   }

 

 end:

-  free(dictionary);

-  free(input);

-  free(output);

+  BrFree(memsize, dictionary);

+  BrFree(memsize, input);

+  BrFree(memsize, output);

   BrotliDestroyState(s);

   return (result == BROTLI_RESULT_SUCCESS) ? 1 : 0;

 }

 

 static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,

-    const char *dictionary_path) {

-  BrotliEncoderState* s = BrotliEncoderCreateInstance(0, 0, 0);

-  uint8_t* buffer = (uint8_t*)malloc(kFileBufferSize << 1);

+    const char *dictionary_path, void *memsize) {

+  BrotliEncoderState* s = BrotliEncoderCreateInstance(BrAlloc, BrFree, memsize);

+  uint8_t* buffer = (uint8_t*)BrAlloc(memsize, kFileBufferSize << 1);

   uint8_t* input = buffer;

   uint8_t* output = buffer + kFileBufferSize;

   size_t available_in = 0;

@@ -360,9 +379,9 @@ static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
   BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);

   if (dictionary_path != NULL) {

     size_t dictionary_size = 0;

-    uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size);

+    uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);

     BrotliEncoderSetCustomDictionary(s, dictionary_size, dictionary);

-    free(dictionary);

+    BrFree(memsize, dictionary);

   }

 

   while (1) {

@@ -392,7 +411,7 @@ static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
   }

 

 finish:

-  free(buffer);

+  BrFree(memsize, buffer);

   BrotliEncoderDestroyInstance(s);

 

   if (!is_ok) {

@@ -409,29 +428,41 @@ finish:
   return 1;

 }

 

+#define GAP_MEM_BLOCK  4096

+

 int main(int argc, char** argv) {

   char *input_path = 0;

   char *output_path = 0;

   char *dictionary_path = 0;

   int force = 0;

   int quality = 11;

+  int gmem = 1;

   int decompress = 0;

   int repeat = 1;

   int verbose = 0;

   int lgwin = 0;

   clock_t clock_start;

   int i;

+  int64_t originsize = 0;

+  int64_t msize = 0;

   ParseArgv(argc, argv, &input_path, &output_path, &dictionary_path, &force,

-            &quality, &decompress, &repeat, &verbose, &lgwin);

+            &quality, &gmem, &decompress, &repeat, &verbose, &lgwin);

   clock_start = clock();

   for (i = 0; i < repeat; ++i) {

     FILE* fin = OpenInputFile(input_path);

     FILE* fout = OpenOutputFile(output_path, force || repeat);

     int is_ok = 0;

     if (decompress) {

-      is_ok = Decompress(fin, fout, dictionary_path);

+      if (fseek(fin, 16, SEEK_SET) != 0) {

+        fclose(fin);

+        return -1;

+      }

+      is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);

     } else {

-      is_ok = Compress(quality, lgwin, fin, fout, dictionary_path);

+      originsize = FileSize(input_path);  /* get original file size */

+      fwrite(&originsize, 1, sizeof(int64_t), fout); /* add in original binary file size */

+      fwrite(&msize, 1, sizeof(int64_t), fout); /* add in dummy decompression required memory size */

+      is_ok = Compress(quality, lgwin, fin, fout, dictionary_path, (void *)&msize);

     }

     if (!is_ok) {

       unlink(output_path);

@@ -445,6 +476,41 @@ int main(int argc, char** argv) {
       perror("fclose");

       exit(1);

     }

+    /* after compression operation then execute decompression operation

+       to get decompression required memory size. */

+    if (decompress == 0) {

+      fin = OpenInputFile(output_path);

+      fout = tmpfile ();

+      msize = 0;

+      if (fseek(fin, 16, SEEK_SET) != 0) {

+        fclose(fin);

+        return -1;

+      }

+      is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);

+      if (!is_ok) {

+        exit(1);

+      }

+      if (fclose(fin) != 0) {

+        perror("fclose");

+        exit(1);

+      }

+      if (fclose(fout) != 0) {

+        perror("fclose");

+        exit(1);

+      }

+      fout = fopen(output_path, "rb+");  /* open output_path file and add in head info */

+      /* seek to the offset of decompression required memory size */

+      if (fseek(fout, 8, SEEK_SET) != 0) {

+        fclose(fout);

+        return -1;

+      }

+      msize += gmem * GAP_MEM_BLOCK;  /* there is a memory gap between IA32 and X64 environment*/

+      fwrite(&msize, 1, sizeof(int64_t), fout); /* update final decompression required memory size */

+      if (fclose(fout) != 0) {

+        perror("fclose");

+        exit(1);

+      }

+    }

   }

   if (verbose) {

     clock_t clock_end = clock();

diff --git a/BaseTools/Source/C/GNUmakefile b/BaseTools/Source/C/GNUmakefile
index 66baabb3f0..83e188c1a0 100644
--- a/BaseTools/Source/C/GNUmakefile
+++ b/BaseTools/Source/C/GNUmakefile
@@ -52,6 +52,7 @@ LIBRARIES = Common
 APPLICATIONS = \

   GnuGenBootSector \

   BootSectImage \

+  BrotliCompress \

   EfiLdrImage \

   EfiRom \

   GenFfs \

diff --git a/BaseTools/Source/C/Makefile b/BaseTools/Source/C/Makefile
index 3005a175b8..77d3b979c9 100644
--- a/BaseTools/Source/C/Makefile
+++ b/BaseTools/Source/C/Makefile
@@ -17,6 +17,7 @@ ARCH = IA32
 LIBRARIES = Common

 APPLICATIONS = \

   BootSectImage \

+  BrotliCompress \

   EfiLdrImage \

   EfiRom \

   GenBootSector \