Migrate to github actions
Not all combinations are migrated to the initial configuration; corresponding TODOs added.
Drive-by: additional combinations uncovered minor portability problems -> fixed
Drive-by: remove no-longer used "script" files.
diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
new file mode 100644
index 0000000..9324847
--- /dev/null
+++ b/.github/workflows/build_test.yml
@@ -0,0 +1,382 @@
+# Copyright 2021 Google Inc. All Rights Reserved.
+#
+# Distributed under MIT license.
+# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+
+# Workflow for building and running tests under Ubuntu
+
+name: Build/Test
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ types: [opened, reopened, labeled, synchronize]
+
+jobs:
+ ubuntu_build:
+ name: Build and test ${{ matrix.name }}
+ runs-on: ${{ matrix.os || 'ubuntu-latest' }}
+ defaults:
+ run:
+ shell: bash
+ strategy:
+ matrix:
+ include:
+ # TODO: consider running this combination in docker
+ #- name: cmake:gcc4.4
+
+ # TODO: consider running this combination in docker
+ #- name: cmake:gcc7
+
+ - name: cmake:gcc9
+ build_system: cmake
+ c_compiler: gcc-9
+ cxx_compiler: g++-9
+ os: ubuntu-18.04
+
+ # TODO: consider running this combination in docker
+ #- name: cmake:clang3.5
+
+ - name: cmake:clang12
+ build_system: cmake
+ c_compiler: clang-12
+ cxx_compiler: clang++12
+
+ - name: cmake:clang12:asan
+ build_system: cmake
+ sanitizer: address
+ c_compiler: clang-12
+ cxx_compiler: clang++12
+
+ - name: cmake:clang12:tsan
+ build_system: cmake
+ sanitizer: thread
+ c_compiler: clang-12
+ cxx_compiler: clang++12
+
+ - name: cmake:clang12:ubsan
+ build_system: cmake
+ sanitizer: undefined
+ c_compiler: clang-12
+ cxx_compiler: clang++-12
+ c_flags: -fno-sanitize-recover=undefined,integer
+
+ - name: cmake:qemu-arm-neon-gcc
+ build_system: cmake
+ c_compiler: arm-linux-gnueabihf-gcc
+ cxx_compiler: arm-linux-gnueabihf-g++
+ c_flags: -march=armv7-a -mfloat-abi=hard -mfpu=neon
+ extra_apt_pkgs: gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user
+
+ - name: cmake-osx:clang
+ build_system: cmake
+ c_compiler: clang
+ cxx_compiler: clang++
+ os: macos-latest
+
+ - name: cmake-osx:gcc
+ build_system: cmake
+ c_compiler: gcc
+ cxx_compiler: g++
+ os: macos-latest
+
+ - name: cmake-win64:msvc2017-rel
+ build_system: cmake
+ cmake_generator: Visual Studio 15 2017 Win64
+ cmake_config: Release
+ os: windows-2016
+
+ - name: cmake-win64:msvc2017-dbg
+ build_system: cmake
+ cmake_generator: Visual Studio 15 2017 Win64
+ cmake_config: Debug
+ os: windows-2016
+
+ # TODO: consider running this combination in docker
+ #- name: autotools:gcc5
+
+ - name: autotools:gcc10
+ build_system: autotools
+ c_compiler: gcc-10
+ cxx_compiler: g++-10
+
+ - name: autotools:clang12
+ build_system: autotools
+ c_compiler: clang-12
+ cxx_compiler: clang++12
+
+ - name: fuzz:clang12
+ build_system: fuzz
+ c_compiler: clang-12
+ cxx_compiler: clang++12
+
+ # TODO: consider running this combination in docker
+ #- name: python2.7:gcc5
+
+ - name: python27:clang9
+ build_system: python
+ c_compiler: clang-9
+ python_version: 2.7
+ cxx_compiler: clang++-9
+ os: ubuntu-18.04
+
+ # TODO: consider running this combination in docker
+ #- name: python27-win
+ # build_system: python
+ # python_version: 2.7
+ # # TODO: investigate why win-builds can't run tests
+ # py_setuptools_cmd: build_ext
+ # os: windows-2016
+
+ # TODO: consider running this combination in docker
+ #- name: python3.6:gcc5
+
+ # TODO: consider running this combination in docker
+ #- name: python3.7:gcc5
+
+ # TODO: consider running this combination in docker
+ #- name: python3.8:gcc5
+
+ - name: python39:clang12
+ build_system: python
+ python_version: 3.9
+ c_compiler: clang-12
+ cxx_compiler: clang++-12
+
+ - name: python39-win
+ build_system: python
+ python_version: 3.9
+ # TODO: investigate why win-builds can't run tests
+ py_setuptools_cmd: build_ext
+ os: windows-2019
+
+ - name: maven
+ build_system: maven
+
+ - name: bazel:root
+ build_system: bazel
+ bazel_project: .
+
+ - name: bazel:go
+ build_system: bazel
+ bazel_project: go
+
+ - name: bazel:java
+ build_system: bazel
+ bazel_project: java
+
+ - name: bazel:js
+ build_system: bazel
+ bazel_project: js
+
+ - name: bazel:research
+ build_system: bazel
+ bazel_project: research
+
+ - name: bazel-osx:root
+ build_system: bazel
+ bazel_project: .
+ os: macos-latest
+
+ - name: bazel-osx:go
+ build_system: bazel
+ bazel_project: go
+ os: macos-latest
+
+ - name: bazel-osx:java
+ build_system: bazel
+ bazel_project: java
+ os: macos-latest
+
+ - name: bazel-osx:js
+ build_system: bazel
+ bazel_project: js
+ os: macos-latest
+
+ - name: bazel-osx:research
+ build_system: bazel
+ bazel_project: research
+ os: macos-latest
+
+ - name: bazel-win:root
+ build_system: bazel
+ bazel_project: .
+ os: windows-latest
+
+ # TODO: use single dll on windows, otherwise it fails to link
+ #- name: bazel-win:go
+ # build_system: bazel
+ # bazel_project: go
+ # os: windows-latest
+
+ - name: bazel-win:java
+ build_system: bazel
+ bazel_project: java
+ os: windows-latest
+
+ # TODO: blocked by Bazel Closure rules issue
+ #- name: bazel-win:js
+ # build_system: bazel
+ # bazel_project: js
+ # os: windows-latest
+
+ - name: bazel-win:research
+ build_system: bazel
+ bazel_project: research
+ os: windows-latest
+
+ - name: make
+ build_system: make
+
+ - name: make-osx
+ build_system: make
+ os: macos-latest
+
+ # TODO: add 32/64-bit x MSYS2/mingw/Cygwin toolchain support.
+ - name: make-win
+ build_system: make
+ os: windows-latest
+
+ env:
+ CC: ${{ matrix.c_compiler || 'gcc' }}
+ CXX: ${{ matrix.cxx_compiler || 'gcc' }}
+
+ steps:
+ - name: Install extra deps @ Ubuntu
+ if: ${{ runner.os == 'Linux' }}
+ # Already installed: bazel, clang{10-12}, cmake, gcc{9,10}, java{8,11}, maven, python{2.7,3.5-3.9}
+ run: |
+ EXTRA_PACKAGES="${{ matrix.extra_apt_pkgs || '' }}"
+ sudo apt update
+ sudo apt install -y ${EXTRA_PACKAGES}
+
+ - name: Checkout the source
+ uses: actions/checkout@v2
+ with:
+ submodules: false
+ fetch-depth: 1
+
+ #- name: Checkout VC9 for Python
+ # if: ${{ runner.os == 'Windows' && matrix.build_system == 'python' && matrix.python_version == '2.7' }}
+ # uses: actions/checkout@v2
+ # with:
+ # repository: reider-roque/sulley-win-installer
+ # path: third_party/VCForPython27
+
+ - name: Configure / Build / Test with CMake
+ if: ${{ matrix.build_system == 'cmake' }}
+ run: |
+ export ASAN_OPTIONS=detect_leaks=0
+ declare -a CMAKE_OPTIONS=()
+ CMAKE_OPTIONS+=("-DCMAKE_VERBOSE_MAKEFILE=ON")
+ [ ! -z '${{ matrix.c_compiler || '' }}' ] && CMAKE_OPTIONS+=(-DCMAKE_C_COMPILER='${{ matrix.c_compiler }}')
+ [ ! -z '${{ matrix.cxx_compiler || '' }}' ] && CMAKE_OPTIONS+=(-DCMAKE_CXX_COMPILER='${{ matrix.cxx_compiler }}')
+ [ ! -z '${{ matrix.sanitizer || '' }}' ] && CMAKE_OPTIONS+=(-DENABLE_SANITIZER='${{ matrix.sanitizer }}')
+ [ ! -z '${{ matrix.cmake_generator || '' }}' ] && export CMAKE_GENERATOR='${{ matrix.cmake_generator }}'
+ declare -a CMAKE_BUILD_OPTIONS=()
+ [ ! -z '${{ matrix.cmake_config || '' }}' ] && CMAKE_BUILD_OPTIONS+=(--config '${{ matrix.cmake_config }}')
+ declare -a CMAKE_TEST_OPTIONS=()
+ [ ! -z '${{ matrix.cmake_config || '' }}' ] && CMAKE_TEST_OPTIONS+=(-C '${{ matrix.cmake_config }}')
+
+ cmake -B out . ${CMAKE_OPTIONS[*]} -DCMAKE_C_FLAGS='${{ matrix.c_flags || '' }}'
+ cmake --build out ${CMAKE_BUILD_OPTIONS[*]}
+ cd out; ctest ${CMAKE_TEST_OPTIONS[*]}; cd ..
+
+ - name: Quick Fuzz
+ if: ${{ matrix.build_system == 'fuzz' }}
+ run: |
+ export ASAN_OPTIONS=detect_leaks=0
+ ./c/fuzz/test_fuzzer.sh
+
+ - name: Build with Bazel
+ if: ${{ matrix.build_system == 'bazel' }}
+ run: |
+ cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }}
+ bazel build -c opt ...:all
+
+ - name: Fix symlinks for Bazel (Windows)
+ if: ${{ matrix.build_system == 'bazel' && runner.os == 'Windows' && matrix.bazel_project == 'java' }}
+ shell: python
+ run: |
+ import fnmatch
+ import os
+ import os.path
+ from shutil import copyfile
+ os.chdir('${{ matrix.bazel_project }}')
+ print('Searching for manifests in ' + os.getcwd())
+ matches = []
+ for root, dirnames, filenames in os.walk('bazel-bin\\org\\brotli'):
+ for filename in fnmatch.filter(filenames, '*.runfiles_manifest'):
+ matches.append(os.path.join(root, filename))
+ for match in matches:
+ print('Scanning manifest ' + match)
+ runfiles = match[:-len('_manifest')]
+ with open(match) as manifest:
+ for entry in manifest:
+ entry = entry.strip()
+ if not entry.startswith("org_brotli_java"):
+ continue
+ if entry.startswith('org_brotli_java/external'):
+ continue
+ (alias, space, link) = entry.partition(' ')
+ if alias.endswith('.jar') or alias.endswith('.exe'):
+ continue
+ link = link.replace('/', '\\')
+ alias = alias.replace('/', '\\')
+ dst = os.path.join(runfiles, alias)
+ if not os.path.exists(dst):
+ print(link + ' -> ' + dst)
+ parent = os.path.dirname(dst)
+ if not os.path.exists(parent):
+ os.makedirs(parent)
+ copyfile(link, dst)
+ print('Finished resolving symlinks')
+
+ - name: Test with Bazel
+ if: ${{ matrix.build_system == 'bazel' }}
+ run: |
+ cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }}
+ bazel query "tests(...)" --output=label > ${RUNNER_TEMP}/tests.lst
+ [ -s ${RUNNER_TEMP}/tests.lst ] && bazel test -c opt ...:all
+ bazel clean
+
+ - name: Build / Test with Maven
+ if: ${{ matrix.build_system == 'maven' }}
+ run: |
+ export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
+ cd java/org/brotli
+ mvn -B install
+ cd integration
+ mvn -B verify
+
+ - name: Build / Test with Autotools
+ if: ${{ matrix.build_system == 'autotools' }}
+ run: |
+ ./bootstrap && ./configure && make
+
+ - name: Build / Test with Make
+ if: ${{ matrix.build_system == 'make' }}
+ run: |
+ make brotli
+ make test
+
+ - uses: actions/setup-python@v2
+ if: ${{ matrix.build_system == 'python' }}
+ with:
+ python-version: ${{ matrix.python_version }}
+
+ # TODO: investigate, why msiexec hangs
+ #- name: Install VC9 for Python
+ # if: ${{ runner.os == 'Windows' && matrix.build_system == 'python' && matrix.python_version == '2.7' }}
+ # run: |
+ # echo "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf third_party/VCForPython27/VCForPython27.msi" | sha256sum --check --status
+ # msiexec ALLUSERS=1 /qn /norestart /i third_party/VCForPython27/VCForPython27.msi /l*v ${RUNNER_TEMP}/msiexec.log
+ # cat ${RUNNER_TEMP}/msiexec.log
+
+ - name: Build / Test with Python
+ if: ${{ matrix.build_system == 'python' }}
+ run: |
+ python -VV
+ python -c "import sys; sys.exit('Invalid python version') if '.'.join(map(str,sys.version_info[0:2])) != '${{ matrix.python_version }}' else True"
+ python setup.py ${{ matrix.py_setuptools_cmd || 'test'}}
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index e97adc3..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,267 +0,0 @@
-language: c
-sudo: false
-branches:
- only:
- - master
-
-cache:
- directories:
- - $HOME/Library/Caches/Homebrew
- - /usr/local/Homebrew
-
-before_cache:
- - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then brew cleanup; fi
- - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then find /usr/local/Homebrew \! -regex ".+\.git.+" -delete; fi
-
-matrix:
- include:
- ###
- ## Linux builds using various versions of GCC.
- ###
- - os: linux
- env: BUILD_SYSTEM=cmake C_COMPILER=gcc-7 CXX_COMPILER=g++-7
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-7
- - g++-7
- - os: linux
- env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.4 CXX_COMPILER=g++-4.4
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-4.4
- - g++-4.4
-
- ###
- ## Test that Autotools build works.
- ###
- - os: linux
- env: BUILD_SYSTEM=autotools C_COMPILER=gcc-5 CXX_COMPILER=g++-5
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-5
- - g++-5
-
- ###
- ## Test that fuzzer is compiling / working.
- ###
- - os: linux
- env: BUILD_SYSTEM=fuzz C_COMPILER=clang-7 CXX_COMPILER=clang++-7 ASAN_OPTIONS=detect_leaks=0
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- - llvm-toolchain-xenial-7
- packages:
- - clang-7
-
- ###
- ## clang on Linux
- ###
- - os: linux
- env: BUILD_SYSTEM=cmake C_COMPILER=clang-7 CXX_COMPILER=clang++-7
- addons:
- apt:
- sources:
- - llvm-toolchain-xenial-7
- - ubuntu-toolchain-r-test
- packages:
- - clang-7
- - os: linux
- env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.5 CXX_COMPILER=clang++-3.5
- addons:
- apt:
- sources:
- - llvm-toolchain-trusty-3.5
- - ubuntu-toolchain-r-test
- packages:
- - clang-3.5
-
- ###
- ## testing arm via qemu on Linux
- ###
- - os: linux
- env: BUILD_SYSTEM=cmake C_COMPILER=arm-linux-gnueabihf-gcc CXX_COMPILER=arm-linux-gnueabihf-g++ CFLAGS="-march=armv7-a -mfloat-abi=hard -mfpu=neon"
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - qemu
- - gcc-arm-linux-gnueabihf
- - libc6-dev-armhf-cross
-
- ###
- ## PGI Community Edition on Linux
- ###
- # Installer is currently broken
- #- os: linux
- # env: BUILD_SYSTEM=cmake C_COMPILER=pgcc CXX_COMPILER=pgc++
-
- ###
- ## Python builds on Linux
- ###
- - os: linux
- language: python
- python: 2.7
- env: BUILD_SYSTEM=python C_COMPILER=gcc-5 CXX_COMPILER=g++-5
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-5
- - g++-5
- - os: linux
- language: python
- python: 3.6
- env: BUILD_SYSTEM=python C_COMPILER=gcc-5 CXX_COMPILER=g++-5
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-5
- - g++-5
-
- - os: linux
- language: python
- python: 3.7
- env: BUILD_SYSTEM=python C_COMPILER=gcc-5 CXX_COMPILER=g++-5
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-5
- - g++-5
-
- - os: linux
- language: python
- python: 3.8
- env: BUILD_SYSTEM=python C_COMPILER=gcc-5 CXX_COMPILER=g++-5
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-5
- - g++-5
-
- ###
- ## CMake on OS X
- ##
- ## These all work, but it seems unnecessary to actually build them
- ## all since we already test all these versions of GCC on Linux.
- ## We'll just test 4.4 and the most recent version.
- ###
- - os: osx
- osx_image: xcode12.2
- env: BUILD_SYSTEM=cmake C_COMPILER=gcc CXX_COMPILER=g++
- #- os: osx
- # osx_image: xcode12.2
- # env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
- - os: osx
- osx_image: xcode12.2
- env: BUILD_SYSTEM=cmake
-
- ###
- ## Python 2.7 OS X build (using the system /usr/bin/python)
- ###
- - os: osx
- osx_image: xcode12.2
- env: BUILD_SYSTEM=python C_COMPILER=gcc CXX_COMPILER=g++
-
- ###
- ## Sanitizers
- ###
- - os: linux
- env: BUILD_SYSTEM=cmake C_COMPILER=clang-7 CXX_COMPILER=clang++-7 SANITIZER=address ASAN_OPTIONS=detect_leaks=0
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- - llvm-toolchain-xenial-7
- packages:
- - clang-7
- - os: linux
- env: BUILD_SYSTEM=cmake C_COMPILER=clang-7 CXX_COMPILER=clang++-7 SANITIZER=thread
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- - llvm-toolchain-xenial-7
- packages:
- - clang-7
- - os: linux
- env: BUILD_SYSTEM=cmake C_COMPILER=clang-7 CXX_COMPILER=clang++-7 SANITIZER=undefined CFLAGS="-fno-sanitize-recover=undefined,integer"
- addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- - llvm-toolchain-xenial-7
- packages:
- - clang-7
-
- - os: linux
- env: BUILD_SYSTEM=maven
- jdk:
- # maven + jdk11 + javadoc == trouble
- - openjdk10
- language: java
-
- - os: linux
- sudo: required
- language: java
- jdk: oraclejdk9
- env: BUILD_SYSTEM=bazel
- addons:
- apt:
- sources:
- - sourceline: "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8"
- key_url: "https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg"
- - ubuntu-toolchain-r-test
- packages:
- - bazel
-
- - os: osx
- osx_image: xcode12.2
- env: BUILD_SYSTEM=bazel
- language: java
-
-before_install:
-###
-## If we use the matrix to set CC/CXX Travis, overwrites the values,
-## so instead we use C/CXX_COMPILER, then copy the values to CC/CXX
-## here (after Travis has set CC/CXX).
-###
-- if [ -n "${C_COMPILER}" ]; then export CC="${C_COMPILER}"; fi
-- if [ -n "${CXX_COMPILER}" ]; then export CXX="${CXX_COMPILER}"; fi
-- scripts/.travis.sh before_install
-install:
-- scripts/.travis.sh install
-script:
-- scripts/.travis.sh script
-after_success:
-- scripts/.travis.sh after_success
-
-before_deploy:
-- scripts/.travis.sh before_deploy
-
-deploy:
-- provider: bintray
- file: "scripts/.bintray.json"
- user: "eustas"
- key:
- secure: "Kbam/lTAdz72fZivbs6riJT+Y4PbuKP7r6t5PAWxJxAAykjwnYTRe3zF472g9HCE14KYMsdB+KSYSgg6TGJnqGC9gL9xhhGU9U/WmA+vbMWS/MSnMWpK9IRpp77pM2i2NKZD4v33JuEwKFCBJP3Vj6QQ5Qd1NKdobuXJyznhgnw="
- on:
- condition: "${BUILD_SYSTEM} = bazel"
- skip_cleanup: true
diff --git a/configure.ac b/configure.ac
index 87d579d..9a3b285 100644
--- a/configure.ac
+++ b/configure.ac
@@ -11,4 +11,4 @@
AC_CONFIG_FILES([Makefile scripts/libbrotlicommon.pc scripts/libbrotlidec.pc scripts/libbrotlienc.pc])
-AC_OUTPUT
\ No newline at end of file
+AC_OUTPUT
diff --git a/docs/brotli.1 b/docs/brotli.1
index 7242a32..a3c9a13 100644
--- a/docs/brotli.1
+++ b/docs/brotli.1
@@ -1,4 +1,4 @@
-.TH "BROTLI" "1" "February 2018" "brotli 1.0.0" "User commands"
+.TH "BROTLI" "1" "August 2021" "" "User commands"
.SH "NAME"
\fBbrotli\fR \- brotli, unbrotli \- compress or decompress files
.SH SYNOPSIS
@@ -108,6 +108,10 @@
windows size improve density; decoder might require up to window size
memory to operate
.IP \(bu 2
+\fB\-D FILE\fP, \fB\-\-dictionary=FILE\fP:
+ use FILE as raw (LZ77) dictionary; same dictionary MUST be used both for
+ compression and decompression
+.IP \(bu 2
\fB\-S SUF\fP, \fB\-\-suffix=SUF\fP:
output file suffix (default: \fB\|\.br\fP)
.IP \(bu 2
diff --git a/docs/constants.h.3 b/docs/constants.h.3
index edf071c..1b29f28 100644
--- a/docs/constants.h.3
+++ b/docs/constants.h.3
@@ -1,4 +1,4 @@
-.TH "constants.h" 3 "Wed May 13 2020" "Brotli" \" -*- nroff -*-
+.TH "constants.h" 3 "August 2021" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME
diff --git a/docs/decode.h.3 b/docs/decode.h.3
index 7b8581c..14abda6 100644
--- a/docs/decode.h.3
+++ b/docs/decode.h.3
@@ -1,4 +1,4 @@
-.TH "decode.h" 3 "Thu Feb 22 2018" "Brotli" \" -*- nroff -*-
+.TH "decode.h" 3 "August 2021" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME
@@ -36,6 +36,10 @@
.in +1c
.ti -1c
+.RI "\fBBROTLI_BOOL\fP \fBBrotliDecoderAttachDictionary\fP (\fBBrotliDecoderState\fP *state, BrotliSharedDictionaryType type, size_t data_size, const uint8_t data[data_size])"
+.br
+.RI "\fIAdds LZ77 prefix dictionary, adds or replaces built-in static dictionary and transforms\&. \fP"
+.ti -1c
.RI "\fBBrotliDecoderState\fP * \fBBrotliDecoderCreateInstance\fP (\fBbrotli_alloc_func\fP alloc_func, \fBbrotli_free_func\fP free_func, void *opaque)"
.br
.RI "\fICreates an instance of \fBBrotliDecoderState\fP and initializes it\&. \fP"
@@ -167,6 +171,35 @@
Partially done; should be called again with more output\&.
.SH "Function Documentation"
.PP
+.SS "\fBBROTLI_BOOL\fP BrotliDecoderAttachDictionary (\fBBrotliDecoderState\fP * state, BrotliSharedDictionaryType type, size_t data_size, const uint8_t data[data_size])"
+
+.PP
+Adds LZ77 prefix dictionary, adds or replaces built-in static dictionary and transforms\&. Attached dictionary ownership is not transferred\&. Data provided to this method should be kept accessible until decoding is finished and decoder instance is destroyed\&.
+.PP
+\fBNote:\fP
+.RS 4
+Dictionaries could NOT be attached after actual decoding is started\&.
+.RE
+.PP
+\fBParameters:\fP
+.RS 4
+\fIstate\fP decoder instance
+.br
+\fItype\fP dictionary data format
+.br
+\fIdata_size\fP length of memory region pointed by \fCdata\fP
+.br
+\fIdata\fP dictionary data in format corresponding to \fCtype\fP
+.RE
+.PP
+\fBReturns:\fP
+.RS 4
+\fBBROTLI_FALSE\fP if dictionary is corrupted, or dictionary count limit is reached
+.PP
+\fBBROTLI_TRUE\fP if dictionary is accepted / attached
+.RE
+.PP
+
.SS "\fBBrotliDecoderState\fP* BrotliDecoderCreateInstance (\fBbrotli_alloc_func\fP alloc_func, \fBbrotli_free_func\fP free_func, void * opaque)"
.PP
diff --git a/docs/encode.h.3 b/docs/encode.h.3
index 0cab816..e31b6b4 100644
--- a/docs/encode.h.3
+++ b/docs/encode.h.3
@@ -1,4 +1,4 @@
-.TH "encode.h" 3 "Thu Feb 22 2018" "Brotli" \" -*- nroff -*-
+.TH "encode.h" 3 "August 2021" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME
@@ -76,6 +76,10 @@
.in +1c
.ti -1c
+.RI "\fBBROTLI_BOOL\fP \fBBrotliEncoderAttachPreparedDictionary\fP (\fBBrotliEncoderState\fP *state, const BrotliEncoderPreparedDictionary *dictionary)"
+.br
+.RI "\fIAttaches a prepared dictionary of any type to the encoder\&. \fP"
+.ti -1c
.RI "\fBBROTLI_BOOL\fP \fBBrotliEncoderCompress\fP (int quality, int lgwin, \fBBrotliEncoderMode\fP mode, size_t input_size, const uint8_t input_buffer[input_size], size_t *encoded_size, uint8_t encoded_buffer[*encoded_size])"
.br
.RI "\fIPerforms one-shot memory-to-memory compression\&. \fP"
@@ -104,6 +108,10 @@
.br
.RI "\fICalculates the output size bound for the given \fCinput_size\fP\&. \fP"
.ti -1c
+.RI "BrotliEncoderPreparedDictionary * \fBBrotliEncoderPrepareDictionary\fP (BrotliSharedDictionaryType type, size_t data_size, const uint8_t data[data_size], int quality, \fBbrotli_alloc_func\fP alloc_func, \fBbrotli_free_func\fP free_func, void *opaque)"
+.br
+.RI "\fIPrepares a shared dictionary from the given file format for the encoder\&. \fP"
+.ti -1c
.RI "\fBBROTLI_BOOL\fP \fBBrotliEncoderSetParameter\fP (\fBBrotliEncoderState\fP *state, \fBBrotliEncoderParameter\fP param, uint32_t value)"
.br
.RI "\fISets the specified parameter to the given encoder instance\&. \fP"
@@ -318,6 +326,19 @@
Range is not artificially limited, but all the values greater or equal to maximal window size have the same effect\&. Values greater than 2**30 are not allowed\&.
.SH "Function Documentation"
.PP
+.SS "\fBBROTLI_BOOL\fP BrotliEncoderAttachPreparedDictionary (\fBBrotliEncoderState\fP * state, const BrotliEncoderPreparedDictionary * dictionary)"
+
+.PP
+Attaches a prepared dictionary of any type to the encoder\&. Can be used multiple times to attach multiple dictionaries\&. The dictionary type was determined by BrotliEncoderPrepareDictionary\&. Multiple raw prefix dictionaries and/or max 1 serialized dictionary with custom words can be attached\&.
+.PP
+\fBReturns:\fP
+.RS 4
+\fBBROTLI_FALSE\fP in case of error
+.PP
+\fBBROTLI_TRUE\fP otherwise
+.RE
+.PP
+
.SS "\fBBROTLI_BOOL\fP BrotliEncoderCompress (int quality, int lgwin, \fBBrotliEncoderMode\fP mode, size_t input_size, const uint8_t input_buffer[input_size], size_t * encoded_size, uint8_t encoded_buffer[*encoded_size])"
.PP
@@ -509,6 +530,29 @@
.RE
.PP
+.SS "BrotliEncoderPreparedDictionary* BrotliEncoderPrepareDictionary (BrotliSharedDictionaryType type, size_t data_size, const uint8_t data[data_size], int quality, \fBbrotli_alloc_func\fP alloc_func, \fBbrotli_free_func\fP free_func, void * opaque)"
+
+.PP
+Prepares a shared dictionary from the given file format for the encoder\&. \fCalloc_func\fP and \fCfree_func\fP \fBMUST\fP be both zero or both non-zero\&. In the case they are both zero, default memory allocators are used\&. \fCopaque\fP is passed to \fCalloc_func\fP and \fCfree_func\fP when they are called\&. \fCfree_func\fP has to return without doing anything when asked to free a NULL pointer\&.
+.PP
+\fBParameters:\fP
+.RS 4
+\fItype\fP type of dictionary stored in data
+.br
+\fIdata_size\fP size of \fCdata\fP buffer
+.br
+\fIdata\fP pointer to the dictionary data
+.br
+\fIquality\fP the maximum Brotli quality to prepare the dictionary for, use BROTLI_MAX_QUALITY by default
+.br
+\fIalloc_func\fP custom memory allocation function
+.br
+\fIfree_func\fP custom memory free function
+.br
+\fIopaque\fP custom memory manager handle
+.RE
+.PP
+
.SS "\fBBROTLI_BOOL\fP BrotliEncoderSetParameter (\fBBrotliEncoderState\fP * state, \fBBrotliEncoderParameter\fP param, uint32_t value)"
.PP
diff --git a/docs/types.h.3 b/docs/types.h.3
index bef9313..3d8a98c 100644
--- a/docs/types.h.3
+++ b/docs/types.h.3
@@ -1,4 +1,4 @@
-.TH "types.h" 3 "Thu Feb 22 2018" "Brotli" \" -*- nroff -*-
+.TH "types.h" 3 "August 2021" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME
diff --git a/go/WORKSPACE b/go/WORKSPACE
index 8fb62e5..03d38da 100644
--- a/go/WORKSPACE
+++ b/go/WORKSPACE
@@ -20,4 +20,4 @@
go_rules_dependencies()
-go_register_toolchains(version = "1.16")
\ No newline at end of file
+go_register_toolchains(version = "1.16")
diff --git a/java/org/brotli/dec/pom.xml b/java/org/brotli/dec/pom.xml
index 8701874..e7f9fd9 100644
--- a/java/org/brotli/dec/pom.xml
+++ b/java/org/brotli/dec/pom.xml
@@ -76,6 +76,9 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version>
+ <configuration>
+ <source>8</source>
+ </configuration>
<executions>
<execution>
<id>attach-javadocs</id>
diff --git a/research/BUILD.libdivsufsort b/research/BUILD.libdivsufsort
index a6deb5b..aeea923 100644
--- a/research/BUILD.libdivsufsort
+++ b/research/BUILD.libdivsufsort
@@ -38,7 +38,8 @@
srcs = ["include/config.h.cmake"],
outs = ["include/config.h"],
cmd = ("awk '{ " +
- "gsub(/@HAVE_IO_H 1@/, \"HAVE_IO_H 0\"); " +
+ "gsub(/HAVE_IO_H 1/, \"HAVE_IO_H 0\"); " +
+ "gsub(/HAVE_STRINGS_H 1/, \"HAVE_STRINGS_H 0\"); " +
commom_awk_replaces +
"print; }' $(<) > $(@)"),
)
diff --git a/research/WORKSPACE b/research/WORKSPACE
index 358f87d..343c3bb 100644
--- a/research/WORKSPACE
+++ b/research/WORKSPACE
@@ -12,6 +12,7 @@
build_file = "@//:BUILD.libdivsufsort",
commit = "5f60d6f026c30fb4ac296f696b3c8b0eb71bd428",
remote = "https://github.com/y-256/libdivsufsort",
+ shallow_since = "1445958113 +0900",
)
new_git_repository(
@@ -27,4 +28,5 @@
""",
commit = "ca7cb332011ec37a8436487f210f396b84bd8273",
remote = "https://github.com/hillbig/esaxx",
+ shallow_since = "1391400691 +0900",
)
diff --git a/research/brotli_decoder.c b/research/brotli_decoder.c
index 4b0bc4a..3febcbd 100644
--- a/research/brotli_decoder.c
+++ b/research/brotli_decoder.c
@@ -6,7 +6,17 @@
#include <stdio.h>
#include <stdlib.h>
+
+#if !defined(_WIN32)
#include <unistd.h>
+#else
+#include <io.h>
+#define fdopen _fdopen
+#if !defined(__MINGW32__)
+#define STDIN_FILENO _fileno(stdin)
+#define STDOUT_FILENO _fileno(stdout)
+#endif
+#endif
#include <brotli/decode.h>
diff --git a/research/deorummolae.cc b/research/deorummolae.cc
index f12acd1..23ac1aa 100644
--- a/research/deorummolae.cc
+++ b/research/deorummolae.cc
@@ -5,6 +5,10 @@
#include "third_party/esaxx/sais.hxx"
+#if defined(_MSC_VER)
+#include <intrin.h> /* __popcnt64 */
+#endif
+
/* Used for quick SA-entry to file mapping. Each file is padded to size that
is a multiple of chunk size. */
#define CHUNK_SIZE 64
@@ -30,7 +34,11 @@
typedef int32_t TextSaIdx;
static size_t popcount(uint64_t u) {
+#if defined(_MSC_VER)
+ return static_cast<size_t>(__popcnt64(u));
+#else
return static_cast<size_t>(__builtin_popcountll(u));
+#endif
}
/* Condense terminators and pad file entries. */
diff --git a/research/dictionary_generator.cc b/research/dictionary_generator.cc
index 715723f..7df55ad 100644
--- a/research/dictionary_generator.cc
+++ b/research/dictionary_generator.cc
@@ -3,7 +3,9 @@
#include <cstdio>
#include <cstring>
#include <fstream>
+#if !defined(_MSC_VER)
#include <glob.h>
+#endif
#include <vector>
#include "./deorummolae.h"
@@ -268,14 +270,19 @@
}
}
+ bool ok = true;
+#if defined(_MSC_VER)
+ const char* resolved_path = argv[i];
+#else
glob_t resolved_paths;
memset(&resolved_paths, 0, sizeof(resolved_paths));
- bool ok = true;
if (glob(argv[i], GLOB_TILDE, NULL, &resolved_paths) == 0) {
for(size_t j = 0; j < resolved_paths.gl_pathc; ++j) {
- std::string content = readFile(resolved_paths.gl_pathv[j]);
+ const char* resolved_path = resolved_paths.gl_pathv[j];
+#endif
+ std::string content = readFile(resolved_path);
if (chunkLen == 0) {
- paths.emplace_back(resolved_paths.gl_pathv[j]);
+ paths.emplace_back(resolved_path);
data.insert(data.end(), content.begin(), content.end());
total += content.size();
sizes.push_back(content.size());
@@ -293,11 +300,13 @@
total += chunk.size();
sizes.push_back(chunk.size());
}
+#if !defined(_MSC_VER)
}
} else {
ok = false;
}
globfree(&resolved_paths);
+#endif
if (!ok) exit(1);
}
diff --git a/scripts/.bintray.json b/scripts/.bintray.json
deleted file mode 100644
index 17f99ed..0000000
--- a/scripts/.bintray.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "package": {
- "name": "bin",
- "repo": "brotli",
- "subject": "eustas"
- },
-
- "version": {"name": "latest"},
-
- "files": [
- {
- "includePattern": "brotli.zip",
- "uploadPattern": "brotli-${TRAVIS_OS_NAME}-${RELEASE_DATE}.zip",
- "matrix_params": {"override": 1}
- }
- ],
-
- "publish": true
-}
diff --git a/scripts/.configure-custom.sh b/scripts/.configure-custom.sh
deleted file mode 100644
index f1f641a..0000000
--- a/scripts/.configure-custom.sh
+++ /dev/null
@@ -1 +0,0 @@
-#!/usr/bin/env bash
diff --git a/scripts/.travis.sh b/scripts/.travis.sh
deleted file mode 100755
index b5ad13a..0000000
--- a/scripts/.travis.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/bash
-
-case "$1" in
- "before_install")
- ;;
- "install")
- case "${TRAVIS_OS_NAME}" in
- "osx")
- brew update >/dev/null
- brew install binutils
-
- case "${CC}" in
- "gcc-"*)
- which ${CC} || brew install $(echo "${CC}" | sed 's/\-/@/') || brew link --overwrite $(echo "${CC}" | sed 's/\-/@/')
- ;;
- esac
-
- case "${BUILD_SYSTEM}" in
- "bazel")
- brew cask install homebrew/cask-versions/adoptopenjdk8
- brew install bazel
- ;;
- esac
- ;;
- "linux")
- case "${CC}" in
- "pgcc")
- wget 'https://raw.githubusercontent.com/nemequ/pgi-travis/de6212d94fd0e7d07a6ef730c23548c337c436a7/install-pgi.sh'
- echo 'acd3ef995ad93cfb87d26f65147395dcbedd4c3c844ee6ec39616f1a347c8df5 install-pgi.sh' | sha256sum -c --strict || exit 1
- /bin/sh install-pgi.sh
- ;;
- esac
- ;;
- esac
- ;;
- "script")
- case "${BUILD_SYSTEM}" in
- "cmake")
- mkdir builddir && cd builddir
- CMAKE_FLAGS=
- if [ "${CROSS_COMPILE}" = "yes" ]; then
- CMAKE_FLAGS="-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_RC_COMPILER=${RC_COMPILER}"
- fi
- cmake ${CMAKE_FLAGS} -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DENABLE_SANITIZER="${SANITIZER}" -DCMAKE_C_FLAGS="${CFLAGS}" .. || exit 1
- make VERBOSE=1 || exit 1
- ctest -V || exit 1
- ;;
- "python")
- python setup.py test
- ;;
- "maven")
- cd java/org/brotli
- mvn install && cd integration && mvn verify
- ;;
- "autotools")
- ./bootstrap && ./configure && make
- ;;
- "fuzz")
- ./c/fuzz/test_fuzzer.sh
- ;;
- "bazel")
- bazel build -c opt ...:all &&
- cd go && bazel test -c opt ...:all && cd .. &&
- cd java && bazel test -c opt ...:all && cd .. &&
- cd js && bazel test -c opt ...:all && cd .. &&
- cd research && bazel build -c opt ...:all && cd ..
- ;;
- esac
- ;;
- "after_success")
- ;;
- "before_deploy")
- case "${BUILD_SYSTEM}" in
- "bazel")
- export RELEASE_DATE=`date +%Y-%m-%d`
- perl -p -i -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' scripts/.bintray.json
- zip -j9 brotli.zip bazel-bin/libbrotli*.a bazel-bin/libbrotli*.so bazel-bin/brotli
- ;;
- esac
- ;;
-esac
diff --git a/scripts/appveyor.yml b/scripts/appveyor.yml
deleted file mode 100644
index e52f65a..0000000
--- a/scripts/appveyor.yml
+++ /dev/null
@@ -1,111 +0,0 @@
-branches:
- only:
- - master
-
-environment:
- matrix:
- - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
- BUILD_SYSTEM: CMake
- GENERATOR: Visual Studio 15 2017 Win64
- CONFIG: Release
-
- - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
- BUILD_SYSTEM: CMake
- GENERATOR: Visual Studio 15 2017 Win64
- CONFIG: Debug
-
- - BUILD_SYSTEM: CMake
- GENERATOR: Visual Studio 14 2015 Win64
- CONFIG: Debug
-
- - BUILD_SYSTEM: CMake
- GENERATOR: Visual Studio 14 2015
- CONFIG: Debug
-
- - BUILD_SYSTEM: Python
- PYTHON: "C:\\Python27"
- PYTHON_VERSION: "2.7.x"
- PYTHON_ARCH: "32"
-
- - BUILD_SYSTEM: Python
- PYTHON: "C:\\Python36-x64"
- PYTHON_VERSION: "3.6.x"
- PYTHON_ARCH: "64"
-
- - BUILD_SYSTEM: make
- ARCH: "i686"
-
- - BUILD_SYSTEM: make
- ARCH: "x86_64"
-
- - BUILD_SYSTEM: bazel
-
-install:
-- IF "%BUILD_SYSTEM%"=="Python" (
- SET "PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%" &&
- python -m pip install --upgrade pip &&
- pip install --upgrade setuptools
- )
-- IF "%BUILD_SYSTEM%"=="make" (
- IF "%ARCH%"=="i686" (
- SET "TOOLCHAIN=i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32"
- ) ELSE (
- SET "TOOLCHAIN=x86_64-7.3.0-posix-seh-rt_v5-rev0\mingw64"
- )
- )
-- IF "%BUILD_SYSTEM%"=="bazel" (
- appveyor DownloadFile https://github.com/bazelbuild/bazel/releases/download/2.2.0/bazel-2.2.0-windows-x86_64.exe -FileName bazel.exe
- )
-
-before_build:
-- FOR /f %%i in ('C:\cygwin64\bin\date.exe +%%Y-%%m-%%d') DO SET "RELEASE_DATE=%%i"
-- IF "%BUILD_SYSTEM%"=="CMake" ( mkdir builddir && cd builddir && cmake -G "%GENERATOR%" .. )
-- IF "%BUILD_SYSTEM%"=="make" (
- SET "CC=gcc" &&
- SET "PATH=C:\mingw-w64\%TOOLCHAIN%\bin;%PATH%" &&
- COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\%TOOLCHAIN%\bin\make.exe
- )
-- SET "ROOT=%APPVEYOR_BUILD_FOLDER%"
-
-build_script:
-- IF "%BUILD_SYSTEM%"=="CMake" ( cmake --build . --config %CONFIG% )
-- IF "%BUILD_SYSTEM%"=="Python" ( python setup.py build_ext )
-- IF "%BUILD_SYSTEM%"=="make" (
- sh -c "make brotli" &&
- cd bin && 7z a -tzip -mx9 brotli-win-%ARCH%-%RELEASE_DATE%.zip brotli.exe &&
- appveyor PushArtifact brotli-win-%ARCH%-%RELEASE_DATE%.zip && cd ..
- )
-- IF "%BUILD_SYSTEM%"=="bazel" (
- cd java &&
- %ROOT%\bazel.exe --batch build -c opt org/brotli/wrapper/...:all &&
- python %ROOT%\scripts\fix-win-bazel-build.py &&
- cd bazel-bin &&
- 7z a -tzip -mx9 brotli-win-bazel-jni-%RELEASE_DATE%.zip brotli_jni.dll &&
- appveyor PushArtifact brotli-win-bazel-jni-%RELEASE_DATE%.zip &&
- cd .. &&
- cd ..
- )
-
-test_script:
-- IF "%BUILD_SYSTEM%"=="CMake" ( ctest --output-on-failure --interactive-debug-mode 0 -C %CONFIG% )
-- IF "%BUILD_SYSTEM%"=="Python" ( python setup.py test )
-- IF "%BUILD_SYSTEM%"=="make" ( sh -c "make test" )
-- IF "%BUILD_SYSTEM%"=="bazel" (
- cd java &&
- %ROOT%\bazel.exe --batch test -c opt --test_output streamed org/brotli/wrapper/...:all &&
- cd ..
- )
-
-deploy:
-- provider: BinTray
- on:
- branch: master
- username: eustas
- api_key:
- secure: B3rM8JYeIHIw6BfOjHggAeyTmxEf8ZDYmyF9gCwqyWaxyxECD9BuN50SiV2KE/RI
- subject: eustas
- repo: brotli
- package: bin
- version: latest
- publish: true
- override: true
diff --git a/scripts/fix-win-bazel-build.py b/scripts/fix-win-bazel-build.py
deleted file mode 100644
index 7a9b211..0000000
--- a/scripts/fix-win-bazel-build.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import fnmatch
-import os
-import os.path
-from shutil import copyfile
-
-print('Searching for manifests...')
-
-matches = []
-for root, dirnames, filenames in os.walk('bazel-bin\\org\\brotli'):
- for filename in fnmatch.filter(filenames, '*.runfiles_manifest'):
- matches.append(os.path.join(root, filename))
-
-for match in matches:
- print('Scanning manifest ' + match)
- runfiles = match[:-len('_manifest')]
- with open(match) as manifest:
- for entry in manifest:
- entry = entry.strip()
- if not entry.startswith("org_brotli_java"):
- continue
- if entry.startswith('org_brotli_java/external'):
- continue
- (alias, space, link) = entry.partition(' ')
- if alias.endswith('.jar') or alias.endswith('.exe'):
- continue
- link = link.replace('/', '\\')
- alias = alias.replace('/', '\\')
- dst = os.path.join(runfiles, alias)
- if not os.path.exists(dst):
- print(link + ' -> ' + dst)
- parent = os.path.dirname(dst)
- if not os.path.exists(parent):
- os.makedirs(parent)
- copyfile(link, dst)
-
-print('Finished resolving symlinks')
