diff options
Diffstat (limited to 'Utilities')
357 files changed, 12787 insertions, 6305 deletions
diff --git a/Utilities/IWYU/mapping.imp b/Utilities/IWYU/mapping.imp index 892cdcb1e..5af8b85dd 100644 --- a/Utilities/IWYU/mapping.imp +++ b/Utilities/IWYU/mapping.imp @@ -52,6 +52,9 @@ #{ symbol: [ "std::pair", private, "<map>", public ] }, #{ symbol: [ "std::pair", private, "<set>", public ] }, + # HACK: iwyu wrongly thinks that <system_error> is needed for std::hash + { symbol: [ "std::hash", private, "<functional>", public ] }, + # __decay_and_strip is used internally in the C++11 standard library. # IWYU does not classify it as internal and suggests to add <type_traits>. # To ignore it, we simply map it to a file that is included anyway. diff --git a/Utilities/Release/CMakeLogo.ico b/Utilities/Release/CMakeLogo.ico Binary files differindex e13bb156f..c1006122b 100644 --- a/Utilities/Release/CMakeLogo.ico +++ b/Utilities/Release/CMakeLogo.ico diff --git a/Utilities/Release/create-cmake-release.cmake b/Utilities/Release/create-cmake-release.cmake index 3af1b035f..b3cc35232 100644 --- a/Utilities/Release/create-cmake-release.cmake +++ b/Utilities/Release/create-cmake-release.cmake @@ -54,7 +54,7 @@ cd \${name}-build && -DCMAKE_INSTALL_PREFIX=\"\$inst/\" \\ -DCMAKE_DOC_DIR=doc/cmake \\ -DSPHINX_EXECUTABLE=\"${SPHINX_EXECUTABLE}\" \\ - -DSPHINX_HTML=ON -DSPHINX_MAN=ON && + -DSPHINX_HTML=ON -DSPHINX_MAN=ON -DSPHINX_QTHELP=ON && make install && cd .. && tar czf \${name}.tar.gz \${name} || diff --git a/Utilities/Release/linux64_release.cmake b/Utilities/Release/linux64_release.cmake index 04a74ac80..dc3412020 100644 --- a/Utilities/Release/linux64_release.cmake +++ b/Utilities/Release/linux64_release.cmake @@ -3,8 +3,8 @@ set(BOOTSTRAP_ARGS "--docdir=doc/cmake") set(HOST linux64) set(MAKE_PROGRAM "make") set(CPACK_BINARY_GENERATORS "STGZ TGZ") -set(CC /opt/gcc-6.1.0/bin/gcc) -set(CXX /opt/gcc-6.1.0/bin/g++) +set(CC /opt/gcc-8.2.0/bin/gcc) +set(CXX /opt/gcc-8.2.0/bin/g++) set(CFLAGS "") set(CXXFLAGS "") set(qt_prefix "/home/kitware/qt-5.7.0") @@ -34,12 +34,16 @@ OPENSSL_INCLUDE_DIR:PATH=/home/kitware/openssl-1.1.0h/include OPENSSL_SSL_LIBRARY:FILEPATH=/home/kitware/openssl-1.1.0h/lib/libssl.a PYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3 CPACK_SYSTEM_NAME:STRING=Linux-x86_64 +BUILD_CursesDialog:BOOL=ON BUILD_QtDialog:BOOL=TRUE CMAKE_SKIP_BOOTSTRAP_TEST:STRING=TRUE CMake_GUI_DISTRIBUTE_WITH_Qt_LGPL:STRING=3 CMAKE_PREFIX_PATH:STRING=${qt_prefix} CMake_QT_STATIC_QXcbIntegrationPlugin_LIBRARIES:STRING=${qt_xcb_libs} ") +set(ENV [[ +export CMAKE_PREFIX_PATH=/opt/binutils-2.31 +]]) # Exclude Qt5 tests because our Qt5 is static. set(EXTRA_CTEST_ARGS "-E Qt5") diff --git a/Utilities/Release/osx_release.cmake b/Utilities/Release/osx_release.cmake index c69eb1187..be11d470b 100644 --- a/Utilities/Release/osx_release.cmake +++ b/Utilities/Release/osx_release.cmake @@ -19,6 +19,7 @@ CMAKE_OSX_ARCHITECTURES:STRING=x86_64 CMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7 CMAKE_SKIP_BOOTSTRAP_TEST:STRING=TRUE CPACK_SYSTEM_NAME:STRING=Darwin-x86_64 +BUILD_CursesDialog:BOOL=ON BUILD_QtDialog:BOOL=TRUE CMake_GUI_DISTRIBUTE_WITH_Qt_LGPL:STRING=3 CMake_INSTALL_DEPENDENCIES:BOOL=ON diff --git a/Utilities/Release/upload_release.cmake b/Utilities/Release/upload_release.cmake index 1edcd65b1..bbc74379d 100644 --- a/Utilities/Release/upload_release.cmake +++ b/Utilities/Release/upload_release.cmake @@ -1,6 +1,6 @@ set(CTEST_RUN_CURRENT_SCRIPT 0) if(NOT VERSION) - set(VERSION 3.12) + set(VERSION 3.13) endif() if(NOT DEFINED PROJECT_PREFIX) set(PROJECT_PREFIX cmake-${VERSION}) diff --git a/Utilities/Release/win32_release.cmake b/Utilities/Release/win32_release.cmake index f9e35a543..2e817d925 100644 --- a/Utilities/Release/win32_release.cmake +++ b/Utilities/Release/win32_release.cmake @@ -39,6 +39,6 @@ get_filename_component(path "${CMAKE_CURRENT_LIST_FILE}" PATH) set(GIT_EXTRA "git config core.autocrlf true") if(CMAKE_CREATE_VERSION STREQUAL "nightly") # Some tests fail spuriously too often. - set(EXTRA_CTEST_ARGS "-E 'ConsoleBuf'") + set(EXTRA_CTEST_ARGS "-E 'ConsoleBuf|Module.ExternalData'") endif() include(${path}/release_cmake.cmake) diff --git a/Utilities/Release/win64_release.cmake b/Utilities/Release/win64_release.cmake index 02e409652..33af830c4 100644 --- a/Utilities/Release/win64_release.cmake +++ b/Utilities/Release/win64_release.cmake @@ -39,6 +39,6 @@ get_filename_component(path "${CMAKE_CURRENT_LIST_FILE}" PATH) set(GIT_EXTRA "git config core.autocrlf true") if(CMAKE_CREATE_VERSION STREQUAL "nightly") # Some tests fail spuriously too often. - set(EXTRA_CTEST_ARGS "-E 'ConsoleBuf'") + set(EXTRA_CTEST_ARGS "-E 'ConsoleBuf|Module.ExternalData'") endif() include(${path}/release_cmake.cmake) diff --git a/Utilities/Scripts/BoostScanDeps.cmake b/Utilities/Scripts/BoostScanDeps.cmake index bd2e6a879..28a94ce89 100644 --- a/Utilities/Scripts/BoostScanDeps.cmake +++ b/Utilities/Scripts/BoostScanDeps.cmake @@ -51,12 +51,12 @@ function(_Boost_FIND_COMPONENT_DEPENDENCIES component includedir _ret_libs) # Special-case since it is part of mpi; look only in boost/mpi/python* if(component STREQUAL "mpi_python") - set(_boost_DEPS "python") + set(_boost_DEPS "python\${component_python_version}") set(library_component TRUE) set(_boost_unprocessed_headers ${_boost_mpi_python_headers}) # Special-case since it is part of python; look only in boost/python/numpy* elseif(component STREQUAL "numpy") - set(_boost_DEPS "python") + set(_boost_DEPS "python\${component_python_version}") set(library_component TRUE) set(_boost_unprocessed_headers ${_boost_python_numpy_headers}) # Special-case since it is a serialization variant; look in boost/serialization diff --git a/Utilities/Scripts/update-curl.bash b/Utilities/Scripts/update-curl.bash index b02180f09..fb46052e8 100755 --- a/Utilities/Scripts/update-curl.bash +++ b/Utilities/Scripts/update-curl.bash @@ -8,7 +8,7 @@ readonly name="curl" readonly ownership="Curl Upstream <curl-library@cool.haxx.se>" readonly subtree="Utilities/cmcurl" readonly repo="https://github.com/curl/curl.git" -readonly tag="curl-7_60_0" +readonly tag="curl-7_62_0" readonly shortlog=false readonly paths=" CMake/* diff --git a/Utilities/Scripts/update-libarchive.bash b/Utilities/Scripts/update-libarchive.bash index 7534f9447..31886584f 100755 --- a/Utilities/Scripts/update-libarchive.bash +++ b/Utilities/Scripts/update-libarchive.bash @@ -8,7 +8,7 @@ readonly name="LibArchive" readonly ownership="LibArchive Upstream <libarchive-discuss@googlegroups.com>" readonly subtree="Utilities/cmlibarchive" readonly repo="https://github.com/libarchive/libarchive.git" -readonly tag="v3.3.2" +readonly tag="v3.3.3" readonly shortlog=false readonly paths=" CMakeLists.txt diff --git a/Utilities/Scripts/update-liblzma.bash b/Utilities/Scripts/update-liblzma.bash index 088eb91e0..fdf66b3ec 100755 --- a/Utilities/Scripts/update-liblzma.bash +++ b/Utilities/Scripts/update-liblzma.bash @@ -7,8 +7,8 @@ shopt -s dotglob readonly name="liblzma" readonly ownership="liblzma upstream <xz-devel@tukaani.org>" readonly subtree="Utilities/cmliblzma" -readonly repo="http://git.tukaani.org/xz.git" -readonly tag="v5.0.8" +readonly repo="https://git.tukaani.org/xz.git" +readonly tag="v5.2.4" readonly shortlog=false readonly paths=" COPYING @@ -24,6 +24,10 @@ extract_source () { mv src/common . mv src/liblzma . rmdir src + rm liblzma/Makefile.* + rm liblzma/*/Makefile.* + rm liblzma/liblzma.map + rm liblzma/validate_map.sh popd } diff --git a/Utilities/Sphinx/cmake.py b/Utilities/Sphinx/cmake.py index b0c8f7186..ebf44da45 100644 --- a/Utilities/Sphinx/cmake.py +++ b/Utilities/Sphinx/cmake.py @@ -144,6 +144,7 @@ class _cmake_index_entry: _cmake_index_objs = { 'command': _cmake_index_entry('command'), + 'cpack_gen': _cmake_index_entry('cpack generator'), 'envvar': _cmake_index_entry('envvar'), 'generator': _cmake_index_entry('generator'), 'manual': _cmake_index_entry('manual'), @@ -325,6 +326,7 @@ class CMakeDomain(Domain): label = 'CMake' object_types = { 'command': ObjType('command', 'command'), + 'cpack_gen': ObjType('cpack_gen', 'cpack_gen'), 'envvar': ObjType('envvar', 'envvar'), 'generator': ObjType('generator', 'generator'), 'variable': ObjType('variable', 'variable'), @@ -358,6 +360,7 @@ class CMakeDomain(Domain): } roles = { 'command': CMakeXRefRole(fix_parens = True, lowercase = True), + 'cpack_gen': CMakeXRefRole(), 'envvar': CMakeXRefRole(), 'generator': CMakeXRefRole(), 'variable': CMakeXRefRole(), diff --git a/Utilities/Sphinx/create_identifiers.py b/Utilities/Sphinx/create_identifiers.py index e63895004..6716b480d 100755 --- a/Utilities/Sphinx/create_identifiers.py +++ b/Utilities/Sphinx/create_identifiers.py @@ -21,6 +21,7 @@ newlines = [] for line in lines: mapping = (("command", "command"), + ("cpack generator", "cpack_gen"), ("envvar", "envvar"), ("variable", "variable"), ("generator", "generator"), diff --git a/Utilities/Sphinx/static/cmake-favicon.ico b/Utilities/Sphinx/static/cmake-favicon.ico Binary files differindex fce8f922f..ae529f54e 100644 --- a/Utilities/Sphinx/static/cmake-favicon.ico +++ b/Utilities/Sphinx/static/cmake-favicon.ico diff --git a/Utilities/Sphinx/static/cmake-logo-16.png b/Utilities/Sphinx/static/cmake-logo-16.png Binary files differindex 2039c25b0..9c95b75f3 100644 --- a/Utilities/Sphinx/static/cmake-logo-16.png +++ b/Utilities/Sphinx/static/cmake-logo-16.png diff --git a/Utilities/cmcurl/CMake/CMakeConfigurableFile.in b/Utilities/cmcurl/CMake/CMakeConfigurableFile.in index 4cf74a12b..df2c382e9 100644 --- a/Utilities/cmcurl/CMake/CMakeConfigurableFile.in +++ b/Utilities/cmcurl/CMake/CMakeConfigurableFile.in @@ -1,2 +1 @@ @CMAKE_CONFIGURABLE_FILE_CONTENT@ - diff --git a/Utilities/cmcurl/CMake/CurlSymbolHiding.cmake b/Utilities/cmcurl/CMake/CurlSymbolHiding.cmake index 9f7d29633..15ba46e46 100644 --- a/Utilities/cmcurl/CMake/CurlSymbolHiding.cmake +++ b/Utilities/cmcurl/CMake/CurlSymbolHiding.cmake @@ -4,57 +4,57 @@ option(CURL_HIDDEN_SYMBOLS "Set to ON to hide libcurl internal symbols (=hide al mark_as_advanced(CURL_HIDDEN_SYMBOLS) if(CURL_HIDDEN_SYMBOLS) - set(SUPPORTS_SYMBOL_HIDING FALSE) + set(SUPPORTS_SYMBOL_HIDING FALSE) - if(CMAKE_C_COMPILER_ID MATCHES "Clang") - set(SUPPORTS_SYMBOL_HIDING TRUE) - set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") - set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") - elseif(CMAKE_COMPILER_IS_GNUCC) - if(NOT CMAKE_VERSION VERSION_LESS 2.8.10) - set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION}) - else() - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion - OUTPUT_VARIABLE GCC_VERSION) - endif() - if(NOT GCC_VERSION VERSION_LESS 3.4) - # note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact - set(SUPPORTS_SYMBOL_HIDING TRUE) - set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") - set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") - endif() - elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0) - set(SUPPORTS_SYMBOL_HIDING TRUE) - set(_SYMBOL_EXTERN "__global") - set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden") - elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) - # note: this should probably just check for version 9.1.045 but I'm not 100% sure - # so let's to it the same way autotools do. - set(SUPPORTS_SYMBOL_HIDING TRUE) - set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") - set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") - check_c_source_compiles("#include <stdio.h> - int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug) - if(NOT _no_bug) - set(SUPPORTS_SYMBOL_HIDING FALSE) - set(_SYMBOL_EXTERN "") - set(_CFLAG_SYMBOLS_HIDE "") - endif() - elseif(MSVC) - set(SUPPORTS_SYMBOL_HIDING TRUE) + if(CMAKE_C_COMPILER_ID MATCHES "Clang") + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") + elseif(CMAKE_COMPILER_IS_GNUCC) + if(NOT CMAKE_VERSION VERSION_LESS 2.8.10) + set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION}) + else() + execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion + OUTPUT_VARIABLE GCC_VERSION) + endif() + if(NOT GCC_VERSION VERSION_LESS 3.4) + # note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") endif() + elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0) + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__global") + set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden") + elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) + # note: this should probably just check for version 9.1.045 but I'm not 100% sure + # so let's to it the same way autotools do. + set(SUPPORTS_SYMBOL_HIDING TRUE) + set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))") + set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden") + check_c_source_compiles("#include <stdio.h> + int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug) + if(NOT _no_bug) + set(SUPPORTS_SYMBOL_HIDING FALSE) + set(_SYMBOL_EXTERN "") + set(_CFLAG_SYMBOLS_HIDE "") + endif() + elseif(MSVC) + set(SUPPORTS_SYMBOL_HIDING TRUE) + endif() - set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING}) + set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING}) elseif(MSVC) - if(NOT CMAKE_VERSION VERSION_LESS 3.7) - set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken - set(HIDES_CURL_PRIVATE_SYMBOLS FALSE) - else() - message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.") - set(HIDES_CURL_PRIVATE_SYMBOLS TRUE) - endif() -elseif() + if(NOT CMAKE_VERSION VERSION_LESS 3.7) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken set(HIDES_CURL_PRIVATE_SYMBOLS FALSE) + else() + message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.") + set(HIDES_CURL_PRIVATE_SYMBOLS TRUE) + endif() +elseif() + set(HIDES_CURL_PRIVATE_SYMBOLS FALSE) endif() set(CURL_CFLAG_SYMBOLS_HIDE ${_CFLAG_SYMBOLS_HIDE}) diff --git a/Utilities/cmcurl/CMake/CurlTests.c b/Utilities/cmcurl/CMake/CurlTests.c index bc36c8ef7..9388c835b 100644 --- a/Utilities/cmcurl/CMake/CurlTests.c +++ b/Utilities/cmcurl/CMake/CurlTests.c @@ -507,30 +507,30 @@ main () #ifdef HAVE_GLIBC_STRERROR_R #include <string.h> #include <errno.h> + +void check(char c) {} + int main () { - char buffer[1024]; /* big enough to play with */ - char *string = - strerror_r(EACCES, buffer, sizeof(buffer)); - /* this should've returned a string */ - if(!string || !string[0]) - return 99; - return 0; + char buffer[1024]; + /* This will not compile if strerror_r does not return a char* */ + check(strerror_r(EACCES, buffer, sizeof(buffer))[0]); + return 0; } #endif #ifdef HAVE_POSIX_STRERROR_R #include <string.h> #include <errno.h> + +/* float, because a pointer can't be implicitly cast to float */ +void check(float f) {} + int main () { - char buffer[1024]; /* big enough to play with */ - int error = - strerror_r(EACCES, buffer, sizeof(buffer)); - /* This should've returned zero, and written an error string in the - buffer.*/ - if(!buffer[0] || error) - return 99; - return 0; + char buffer[1024]; + /* This will not compile if strerror_r does not return an int */ + check(strerror_r(EACCES, buffer, sizeof(buffer))); + return 0; } #endif #ifdef HAVE_FSETXATTR_6 @@ -549,3 +549,19 @@ main() { return 0; } #endif +#ifdef HAVE_CLOCK_GETTIME_MONOTONIC +#include <time.h> +int +main() { + struct timespec ts = {0, 0}; + clock_gettime(CLOCK_MONOTONIC, &ts); + return 0; +} +#endif +#ifdef HAVE_BUILTIN_AVAILABLE +int +main() { + if(__builtin_available(macOS 10.12, *)) {} + return 0; +} +#endif diff --git a/Utilities/cmcurl/CMake/FindCARES.cmake b/Utilities/cmcurl/CMake/FindCARES.cmake index c4ab5f132..723044a64 100644 --- a/Utilities/cmcurl/CMake/FindCARES.cmake +++ b/Utilities/cmcurl/CMake/FindCARES.cmake @@ -7,36 +7,36 @@ # also defined, but not for general use are # CARES_LIBRARY, where to find the c-ares library. -FIND_PATH(CARES_INCLUDE_DIR ares.h +find_path(CARES_INCLUDE_DIR ares.h /usr/local/include /usr/include ) -SET(CARES_NAMES ${CARES_NAMES} cares) -FIND_LIBRARY(CARES_LIBRARY +set(CARES_NAMES ${CARES_NAMES} cares) +find_library(CARES_LIBRARY NAMES ${CARES_NAMES} PATHS /usr/lib /usr/local/lib ) -IF (CARES_LIBRARY AND CARES_INCLUDE_DIR) - SET(CARES_LIBRARIES ${CARES_LIBRARY}) - SET(CARES_FOUND "YES") -ELSE (CARES_LIBRARY AND CARES_INCLUDE_DIR) - SET(CARES_FOUND "NO") -ENDIF (CARES_LIBRARY AND CARES_INCLUDE_DIR) +if(CARES_LIBRARY AND CARES_INCLUDE_DIR) + set(CARES_LIBRARIES ${CARES_LIBRARY}) + set(CARES_FOUND "YES") +else() + set(CARES_FOUND "NO") +endif() -IF (CARES_FOUND) - IF (NOT CARES_FIND_QUIETLY) - MESSAGE(STATUS "Found c-ares: ${CARES_LIBRARIES}") - ENDIF (NOT CARES_FIND_QUIETLY) -ELSE (CARES_FOUND) - IF (CARES_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find c-ares library") - ENDIF (CARES_FIND_REQUIRED) -ENDIF (CARES_FOUND) +if(CARES_FOUND) + if(NOT CARES_FIND_QUIETLY) + message(STATUS "Found c-ares: ${CARES_LIBRARIES}") + endif() +else() + if(CARES_FIND_REQUIRED) + message(FATAL_ERROR "Could not find c-ares library") + endif() +endif() -MARK_AS_ADVANCED( +mark_as_advanced( CARES_LIBRARY CARES_INCLUDE_DIR ) diff --git a/Utilities/cmcurl/CMake/FindGSS.cmake b/Utilities/cmcurl/CMake/FindGSS.cmake index 60dcb73c9..7a637fc29 100644 --- a/Utilities/cmcurl/CMake/FindGSS.cmake +++ b/Utilities/cmcurl/CMake/FindGSS.cmake @@ -28,211 +28,209 @@ set(_GSS_ROOT_HINTS # try to find library using system pkg-config if user didn't specify root dir if(NOT GSS_ROOT_DIR AND NOT "$ENV{GSS_ROOT_DIR}") - if(UNIX) - find_package(PkgConfig QUIET) - pkg_search_module(_GSS_PKG ${_MIT_MODNAME} ${_HEIMDAL_MODNAME}) - list(APPEND _GSS_ROOT_HINTS "${_GSS_PKG_PREFIX}") - elseif(WIN32) - list(APPEND _GSS_ROOT_HINTS "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MIT\\Kerberos;InstallDir]") - endif() + if(UNIX) + find_package(PkgConfig QUIET) + pkg_search_module(_GSS_PKG ${_MIT_MODNAME} ${_HEIMDAL_MODNAME}) + list(APPEND _GSS_ROOT_HINTS "${_GSS_PKG_PREFIX}") + elseif(WIN32) + list(APPEND _GSS_ROOT_HINTS "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MIT\\Kerberos;InstallDir]") + endif() endif() if(NOT _GSS_FOUND) #not found by pkg-config. Let's take more traditional approach. - find_file(_GSS_CONFIGURE_SCRIPT - NAMES - "krb5-config" - HINTS - ${_GSS_ROOT_HINTS} - PATH_SUFFIXES - bin - NO_CMAKE_PATH - NO_CMAKE_ENVIRONMENT_PATH + find_file(_GSS_CONFIGURE_SCRIPT + NAMES + "krb5-config" + HINTS + ${_GSS_ROOT_HINTS} + PATH_SUFFIXES + bin + NO_CMAKE_PATH + NO_CMAKE_ENVIRONMENT_PATH + ) + + # if not found in user-supplied directories, maybe system knows better + find_file(_GSS_CONFIGURE_SCRIPT + NAMES + "krb5-config" + PATH_SUFFIXES + bin + ) + + if(_GSS_CONFIGURE_SCRIPT) + execute_process( + COMMAND ${_GSS_CONFIGURE_SCRIPT} "--cflags" "gssapi" + OUTPUT_VARIABLE _GSS_CFLAGS + RESULT_VARIABLE _GSS_CONFIGURE_FAILED + ) + message(STATUS "CFLAGS: ${_GSS_CFLAGS}") + if(NOT _GSS_CONFIGURE_FAILED) # 0 means success + # should also work in an odd case when multiple directories are given + string(STRIP "${_GSS_CFLAGS}" _GSS_CFLAGS) + string(REGEX REPLACE " +-I" ";" _GSS_CFLAGS "${_GSS_CFLAGS}") + string(REGEX REPLACE " +-([^I][^ \\t;]*)" ";-\\1"_GSS_CFLAGS "${_GSS_CFLAGS}") + + foreach(_flag ${_GSS_CFLAGS}) + if(_flag MATCHES "^-I.*") + string(REGEX REPLACE "^-I" "" _val "${_flag}") + list(APPEND _GSS_INCLUDE_DIR "${_val}") + else() + list(APPEND _GSS_COMPILER_FLAGS "${_flag}") + endif() + endforeach() + endif() + + execute_process( + COMMAND ${_GSS_CONFIGURE_SCRIPT} "--libs" "gssapi" + OUTPUT_VARIABLE _GSS_LIB_FLAGS + RESULT_VARIABLE _GSS_CONFIGURE_FAILED ) + message(STATUS "LDFLAGS: ${_GSS_LIB_FLAGS}") + + if(NOT _GSS_CONFIGURE_FAILED) # 0 means success + # this script gives us libraries and link directories. Blah. We have to deal with it. + string(STRIP "${_GSS_LIB_FLAGS}" _GSS_LIB_FLAGS) + string(REGEX REPLACE " +-(L|l)" ";-\\1" _GSS_LIB_FLAGS "${_GSS_LIB_FLAGS}") + string(REGEX REPLACE " +-([^Ll][^ \\t;]*)" ";-\\1"_GSS_LIB_FLAGS "${_GSS_LIB_FLAGS}") + + foreach(_flag ${_GSS_LIB_FLAGS}) + if(_flag MATCHES "^-l.*") + string(REGEX REPLACE "^-l" "" _val "${_flag}") + list(APPEND _GSS_LIBRARIES "${_val}") + elseif(_flag MATCHES "^-L.*") + string(REGEX REPLACE "^-L" "" _val "${_flag}") + list(APPEND _GSS_LINK_DIRECTORIES "${_val}") + else() + list(APPEND _GSS_LINKER_FLAGS "${_flag}") + endif() + endforeach() + endif() - # if not found in user-supplied directories, maybe system knows better - find_file(_GSS_CONFIGURE_SCRIPT - NAMES - "krb5-config" - PATH_SUFFIXES - bin + execute_process( + COMMAND ${_GSS_CONFIGURE_SCRIPT} "--version" + OUTPUT_VARIABLE _GSS_VERSION + RESULT_VARIABLE _GSS_CONFIGURE_FAILED ) - if(_GSS_CONFIGURE_SCRIPT) - execute_process( - COMMAND ${_GSS_CONFIGURE_SCRIPT} "--cflags" "gssapi" - OUTPUT_VARIABLE _GSS_CFLAGS - RESULT_VARIABLE _GSS_CONFIGURE_FAILED - ) -message(STATUS "CFLAGS: ${_GSS_CFLAGS}") - if(NOT _GSS_CONFIGURE_FAILED) # 0 means success - # should also work in an odd case when multiple directories are given - string(STRIP "${_GSS_CFLAGS}" _GSS_CFLAGS) - string(REGEX REPLACE " +-I" ";" _GSS_CFLAGS "${_GSS_CFLAGS}") - string(REGEX REPLACE " +-([^I][^ \\t;]*)" ";-\\1"_GSS_CFLAGS "${_GSS_CFLAGS}") - - foreach(_flag ${_GSS_CFLAGS}) - if(_flag MATCHES "^-I.*") - string(REGEX REPLACE "^-I" "" _val "${_flag}") - list(APPEND _GSS_INCLUDE_DIR "${_val}") - else() - list(APPEND _GSS_COMPILER_FLAGS "${_flag}") - endif() - endforeach() - endif() + # older versions may not have the "--version" parameter. In this case we just don't care. + if(_GSS_CONFIGURE_FAILED) + set(_GSS_VERSION 0) + endif() - execute_process( - COMMAND ${_GSS_CONFIGURE_SCRIPT} "--libs" "gssapi" - OUTPUT_VARIABLE _GSS_LIB_FLAGS - RESULT_VARIABLE _GSS_CONFIGURE_FAILED - ) -message(STATUS "LDFLAGS: ${_GSS_LIB_FLAGS}") - if(NOT _GSS_CONFIGURE_FAILED) # 0 means success - # this script gives us libraries and link directories. Blah. We have to deal with it. - string(STRIP "${_GSS_LIB_FLAGS}" _GSS_LIB_FLAGS) - string(REGEX REPLACE " +-(L|l)" ";-\\1" _GSS_LIB_FLAGS "${_GSS_LIB_FLAGS}") - string(REGEX REPLACE " +-([^Ll][^ \\t;]*)" ";-\\1"_GSS_LIB_FLAGS "${_GSS_LIB_FLAGS}") - - foreach(_flag ${_GSS_LIB_FLAGS}) - if(_flag MATCHES "^-l.*") - string(REGEX REPLACE "^-l" "" _val "${_flag}") - list(APPEND _GSS_LIBRARIES "${_val}") - elseif(_flag MATCHES "^-L.*") - string(REGEX REPLACE "^-L" "" _val "${_flag}") - list(APPEND _GSS_LINK_DIRECTORIES "${_val}") - else() - list(APPEND _GSS_LINKER_FLAGS "${_flag}") - endif() - endforeach() - endif() + execute_process( + COMMAND ${_GSS_CONFIGURE_SCRIPT} "--vendor" + OUTPUT_VARIABLE _GSS_VENDOR + RESULT_VARIABLE _GSS_CONFIGURE_FAILED + ) + # older versions may not have the "--vendor" parameter. In this case we just don't care. + if(_GSS_CONFIGURE_FAILED) + set(GSS_FLAVOUR "Heimdal") # most probably, shouldn't really matter + else() + if(_GSS_VENDOR MATCHES ".*H|heimdal.*") + set(GSS_FLAVOUR "Heimdal") + else() + set(GSS_FLAVOUR "MIT") + endif() + endif() - execute_process( - COMMAND ${_GSS_CONFIGURE_SCRIPT} "--version" - OUTPUT_VARIABLE _GSS_VERSION - RESULT_VARIABLE _GSS_CONFIGURE_FAILED - ) + else() # either there is no config script or we are on platform that doesn't provide one (Windows?) - # older versions may not have the "--version" parameter. In this case we just don't care. - if(_GSS_CONFIGURE_FAILED) - set(_GSS_VERSION 0) - endif() + find_path(_GSS_INCLUDE_DIR + NAMES + "gssapi/gssapi.h" + HINTS + ${_GSS_ROOT_HINTS} + PATH_SUFFIXES + include + inc + ) + if(_GSS_INCLUDE_DIR) #jay, we've found something + set(CMAKE_REQUIRED_INCLUDES "${_GSS_INCLUDE_DIR}") + check_include_files( "gssapi/gssapi_generic.h;gssapi/gssapi_krb5.h" _GSS_HAVE_MIT_HEADERS) - execute_process( - COMMAND ${_GSS_CONFIGURE_SCRIPT} "--vendor" - OUTPUT_VARIABLE _GSS_VENDOR - RESULT_VARIABLE _GSS_CONFIGURE_FAILED - ) + if(_GSS_HAVE_MIT_HEADERS) + set(GSS_FLAVOUR "MIT") + else() + # prevent compiling the header - just check if we can include it + set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D__ROKEN_H__") + check_include_file( "roken.h" _GSS_HAVE_ROKEN_H) + + check_include_file( "heimdal/roken.h" _GSS_HAVE_HEIMDAL_ROKEN_H) + if(_GSS_HAVE_ROKEN_H OR _GSS_HAVE_HEIMDAL_ROKEN_H) + set(GSS_FLAVOUR "Heimdal") + endif() + set(CMAKE_REQUIRED_DEFINITIONS "") + endif() + else() + # I'm not convienced if this is the right way but this is what autotools do at the moment + find_path(_GSS_INCLUDE_DIR + NAMES + "gssapi.h" + HINTS + ${_GSS_ROOT_HINTS} + PATH_SUFFIXES + include + inc + ) + + if(_GSS_INCLUDE_DIR) + set(GSS_FLAVOUR "Heimdal") + endif() + endif() + + # if we have headers, check if we can link libraries + if(GSS_FLAVOUR) + set(_GSS_LIBDIR_SUFFIXES "") + set(_GSS_LIBDIR_HINTS ${_GSS_ROOT_HINTS}) + get_filename_component(_GSS_CALCULATED_POTENTIAL_ROOT "${_GSS_INCLUDE_DIR}" PATH) + list(APPEND _GSS_LIBDIR_HINTS ${_GSS_CALCULATED_POTENTIAL_ROOT}) - # older versions may not have the "--vendor" parameter. In this case we just don't care. - if(_GSS_CONFIGURE_FAILED) - set(GSS_FLAVOUR "Heimdal") # most probably, shouldn't really matter + if(WIN32) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + list(APPEND _GSS_LIBDIR_SUFFIXES "lib/AMD64") + if(GSS_FLAVOUR STREQUAL "MIT") + set(_GSS_LIBNAME "gssapi64") + else() + set(_GSS_LIBNAME "libgssapi") + endif() else() - if(_GSS_VENDOR MATCHES ".*H|heimdal.*") - set(GSS_FLAVOUR "Heimdal") - else() - set(GSS_FLAVOUR "MIT") - endif() + list(APPEND _GSS_LIBDIR_SUFFIXES "lib/i386") + if(GSS_FLAVOUR STREQUAL "MIT") + set(_GSS_LIBNAME "gssapi32") + else() + set(_GSS_LIBNAME "libgssapi") + endif() endif() - - else() # either there is no config script or we are on platform that doesn't provide one (Windows?) - - find_path(_GSS_INCLUDE_DIR - NAMES - "gssapi/gssapi.h" - HINTS - ${_GSS_ROOT_HINTS} - PATH_SUFFIXES - include - inc - ) - - if(_GSS_INCLUDE_DIR) #jay, we've found something - set(CMAKE_REQUIRED_INCLUDES "${_GSS_INCLUDE_DIR}") - check_include_files( "gssapi/gssapi_generic.h;gssapi/gssapi_krb5.h" _GSS_HAVE_MIT_HEADERS) - - if(_GSS_HAVE_MIT_HEADERS) - set(GSS_FLAVOUR "MIT") - else() - # prevent compiling the header - just check if we can include it - set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D__ROKEN_H__") - check_include_file( "roken.h" _GSS_HAVE_ROKEN_H) - - check_include_file( "heimdal/roken.h" _GSS_HAVE_HEIMDAL_ROKEN_H) - if(_GSS_HAVE_ROKEN_H OR _GSS_HAVE_HEIMDAL_ROKEN_H) - set(GSS_FLAVOUR "Heimdal") - endif() - set(CMAKE_REQUIRED_DEFINITIONS "") - endif() + else() + list(APPEND _GSS_LIBDIR_SUFFIXES "lib;lib64") # those suffixes are not checked for HINTS + if(GSS_FLAVOUR STREQUAL "MIT") + set(_GSS_LIBNAME "gssapi_krb5") else() - # I'm not convienced if this is the right way but this is what autotools do at the moment - find_path(_GSS_INCLUDE_DIR - NAMES - "gssapi.h" - HINTS - ${_GSS_ROOT_HINTS} - PATH_SUFFIXES - include - inc - ) - - if(_GSS_INCLUDE_DIR) - set(GSS_FLAVOUR "Heimdal") - endif() + set(_GSS_LIBNAME "gssapi") endif() + endif() - # if we have headers, check if we can link libraries - if(GSS_FLAVOUR) - set(_GSS_LIBDIR_SUFFIXES "") - set(_GSS_LIBDIR_HINTS ${_GSS_ROOT_HINTS}) - get_filename_component(_GSS_CALCULATED_POTENTIAL_ROOT "${_GSS_INCLUDE_DIR}" PATH) - list(APPEND _GSS_LIBDIR_HINTS ${_GSS_CALCULATED_POTENTIAL_ROOT}) - - if(WIN32) - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - list(APPEND _GSS_LIBDIR_SUFFIXES "lib/AMD64") - if(GSS_FLAVOUR STREQUAL "MIT") - set(_GSS_LIBNAME "gssapi64") - else() - set(_GSS_LIBNAME "libgssapi") - endif() - else() - list(APPEND _GSS_LIBDIR_SUFFIXES "lib/i386") - if(GSS_FLAVOUR STREQUAL "MIT") - set(_GSS_LIBNAME "gssapi32") - else() - set(_GSS_LIBNAME "libgssapi") - endif() - endif() - else() - list(APPEND _GSS_LIBDIR_SUFFIXES "lib;lib64") # those suffixes are not checked for HINTS - if(GSS_FLAVOUR STREQUAL "MIT") - set(_GSS_LIBNAME "gssapi_krb5") - else() - set(_GSS_LIBNAME "gssapi") - endif() - endif() - - find_library(_GSS_LIBRARIES - NAMES - ${_GSS_LIBNAME} - HINTS - ${_GSS_LIBDIR_HINTS} - PATH_SUFFIXES - ${_GSS_LIBDIR_SUFFIXES} - ) - - endif() + find_library(_GSS_LIBRARIES + NAMES + ${_GSS_LIBNAME} + HINTS + ${_GSS_LIBDIR_HINTS} + PATH_SUFFIXES + ${_GSS_LIBDIR_SUFFIXES} + ) endif() + endif() else() - if(_GSS_PKG_${_MIT_MODNAME}_VERSION) - set(GSS_FLAVOUR "MIT") - set(_GSS_VERSION _GSS_PKG_${_MIT_MODNAME}_VERSION) - else() - set(GSS_FLAVOUR "Heimdal") - set(_GSS_VERSION _GSS_PKG_${_MIT_HEIMDAL}_VERSION) - endif() + if(_GSS_PKG_${_MIT_MODNAME}_VERSION) + set(GSS_FLAVOUR "MIT") + set(_GSS_VERSION _GSS_PKG_${_MIT_MODNAME}_VERSION) + else() + set(GSS_FLAVOUR "Heimdal") + set(_GSS_VERSION _GSS_PKG_${_MIT_HEIMDAL}_VERSION) + endif() endif() set(GSS_INCLUDE_DIR ${_GSS_INCLUDE_DIR}) @@ -243,36 +241,34 @@ set(GSS_COMPILER_FLAGS ${_GSS_COMPILER_FLAGS}) set(GSS_VERSION ${_GSS_VERSION}) if(GSS_FLAVOUR) + if(NOT GSS_VERSION AND GSS_FLAVOUR STREQUAL "Heimdal") + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(HEIMDAL_MANIFEST_FILE "Heimdal.Application.amd64.manifest") + else() + set(HEIMDAL_MANIFEST_FILE "Heimdal.Application.x86.manifest") + endif() - if(NOT GSS_VERSION AND GSS_FLAVOUR STREQUAL "Heimdal") - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(HEIMDAL_MANIFEST_FILE "Heimdal.Application.amd64.manifest") - else() - set(HEIMDAL_MANIFEST_FILE "Heimdal.Application.x86.manifest") - endif() - - if(EXISTS "${GSS_INCLUDE_DIR}/${HEIMDAL_MANIFEST_FILE}") - file(STRINGS "${GSS_INCLUDE_DIR}/${HEIMDAL_MANIFEST_FILE}" heimdal_version_str - REGEX "^.*version=\"[0-9]\\.[^\"]+\".*$") + if(EXISTS "${GSS_INCLUDE_DIR}/${HEIMDAL_MANIFEST_FILE}") + file(STRINGS "${GSS_INCLUDE_DIR}/${HEIMDAL_MANIFEST_FILE}" heimdal_version_str + REGEX "^.*version=\"[0-9]\\.[^\"]+\".*$") - string(REGEX MATCH "[0-9]\\.[^\"]+" - GSS_VERSION "${heimdal_version_str}") - endif() + string(REGEX MATCH "[0-9]\\.[^\"]+" + GSS_VERSION "${heimdal_version_str}") + endif() - if(NOT GSS_VERSION) - set(GSS_VERSION "Heimdal Unknown") - endif() - elseif(NOT GSS_VERSION AND GSS_FLAVOUR STREQUAL "MIT") - get_filename_component(_MIT_VERSION "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MIT\\Kerberos\\SDK\\CurrentVersion;VersionString]" NAME CACHE) - if(WIN32 AND _MIT_VERSION) - set(GSS_VERSION "${_MIT_VERSION}") - else() - set(GSS_VERSION "MIT Unknown") - endif() + if(NOT GSS_VERSION) + set(GSS_VERSION "Heimdal Unknown") endif() + elseif(NOT GSS_VERSION AND GSS_FLAVOUR STREQUAL "MIT") + get_filename_component(_MIT_VERSION "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MIT\\Kerberos\\SDK\\CurrentVersion;VersionString]" NAME CACHE) + if(WIN32 AND _MIT_VERSION) + set(GSS_VERSION "${_MIT_VERSION}") + else() + set(GSS_VERSION "MIT Unknown") + endif() + endif() endif() - include(FindPackageHandleStandardArgs) set(_GSS_REQUIRED_VARS GSS_LIBRARIES GSS_FLAVOUR) diff --git a/Utilities/cmcurl/CMake/FindLibSSH2.cmake b/Utilities/cmcurl/CMake/FindLibSSH2.cmake index 12a7c612b..84822dba7 100644 --- a/Utilities/cmcurl/CMake/FindLibSSH2.cmake +++ b/Utilities/cmcurl/CMake/FindLibSSH2.cmake @@ -5,14 +5,14 @@ # LIBSSH2_INCLUDE_DIR - the libssh2 include directory # LIBSSH2_LIBRARY - the libssh2 library name -if (LIBSSH2_INCLUDE_DIR AND LIBSSH2_LIBRARY) +if(LIBSSH2_INCLUDE_DIR AND LIBSSH2_LIBRARY) set(LibSSH2_FIND_QUIETLY TRUE) -endif (LIBSSH2_INCLUDE_DIR AND LIBSSH2_LIBRARY) +endif() -FIND_PATH(LIBSSH2_INCLUDE_DIR libssh2.h +find_path(LIBSSH2_INCLUDE_DIR libssh2.h ) -FIND_LIBRARY(LIBSSH2_LIBRARY NAMES ssh2 +find_library(LIBSSH2_LIBRARY NAMES ssh2 ) if(LIBSSH2_INCLUDE_DIR) @@ -27,9 +27,9 @@ if(LIBSSH2_INCLUDE_DIR) string(REGEX REPLACE "^0(.+)" "\\1" LIBSSH2_VERSION_PATCH "${LIBSSH2_VERSION_PATCH}") set(LIBSSH2_VERSION "${LIBSSH2_VERSION_MAJOR}.${LIBSSH2_VERSION_MINOR}.${LIBSSH2_VERSION_PATCH}") -endif(LIBSSH2_INCLUDE_DIR) +endif() include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibSSH2 DEFAULT_MSG LIBSSH2_INCLUDE_DIR LIBSSH2_LIBRARY ) +find_package_handle_standard_args(LibSSH2 DEFAULT_MSG LIBSSH2_INCLUDE_DIR LIBSSH2_LIBRARY ) -MARK_AS_ADVANCED(LIBSSH2_INCLUDE_DIR LIBSSH2_LIBRARY LIBSSH2_VERSION_MAJOR LIBSSH2_VERSION_MINOR LIBSSH2_VERSION_PATCH LIBSSH2_VERSION) +mark_as_advanced(LIBSSH2_INCLUDE_DIR LIBSSH2_LIBRARY LIBSSH2_VERSION_MAJOR LIBSSH2_VERSION_MINOR LIBSSH2_VERSION_PATCH LIBSSH2_VERSION) diff --git a/Utilities/cmcurl/CMake/FindNGHTTP2.cmake b/Utilities/cmcurl/CMake/FindNGHTTP2.cmake index 4e566cf02..348b9612d 100644 --- a/Utilities/cmcurl/CMake/FindNGHTTP2.cmake +++ b/Utilities/cmcurl/CMake/FindNGHTTP2.cmake @@ -14,5 +14,5 @@ find_package_handle_standard_args(NGHTTP2 "Could NOT find NGHTTP2" ) -set(NGHTTP2_INCLUDE_DIRS ${NGHTTP2_INCLUDE_DIR} ) +set(NGHTTP2_INCLUDE_DIRS ${NGHTTP2_INCLUDE_DIR}) set(NGHTTP2_LIBRARIES ${NGHTTP2_LIBRARY}) diff --git a/Utilities/cmcurl/CMake/Macros.cmake b/Utilities/cmcurl/CMake/Macros.cmake index 82aadca9d..7f7134515 100644 --- a/Utilities/cmcurl/CMake/Macros.cmake +++ b/Utilities/cmcurl/CMake/Macros.cmake @@ -5,35 +5,35 @@ # multiple times with a sequence of possibly dependent libraries in # order of least-to-most-dependent. Some libraries depend on others # to link correctly. -macro(CHECK_LIBRARY_EXISTS_CONCAT LIBRARY SYMBOL VARIABLE) +macro(check_library_exists_concat LIBRARY SYMBOL VARIABLE) check_library_exists("${LIBRARY};${CURL_LIBS}" ${SYMBOL} "${CMAKE_LIBRARY_PATH}" ${VARIABLE}) if(${VARIABLE}) set(CURL_LIBS ${LIBRARY} ${CURL_LIBS}) - endif(${VARIABLE}) -endmacro(CHECK_LIBRARY_EXISTS_CONCAT) + endif() +endmacro() # Check if header file exists and add it to the list. # This macro is intended to be called multiple times with a sequence of # possibly dependent header files. Some headers depend on others to be # compiled correctly. -macro(CHECK_INCLUDE_FILE_CONCAT FILE VARIABLE) +macro(check_include_file_concat FILE VARIABLE) check_include_files("${CURL_INCLUDES};${FILE}" ${VARIABLE}) if(${VARIABLE}) set(CURL_INCLUDES ${CURL_INCLUDES} ${FILE}) set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -D${VARIABLE}") - endif(${VARIABLE}) -endmacro(CHECK_INCLUDE_FILE_CONCAT) + endif() +endmacro() # For other curl specific tests, use this macro. -macro(CURL_INTERNAL_TEST CURL_TEST) +macro(curl_internal_test CURL_TEST) if(NOT DEFINED "${CURL_TEST}") set(MACRO_CHECK_FUNCTION_DEFINITIONS "-D${CURL_TEST} ${CURL_TEST_DEFINES} ${CMAKE_REQUIRED_FLAGS}") if(CMAKE_REQUIRED_LIBRARIES) set(CURL_TEST_ADD_LIBRARIES "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}") - endif(CMAKE_REQUIRED_LIBRARIES) + endif() message(STATUS "Performing Curl Test ${CURL_TEST}") try_compile(${CURL_TEST} @@ -48,53 +48,17 @@ macro(CURL_INTERNAL_TEST CURL_TEST) file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log "Performing Curl Test ${CURL_TEST} passed with the following output:\n" "${OUTPUT}\n") - else(${CURL_TEST}) + else() message(STATUS "Performing Curl Test ${CURL_TEST} - Failed") set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}") file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log "Performing Curl Test ${CURL_TEST} failed with the following output:\n" "${OUTPUT}\n") - endif(${CURL_TEST}) - endif() -endmacro(CURL_INTERNAL_TEST) - -macro(CURL_INTERNAL_TEST_RUN CURL_TEST) - if(NOT DEFINED "${CURL_TEST}_COMPILE") - set(MACRO_CHECK_FUNCTION_DEFINITIONS - "-D${CURL_TEST} ${CMAKE_REQUIRED_FLAGS}") - if(CMAKE_REQUIRED_LIBRARIES) - set(CURL_TEST_ADD_LIBRARIES - "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}") - endif(CMAKE_REQUIRED_LIBRARIES) - - message(STATUS "Performing Curl Test ${CURL_TEST}") - try_run(${CURL_TEST} ${CURL_TEST}_COMPILE - ${CMAKE_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/CMake/CurlTests.c - CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS} - "${CURL_TEST_ADD_LIBRARIES}" - OUTPUT_VARIABLE OUTPUT) - if(${CURL_TEST}_COMPILE AND NOT ${CURL_TEST}) - set(${CURL_TEST} 1 CACHE INTERNAL "Curl test ${FUNCTION}") - message(STATUS "Performing Curl Test ${CURL_TEST} - Success") - else(${CURL_TEST}_COMPILE AND NOT ${CURL_TEST}) - message(STATUS "Performing Curl Test ${CURL_TEST} - Failed") - set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}") - file(APPEND "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log" - "Performing Curl Test ${CURL_TEST} failed with the following output:\n" - "${OUTPUT}") - if(${CURL_TEST}_COMPILE) - file(APPEND - "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log" - "There was a problem running this test\n") - endif(${CURL_TEST}_COMPILE) - file(APPEND "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log" - "\n\n") - endif(${CURL_TEST}_COMPILE AND NOT ${CURL_TEST}) + endif() endif() -endmacro(CURL_INTERNAL_TEST_RUN) +endmacro() -macro(CURL_NROFF_CHECK) +macro(curl_nroff_check) find_program(NROFF NAMES gnroff nroff) if(NROFF) # Need a way to write to stdin, this will do @@ -121,4 +85,4 @@ macro(CURL_NROFF_CHECK) else() message(WARNING "Found no *nroff program") endif() -endmacro(CURL_NROFF_CHECK) +endmacro() diff --git a/Utilities/cmcurl/CMake/OtherTests.cmake b/Utilities/cmcurl/CMake/OtherTests.cmake index 989f04eb2..ce6d3e13a 100644 --- a/Utilities/cmcurl/CMake/OtherTests.cmake +++ b/Utilities/cmcurl/CMake/OtherTests.cmake @@ -5,8 +5,8 @@ set(_source_epilogue "#undef inline") macro(add_header_include check header) if(${check}) set(_source_epilogue "${_source_epilogue}\n#include <${header}>") - endif(${check}) -endmacro(add_header_include) + endif() +endmacro() set(signature_call_conv) if(HAVE_WINDOWS_H) @@ -19,10 +19,10 @@ if(HAVE_WINDOWS_H) if(HAVE_LIBWS2_32) set(CMAKE_REQUIRED_LIBRARIES ws2_32) endif() -else(HAVE_WINDOWS_H) +else() add_header_include(HAVE_SYS_TYPES_H "sys/types.h") add_header_include(HAVE_SYS_SOCKET_H "sys/socket.h") -endif(HAVE_WINDOWS_H) +endif() check_c_source_compiles("${_source_epilogue} int main(void) { @@ -64,13 +64,13 @@ if(curl_cv_recv) set(RECV_TYPE_RETV "${recv_retv}") set(HAVE_RECV 1) set(curl_cv_func_recv_done 1) - endif(curl_cv_func_recv_test) - endif(NOT curl_cv_func_recv_done) - endforeach(recv_arg4) - endforeach(recv_arg3) - endforeach(recv_arg2) - endforeach(recv_arg1) - endforeach(recv_retv) + endif() + endif() + endforeach() + endforeach() + endforeach() + endforeach() + endforeach() else() string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG1 "${curl_cv_func_recv_args}") string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG2 "${curl_cv_func_recv_args}") @@ -81,10 +81,10 @@ if(curl_cv_recv) if("${curl_cv_func_recv_args}" STREQUAL "unknown") message(FATAL_ERROR "Cannot find proper types to use for recv args") - endif("${curl_cv_func_recv_args}" STREQUAL "unknown") -else(curl_cv_recv) + endif() +else() message(FATAL_ERROR "Unable to link function recv") -endif(curl_cv_recv) +endif() set(curl_cv_func_recv_args "${curl_cv_func_recv_args}" CACHE INTERNAL "Arguments for recv") set(HAVE_RECV 1) @@ -130,13 +130,13 @@ if(curl_cv_send) set(SEND_TYPE_RETV "${send_retv}") set(HAVE_SEND 1) set(curl_cv_func_send_done 1) - endif(curl_cv_func_send_test) - endif(NOT curl_cv_func_send_done) - endforeach(send_arg4) - endforeach(send_arg3) - endforeach(send_arg2) - endforeach(send_arg1) - endforeach(send_retv) + endif() + endif() + endforeach() + endforeach() + endforeach() + endforeach() + endforeach() else() string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG1 "${curl_cv_func_send_args}") string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG2 "${curl_cv_func_send_args}") @@ -148,11 +148,11 @@ if(curl_cv_send) if("${curl_cv_func_send_args}" STREQUAL "unknown") message(FATAL_ERROR "Cannot find proper types to use for send args") - endif("${curl_cv_func_send_args}" STREQUAL "unknown") + endif() set(SEND_QUAL_ARG2 "const") -else(curl_cv_send) +else() message(FATAL_ERROR "Unable to link function send") -endif(curl_cv_send) +endif() set(curl_cv_func_send_args "${curl_cv_func_send_args}" CACHE INTERNAL "Arguments for send") set(HAVE_SEND 1) @@ -184,7 +184,7 @@ if(NOT APPLE) set(CMAKE_REQUIRED_FLAGS) if(HAVE_SYS_POLL_H) set(CMAKE_REQUIRED_FLAGS "-DHAVE_SYS_POLL_H") - endif(HAVE_SYS_POLL_H) + endif() check_c_source_runs(" #ifdef HAVE_SYS_POLL_H # include <sys/poll.h> @@ -199,7 +199,7 @@ set(CMAKE_REQUIRED_FLAGS) if(HAVE_SIGNAL_H) set(CMAKE_REQUIRED_FLAGS "-DHAVE_SIGNAL_H") set(CMAKE_EXTRA_INCLUDE_FILES "signal.h") -endif(HAVE_SIGNAL_H) +endif() check_type_size("sig_atomic_t" SIZEOF_SIG_ATOMIC_T) if(HAVE_SIZEOF_SIG_ATOMIC_T) check_c_source_compiles(" @@ -213,8 +213,8 @@ if(HAVE_SIZEOF_SIG_ATOMIC_T) }" HAVE_SIG_ATOMIC_T_NOT_VOLATILE) if(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE) set(HAVE_SIG_ATOMIC_T_VOLATILE 1) - endif(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE) -endif(HAVE_SIZEOF_SIG_ATOMIC_T) + endif() +endif() if(HAVE_WINDOWS_H) set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h) @@ -222,11 +222,10 @@ else() set(CMAKE_EXTRA_INCLUDE_FILES) if(HAVE_SYS_SOCKET_H) set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h) - endif(HAVE_SYS_SOCKET_H) + endif() endif() check_type_size("struct sockaddr_storage" SIZEOF_STRUCT_SOCKADDR_STORAGE) if(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE) set(HAVE_STRUCT_SOCKADDR_STORAGE 1) -endif(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE) - +endif() diff --git a/Utilities/cmcurl/CMake/Platforms/WindowsCache.cmake b/Utilities/cmcurl/CMake/Platforms/WindowsCache.cmake index 53d0a5e88..2dbe1bbbd 100644 --- a/Utilities/cmcurl/CMake/Platforms/WindowsCache.cmake +++ b/Utilities/cmcurl/CMake/Platforms/WindowsCache.cmake @@ -115,8 +115,7 @@ if(NOT UNIX) set(HAVE_SIGACTION 0) set(HAVE_MACRO_SIGSETJMP 0) - else(WIN32) + else() message("This file should be included on Windows platform only") - endif(WIN32) -endif(NOT UNIX) - + endif() +endif() diff --git a/Utilities/cmcurl/CMake/Utilities.cmake b/Utilities/cmcurl/CMake/Utilities.cmake index 005b166ff..5cb1d4497 100644 --- a/Utilities/cmcurl/CMake/Utilities.cmake +++ b/Utilities/cmcurl/CMake/Utilities.cmake @@ -1,44 +1,13 @@ # File containing various utilities -# Converts a CMake list to a string containing elements separated by spaces -function(TO_LIST_SPACES _LIST_NAME OUTPUT_VAR) - set(NEW_LIST_SPACE) - foreach(ITEM ${${_LIST_NAME}}) - set(NEW_LIST_SPACE "${NEW_LIST_SPACE} ${ITEM}") - endforeach() - string(STRIP ${NEW_LIST_SPACE} NEW_LIST_SPACE) - set(${OUTPUT_VAR} "${NEW_LIST_SPACE}" PARENT_SCOPE) -endfunction() - -# Appends a lis of item to a string which is a space-separated list, if they don't already exist. -function(LIST_SPACES_APPEND_ONCE LIST_NAME) - string(REPLACE " " ";" _LIST ${${LIST_NAME}}) - list(APPEND _LIST ${ARGN}) - list(REMOVE_DUPLICATES _LIST) - to_list_spaces(_LIST NEW_LIST_SPACE) - set(${LIST_NAME} "${NEW_LIST_SPACE}" PARENT_SCOPE) -endfunction() - -# Convenience function that does the same as LIST(FIND ...) but with a TRUE/FALSE return value. -# Ex: IN_STR_LIST(MY_LIST "Searched item" WAS_FOUND) -function(IN_STR_LIST LIST_NAME ITEM_SEARCHED RETVAL) - list(FIND ${LIST_NAME} ${ITEM_SEARCHED} FIND_POS) - if(${FIND_POS} EQUAL -1) - set(${RETVAL} FALSE PARENT_SCOPE) - else() - set(${RETVAL} TRUE PARENT_SCOPE) - endif() -endfunction() - # Returns a list of arguments that evaluate to true -function(collect_true output_var output_count_var) - set(${output_var}) +function(count_true output_count_var) + set(lst) foreach(option_var IN LISTS ARGN) if(${option_var}) - list(APPEND ${output_var} ${option_var}) + list(APPEND lst ${option_var}) endif() endforeach() - set(${output_var} ${${output_var}} PARENT_SCOPE) - list(LENGTH ${output_var} ${output_count_var}) - set(${output_count_var} ${${output_count_var}} PARENT_SCOPE) + list(LENGTH lst lst_len) + set(${output_count_var} ${lst_len} PARENT_SCOPE) endfunction() diff --git a/Utilities/cmcurl/CMake/cmake_uninstall.cmake.in b/Utilities/cmcurl/CMake/cmake_uninstall.cmake.in index d00a51665..db8e5367d 100644 --- a/Utilities/cmcurl/CMake/cmake_uninstall.cmake.in +++ b/Utilities/cmcurl/CMake/cmake_uninstall.cmake.in @@ -1,11 +1,11 @@ if(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") message(FATAL_ERROR "Cannot find install manifest: @CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") -endif(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") +endif() -if (NOT DEFINED CMAKE_INSTALL_PREFIX) - set (CMAKE_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@") -endif () - message(${CMAKE_INSTALL_PREFIX}) +if(NOT DEFINED CMAKE_INSTALL_PREFIX) + set(CMAKE_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@") +endif() +message(${CMAKE_INSTALL_PREFIX}) file(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) string(REGEX REPLACE "\n" ";" files "${files}") @@ -19,8 +19,8 @@ foreach(file ${files}) ) if(NOT "${rm_retval}" STREQUAL 0) message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}") - endif(NOT "${rm_retval}" STREQUAL 0) - else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") + endif() + else() message(STATUS "File $ENV{DESTDIR}${file} does not exist.") - endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") -endforeach(file) + endif() +endforeach() diff --git a/Utilities/cmcurl/CMake/curl-config.cmake b/Utilities/cmcurl/CMake/curl-config.cmake deleted file mode 100644 index 119332cdd..000000000 --- a/Utilities/cmcurl/CMake/curl-config.cmake +++ /dev/null @@ -1,59 +0,0 @@ - -get_filename_component(_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) - -if(NOT CURL_FIND_COMPONENTS) - set(CURL_FIND_COMPONENTS curl libcurl) - if(CURL_FIND_REQUIRED) - set(CURL_FIND_REQUIRED_curl TRUE) - set(CURL_FIND_REQUIRED_libcurl TRUE) - endif() -endif() - -set(_curl_missing_components) -foreach(_comp ${CURL_FIND_COMPONENTS}) - if(EXISTS "${_DIR}/${_comp}-target.cmake") - include("${_DIR}/${_comp}-target.cmake") - set(CURL_${_comp}_FOUND TRUE) - else() - set(CURL_${_comp}_FOUND FALSE) - if(CURL_FIND_REQUIRED_${_comp}) - set(CURL_FOUND FALSE) - list(APPEND _curl_missing_components ${_comp}) - endif() - endif() -endforeach() - -if(_curl_missing_components) - set(CURL_NOT_FOUND_MESSAGE "Following required components not found: " ${_curl_missing_components}) -else() - if(TARGET CURL::libcurl) - string(TOUPPER "${CMAKE_BUILD_TYPE}" _curl_current_config) - if(NOT _curl_current_config) - set(_curl_current_config "NOCONFIG") - endif() - get_target_property(_curl_configurations CURL::libcurl IMPORTED_CONFIGURATIONS) - list(FIND _curl_configurations "${_curl_current_config}" _i) - if(_i LESS 0) - set(_curl_config "RELEASE") - list(FIND _curl_configurations "${_curl_current_config}" _i) - if(_i LESS 0) - set(_curl_config "NOCONFIG") - list(FIND _curl_configurations "${_curl_current_config}" _i) - endif() - endif() - - if(_i LESS 0) - set(_curl_current_config "") # let CMake pick config at random - else() - set(_curl_current_config "_${_curl_current_config}") - endif() - - get_target_property(CURL_INCLUDE_DIRS CURL::libcurl INTERFACE_INCLUDE_DIRECTORIES) - get_target_property(CURL_LIBRARIES CURL::libcurl "LOCATION${_curl_current_config}") - set(_curl_current_config) - set(_curl_configurations) - set(_i) - endif() -endif() - -unset(_curl_missing_components) diff --git a/Utilities/cmcurl/CMake/curl-config.cmake.in b/Utilities/cmcurl/CMake/curl-config.cmake.in new file mode 100644 index 000000000..1294e173a --- /dev/null +++ b/Utilities/cmcurl/CMake/curl-config.cmake.in @@ -0,0 +1,12 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +if(@USE_OPENSSL@) + find_dependency(OpenSSL @OPENSSL_VERSION_MAJOR@) +endif() +if(@USE_ZLIB@) + find_dependency(ZLIB @ZLIB_VERSION_MAJOR@) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake") +check_required_components("@PROJECT_NAME@") diff --git a/Utilities/cmcurl/CMakeLists.txt b/Utilities/cmcurl/CMakeLists.txt index 012821415..27b7507f6 100644 --- a/Utilities/cmcurl/CMakeLists.txt +++ b/Utilities/cmcurl/CMakeLists.txt @@ -2,6 +2,7 @@ set(BUILD_CURL_EXE OFF CACHE INTERNAL "No curl exe") set(BUILD_DASHBOARD_REPORTS OFF CACHE INTERNAL "No curl dashboard reports") set(BUILD_RELEASE_DEBUG_DIRS OFF CACHE INTERNAL "No curl release/debug dirs") +set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "Build shared libraries") set(CMAKE_USE_GSSAPI OFF CACHE INTERNAL "Disable curl gssapi") set(CMAKE_USE_LIBSSH2 OFF CACHE INTERNAL "Disable curl libssh2") set(CMAKE_USE_OPENLDAP OFF CACHE INTERNAL "No curl OpenLDAP") @@ -23,7 +24,6 @@ set(CURL_DISABLE_TELNET ON CACHE INTERNAL "Disable curl telnet protocol?") set(CURL_DISABLE_TFTP ON CACHE INTERNAL "Disable curl tftp protocol?") set(CURL_DISABLE_VERBOSE_STRINGS OFF CACHE INTERNAL "Do not disable curl verbosity") set(CURL_HIDDEN_SYMBOLS OFF CACHE INTERNAL "No curl hidden symbols") -set(CURL_STATICLIB ON CACHE INTERNAL "Static curl") set(CURL_WERROR OFF CACHE INTERNAL "Turn compiler warnings into errors") set(DISABLED_THREADSAFE OFF CACHE INTERNAL "Curl can use thread-safe functions") set(ENABLE_ARES OFF CACHE INTERNAL "No curl c-ares support") @@ -123,26 +123,26 @@ endif() # To check: # (From Daniel Stenberg) The cmake build selected to run gcc with -fPIC on my box while the plain configure script did not. # (From Daniel Stenberg) The gcc command line use neither -g nor any -O options. As a developer, I also treasure our configure scripts's --enable-debug option that sets a long range of "picky" compiler options. -cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}") include(Utilities) include(Macros) include(CMakeDependentOption) include(CheckCCompilerFlag) -project( CURL C ) +project(CURL C) if(0) # This code not needed for building within CMake. message(WARNING "the curl cmake build system is poorly maintained. Be aware") endif() -file (READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS) -string (REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*" +file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS) +string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*" CURL_VERSION ${CURL_VERSION_H_CONTENTS}) -string (REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION}) -string (REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+" +string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION}) +string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+" CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS}) -string (REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM}) +string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM}) include_regular_expression("^.*$") # Sukender: Is it necessary? @@ -160,19 +160,19 @@ set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}") set(OS "\"${CMAKE_SYSTEM_NAME}\"") include_directories(${PROJECT_BINARY_DIR}/include/curl) -include_directories( ${CURL_SOURCE_DIR}/include ) +include_directories(${CURL_SOURCE_DIR}/include) option(CURL_WERROR "Turn compiler warnings into errors" OFF) option(PICKY_COMPILER "Enable picky compiler options" ON) option(BUILD_CURL_EXE "Set to ON to build curl executable." ON) -option(CURL_STATICLIB "Set to ON to build libcurl with static linking." OFF) +option(BUILD_SHARED_LIBS "Build shared libraries" ON) option(ENABLE_ARES "Set to ON to enable c-ares support" OFF) if(WIN32) option(CURL_STATIC_CRT "Set to ON to build libcurl with static CRT on Windows (/MT)." OFF) option(ENABLE_INET_PTON "Set to OFF to prevent usage of inet_pton when building against modern SDKs while still requiring compatibility with older Windows versions, such as Windows XP, Windows Server 2003 etc." ON) endif() -CMAKE_DEPENDENT_OPTION(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" +cmake_dependent_option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON "NOT ENABLE_ARES" OFF) @@ -180,19 +180,19 @@ option(ENABLE_DEBUG "Set to ON to enable curl debug features" OFF) option(ENABLE_CURLDEBUG "Set to ON to build with TrackMemory feature enabled" OFF) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) - if (PICKY_COMPILER) - foreach (_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers) + if(PICKY_COMPILER) + foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format) # surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new # test result in. - CHECK_C_COMPILER_FLAG(${_CCOPT} OPT${_CCOPT}) + check_c_compiler_flag(${_CCOPT} OPT${_CCOPT}) if(OPT${_CCOPT}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}") endif() endforeach() - endif(PICKY_COMPILER) -endif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) + endif() +endif() -if (ENABLE_DEBUG) +if(ENABLE_DEBUG) # DEBUGBUILD will be defined only for Debug builds if(NOT CMAKE_VERSION VERSION_LESS 3.0) set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS $<$<CONFIG:Debug>:DEBUGBUILD>) @@ -202,13 +202,15 @@ if (ENABLE_DEBUG) set(ENABLE_CURLDEBUG ON) endif() -if (ENABLE_CURLDEBUG) +if(ENABLE_CURLDEBUG) set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS CURLDEBUG) endif() if(0) # This code not needed for building within CMake. # For debug libs and exes, add "-d" postfix -set(CMAKE_DEBUG_POSTFIX "-d" CACHE STRING "Set debug library postfix") +if(NOT DEFINED CMAKE_DEBUG_POSTFIX) + set(CMAKE_DEBUG_POSTFIX "-d") +endif() endif() # initialize CURL_LIBS @@ -217,7 +219,7 @@ set(CURL_LIBS "") if(ENABLE_ARES) set(USE_ARES 1) find_package(CARES REQUIRED) - list(APPEND CURL_LIBS ${CARES_LIBRARY} ) + list(APPEND CURL_LIBS ${CARES_LIBRARY}) set(CURL_LIBS ${CURL_LIBS} ${CARES_LIBRARY}) endif() @@ -297,11 +299,11 @@ if(ENABLE_IPV6 AND NOT WIN32) endif() if(0) # This code not needed for building within CMake. -CURL_NROFF_CHECK() # Required for building manual, docs, tests +curl_nroff_check() find_package(Perl) -CMAKE_DEPENDENT_OPTION(ENABLE_MANUAL "to provide the built-in manual" +cmake_dependent_option(ENABLE_MANUAL "to provide the built-in manual" ON "NROFF_USEFUL;PERL_FOUND" OFF) @@ -326,27 +328,27 @@ endif() # Disable warnings on Borland to avoid changing 3rd party code. if(BORLAND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w-") -endif(BORLAND) +endif() # If we are on AIX, do the _ALL_SOURCE magic if(${CMAKE_SYSTEM_NAME} MATCHES AIX) set(_ALL_SOURCE 1) -endif(${CMAKE_SYSTEM_NAME} MATCHES AIX) +endif() # Include all the necessary files for macros -include (CheckFunctionExists) -include (CheckIncludeFile) -include (CheckIncludeFiles) -include (CheckLibraryExists) -include (CheckSymbolExists) -include (CheckTypeSize) -include (CheckCSourceCompiles) +include(CheckFunctionExists) +include(CheckIncludeFile) +include(CheckIncludeFiles) +include(CheckLibraryExists) +include(CheckSymbolExists) +include(CheckTypeSize) +include(CheckCSourceCompiles) # On windows preload settings if(WIN32) set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D_WINSOCKAPI_=") include(${CMAKE_CURRENT_SOURCE_DIR}/CMake/Platforms/WindowsCache.cmake) -endif(WIN32) +endif() if(ENABLE_THREADED_RESOLVER) find_package(Threads REQUIRED) @@ -375,13 +377,13 @@ if(BEOS) set(NOT_NEED_LIBNSL 1) check_library_exists_concat("bind" gethostbyname HAVE_LIBBIND) check_library_exists_concat("bnetapi" closesocket HAVE_LIBBNETAPI) -endif(BEOS) +endif() check_library_exists_concat("network" recv HAVE_LIBNETWORK) if(NOT NOT_NEED_LIBNSL) check_library_exists_concat("nsl" gethostbyname HAVE_LIBNSL) -endif(NOT NOT_NEED_LIBNSL) +endif() check_function_exists(gethostname HAVE_GETHOSTNAME) @@ -409,14 +411,14 @@ if(WIN32 OR CMAKE_USE_DARWINSSL OR CMAKE_USE_WINSSL OR CMAKE_USE_MBEDTLS) set(openssl_default OFF) endif() -collect_true(enabled_ssl_options enabled_ssl_options_count +count_true(enabled_ssl_options_count CMAKE_USE_WINSSL CMAKE_USE_DARWINSSL CMAKE_USE_OPENSSL CMAKE_USE_MBEDTLS ) -if(enabled_ssl_options_count GREATER 1) - message(FATAL_ERROR "Multiple SSL options specified: ${enabled_ssl_options}. Please pick at most one and disable the rest.") +if(enabled_ssl_options_count GREATER "1") + set(CURL_WITH_MULTI_SSL ON) endif() if(CMAKE_USE_WINSSL) @@ -454,9 +456,9 @@ if(CMAKE_USE_OPENSSL) set(HAVE_LIBSSL ON) list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES}) include_directories(${OPENSSL_INCLUDE_DIR}) + set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR}) check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H) - check_include_file("openssl/engine.h" HAVE_OPENSSL_ENGINE_H) check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H) check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H) check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H) @@ -585,7 +587,6 @@ if(NOT CURL_DISABLE_LDAP) endif() endif() endif() - endif() # No ldap, no ldaps. @@ -611,15 +612,23 @@ if(0) # This code not needed for building within CMake. option(CURL_ZLIB "Set to ON to enable building curl with zlib support." ON) set(HAVE_LIBZ OFF) set(HAVE_ZLIB_H OFF) -set(HAVE_ZLIB OFF) +set(USE_ZLIB OFF) if(CURL_ZLIB) find_package(ZLIB QUIET) if(ZLIB_FOUND) set(HAVE_ZLIB_H ON) - set(HAVE_ZLIB ON) set(HAVE_LIBZ ON) - list(APPEND CURL_LIBS ${ZLIB_LIBRARIES}) - include_directories(${ZLIB_INCLUDE_DIRS}) + set(USE_ZLIB ON) + + # Depend on ZLIB via imported targets if supported by the running + # version of CMake. This allows our dependents to get our dependencies + # transitively. + if(NOT CMAKE_VERSION VERSION_LESS 3.4) + list(APPEND CURL_LIBS ZLIB::ZLIB) + else() + list(APPEND CURL_LIBS ${ZLIB_LIBRARIES}) + include_directories(${ZLIB_INCLUDE_DIRS}) + endif() list(APPEND CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIRS}) endif() endif() @@ -677,9 +686,8 @@ if(CMAKE_USE_LIBSSH2) check_function_exists(libssh2_scp_send64 HAVE_LIBSSH2_SCP_SEND64) check_function_exists(libssh2_session_handshake HAVE_LIBSSH2_SESSION_HANDSHAKE) set(CMAKE_EXTRA_INCLUDE_FILES "") - - endif(LIBSSH2_FOUND) -endif(CMAKE_USE_LIBSSH2) + endif() +endif() option(CMAKE_USE_GSSAPI "Use GSSAPI implementation (right now only Heimdal is supported with CMake build)" OFF) mark_as_advanced(CMAKE_USE_GSSAPI) @@ -761,68 +769,67 @@ set(CURL_CA_PATH "auto" CACHE STRING "Location of default CA path. Set 'none' to disable or 'auto' for auto-detection. Defaults to 'auto'.") if("${CURL_CA_BUNDLE}" STREQUAL "") - message(FATAL_ERROR "Invalid value of CURL_CA_BUNDLE. Use 'none', 'auto' or file path.") + message(FATAL_ERROR "Invalid value of CURL_CA_BUNDLE. Use 'none', 'auto' or file path.") elseif("${CURL_CA_BUNDLE}" STREQUAL "none") - unset(CURL_CA_BUNDLE CACHE) + unset(CURL_CA_BUNDLE CACHE) elseif("${CURL_CA_BUNDLE}" STREQUAL "auto") - unset(CURL_CA_BUNDLE CACHE) - set(CURL_CA_BUNDLE_AUTODETECT TRUE) + unset(CURL_CA_BUNDLE CACHE) + set(CURL_CA_BUNDLE_AUTODETECT TRUE) else() - set(CURL_CA_BUNDLE_SET TRUE) + set(CURL_CA_BUNDLE_SET TRUE) endif() if("${CURL_CA_PATH}" STREQUAL "") - message(FATAL_ERROR "Invalid value of CURL_CA_PATH. Use 'none', 'auto' or directory path.") + message(FATAL_ERROR "Invalid value of CURL_CA_PATH. Use 'none', 'auto' or directory path.") elseif("${CURL_CA_PATH}" STREQUAL "none") - unset(CURL_CA_PATH CACHE) + unset(CURL_CA_PATH CACHE) elseif("${CURL_CA_PATH}" STREQUAL "auto") - unset(CURL_CA_PATH CACHE) - set(CURL_CA_PATH_AUTODETECT TRUE) + unset(CURL_CA_PATH CACHE) + set(CURL_CA_PATH_AUTODETECT TRUE) else() - set(CURL_CA_PATH_SET TRUE) + set(CURL_CA_PATH_SET TRUE) endif() if(CURL_CA_BUNDLE_SET AND CURL_CA_PATH_AUTODETECT) - # Skip autodetection of unset CA path because CA bundle is set explicitly + # Skip autodetection of unset CA path because CA bundle is set explicitly elseif(CURL_CA_PATH_SET AND CURL_CA_BUNDLE_AUTODETECT) - # Skip autodetection of unset CA bundle because CA path is set explicitly + # Skip autodetection of unset CA bundle because CA path is set explicitly elseif(CURL_CA_PATH_AUTODETECT OR CURL_CA_BUNDLE_AUTODETECT) - # first try autodetecting a CA bundle, then a CA path - - if(CURL_CA_BUNDLE_AUTODETECT) - set(SEARCH_CA_BUNDLE_PATHS - /etc/ssl/certs/ca-certificates.crt - /etc/pki/tls/certs/ca-bundle.crt - /usr/share/ssl/certs/ca-bundle.crt - /usr/local/share/certs/ca-root-nss.crt - /etc/ssl/cert.pem) - - foreach(SEARCH_CA_BUNDLE_PATH ${SEARCH_CA_BUNDLE_PATHS}) - if(EXISTS "${SEARCH_CA_BUNDLE_PATH}") - message(STATUS "Found CA bundle: ${SEARCH_CA_BUNDLE_PATH}") - set(CURL_CA_BUNDLE "${SEARCH_CA_BUNDLE_PATH}") - set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set") - break() - endif() - endforeach() - endif() + # first try autodetecting a CA bundle, then a CA path + + if(CURL_CA_BUNDLE_AUTODETECT) + set(SEARCH_CA_BUNDLE_PATHS + /etc/ssl/certs/ca-certificates.crt + /etc/pki/tls/certs/ca-bundle.crt + /usr/share/ssl/certs/ca-bundle.crt + /usr/local/share/certs/ca-root-nss.crt + /etc/ssl/cert.pem) + + foreach(SEARCH_CA_BUNDLE_PATH ${SEARCH_CA_BUNDLE_PATHS}) + if(EXISTS "${SEARCH_CA_BUNDLE_PATH}") + message(STATUS "Found CA bundle: ${SEARCH_CA_BUNDLE_PATH}") + set(CURL_CA_BUNDLE "${SEARCH_CA_BUNDLE_PATH}") + set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set") + break() + endif() + endforeach() + endif() - if(CURL_CA_PATH_AUTODETECT AND (NOT CURL_CA_PATH_SET)) - if(EXISTS "/etc/ssl/certs") - set(CURL_CA_PATH "/etc/ssl/certs") - set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set") - endif() + if(CURL_CA_PATH_AUTODETECT AND (NOT CURL_CA_PATH_SET)) + if(EXISTS "/etc/ssl/certs") + set(CURL_CA_PATH "/etc/ssl/certs") + set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set") endif() + endif() endif() if(CURL_CA_PATH_SET AND NOT USE_OPENSSL AND NOT USE_MBEDTLS) - message(FATAL_ERROR - "CA path only supported by OpenSSL, GnuTLS or mbed TLS. " - "Set CURL_CA_PATH=none or enable one of those TLS backends.") + message(FATAL_ERROR + "CA path only supported by OpenSSL, GnuTLS or mbed TLS. " + "Set CURL_CA_PATH=none or enable one of those TLS backends.") endif() endif() - # Check for header files if(NOT UNIX) check_include_file_concat("windows.h" HAVE_WINDOWS_H) @@ -920,7 +927,7 @@ check_type_size("time_t" SIZEOF_TIME_T) if(HAVE_SIZEOF_LONG_LONG) set(HAVE_LONGLONG 1) set(HAVE_LL 1) -endif(HAVE_SIZEOF_LONG_LONG) +endif() find_file(RANDOM_FILE urandom /dev) mark_as_advanced(RANDOM_FILE) @@ -954,7 +961,7 @@ check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI) check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM) if(NOT HAVE_STRNCMPI) set(HAVE_STRCMPI) -endif(NOT HAVE_STRNCMPI) +endif() check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR) check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R) check_symbol_exists(gettimeofday "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY) @@ -970,6 +977,7 @@ check_symbol_exists(sigsetjmp "${CURL_INCLUDES}" HAVE_SIGSETJMP) check_symbol_exists(getpass_r "${CURL_INCLUDES}" HAVE_GETPASS_R) check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT) check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID) +check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R) check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID) check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME) check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R) @@ -982,7 +990,7 @@ check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC) check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO) if(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO) set(HAVE_SIGNAL 1) -endif(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO) +endif() check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME) check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL) check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64) @@ -1016,35 +1024,35 @@ if(WIN32) add_definitions(-D_WIN32_WINNT=0x0501) endif() else() - check_function_exists(inet_pton HAVE_INET_PTON) + check_function_exists(inet_pton HAVE_INET_PTON) endif() check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR) if(HAVE_FSETXATTR) foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6) - curl_internal_test_run(${CURL_TEST}) - endforeach(CURL_TEST) -endif(HAVE_FSETXATTR) + curl_internal_test(${CURL_TEST}) + endforeach() +endif() # sigaction and sigsetjmp are special. Use special mechanism for # detecting those, but only if previous attempt failed. if(HAVE_SIGNAL_H) check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION) -endif(HAVE_SIGNAL_H) +endif() if(NOT HAVE_SIGSETJMP) if(HAVE_SETJMP_H) check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP) if(HAVE_MACRO_SIGSETJMP) set(HAVE_SIGSETJMP 1) - endif(HAVE_MACRO_SIGSETJMP) - endif(HAVE_SETJMP_H) -endif(NOT HAVE_SIGSETJMP) + endif() + endif() +endif() # If there is no stricmp(), do not allow LDAP to parse URLs if(NOT HAVE_STRICMP) set(HAVE_LDAP_URL_PARSE 1) -endif(NOT HAVE_STRICMP) +endif() # Do curl specific tests foreach(CURL_TEST @@ -1071,7 +1079,6 @@ foreach(CURL_TEST HAVE_GETHOSTBYNAME_R_3_REENTRANT HAVE_GETHOSTBYNAME_R_5_REENTRANT HAVE_GETHOSTBYNAME_R_6_REENTRANT - HAVE_SOCKLEN_T HAVE_IN_ADDR_T HAVE_BOOL_T STDC_HEADERS @@ -1082,12 +1089,12 @@ foreach(CURL_TEST HAVE_FILE_OFFSET_BITS ) curl_internal_test(${CURL_TEST}) -endforeach(CURL_TEST) +endforeach() if(HAVE_FILE_OFFSET_BITS) set(_FILE_OFFSET_BITS 64) set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64") -endif(HAVE_FILE_OFFSET_BITS) +endif() check_type_size("off_t" SIZEOF_OFF_T) # include this header to get the type @@ -1102,8 +1109,8 @@ foreach(CURL_TEST HAVE_GLIBC_STRERROR_R HAVE_POSIX_STRERROR_R ) - curl_internal_test_run(${CURL_TEST}) -endforeach(CURL_TEST) + curl_internal_test(${CURL_TEST}) +endforeach() # Check for reentrant foreach(CURL_TEST @@ -1117,9 +1124,9 @@ foreach(CURL_TEST if(NOT ${CURL_TEST}) if(${CURL_TEST}_REENTRANT) set(NEED_REENTRANT 1) - endif(${CURL_TEST}_REENTRANT) - endif(NOT ${CURL_TEST}) -endforeach(CURL_TEST) + endif() + endif() +endforeach() if(NEED_REENTRANT) foreach(CURL_TEST @@ -1132,32 +1139,38 @@ if(NEED_REENTRANT) set(${CURL_TEST} 0) if(${CURL_TEST}_REENTRANT) set(${CURL_TEST} 1) - endif(${CURL_TEST}_REENTRANT) - endforeach(CURL_TEST) -endif(NEED_REENTRANT) + endif() + endforeach() +endif() if(HAVE_INET_NTOA_R_DECL_REENTRANT) set(HAVE_INET_NTOA_R_DECL 1) set(NEED_REENTRANT 1) -endif(HAVE_INET_NTOA_R_DECL_REENTRANT) +endif() + +# Check clock_gettime(CLOCK_MONOTONIC, x) support +curl_internal_test(HAVE_CLOCK_GETTIME_MONOTONIC) + +# Check compiler support of __builtin_available() +curl_internal_test(HAVE_BUILTIN_AVAILABLE) # Some other minor tests if(NOT HAVE_IN_ADDR_T) set(in_addr_t "unsigned long") -endif(NOT HAVE_IN_ADDR_T) +endif() # Fix libz / zlib.h if(NOT CURL_SPECIAL_LIBZ) if(NOT HAVE_LIBZ) set(HAVE_ZLIB_H 0) - endif(NOT HAVE_LIBZ) + endif() if(NOT HAVE_ZLIB_H) set(HAVE_LIBZ 0) - endif(NOT HAVE_ZLIB_H) -endif(NOT CURL_SPECIAL_LIBZ) + endif() +endif() # Check for nonblocking set(HAVE_DISABLED_NONBLOCKING 1) @@ -1166,16 +1179,13 @@ if(HAVE_FIONBIO OR HAVE_IOCTLSOCKET_CASE OR HAVE_O_NONBLOCK) set(HAVE_DISABLED_NONBLOCKING) -endif(HAVE_FIONBIO OR - HAVE_IOCTLSOCKET OR - HAVE_IOCTLSOCKET_CASE OR - HAVE_O_NONBLOCK) +endif() if(RETSIGTYPE_TEST) set(RETSIGTYPE void) -else(RETSIGTYPE_TEST) +else() set(RETSIGTYPE int) -endif(RETSIGTYPE_TEST) +endif() if(CMAKE_COMPILER_IS_GNUCC AND APPLE) include(CheckCCompilerFlag) @@ -1185,30 +1195,12 @@ if(CMAKE_COMPILER_IS_GNUCC AND APPLE) get_source_file_property(MPRINTF_COMPILE_FLAGS mprintf.c COMPILE_FLAGS) if(MPRINTF_COMPILE_FLAGS) set(MPRINTF_COMPILE_FLAGS "${MPRINTF_COMPILE_FLAGS} -Wno-long-double") - else(MPRINTF_COMPILE_FLAGS) + else() set(MPRINTF_COMPILE_FLAGS "-Wno-long-double") - endif(MPRINTF_COMPILE_FLAGS) + endif() set_source_files_properties(mprintf.c PROPERTIES COMPILE_FLAGS ${MPRINTF_COMPILE_FLAGS}) - endif(HAVE_C_FLAG_Wno_long_double) -endif(CMAKE_COMPILER_IS_GNUCC AND APPLE) - -if(HAVE_SOCKLEN_T) - set(CURL_HAVE_SOCKLEN_T 1) - set(CURL_TYPEOF_CURL_SOCKLEN_T "socklen_t") - if(WIN32) - set(CMAKE_EXTRA_INCLUDE_FILES "winsock2.h;ws2tcpip.h") - elseif(HAVE_SYS_SOCKET_H) - set(CMAKE_EXTRA_INCLUDE_FILES "sys/socket.h") - endif() - check_type_size("socklen_t" CURL_SIZEOF_CURL_SOCKLEN_T) - set(CMAKE_EXTRA_INCLUDE_FILES) - if(NOT HAVE_CURL_SIZEOF_CURL_SOCKLEN_T) - message(FATAL_ERROR - "Check for sizeof socklen_t failed, see CMakeFiles/CMakerror.log") endif() -else() - set(CURL_HAVE_SOCKLEN_T 0) endif() # TODO test which of these headers are required @@ -1226,19 +1218,25 @@ include(CMake/OtherTests.cmake) add_definitions(-DHAVE_CONFIG_H) -# For windows, all compilers used by cmake should support large files +# For Windows, all compilers used by CMake should support large files if(WIN32) set(USE_WIN32_LARGE_FILES ON) -endif(WIN32) + + # Use the manifest embedded in the Windows Resource + set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -DCURL_EMBED_MANIFEST") +endif() if(MSVC) + # Disable default manifest added by CMake + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") + add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) if(CMAKE_C_FLAGS MATCHES "/W[0-4]") string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") - else(CMAKE_C_FLAGS MATCHES "/W[0-4]") + else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4") - endif(CMAKE_C_FLAGS MATCHES "/W[0-4]") -endif(MSVC) + endif() +endif() if(CURL_WERROR) if(MSVC_VERSION) @@ -1247,7 +1245,7 @@ if(CURL_WERROR) # this assumes clang or gcc style options set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") endif() -endif(CURL_WERROR) +endif() # Ugly (but functional) way to include "Makefile.inc" by transforming it (= regenerate it). function(TRANSFORM_MAKEFILE_INC INPUT_FILE OUTPUT_FILE) @@ -1265,10 +1263,14 @@ function(TRANSFORM_MAKEFILE_INC INPUT_FILE OUTPUT_FILE) endfunction() -if(WIN32 AND NOT CYGWIN) - set(CURL_INSTALL_CMAKE_DIR CMake) -else() - set(CURL_INSTALL_CMAKE_DIR lib/cmake/curl) +if(0) # This code not needed for building within CMake. +include(GNUInstallDirs) + +set(CURL_INSTALL_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) +set(TARGETS_EXPORT_NAME "${PROJECT_NAME}Targets") +set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") +set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake") +set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") endif() if(USE_MANUAL) @@ -1283,11 +1285,11 @@ endif() #----------------------------------------------------------------------------- # CMake-specific curl code. -add_executable(LIBCURL curltest.c) -target_link_libraries(LIBCURL cmcurl) +add_executable(curltest curltest.c) +target_link_libraries(curltest cmcurl) -if(CMAKE_CURL_TEST_URL) - add_test(curl LIBCURL ${CMAKE_CURL_TEST_URL}) +if(BUILD_TESTING AND CMAKE_CURL_TEST_URL) + add_test(curl curltest ${CMAKE_CURL_TEST_URL}) endif() install(FILES COPYING DESTINATION ${CMAKE_DOC_DIR}/cmcurl) @@ -1381,11 +1383,12 @@ set(CONFIGURE_OPTIONS "") # TODO when to set "-DCURL_STATICLIB" for CPPFLAG_CURL_STATICLIB? set(CPPFLAG_CURL_STATICLIB "") set(CURLVERSION "${CURL_VERSION}") -set(ENABLE_SHARED "yes") -if(CURL_STATICLIB) - set(ENABLE_STATIC "yes") -else() +if(BUILD_SHARED_LIBS) + set(ENABLE_SHARED "yes") set(ENABLE_STATIC "no") +else() + set(ENABLE_SHARED "no") + set(ENABLE_STATIC "yes") endif() set(exec_prefix "\${prefix}") set(includedir "\${prefix}/include") @@ -1409,10 +1412,13 @@ set(REQUIRE_LIB_DEPS "no") set(VERSIONNUM "${CURL_VERSION_NUM}") # Finally generate a "curl-config" matching this config +# Use: +# * ENABLE_SHARED +# * ENABLE_STATIC configure_file("${CURL_SOURCE_DIR}/curl-config.in" "${CURL_BINARY_DIR}/curl-config" @ONLY) install(FILES "${CURL_BINARY_DIR}/curl-config" - DESTINATION bin + DESTINATION ${CMAKE_INSTALL_BINDIR} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE @@ -1422,34 +1428,36 @@ install(FILES "${CURL_BINARY_DIR}/curl-config" configure_file("${CURL_SOURCE_DIR}/libcurl.pc.in" "${CURL_BINARY_DIR}/libcurl.pc" @ONLY) install(FILES "${CURL_BINARY_DIR}/libcurl.pc" - DESTINATION lib/pkgconfig) - -# This needs to be run very last so other parts of the scripts can take advantage of this. -if(NOT CURL_CONFIG_HAS_BEEN_RUN_BEFORE) - set(CURL_CONFIG_HAS_BEEN_RUN_BEFORE 1 CACHE INTERNAL "Flag to track whether this is the first time running CMake or if CMake has been configured before") -endif() + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) # install headers install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/curl" - DESTINATION include + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.h") - include(CMakePackageConfigHelpers) write_basic_package_version_file( - "${PROJECT_BINARY_DIR}/curl-config-version.cmake" + "${version_config}" VERSION ${CURL_VERSION} COMPATIBILITY SameMajorVersion ) -configure_file(CMake/curl-config.cmake - "${PROJECT_BINARY_DIR}/curl-config.cmake" - COPYONLY +# Use: +# * TARGETS_EXPORT_NAME +# * PROJECT_NAME +configure_package_config_file(CMake/curl-config.cmake.in + "${project_config}" + INSTALL_DESTINATION ${CURL_INSTALL_CMAKE_DIR} +) + +install( + EXPORT "${TARGETS_EXPORT_NAME}" + NAMESPACE "${PROJECT_NAME}::" + DESTINATION ${CURL_INSTALL_CMAKE_DIR} ) install( - FILES ${PROJECT_BINARY_DIR}/curl-config.cmake - ${PROJECT_BINARY_DIR}/curl-config-version.cmake + FILES ${version_config} ${project_config} DESTINATION ${CURL_INSTALL_CMAKE_DIR} ) diff --git a/Utilities/cmcurl/curltest.c b/Utilities/cmcurl/curltest.c index 210868e36..f80e758a0 100644 --- a/Utilities/cmcurl/curltest.c +++ b/Utilities/cmcurl/curltest.c @@ -1,159 +1,81 @@ -/* Prevent warnings on Visual Studio */ -struct _RPC_ASYNC_STATE; - #include "curl/curl.h" + +#include <stdio.h> #include <stdlib.h> #include <string.h> -int GetFtpFile(void) +int test_curl(const char* url) { - int retVal = 0; - CURL *curl; - CURLcode res; - curl = curl_easy_init(); - if(curl) - { - /* Get curl 7.9.2 from sunet.se's FTP site: */ - curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); - curl_easy_setopt(curl, CURLOPT_HEADER, 1); - curl_easy_setopt(curl, CURLOPT_URL, - "ftp://public.kitware.com/pub/cmake/cygwin/setup.hint"); - res = curl_easy_perform(curl); - if ( res != 0 ) - { - printf("Error fetching: ftp://public.kitware.com/pub/cmake/cygwin/setup.hint\n"); - retVal = 1; - } - - /* always cleanup */ - curl_easy_cleanup(curl); - } - else - { - printf("Cannot create curl object\n"); - retVal = 1; - } - return retVal; -} - -int GetWebFiles(char *url1, char *url2) -{ - int retVal = 0; - CURL *curl; - CURLcode res; - + CURL* curl; + CURLcode r; char proxy[1024]; int proxy_type = 0; - if ( getenv("HTTP_PROXY") ) - { + if (getenv("HTTP_PROXY")) { proxy_type = 1; - if (getenv("HTTP_PROXY_PORT") ) - { + if (getenv("HTTP_PROXY_PORT")) { sprintf(proxy, "%s:%s", getenv("HTTP_PROXY"), getenv("HTTP_PROXY_PORT")); - } - else - { + } else { sprintf(proxy, "%s", getenv("HTTP_PROXY")); - } - if ( getenv("HTTP_PROXY_TYPE") ) - { + } + if (getenv("HTTP_PROXY_TYPE")) { /* HTTP/SOCKS4/SOCKS5 */ - if ( strcmp(getenv("HTTP_PROXY_TYPE"), "HTTP") == 0 ) - { + if (strcmp(getenv("HTTP_PROXY_TYPE"), "HTTP") == 0) { proxy_type = 1; - } - else if ( strcmp(getenv("HTTP_PROXY_TYPE"), "SOCKS4") == 0 ) - { + } else if (strcmp(getenv("HTTP_PROXY_TYPE"), "SOCKS4") == 0) { proxy_type = 2; - } - else if ( strcmp(getenv("HTTP_PROXY_TYPE"), "SOCKS5") == 0 ) - { + } else if (strcmp(getenv("HTTP_PROXY_TYPE"), "SOCKS5") == 0) { proxy_type = 3; - } } } + } curl = curl_easy_init(); - if(curl) - { - curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); - curl_easy_setopt(curl, CURLOPT_HEADER, 1); - - /* Using proxy */ - if ( proxy_type > 0 ) - { - curl_easy_setopt(curl, CURLOPT_PROXY, proxy); - switch (proxy_type) - { - case 2: - curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4); - break; - case 3: - curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); - break; - default: - curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); - } - } - - /* get the first document */ - curl_easy_setopt(curl, CURLOPT_URL, url1); - res = curl_easy_perform(curl); - if ( res != 0 ) - { - printf("Error fetching: %s\n", url1); - retVal = 1; - } + if (!curl) { + fprintf(stderr, "curl_easy_init failed\n"); + return 1; + } + + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(curl, CURLOPT_HEADER, 1); + + if (proxy_type > 0) { + curl_easy_setopt(curl, CURLOPT_PROXY, proxy); + switch (proxy_type) { + case 2: + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4); + break; + case 3: + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); + break; + default: + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + } + } - /* get another document from the same server using the same - connection */ - /* avoid warnings about url2 since below block is commented out: */ - (void) url2; - /* - curl_easy_setopt(curl, CURLOPT_URL, url2); - res = curl_easy_perform(curl); - if ( res != 0 ) - { - printf("Error fetching: %s\n", url2); - retVal = 1; - } - */ + curl_easy_setopt(curl, CURLOPT_URL, url); + r = curl_easy_perform(curl); + curl_easy_cleanup(curl); - /* always cleanup */ - curl_easy_cleanup(curl); - } - else - { - printf("Cannot create curl object\n"); - retVal = 1; - } + if (r != CURLE_OK) { + fprintf(stderr, "error: fetching '%s' failed: %s\n", url, + curl_easy_strerror(r)); + return 1; + } - return retVal; + return 0; } - -int main(int argc, char **argv) +int main(int argc, const char* argv[]) { - int retVal = 0; - + int r; curl_global_init(CURL_GLOBAL_DEFAULT); - - if(argc>1) - { - retVal += GetWebFiles(argv[1], 0); - } - else - { - printf("error: first argument should be a url to download\n"); - retVal = 1; - } - - /* Do not check the output of FTP socks5 cannot handle FTP yet */ - /* GetFtpFile(); */ - /* do not test ftp right now because we don't enable that port */ - + if (argc == 2) { + r = test_curl(argv[1]); + } else { + fprintf(stderr, "error: no URL given as first argument\n"); + r = 1; + } curl_global_cleanup(); - - return retVal; + return r; } diff --git a/Utilities/cmcurl/include/curl/curl.h b/Utilities/cmcurl/include/curl/curl.h index 10e5c494b..63bb291b3 100644 --- a/Utilities/cmcurl/include/curl/curl.h +++ b/Utilities/cmcurl/include/curl/curl.h @@ -146,7 +146,8 @@ typedef enum { CURLSSLBACKEND_SCHANNEL = 8, CURLSSLBACKEND_DARWINSSL = 9, CURLSSLBACKEND_AXTLS = 10, - CURLSSLBACKEND_MBEDTLS = 11 + CURLSSLBACKEND_MBEDTLS = 11, + CURLSSLBACKEND_MESALINK = 12 } curl_sslbackend; /* aliases for library clones and renames */ @@ -517,8 +518,7 @@ typedef enum { CURLE_UNKNOWN_OPTION, /* 48 - User specified an unknown option */ CURLE_TELNET_OPTION_SYNTAX, /* 49 - Malformed telnet option */ CURLE_OBSOLETE50, /* 50 - NOT USED */ - CURLE_PEER_FAILED_VERIFICATION, /* 51 - peer's certificate or fingerprint - wasn't verified fine */ + CURLE_OBSOLETE51, /* 51 - NOT USED */ CURLE_GOT_NOTHING, /* 52 - when this is a specific error */ CURLE_SSL_ENGINE_NOTFOUND, /* 53 - SSL crypto engine not found */ CURLE_SSL_ENGINE_SETFAILED, /* 54 - can not set SSL crypto engine as @@ -528,7 +528,8 @@ typedef enum { CURLE_OBSOLETE57, /* 57 - NOT IN USE */ CURLE_SSL_CERTPROBLEM, /* 58 - problem with the local certificate */ CURLE_SSL_CIPHER, /* 59 - couldn't use specified cipher */ - CURLE_SSL_CACERT, /* 60 - problem with the CA cert (path?) */ + CURLE_PEER_FAILED_VERIFICATION, /* 60 - peer's certificate or fingerprint + wasn't verified fine */ CURLE_BAD_CONTENT_ENCODING, /* 61 - Unrecognized/bad encoding */ CURLE_LDAP_INVALID_URL, /* 62 - Invalid LDAP URL */ CURLE_FILESIZE_EXCEEDED, /* 63 - Maximum file size exceeded */ @@ -584,6 +585,9 @@ typedef enum { CURL_LAST /* never use! */ } CURLcode; +/* added in 7.62.0 */ +#define CURLE_SSL_CACERT CURLE_PEER_FAILED_VERIFICATION + #ifndef CURL_NO_OLDIES /* define this to test if your app builds with all the obsolete stuff removed! */ @@ -691,6 +695,7 @@ typedef enum { * CURLAUTH_NTLM - HTTP NTLM authentication * CURLAUTH_DIGEST_IE - HTTP Digest authentication with IE flavour * CURLAUTH_NTLM_WB - HTTP NTLM authentication delegated to winbind helper + * CURLAUTH_BEARER - HTTP Bearer token authentication * CURLAUTH_ONLY - Use together with a single other type to force no * authentication or just that single type * CURLAUTH_ANY - All fine types set @@ -708,6 +713,7 @@ typedef enum { #define CURLAUTH_NTLM (((unsigned long)1)<<3) #define CURLAUTH_DIGEST_IE (((unsigned long)1)<<4) #define CURLAUTH_NTLM_WB (((unsigned long)1)<<5) +#define CURLAUTH_BEARER (((unsigned long)1)<<6) #define CURLAUTH_ONLY (((unsigned long)1)<<31) #define CURLAUTH_ANY (~CURLAUTH_DIGEST_IE) #define CURLAUTH_ANYSAFE (~(CURLAUTH_BASIC|CURLAUTH_DIGEST_IE)) @@ -798,6 +804,9 @@ typedef enum { this value, keep them in sync. */ #define CURL_HET_DEFAULT 200L +/* The default connection upkeep interval in milliseconds. */ +#define CURL_UPKEEP_INTERVAL_DEFAULT 60000L + #ifndef CURL_NO_OLDIES /* define this to test if your app builds with all the obsolete stuff removed! */ @@ -1847,6 +1856,22 @@ typedef enum { /* shuffle addresses before use when DNS returns multiple */ CINIT(DNS_SHUFFLE_ADDRESSES, LONG, 275), + /* Specify which TLS 1.3 ciphers suites to use */ + CINIT(TLS13_CIPHERS, STRINGPOINT, 276), + CINIT(PROXY_TLS13_CIPHERS, STRINGPOINT, 277), + + /* Disallow specifying username/login in URL. */ + CINIT(DISALLOW_USERNAME_IN_URL, LONG, 278), + + /* DNS-over-HTTPS URL */ + CINIT(DOH_URL, STRINGPOINT, 279), + + /* Preferred buffer size to use for uploads */ + CINIT(UPLOAD_BUFFERSIZE, LONG, 280), + + /* Time in ms between connection upkeep calls for long-lived connections. */ + CINIT(UPKEEP_INTERVAL_MS, LONG, 281), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; @@ -2527,7 +2552,17 @@ typedef enum { CURLINFO_SCHEME = CURLINFO_STRING + 49, /* Fill in new entries below here! */ - CURLINFO_LASTONE = 49 + /* Preferably these would be defined conditionally based on the + sizeof curl_off_t being 64-bits */ + CURLINFO_TOTAL_TIME_T = CURLINFO_OFF_T + 50, + CURLINFO_NAMELOOKUP_TIME_T = CURLINFO_OFF_T + 51, + CURLINFO_CONNECT_TIME_T = CURLINFO_OFF_T + 52, + CURLINFO_PRETRANSFER_TIME_T = CURLINFO_OFF_T + 53, + CURLINFO_STARTTRANSFER_TIME_T = CURLINFO_OFF_T + 54, + CURLINFO_REDIRECT_TIME_T = CURLINFO_OFF_T + 55, + CURLINFO_APPCONNECT_TIME_T = CURLINFO_OFF_T + 56, + + CURLINFO_LASTONE = 56 } CURLINFO; /* CURLINFO_RESPONSE_CODE is the new name for the option previously known as @@ -2570,6 +2605,7 @@ typedef enum { CURL_LOCK_DATA_DNS, CURL_LOCK_DATA_SSL_SESSION, CURL_LOCK_DATA_CONNECT, + CURL_LOCK_DATA_PSL, CURL_LOCK_DATA_LAST } curl_lock_data; @@ -2759,6 +2795,7 @@ CURL_EXTERN CURLcode curl_easy_pause(CURL *handle, int bitmask); stuff before they can be included! */ #include "easy.h" /* nothing in curl is fun without the easy stuff */ #include "multi.h" +#include "urlapi.h" /* the typechecker doesn't work in C++ (yet) */ #if defined(__GNUC__) && defined(__GNUC_MINOR__) && \ diff --git a/Utilities/cmcurl/include/curl/curlver.h b/Utilities/cmcurl/include/curl/curlver.h index 2d1505aa4..3d7b1fbfe 100644 --- a/Utilities/cmcurl/include/curl/curlver.h +++ b/Utilities/cmcurl/include/curl/curlver.h @@ -30,12 +30,12 @@ /* This is the version number of the libcurl package from which this header file origins: */ -#define LIBCURL_VERSION "7.60.0" +#define LIBCURL_VERSION "7.62.0" /* The numeric version number is also available "in parts" by using these defines: */ #define LIBCURL_VERSION_MAJOR 7 -#define LIBCURL_VERSION_MINOR 60 +#define LIBCURL_VERSION_MINOR 62 #define LIBCURL_VERSION_PATCH 0 /* This is the numeric version of the libcurl version number, meant for easier @@ -57,7 +57,7 @@ CURL_VERSION_BITS() macro since curl's own configure script greps for it and needs it to contain the full number. */ -#define LIBCURL_VERSION_NUM 0x073C00 +#define LIBCURL_VERSION_NUM 0x073E00 /* * This is the date and time when the full source package was created. The diff --git a/Utilities/cmcurl/include/curl/easy.h b/Utilities/cmcurl/include/curl/easy.h index 752c5049f..f42a8a969 100644 --- a/Utilities/cmcurl/include/curl/easy.h +++ b/Utilities/cmcurl/include/curl/easy.h @@ -95,6 +95,16 @@ CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen, CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer, size_t buflen, size_t *n); + +/* + * NAME curl_easy_upkeep() + * + * DESCRIPTION + * + * Performs connection upkeep for the given session handle. + */ +CURL_EXTERN CURLcode curl_easy_upkeep(CURL *curl); + #ifdef __cplusplus } #endif diff --git a/Utilities/cmcurl/include/curl/system.h b/Utilities/cmcurl/include/curl/system.h index eac4cfeb6..1e555ec19 100644 --- a/Utilities/cmcurl/include/curl/system.h +++ b/Utilities/cmcurl/include/curl/system.h @@ -298,7 +298,7 @@ # define CURL_PULL_SYS_TYPES_H 1 # define CURL_PULL_SYS_SOCKET_H 1 -#elif defined(__SUNPRO_C) /* Oracle Solaris Studio */ +#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) /* Oracle Solaris Studio */ # if !defined(__LP64) && (defined(__ILP32) || \ defined(__i386) || \ defined(__sparcv8) || \ @@ -320,6 +320,24 @@ # define CURL_PULL_SYS_TYPES_H 1 # define CURL_PULL_SYS_SOCKET_H 1 +#elif defined(__xlc__) /* IBM xlc compiler */ +# if !defined(_LP64) +# define CURL_TYPEOF_CURL_OFF_T long long +# define CURL_FORMAT_CURL_OFF_T "lld" +# define CURL_FORMAT_CURL_OFF_TU "llu" +# define CURL_SUFFIX_CURL_OFF_T LL +# define CURL_SUFFIX_CURL_OFF_TU ULL +# else +# define CURL_TYPEOF_CURL_OFF_T long +# define CURL_FORMAT_CURL_OFF_T "ld" +# define CURL_FORMAT_CURL_OFF_TU "lu" +# define CURL_SUFFIX_CURL_OFF_T L +# define CURL_SUFFIX_CURL_OFF_TU UL +# endif +# define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t +# define CURL_PULL_SYS_TYPES_H 1 +# define CURL_PULL_SYS_SOCKET_H 1 + /* ===================================== */ /* KEEP MSVC THE PENULTIMATE ENTRY */ /* ===================================== */ @@ -344,7 +362,7 @@ /* KEEP GENERIC GCC THE LAST ENTRY */ /* ===================================== */ -#elif defined(__GNUC__) +#elif defined(__GNUC__) && !defined(_SCO_DS) # if !defined(__LP64__) && \ (defined(__ILP32__) || defined(__i386__) || defined(__hppa__) || \ defined(__ppc__) || defined(__powerpc__) || defined(__arm__) || \ diff --git a/Utilities/cmcurl/include/curl/typecheck-gcc.h b/Utilities/cmcurl/include/curl/typecheck-gcc.h index 3a0f253f6..244336204 100644 --- a/Utilities/cmcurl/include/curl/typecheck-gcc.h +++ b/Utilities/cmcurl/include/curl/typecheck-gcc.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -269,6 +269,7 @@ _CURL_WARNING(_curl_easy_getinfo_err_curl_off_t, (option) == CURLOPT_DNS_LOCAL_IP4 || \ (option) == CURLOPT_DNS_LOCAL_IP6 || \ (option) == CURLOPT_DNS_SERVERS || \ + (option) == CURLOPT_DOH_URL || \ (option) == CURLOPT_EGDSOCKET || \ (option) == CURLOPT_FTPPORT || \ (option) == CURLOPT_FTP_ACCOUNT || \ @@ -428,7 +429,7 @@ _CURL_WARNING(_curl_easy_getinfo_err_curl_off_t, * == or whatsoever. */ -/* XXX: should evaluate to true iff expr is a pointer */ +/* XXX: should evaluate to true if expr is a pointer */ #define _curl_is_any_ptr(expr) \ (sizeof(expr) == sizeof(void *)) @@ -500,7 +501,8 @@ _CURL_WARNING(_curl_easy_getinfo_err_curl_off_t, /* evaluates to true if expr can be passed as POST data (void* or char*) */ #define _curl_is_postfields(expr) \ (_curl_is_ptr((expr), void) || \ - _curl_is_arr((expr), char)) + _curl_is_arr((expr), char) || \ + _curl_is_arr((expr), unsigned char)) /* FIXME: the whole callback checking is messy... * The idea is to tolerate char vs. void and const vs. not const diff --git a/Utilities/cmcurl/include/curl/urlapi.h b/Utilities/cmcurl/include/curl/urlapi.h new file mode 100644 index 000000000..90dd56c00 --- /dev/null +++ b/Utilities/cmcurl/include/curl/urlapi.h @@ -0,0 +1,120 @@ +#ifndef __CURL_URLAPI_H +#define __CURL_URLAPI_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* the error codes for the URL API */ +typedef enum { + CURLUE_OK, + CURLUE_BAD_HANDLE, /* 1 */ + CURLUE_BAD_PARTPOINTER, /* 2 */ + CURLUE_MALFORMED_INPUT, /* 3 */ + CURLUE_BAD_PORT_NUMBER, /* 4 */ + CURLUE_UNSUPPORTED_SCHEME, /* 5 */ + CURLUE_URLDECODE, /* 6 */ + CURLUE_OUT_OF_MEMORY, /* 7 */ + CURLUE_USER_NOT_ALLOWED, /* 8 */ + CURLUE_UNKNOWN_PART, /* 9 */ + CURLUE_NO_SCHEME, /* 10 */ + CURLUE_NO_USER, /* 11 */ + CURLUE_NO_PASSWORD, /* 12 */ + CURLUE_NO_OPTIONS, /* 13 */ + CURLUE_NO_HOST, /* 14 */ + CURLUE_NO_PORT, /* 15 */ + CURLUE_NO_QUERY, /* 16 */ + CURLUE_NO_FRAGMENT /* 17 */ +} CURLUcode; + +typedef enum { + CURLUPART_URL, + CURLUPART_SCHEME, + CURLUPART_USER, + CURLUPART_PASSWORD, + CURLUPART_OPTIONS, + CURLUPART_HOST, + CURLUPART_PORT, + CURLUPART_PATH, + CURLUPART_QUERY, + CURLUPART_FRAGMENT +} CURLUPart; + +#define CURLU_DEFAULT_PORT (1<<0) /* return default port number */ +#define CURLU_NO_DEFAULT_PORT (1<<1) /* act as if no port number was set, + if the port number matches the + default for the scheme */ +#define CURLU_DEFAULT_SCHEME (1<<2) /* return default scheme if + missing */ +#define CURLU_NON_SUPPORT_SCHEME (1<<3) /* allow non-supported scheme */ +#define CURLU_PATH_AS_IS (1<<4) /* leave dot sequences */ +#define CURLU_DISALLOW_USER (1<<5) /* no user+password allowed */ +#define CURLU_URLDECODE (1<<6) /* URL decode on get */ +#define CURLU_URLENCODE (1<<7) /* URL encode on set */ +#define CURLU_APPENDQUERY (1<<8) /* append a form style part */ +#define CURLU_GUESS_SCHEME (1<<9) /* legacy curl-style guessing */ + +typedef struct Curl_URL CURLU; + +/* + * curl_url() creates a new CURLU handle and returns a pointer to it. + * Must be freed with curl_url_cleanup(). + */ +CURL_EXTERN CURLU *curl_url(void); + +/* + * curl_url_cleanup() frees the CURLU handle and related resources used for + * the URL parsing. It will not free strings previously returned with the URL + * API. + */ +CURL_EXTERN void curl_url_cleanup(CURLU *handle); + +/* + * curl_url_dup() duplicates a CURLU handle and returns a new copy. The new + * handle must also be freed with curl_url_cleanup(). + */ +CURL_EXTERN CURLU *curl_url_dup(CURLU *in); + +/* + * curl_url_get() extracts a specific part of the URL from a CURLU + * handle. Returns error code. The returned pointer MUST be freed with + * curl_free() afterwards. + */ +CURL_EXTERN CURLUcode curl_url_get(CURLU *handle, CURLUPart what, + char **part, unsigned int flags); + +/* + * curl_url_set() sets a specific part of the URL in a CURLU handle. Returns + * error code. The passed in string will be copied. Passing a NULL instead of + * a part string, clears that part. + */ +CURL_EXTERN CURLUcode curl_url_set(CURLU *handle, CURLUPart what, + const char *part, unsigned int flags); + + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif diff --git a/Utilities/cmcurl/lib/CMakeLists.txt b/Utilities/cmcurl/lib/CMakeLists.txt index 9f19daa79..2a6279c29 100644 --- a/Utilities/cmcurl/lib/CMakeLists.txt +++ b/Utilities/cmcurl/lib/CMakeLists.txt @@ -1,5 +1,13 @@ set(LIB_NAME cmcurl) +if(BUILD_SHARED_LIBS) + set(CURL_STATICLIB NO) +else() + set(CURL_STATICLIB YES) +endif() + +# Use: +# * CURL_STATICLIB configure_file(curl_config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/curl_config.h) @@ -59,14 +67,6 @@ if(USE_ARES) include_directories(${CARES_INCLUDE_DIR}) endif() -if(CURL_STATICLIB) - # Static lib - set(CURL_USER_DEFINED_DYNAMIC_OR_STATIC STATIC) -else() - # DLL / so dynamic lib - set(CURL_USER_DEFINED_DYNAMIC_OR_STATIC SHARED) -endif() - # For windows we want to install OPENSSL_LIBRARIES dlls # and also copy them into the build tree so that testing # can find them. @@ -93,16 +93,19 @@ endif() add_library( ${LIB_NAME} - ${CURL_USER_DEFINED_DYNAMIC_OR_STATIC} ${HHEADERS} ${CSOURCES} ${CMAKE_CURL_SSL_DLLS} ) +if(NOT BUILD_SHARED_LIBS) + set_target_properties(${LIB_NAME} PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB) +endif() + target_link_libraries(${LIB_NAME} ${CURL_LIBS}) if(0) # This code not needed for building within CMake. if(WIN32) - add_definitions( -D_USRDLL ) + add_definitions(-D_USRDLL) endif() endif() @@ -119,27 +122,21 @@ set_target_properties(${LIB_NAME} PROPERTIES PREFIX "") set_target_properties(${LIB_NAME} PROPERTIES IMPORT_PREFIX "") if(WIN32) - if(NOT CURL_STATICLIB) + if(BUILD_SHARED_LIBS) # Add "_imp" as a suffix before the extension to avoid conflicting with the statically linked "libcurl.lib" set_target_properties(${LIB_NAME} PROPERTIES IMPORT_SUFFIX "_imp.lib") - - set_target_properties (${LIB_NAME} PROPERTIES - DEBUG_POSTFIX "-d" - # Note: no postfix for release variants, let user choose what style of release he wants - # MINSIZEREL_POSTFIX "-z" - # RELWITHDEBINFO_POSTFIX "-g" - ) endif() endif() target_include_directories(${LIB_NAME} INTERFACE - $<INSTALL_INTERFACE:include>) + $<INSTALL_INTERFACE:include> + $<BUILD_INTERFACE:${CURL_SOURCE_DIR}/include>) install(TARGETS ${LIB_NAME} - EXPORT libcurl-target - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION bin + EXPORT ${TARGETS_EXPORT_NAME} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) export(TARGETS ${LIB_NAME} @@ -147,10 +144,4 @@ export(TARGETS ${LIB_NAME} NAMESPACE CURL:: ) -install(EXPORT libcurl-target - FILE libcurl-target.cmake - NAMESPACE CURL:: - DESTINATION ${CURL_INSTALL_CMAKE_DIR} -) - endif() diff --git a/Utilities/cmcurl/lib/Makefile.inc b/Utilities/cmcurl/lib/Makefile.inc index 61c23411d..4aa042259 100644 --- a/Utilities/cmcurl/lib/Makefile.inc +++ b/Utilities/cmcurl/lib/Makefile.inc @@ -30,12 +30,12 @@ LIB_VAUTH_HFILES = vauth/vauth.h vauth/digest.h vauth/ntlm.h LIB_VTLS_CFILES = vtls/openssl.c vtls/gtls.c vtls/vtls.c vtls/nss.c \ vtls/polarssl.c vtls/polarssl_threadlock.c vtls/axtls.c \ vtls/cyassl.c vtls/schannel.c vtls/schannel_verify.c \ - vtls/darwinssl.c vtls/gskit.c vtls/mbedtls.c + vtls/darwinssl.c vtls/gskit.c vtls/mbedtls.c vtls/mesalink.c LIB_VTLS_HFILES = vtls/openssl.h vtls/vtls.h vtls/gtls.h \ vtls/nssg.h vtls/polarssl.h vtls/polarssl_threadlock.h vtls/axtls.h \ vtls/cyassl.h vtls/schannel.h vtls/darwinssl.h vtls/gskit.h \ - vtls/mbedtls.h + vtls/mbedtls.h vtls/mesalink.h LIB_CFILES = file.c timeval.c base64.c hostip.c progress.c formdata.c \ cookie.c http.c sendf.c ftp.c url.c dict.c if2ip.c speedcheck.c \ @@ -54,7 +54,8 @@ LIB_CFILES = file.c timeval.c base64.c hostip.c progress.c formdata.c \ http_ntlm.c curl_ntlm_wb.c curl_ntlm_core.c curl_sasl.c rand.c \ curl_multibyte.c hostcheck.c conncache.c pipeline.c dotdot.c \ x509asn1.c http2.c smb.c curl_endian.c curl_des.c system_win32.c \ - mime.c sha256.c setopt.c curl_path.c curl_ctype.c curl_range.c + mime.c sha256.c setopt.c curl_path.c curl_ctype.c curl_range.c psl.c \ + doh.c urlapi.c LIB_HFILES = arpa_telnet.h netrc.h file.h timeval.h hostip.h progress.h \ formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h if2ip.h \ @@ -74,7 +75,7 @@ LIB_HFILES = arpa_telnet.h netrc.h file.h timeval.h hostip.h progress.h \ curl_setup_once.h multihandle.h setup-vms.h pipeline.h dotdot.h \ x509asn1.h http2.h sigpipe.h smb.h curl_endian.h curl_des.h \ curl_printf.h system_win32.h rand.h mime.h curl_sha256.h setopt.h \ - curl_path.h curl_ctype.h curl_range.h + curl_path.h curl_ctype.h curl_range.h psl.h doh.h urlapi-int.h LIB_RCFILES = libcurl.rc diff --git a/Utilities/cmcurl/lib/amigaos.h b/Utilities/cmcurl/lib/amigaos.h index 02bee1610..7c0926cc3 100644 --- a/Utilities/cmcurl/lib/amigaos.h +++ b/Utilities/cmcurl/lib/amigaos.h @@ -36,4 +36,3 @@ void Curl_amiga_cleanup(); #endif #endif /* HEADER_CURL_AMIGAOS_H */ - diff --git a/Utilities/cmcurl/lib/arpa_telnet.h b/Utilities/cmcurl/lib/arpa_telnet.h index ec238729d..232680e7d 100644 --- a/Utilities/cmcurl/lib/arpa_telnet.h +++ b/Utilities/cmcurl/lib/arpa_telnet.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -37,6 +37,7 @@ #define CURL_NEW_ENV_VAR 0 #define CURL_NEW_ENV_VALUE 1 +#ifndef CURL_DISABLE_VERBOSE_STRINGS /* * The telnet options represented as strings */ @@ -53,6 +54,7 @@ static const char * const telnetoptions[]= "TERM SPEED", "LFLOW", "LINEMODE", "XDISPLOC", "OLD-ENVIRON", "AUTHENTICATION", "ENCRYPT", "NEW-ENVIRON" }; +#endif #define CURL_TELOPT_MAXIMUM CURL_TELOPT_NEW_ENVIRON @@ -76,6 +78,7 @@ static const char * const telnetoptions[]= #define CURL_DONT 254 /* DON'T use this option! */ #define CURL_IAC 255 /* Interpret As Command */ +#ifndef CURL_DISABLE_VERBOSE_STRINGS /* * Then those numbers represented as strings: */ @@ -86,6 +89,7 @@ static const char * const telnetcmds[]= "AYT", "EC", "EL", "GA", "SB", "WILL", "WONT", "DO", "DONT", "IAC" }; +#endif #define CURL_TELCMD_MINIMUM CURL_xEOF /* the first one */ #define CURL_TELCMD_MAXIMUM CURL_IAC /* surprise, 255 is the last one! ;-) */ diff --git a/Utilities/cmcurl/lib/asyn-ares.c b/Utilities/cmcurl/lib/asyn-ares.c index aa581a49a..5cfb2602d 100644 --- a/Utilities/cmcurl/lib/asyn-ares.c +++ b/Utilities/cmcurl/lib/asyn-ares.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -312,22 +312,25 @@ CURLcode Curl_resolver_is_resolved(struct connectdata *conn, conn->async.os_specific; CURLcode result = CURLE_OK; - *dns = NULL; + if(dns) + *dns = NULL; waitperform(conn, 0); if(res && !res->num_pending) { - (void)Curl_addrinfo_callback(conn, res->last_status, res->temp_ai); - /* temp_ai ownership is moved to the connection, so we need not free-up - them */ - res->temp_ai = NULL; + if(dns) { + (void)Curl_addrinfo_callback(conn, res->last_status, res->temp_ai); + /* temp_ai ownership is moved to the connection, so we need not free-up + them */ + res->temp_ai = NULL; + } if(!conn->async.dns) { failf(data, "Could not resolve: %s (%s)", conn->async.hostname, ares_strerror(conn->async.status)); result = conn->bits.proxy?CURLE_COULDNT_RESOLVE_PROXY: CURLE_COULDNT_RESOLVE_HOST; } - else + else if(dns) *dns = conn->async.dns; destroy_async_data(&conn->async); @@ -390,7 +393,7 @@ CURLcode Curl_resolver_wait_resolv(struct connectdata *conn, timeout_ms = 1000; waitperform(conn, timeout_ms); - result = Curl_resolver_is_resolved(conn, &temp_entry); + result = Curl_resolver_is_resolved(conn, entry?&temp_entry:NULL); if(result || conn->async.done) break; @@ -472,17 +475,19 @@ static void query_completed_cb(void *arg, /* (struct connectdata *) */ return; res = (struct ResolverResults *)conn->async.os_specific; - res->num_pending--; + if(res) { + res->num_pending--; - if(CURL_ASYNC_SUCCESS == status) { - Curl_addrinfo *ai = Curl_he2ai(hostent, conn->async.port); - if(ai) { - compound_results(res, ai); + if(CURL_ASYNC_SUCCESS == status) { + Curl_addrinfo *ai = Curl_he2ai(hostent, conn->async.port); + if(ai) { + compound_results(res, ai); + } } + /* A successful result overwrites any previous error */ + if(res->last_status != ARES_SUCCESS) + res->last_status = status; } - /* A successful result overwrites any previous error */ - if(res->last_status != ARES_SUCCESS) - res->last_status = status; } /* diff --git a/Utilities/cmcurl/lib/asyn-thread.c b/Utilities/cmcurl/lib/asyn-thread.c index b11fab246..2a59294e5 100644 --- a/Utilities/cmcurl/lib/asyn-thread.c +++ b/Utilities/cmcurl/lib/asyn-thread.c @@ -182,8 +182,6 @@ static struct thread_sync_data *conn_thread_sync_data(struct connectdata *conn) return &(((struct thread_data *)conn->async.os_specific)->tsd); } -#define CONN_THREAD_SYNC_DATA(conn) &(((conn)->async.os_specific)->tsd); - /* Destroy resolver thread synchronization data */ static void destroy_thread_sync_data(struct thread_sync_data * tsd) @@ -481,8 +479,10 @@ CURLcode Curl_resolver_wait_resolv(struct connectdata *conn, DEBUGASSERT(conn && td); /* wait for the thread to resolve the name */ - if(Curl_thread_join(&td->thread_hnd)) - result = getaddrinfo_complete(conn); + if(Curl_thread_join(&td->thread_hnd)) { + if(entry) + result = getaddrinfo_complete(conn); + } else DEBUGASSERT(0); @@ -572,10 +572,10 @@ int Curl_resolver_getsock(struct connectdata *conn, (void)socks; (void)numsocks; ms = Curl_timediff(Curl_now(), reslv->start); - if(ms < 10) - milli = ms/3; + if(ms < 3) + milli = 0; else if(ms <= 50) - milli = 10; + milli = ms/3; else if(ms <= 250) milli = 50; else diff --git a/Utilities/cmcurl/lib/base64.c b/Utilities/cmcurl/lib/base64.c index 204a2273d..6370e4cdf 100644 --- a/Utilities/cmcurl/lib/base64.c +++ b/Utilities/cmcurl/lib/base64.c @@ -49,13 +49,12 @@ static size_t decodeQuantum(unsigned char *dest, const char *src) unsigned long i, x = 0; for(i = 0, s = src; i < 4; i++, s++) { - unsigned long v = 0; - if(*s == '=') { x = (x << 6); padding++; } else { + unsigned long v = 0; p = base64; while(*p && (*p != *s)) { diff --git a/Utilities/cmcurl/lib/conncache.c b/Utilities/cmcurl/lib/conncache.c index b8f544485..6fbf3b1d2 100644 --- a/Utilities/cmcurl/lib/conncache.c +++ b/Utilities/cmcurl/lib/conncache.c @@ -6,7 +6,7 @@ * \___|\___/|_| \_\_____| * * Copyright (C) 2012 - 2016, Linus Nielsen Feltzing, <linus@haxx.se> - * Copyright (C) 2012 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 2012 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -63,10 +63,9 @@ static void conn_llist_dtor(void *user, void *element) { - struct connectdata *data = element; + struct connectdata *conn = element; (void)user; - - data->bundle = NULL; + conn->bundle = NULL; } static CURLcode bundle_create(struct Curl_easy *data, @@ -96,14 +95,13 @@ static void bundle_destroy(struct connectbundle *cb_ptr) } /* Add a connection to a bundle */ -static CURLcode bundle_add_conn(struct connectbundle *cb_ptr, - struct connectdata *conn) +static void bundle_add_conn(struct connectbundle *cb_ptr, + struct connectdata *conn) { Curl_llist_insert_next(&cb_ptr->conn_list, cb_ptr->conn_list.tail, conn, &conn->bundle_node); conn->bundle = cb_ptr; cb_ptr->num_connections++; - return CURLE_OK; } /* Remove a connection from a bundle */ @@ -263,7 +261,7 @@ static void conncache_remove_bundle(struct conncache *connc, CURLcode Curl_conncache_add_conn(struct conncache *connc, struct connectdata *conn) { - CURLcode result; + CURLcode result = CURLE_OK; struct connectbundle *bundle; struct connectbundle *new_bundle = NULL; struct Curl_easy *data = conn->data; @@ -290,19 +288,13 @@ CURLcode Curl_conncache_add_conn(struct conncache *connc, bundle = new_bundle; } - result = bundle_add_conn(bundle, conn); - if(result) { - if(new_bundle) - conncache_remove_bundle(data->state.conn_cache, new_bundle); - goto unlock; - } - + bundle_add_conn(bundle, conn); conn->connection_id = connc->next_connection_id++; connc->num_conn++; DEBUGF(infof(conn->data, "Added connection %ld. " - "The cache now contains %" CURL_FORMAT_CURL_OFF_TU " members\n", - conn->connection_id, (curl_off_t) connc->num_conn)); + "The cache now contains %zu members\n", + conn->connection_id, connc->num_conn)); unlock: CONN_UNLOCK(data); @@ -320,7 +312,7 @@ void Curl_conncache_remove_conn(struct connectdata *conn, bool lock) due to a failed connection attempt, before being added to a bundle */ if(bundle) { if(lock) { - CONN_LOCK(conn->data); + CONN_LOCK(data); } bundle_remove_conn(bundle, conn); if(bundle->num_connections == 0) @@ -328,12 +320,11 @@ void Curl_conncache_remove_conn(struct connectdata *conn, bool lock) conn->bundle = NULL; /* removed from it */ if(connc) { connc->num_conn--; - DEBUGF(infof(conn->data, "The cache now contains %" - CURL_FORMAT_CURL_OFF_TU " members\n", - (curl_off_t) connc->num_conn)); + DEBUGF(infof(data, "The cache now contains %zu members\n", + connc->num_conn)); } if(lock) { - CONN_UNLOCK(conn->data); + CONN_UNLOCK(data); } } } @@ -441,18 +432,11 @@ bool Curl_conncache_return_conn(struct connectdata *conn) infof(data, "Connection cache is full, closing the oldest one.\n"); conn_candidate = Curl_conncache_extract_oldest(data); - if(conn_candidate) { - /* Set the connection's owner correctly */ - conn_candidate->data = data; - /* the winner gets the honour of being disconnected */ - (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); + (void)Curl_disconnect(data, conn_candidate, /* dead_connection */ FALSE); } } - CONN_LOCK(data); - conn->inuse = FALSE; /* Mark the connection unused */ - CONN_UNLOCK(data); return (conn_candidate == conn) ? FALSE : TRUE; @@ -486,7 +470,7 @@ Curl_conncache_extract_bundle(struct Curl_easy *data, while(curr) { conn = curr->ptr; - if(!conn->inuse) { + if(!CONN_INUSE(conn)) { /* Set higher score for the age passed since the connection was used */ score = Curl_timediff(now, conn->now); @@ -501,9 +485,9 @@ Curl_conncache_extract_bundle(struct Curl_easy *data, /* remove it to prevent another thread from nicking it */ bundle_remove_conn(bundle, conn_candidate); data->state.conn_cache->num_conn--; - DEBUGF(infof(data, "The cache now contains %" - CURL_FORMAT_CURL_OFF_TU " members\n", - (curl_off_t) data->state.conn_cache->num_conn)); + DEBUGF(infof(data, "The cache now contains %zu members\n", + data->state.conn_cache->num_conn)); + conn_candidate->data = data; /* associate! */ } return conn_candidate; @@ -544,7 +528,7 @@ Curl_conncache_extract_oldest(struct Curl_easy *data) while(curr) { conn = curr->ptr; - if(!conn->inuse) { + if(!CONN_INUSE(conn)) { /* Set higher score for the age passed since the connection was used */ score = Curl_timediff(now, conn->now); @@ -563,9 +547,9 @@ Curl_conncache_extract_oldest(struct Curl_easy *data) /* remove it to prevent another thread from nicking it */ bundle_remove_conn(bundle_candidate, conn_candidate); connc->num_conn--; - DEBUGF(infof(data, "The cache now contains %" - CURL_FORMAT_CURL_OFF_TU " members\n", - (curl_off_t) connc->num_conn)); + DEBUGF(infof(data, "The cache now contains %zu members\n", + connc->num_conn)); + conn_candidate->data = data; /* associate! */ } CONN_UNLOCK(data); @@ -586,7 +570,7 @@ void Curl_conncache_close_all_connections(struct conncache *connc) pointer */ /* This will remove the connection from the cache */ connclose(conn, "kill all"); - (void)Curl_disconnect(conn, FALSE); + (void)Curl_disconnect(connc->closure_handle, conn, FALSE); sigpipe_restore(&pipe_st); conn = Curl_conncache_find_first_connection(connc); diff --git a/Utilities/cmcurl/lib/conncache.h b/Utilities/cmcurl/lib/conncache.h index d8ad80f96..eedd7a800 100644 --- a/Utilities/cmcurl/lib/conncache.h +++ b/Utilities/cmcurl/lib/conncache.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 2015 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 2015 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * Copyright (C) 2012 - 2014, Linus Nielsen Feltzing, <linus@haxx.se> * * This software is licensed as described in the file COPYING, which @@ -63,7 +63,7 @@ size_t Curl_conncache_bundle_size(struct connectdata *conn); bool Curl_conncache_return_conn(struct connectdata *conn); CURLcode Curl_conncache_add_conn(struct conncache *connc, - struct connectdata *conn); + struct connectdata *conn) WARN_UNUSED_RESULT; void Curl_conncache_remove_conn(struct connectdata *conn, bool lock); bool Curl_conncache_foreach(struct Curl_easy *data, diff --git a/Utilities/cmcurl/lib/connect.c b/Utilities/cmcurl/lib/connect.c index 1a27ae135..41f220268 100644 --- a/Utilities/cmcurl/lib/connect.c +++ b/Utilities/cmcurl/lib/connect.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -1237,8 +1237,6 @@ static int conn_is_conn(struct connectdata *conn, void *param) curl_socket_t Curl_getconnectinfo(struct Curl_easy *data, struct connectdata **connp) { - curl_socket_t sockfd; - DEBUGASSERT(data); /* this works for an easy handle: @@ -1261,15 +1259,15 @@ curl_socket_t Curl_getconnectinfo(struct Curl_easy *data, return CURL_SOCKET_BAD; } - if(connp) + if(connp) { /* only store this if the caller cares for it */ *connp = c; - sockfd = c->sock[FIRSTSOCKET]; + c->data = data; + } + return c->sock[FIRSTSOCKET]; } else return CURL_SOCKET_BAD; - - return sockfd; } /* diff --git a/Utilities/cmcurl/lib/content_encoding.c b/Utilities/cmcurl/lib/content_encoding.c index 7c979efcc..6d4753788 100644 --- a/Utilities/cmcurl/lib/content_encoding.c +++ b/Utilities/cmcurl/lib/content_encoding.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -71,13 +71,13 @@ #define RESERVED 0xE0 /* bits 5..7: reserved */ typedef enum { - ZLIB_UNINIT, /* uninitialized */ - ZLIB_INIT, /* initialized */ - ZLIB_INFLATING, /* Inflating started. */ - ZLIB_GZIP_HEADER, /* reading gzip header */ - ZLIB_GZIP_TRAILER, /* reading gzip trailer */ - ZLIB_GZIP_INFLATING, /* inflating gzip stream */ - ZLIB_INIT_GZIP /* initialized in transparent gzip mode */ + ZLIB_UNINIT, /* uninitialized */ + ZLIB_INIT, /* initialized */ + ZLIB_INFLATING, /* inflating started. */ + ZLIB_EXTERNAL_TRAILER, /* reading external trailer */ + ZLIB_GZIP_HEADER, /* reading gzip header */ + ZLIB_GZIP_INFLATING, /* inflating gzip stream */ + ZLIB_INIT_GZIP /* initialized in transparent gzip mode */ } zlibInitState; /* Writer parameters. */ @@ -150,8 +150,8 @@ static CURLcode process_trailer(struct connectdata *conn, zlib_params *zp) if(result || !zp->trailerlen) result = exit_zlib(conn, z, &zp->zlib_init, result); else { - /* Only occurs for gzip with zlib < 1.2.0.4. */ - zp->zlib_init = ZLIB_GZIP_TRAILER; + /* Only occurs for gzip with zlib < 1.2.0.4 or raw deflate. */ + zp->zlib_init = ZLIB_EXTERNAL_TRAILER; } return result; } @@ -163,7 +163,6 @@ static CURLcode inflate_stream(struct connectdata *conn, z_stream *z = &zp->z; /* zlib state structure */ uInt nread = z->avail_in; Bytef *orig_in = z->next_in; - int status; /* zlib status */ bool done = FALSE; CURLcode result = CURLE_OK; /* Curl_client_write status */ char *decomp; /* Put the decompressed data here. */ @@ -184,13 +183,20 @@ static CURLcode inflate_stream(struct connectdata *conn, /* because the buffer size is fixed, iteratively decompress and transfer to the client via downstream_write function. */ while(!done) { + int status; /* zlib status */ done = TRUE; /* (re)set buffer for decompressed output for every iteration */ z->next_out = (Bytef *) decomp; z->avail_out = DSIZ; +#ifdef Z_BLOCK + /* Z_BLOCK is only available in zlib ver. >= 1.2.0.5 */ status = inflate(z, Z_BLOCK); +#else + /* fallback for zlib ver. < 1.2.0.5 */ + status = inflate(z, Z_SYNC_FLUSH); +#endif /* Flush output data if some. */ if(z->avail_out != DSIZ) { @@ -227,6 +233,7 @@ static CURLcode inflate_stream(struct connectdata *conn, z->next_in = orig_in; z->avail_in = nread; zp->zlib_init = ZLIB_INFLATING; + zp->trailerlen = 4; /* Tolerate up to 4 unknown trailer bytes. */ done = FALSE; break; } @@ -281,6 +288,9 @@ static CURLcode deflate_unencode_write(struct connectdata *conn, z->next_in = (Bytef *) buf; z->avail_in = (uInt) nbytes; + if(zp->zlib_init == ZLIB_EXTERNAL_TRAILER) + return process_trailer(conn, zp); + /* Now uncompress the data */ return inflate_stream(conn, writer, ZLIB_INFLATING); } @@ -526,7 +536,7 @@ static CURLcode gzip_unencode_write(struct connectdata *conn, } break; - case ZLIB_GZIP_TRAILER: + case ZLIB_EXTERNAL_TRAILER: z->next_in = (Bytef *) buf; z->avail_in = (uInt) nbytes; return process_trailer(conn, zp); @@ -755,7 +765,6 @@ char *Curl_all_content_encodings(void) const content_encoding * const *cep; const content_encoding *ce; char *ace; - char *p; for(cep = encodings; *cep; cep++) { ce = *cep; @@ -768,7 +777,7 @@ char *Curl_all_content_encodings(void) ace = malloc(len); if(ace) { - p = ace; + char *p = ace; for(cep = encodings; *cep; cep++) { ce = *cep; if(!strcasecompare(ce->name, CONTENT_ENCODING_DEFAULT)) { @@ -915,10 +924,9 @@ void Curl_unencode_cleanup(struct connectdata *conn) static const content_encoding *find_encoding(const char *name, size_t len) { const content_encoding * const *cep; - const content_encoding *ce; for(cep = encodings; *cep; cep++) { - ce = *cep; + const content_encoding *ce = *cep; if((strncasecompare(name, ce->name, len) && !ce->name[len]) || (ce->alias && strncasecompare(name, ce->alias, len) && !ce->alias[len])) return ce; diff --git a/Utilities/cmcurl/lib/cookie.c b/Utilities/cmcurl/lib/cookie.c index 2c028b381..a342c61de 100644 --- a/Utilities/cmcurl/lib/cookie.c +++ b/Utilities/cmcurl/lib/cookie.c @@ -40,7 +40,7 @@ struct Cookie *Curl_cookie_add(struct Curl_easy *data, received from a server. The function need to replace previously stored lines that this new - line superceeds. + line supersedes. It may remove lines that are expired. @@ -84,12 +84,9 @@ Example set of cookies: #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES) -#ifdef USE_LIBPSL -# include <libpsl.h> -#endif - #include "urldata.h" #include "cookie.h" +#include "psl.h" #include "strtok.h" #include "sendf.h" #include "slist.h" @@ -253,9 +250,9 @@ static const char *get_top_domain(const char * const domain, size_t *outlen) len = strlen(domain); last = memrchr(domain, '.', len); if(last) { - first = memrchr(domain, '.', (size_t) (last - domain)); + first = memrchr(domain, '.', (last - domain)); if(first) - len -= (size_t) (++first - domain); + len -= (++first - domain); } if(outlen) @@ -379,13 +376,13 @@ static void strstore(char **str, const char *newstr) */ static void remove_expired(struct CookieInfo *cookies) { - struct Cookie *co, *nx, *pv; + struct Cookie *co, *nx; curl_off_t now = (curl_off_t)time(NULL); unsigned int i; for(i = 0; i < COOKIE_HASH_SIZE; i++) { + struct Cookie *pv = NULL; co = cookies->cookies[i]; - pv = NULL; while(co) { nx = co->next; if(co->expires && co->expires < now) { @@ -406,6 +403,12 @@ static void remove_expired(struct CookieInfo *cookies) } } +/* Make sure domain contains a dot or is localhost. */ +static bool bad_domain(const char *domain) +{ + return !strchr(domain, '.') && !strcasecompare(domain, "localhost"); +} + /**************************************************************************** * * Curl_cookie_add() @@ -442,10 +445,6 @@ Curl_cookie_add(struct Curl_easy *data, bool badcookie = FALSE; /* cookies are good by default. mmmmm yummy */ size_t myhash; -#ifdef USE_LIBPSL - const psl_ctx_t *psl; -#endif - #ifdef CURL_DISABLE_VERBOSE_STRINGS (void)data; #endif @@ -497,7 +496,7 @@ Curl_cookie_add(struct Curl_easy *data, name + contents. Chrome and Firefox support 4095 or 4096 bytes combo. */ freecookie(co); - infof(data, "oversized cookie dropped, name/val %d + %d bytes\n", + infof(data, "oversized cookie dropped, name/val %zu + %zu bytes\n", nlen, len); return NULL; } @@ -585,13 +584,8 @@ Curl_cookie_add(struct Curl_easy *data, * TLD or otherwise "protected" suffix. To reduce risk, we require a * dot OR the exact host name being "localhost". */ - { - const char *dotp; - /* check for more dots */ - dotp = strchr(whatptr, '.'); - if(!dotp && !strcasecompare("localhost", whatptr)) - domain = ":"; - } + if(bad_domain(whatptr)) + domain = ":"; #endif is_ip = isip(domain ? domain : whatptr); @@ -723,9 +717,9 @@ Curl_cookie_add(struct Curl_easy *data, if(!queryp) endslash = strrchr(path, '/'); else - endslash = memrchr(path, '/', (size_t)(queryp - path)); + endslash = memrchr(path, '/', (queryp - path)); if(endslash) { - size_t pathlen = (size_t)(endslash-path + 1); /* include end slash */ + size_t pathlen = (endslash-path + 1); /* include end slash */ co->path = malloc(pathlen + 1); /* one extra for the zero byte */ if(co->path) { memcpy(co->path, path, pathlen); @@ -880,9 +874,10 @@ Curl_cookie_add(struct Curl_easy *data, } co->livecookie = c->running; + co->creationtime = ++c->lastct; /* now, we have parsed the incoming line, we must now check if this - superceeds an already existing cookie, which it may if the previous have + supersedes an already existing cookie, which it may if the previous have the same domain and path as this */ /* at first, remove expired cookies */ @@ -890,14 +885,21 @@ Curl_cookie_add(struct Curl_easy *data, remove_expired(c); #ifdef USE_LIBPSL - /* Check if the domain is a Public Suffix and if yes, ignore the cookie. - This needs a libpsl compiled with builtin data. */ + /* Check if the domain is a Public Suffix and if yes, ignore the cookie. */ if(domain && co->domain && !isip(co->domain)) { - psl = psl_builtin(); - if(psl && !psl_is_cookie_domain_acceptable(psl, domain, co->domain)) { - infof(data, - "cookie '%s' dropped, domain '%s' must not set cookies for '%s'\n", - co->name, domain, co->domain); + const psl_ctx_t *psl = Curl_psl_use(data); + int acceptable; + + if(psl) { + acceptable = psl_is_cookie_domain_acceptable(psl, domain, co->domain); + Curl_psl_release(data); + } + else + acceptable = !bad_domain(domain); + + if(!acceptable) { + infof(data, "cookie '%s' dropped, domain '%s' must not " + "set cookies for '%s'\n", co->name, domain, co->domain); freecookie(co); return NULL; } @@ -951,6 +953,9 @@ Curl_cookie_add(struct Curl_easy *data, if(replace_old) { co->next = clist->next; /* get the next-pointer first */ + /* when replacing, creationtime is kept from old */ + co->creationtime = clist->creationtime; + /* then free all the old pointers */ free(clist->name); free(clist->value); @@ -1140,12 +1145,24 @@ static int cookie_sort(const void *p1, const void *p2) if(l1 != l2) return (l2 > l1) ? 1 : -1 ; /* avoid size_t <=> int conversions */ - /* 3 - compare cookie names */ - if(c1->name && c2->name) - return strcmp(c1->name, c2->name); + /* 3 - compare cookie name lengths */ + l1 = c1->name ? strlen(c1->name) : 0; + l2 = c2->name ? strlen(c2->name) : 0; - /* sorry, can't be more deterministic */ - return 0; + if(l1 != l2) + return (l2 > l1) ? 1 : -1; + + /* 4 - compare cookie creation time */ + return (c2->creationtime > c1->creationtime) ? 1 : -1; +} + +/* sort cookies only according to creation time */ +static int cookie_sort_ct(const void *p1, const void *p2) +{ + struct Cookie *c1 = *(struct Cookie **)p1; + struct Cookie *c2 = *(struct Cookie **)p2; + + return (c2->creationtime > c1->creationtime) ? 1 : -1; } #define CLONE(field) \ @@ -1174,6 +1191,7 @@ static struct Cookie *dup_cookie(struct Cookie *src) d->secure = src->secure; d->livecookie = src->livecookie; d->httponly = src->httponly; + d->creationtime = src->creationtime; } return d; @@ -1200,7 +1218,6 @@ struct Cookie *Curl_cookie_getlist(struct CookieInfo *c, { struct Cookie *newco; struct Cookie *co; - time_t now = time(NULL); struct Cookie *mainco = NULL; size_t matches = 0; bool is_ip; @@ -1218,11 +1235,8 @@ struct Cookie *Curl_cookie_getlist(struct CookieInfo *c, co = c->cookies[myhash]; while(co) { - /* only process this cookie if it is not expired or had no expire - date AND that if the cookie requires we're secure we must only - continue if we are! */ - if((!co->expires || (co->expires > now)) && - (co->secure?secure:TRUE)) { + /* if the cookie requires we're secure we must only continue if we are! */ + if(co->secure?secure:TRUE) { /* now check if the domain is correct */ if(!co->domain || @@ -1248,12 +1262,8 @@ struct Cookie *Curl_cookie_getlist(struct CookieInfo *c, matches++; } - else { - fail: - /* failure, clear up the allocated chain and return NULL */ - Curl_cookie_freelist(mainco); - return NULL; - } + else + goto fail; } } } @@ -1291,6 +1301,11 @@ struct Cookie *Curl_cookie_getlist(struct CookieInfo *c, } return mainco; /* return the new list */ + +fail: + /* failure, clear up the allocated chain and return NULL */ + Curl_cookie_freelist(mainco); + return NULL; } /***************************************************************************** @@ -1384,9 +1399,8 @@ void Curl_cookie_clearsess(struct CookieInfo *cookies) ****************************************************************************/ void Curl_cookie_cleanup(struct CookieInfo *c) { - unsigned int i; - if(c) { + unsigned int i; free(c->filename); for(i = 0; i < COOKIE_HASH_SIZE; i++) Curl_cookie_freelist(c->cookies[i]); @@ -1439,6 +1453,8 @@ static int cookie_output(struct CookieInfo *c, const char *dumphere) bool use_stdout = FALSE; char *format_ptr; unsigned int i; + unsigned int j; + struct Cookie **array; if((NULL == c) || (0 == c->numcookies)) /* If there are no known cookies, we don't write or even create any @@ -1452,6 +1468,10 @@ static int cookie_output(struct CookieInfo *c, const char *dumphere) if(0 == c->numcookies) return 0; + array = malloc(sizeof(struct Cookie *) * c->numcookies); + if(!array) + return 1; + if(!strcmp("-", dumphere)) { /* use stdout */ out = stdout; @@ -1459,8 +1479,10 @@ static int cookie_output(struct CookieInfo *c, const char *dumphere) } else { out = fopen(dumphere, FOPEN_WRITETEXT); - if(!out) + if(!out) { + free(array); return 1; /* failure */ + } } fputs("# Netscape HTTP Cookie File\n" @@ -1468,22 +1490,32 @@ static int cookie_output(struct CookieInfo *c, const char *dumphere) "# This file was generated by libcurl! Edit at your own risk.\n\n", out); + j = 0; for(i = 0; i < COOKIE_HASH_SIZE; i++) { for(co = c->cookies[i]; co; co = co->next) { if(!co->domain) continue; - format_ptr = get_netscape_format(co); - if(format_ptr == NULL) { - fprintf(out, "#\n# Fatal libcurl error\n"); - if(!use_stdout) - fclose(out); - return 1; - } - fprintf(out, "%s\n", format_ptr); - free(format_ptr); + array[j++] = co; } } + qsort(array, c->numcookies, sizeof(struct Cookie *), cookie_sort_ct); + + for(i = 0; i < j; i++) { + format_ptr = get_netscape_format(array[i]); + if(format_ptr == NULL) { + fprintf(out, "#\n# Fatal libcurl error\n"); + free(array); + if(!use_stdout) + fclose(out); + return 1; + } + fprintf(out, "%s\n", format_ptr); + free(format_ptr); + } + + free(array); + if(!use_stdout) fclose(out); diff --git a/Utilities/cmcurl/lib/cookie.h b/Utilities/cmcurl/lib/cookie.h index 79b5928dc..a9f90ca71 100644 --- a/Utilities/cmcurl/lib/cookie.h +++ b/Utilities/cmcurl/lib/cookie.h @@ -34,7 +34,7 @@ struct Cookie { char *domain; /* domain = <this> */ curl_off_t expires; /* expires = <this> */ char *expirestr; /* the plain text version */ - bool tailmatch; /* weather we do tail-matchning of the domain name */ + bool tailmatch; /* whether we do tail-matching of the domain name */ /* RFC 2109 keywords. Version=1 means 2109-compliant cookie sending */ char *version; /* Version = <value> */ @@ -43,6 +43,7 @@ struct Cookie { bool secure; /* whether the 'secure' keyword was used */ bool livecookie; /* updated from a server, not a stored file */ bool httponly; /* true if the httponly directive is present */ + int creationtime; /* time when the cookie was written */ }; #define COOKIE_HASH_SIZE 256 @@ -55,6 +56,7 @@ struct CookieInfo { bool running; /* state info, for cookie adding information */ long numcookies; /* number of cookies in the "jar" */ bool newsession; /* new session, discard session cookies on load */ + int lastct; /* last creation-time used in the jar */ }; /* This is the maximum line length we accept for a cookie line. RFC 2109 diff --git a/Utilities/cmcurl/lib/curl_addrinfo.c b/Utilities/cmcurl/lib/curl_addrinfo.c index 55d5a3942..fd49679c9 100644 --- a/Utilities/cmcurl/lib/curl_addrinfo.c +++ b/Utilities/cmcurl/lib/curl_addrinfo.c @@ -536,7 +536,8 @@ Curl_addrinfo *Curl_unix2addr(const char *path, bool *longpath, bool abstract) } #endif -#if defined(CURLDEBUG) && defined(HAVE_FREEADDRINFO) +#if defined(CURLDEBUG) && defined(HAVE_GETADDRINFO) && \ + defined(HAVE_FREEADDRINFO) /* * curl_dofreeaddrinfo() * diff --git a/Utilities/cmcurl/lib/curl_config.h.cmake b/Utilities/cmcurl/lib/curl_config.h.cmake index 24d1ca0bd..d5e3a90cf 100644 --- a/Utilities/cmcurl/lib/curl_config.h.cmake +++ b/Utilities/cmcurl/lib/curl_config.h.cmake @@ -118,6 +118,9 @@ /* Define to 1 if bool is an available type. */ #cmakedefine HAVE_BOOL_T 1 +/* Define to 1 if you have the __builtin_available function. */ +#cmakedefine HAVE_BUILTIN_AVAILABLE 1 + /* Define to 1 if you have the clock_gettime function and monotonic timer. */ #cmakedefine HAVE_CLOCK_GETTIME_MONOTONIC 1 @@ -226,6 +229,9 @@ /* Define to 1 if you have the `getpwuid' function. */ #cmakedefine HAVE_GETPWUID 1 +/* Define to 1 if you have the `getpwuid_r' function. */ +#cmakedefine HAVE_GETPWUID_R 1 + /* Define to 1 if you have the `getrlimit' function. */ #cmakedefine HAVE_GETRLIMIT 1 @@ -876,6 +882,12 @@ /* The size of `long', as computed by sizeof. */ @SIZEOF_LONG_CODE@ +/* The size of `long long', as computed by sizeof. */ +@SIZEOF_LONG_LONG_CODE@ + +/* The size of `__int64', as computed by sizeof. */ +@SIZEOF___INT64_CODE@ + /* The size of `off_t', as computed by sizeof. */ @SIZEOF_OFF_T_CODE@ @@ -951,6 +963,9 @@ /* to enable Windows SSL */ #cmakedefine USE_SCHANNEL 1 +/* enable multiple SSL backends */ +#cmakedefine CURL_WITH_MULTI_SSL 1 + /* Define to 1 if using yaSSL in OpenSSL compatibility mode. */ #cmakedefine USE_YASSLEMUL 1 diff --git a/Utilities/cmcurl/lib/curl_fnmatch.c b/Utilities/cmcurl/lib/curl_fnmatch.c index 0179a4f71..fbfd85c43 100644 --- a/Utilities/cmcurl/lib/curl_fnmatch.c +++ b/Utilities/cmcurl/lib/curl_fnmatch.c @@ -30,6 +30,17 @@ /* The last #include file should be: */ #include "memdebug.h" +#ifndef HAVE_FNMATCH + +/* + * TODO: + * + * Make this function match POSIX. Test 1307 includes a set of test patterns + * that returns different results with a POSIX fnmatch() than with this + * implementation and this is considered a bug where POSIX is the guiding + * light. + */ + #define CURLFNM_CHARSET_LEN (sizeof(char) * 256) #define CURLFNM_CHSET_SIZE (CURLFNM_CHARSET_LEN + 15) @@ -334,9 +345,9 @@ static int loop(const unsigned char *pattern, const unsigned char *string, s++; break; } + /* Syntax error in set; mismatch! */ + return CURL_FNMATCH_NOMATCH; - /* Syntax error in set: this must be taken as a regular character. */ - /* FALLTHROUGH */ default: if(*p++ != *s++) return CURL_FNMATCH_NOMATCH; @@ -355,5 +366,31 @@ int Curl_fnmatch(void *ptr, const char *pattern, const char *string) if(!pattern || !string) { return CURL_FNMATCH_FAIL; } - return loop((unsigned char *)pattern, (unsigned char *)string, 5); + return loop((unsigned char *)pattern, (unsigned char *)string, 2); +} +#else +#include <fnmatch.h> +/* + * @unittest: 1307 + */ +int Curl_fnmatch(void *ptr, const char *pattern, const char *string) +{ + int rc; + (void)ptr; /* the argument is specified by the curl_fnmatch_callback + prototype, but not used by Curl_fnmatch() */ + if(!pattern || !string) { + return CURL_FNMATCH_FAIL; + } + rc = fnmatch(pattern, string, 0); + switch(rc) { + case 0: + return CURL_FNMATCH_MATCH; + case FNM_NOMATCH: + return CURL_FNMATCH_NOMATCH; + default: + return CURL_FNMATCH_FAIL; + } + /* not reached */ } + +#endif diff --git a/Utilities/cmcurl/lib/curl_ldap.h b/Utilities/cmcurl/lib/curl_ldap.h index 27d03810f..94c002948 100644 --- a/Utilities/cmcurl/lib/curl_ldap.h +++ b/Utilities/cmcurl/lib/curl_ldap.h @@ -32,4 +32,3 @@ extern const struct Curl_handler Curl_handler_ldaps; #endif #endif /* HEADER_CURL_LDAP_H */ - diff --git a/Utilities/cmcurl/lib/curl_ntlm_core.c b/Utilities/cmcurl/lib/curl_ntlm_core.c index e27cab353..922e85a92 100644 --- a/Utilities/cmcurl/lib/curl_ntlm_core.c +++ b/Utilities/cmcurl/lib/curl_ntlm_core.c @@ -557,8 +557,11 @@ CURLcode Curl_ntlm_core_mk_nt_hash(struct Curl_easy *data, unsigned char *ntbuffer /* 21 bytes */) { size_t len = strlen(password); - unsigned char *pw = len ? malloc(len * 2) : strdup(""); + unsigned char *pw; CURLcode result; + if(len > SIZE_T_MAX/2) /* avoid integer overflow */ + return CURLE_OUT_OF_MEMORY; + pw = len ? malloc(len * 2) : strdup(""); if(!pw) return CURLE_OUT_OF_MEMORY; diff --git a/Utilities/cmcurl/lib/curl_ntlm_wb.c b/Utilities/cmcurl/lib/curl_ntlm_wb.c index 353a65645..a4791eb41 100644 --- a/Utilities/cmcurl/lib/curl_ntlm_wb.c +++ b/Utilities/cmcurl/lib/curl_ntlm_wb.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -249,6 +249,9 @@ done: return CURLE_REMOTE_ACCESS_DENIED; } +/* if larger than this, something is seriously wrong */ +#define MAX_NTLM_WB_RESPONSE 100000 + static CURLcode ntlm_wb_response(struct connectdata *conn, const char *input, curlntlm state) { @@ -289,6 +292,13 @@ static CURLcode ntlm_wb_response(struct connectdata *conn, buf[len_out - 1] = '\0'; break; } + + if(len_out > MAX_NTLM_WB_RESPONSE) { + failf(conn->data, "too large ntlm_wb response!"); + free(buf); + return CURLE_OUT_OF_MEMORY; + } + newbuf = Curl_saferealloc(buf, len_out + NTLM_BUFSIZE); if(!newbuf) return CURLE_OUT_OF_MEMORY; @@ -314,6 +324,8 @@ static CURLcode ntlm_wb_response(struct connectdata *conn, conn->response_header = aprintf("NTLM %.*s", len_out - 4, buf + 3); free(buf); + if(!conn->response_header) + return CURLE_OUT_OF_MEMORY; return CURLE_OK; done: free(buf); @@ -389,6 +401,8 @@ CURLcode Curl_output_ntlm_wb(struct connectdata *conn, conn->response_header); DEBUG_OUT(fprintf(stderr, "**** Header %s\n ", *allocuserpwd)); free(conn->response_header); + if(!*allocuserpwd) + return CURLE_OUT_OF_MEMORY; conn->response_header = NULL; break; case NTLMSTATE_TYPE2: @@ -409,6 +423,8 @@ CURLcode Curl_output_ntlm_wb(struct connectdata *conn, ntlm->state = NTLMSTATE_TYPE3; /* we sent a type-3 */ authp->done = TRUE; Curl_ntlm_wb_cleanup(conn); + if(!*allocuserpwd) + return CURLE_OUT_OF_MEMORY; break; case NTLMSTATE_TYPE3: /* connection is already authenticated, diff --git a/Utilities/cmcurl/lib/curl_path.c b/Utilities/cmcurl/lib/curl_path.c index e843deac7..68f3e44ba 100644 --- a/Utilities/cmcurl/lib/curl_path.c +++ b/Utilities/cmcurl/lib/curl_path.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -39,7 +39,7 @@ CURLcode Curl_getworkingpath(struct connectdata *conn, char *working_path; size_t working_path_len; CURLcode result = - Curl_urldecode(data, data->state.path, 0, &working_path, + Curl_urldecode(data, data->state.up.path, 0, &working_path, &working_path_len, FALSE); if(result) return result; diff --git a/Utilities/cmcurl/lib/curl_path.h b/Utilities/cmcurl/lib/curl_path.h index 5ee4ff367..636c37f20 100644 --- a/Utilities/cmcurl/lib/curl_path.h +++ b/Utilities/cmcurl/lib/curl_path.h @@ -44,4 +44,4 @@ CURLcode Curl_getworkingpath(struct connectdata *conn, char **path); CURLcode Curl_get_pathname(const char **cpp, char **path, char *homedir); -#endif +#endif /* HEADER_CURL_PATH_H */ diff --git a/Utilities/cmcurl/lib/curl_rtmp.c b/Utilities/cmcurl/lib/curl_rtmp.c index 97430647b..f09f2f332 100644 --- a/Utilities/cmcurl/lib/curl_rtmp.c +++ b/Utilities/cmcurl/lib/curl_rtmp.c @@ -37,9 +37,11 @@ /* The last #include file should be: */ #include "memdebug.h" -#ifdef _WIN32 +#if defined(WIN32) && !defined(USE_LWIPSOCK) #define setsockopt(a,b,c,d,e) (setsockopt)(a,b,c,(const char *)d,(int)e) #define SET_RCVTIMEO(tv,s) int tv = s*1000 +#elif defined(LWIP_SO_SNDRCVTIMEO_NONSTANDARD) +#define SET_RCVTIMEO(tv,s) int tv = s*1000 #else #define SET_RCVTIMEO(tv,s) struct timeval tv = {s,0} #endif diff --git a/Utilities/cmcurl/lib/curl_sasl.c b/Utilities/cmcurl/lib/curl_sasl.c index e54e4875e..354bc5448 100644 --- a/Utilities/cmcurl/lib/curl_sasl.c +++ b/Utilities/cmcurl/lib/curl_sasl.c @@ -146,7 +146,6 @@ CURLcode Curl_sasl_parse_url_auth_option(struct SASL *sasl, const char *value, size_t len) { CURLcode result = CURLE_OK; - unsigned int mechbit; size_t mechlen; if(!len) @@ -160,7 +159,7 @@ CURLcode Curl_sasl_parse_url_auth_option(struct SASL *sasl, if(!strncmp(value, "*", len)) sasl->prefmech = SASL_AUTH_DEFAULT; else { - mechbit = Curl_sasl_decode_mech(value, len, &mechlen); + unsigned int mechbit = Curl_sasl_decode_mech(value, len, &mechlen); if(mechbit && mechlen == len) sasl->prefmech |= mechbit; else diff --git a/Utilities/cmcurl/lib/curl_setup.h b/Utilities/cmcurl/lib/curl_setup.h index 7e6fd5e7e..5cdbc5923 100644 --- a/Utilities/cmcurl/lib/curl_setup.h +++ b/Utilities/cmcurl/lib/curl_setup.h @@ -44,6 +44,9 @@ # ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN # endif +# ifndef NOGDI +# define NOGDI +# endif #endif /* @@ -221,7 +224,7 @@ /* * Use getaddrinfo to resolve the IPv4 address literal. If the current network - * interface doesn’t support IPv4, but supports IPv6, NAT64, and DNS64, + * interface doesn't support IPv4, but supports IPv6, NAT64, and DNS64, * performing this task will result in a synthesized IPv6 address. */ #ifdef __APPLE__ @@ -246,6 +249,7 @@ # if defined(_UNICODE) && !defined(UNICODE) # define UNICODE # endif +# include <winerror.h> # include <windows.h> # ifdef HAVE_WINSOCK2_H # include <winsock2.h> @@ -659,7 +663,7 @@ int netware_init(void); #if defined(USE_GNUTLS) || defined(USE_OPENSSL) || defined(USE_NSS) || \ defined(USE_POLARSSL) || defined(USE_AXTLS) || defined(USE_MBEDTLS) || \ defined(USE_CYASSL) || defined(USE_SCHANNEL) || \ - defined(USE_DARWINSSL) || defined(USE_GSKIT) + defined(USE_DARWINSSL) || defined(USE_GSKIT) || defined(USE_MESALINK) #define USE_SSL /* SSL support has been enabled */ #endif @@ -815,4 +819,16 @@ endings either CRLF or LF so 't' is appropriate. #define CURL_SA_FAMILY_T unsigned short #endif +/* Some convenience macros to get the larger/smaller value out of two given. + We prefix with CURL to prevent name collisions. */ +#define CURLMAX(x,y) ((x)>(y)?(x):(y)) +#define CURLMIN(x,y) ((x)<(y)?(x):(y)) + +/* Some versions of the Android SDK is missing the declaration */ +#if defined(HAVE_GETPWUID_R) && defined(HAVE_DECL_GETPWUID_R_MISSING) +struct passwd; +int getpwuid_r(uid_t uid, struct passwd *pwd, char *buf, + size_t buflen, struct passwd **result); +#endif + #endif /* HEADER_CURL_SETUP_H */ diff --git a/Utilities/cmcurl/lib/curl_setup_once.h b/Utilities/cmcurl/lib/curl_setup_once.h index 6d01ea156..413ccea91 100644 --- a/Utilities/cmcurl/lib/curl_setup_once.h +++ b/Utilities/cmcurl/lib/curl_setup_once.h @@ -515,4 +515,3 @@ typedef int sig_atomic_t; #endif /* HEADER_CURL_SETUP_ONCE_H */ - diff --git a/Utilities/cmcurl/lib/curl_sspi.c b/Utilities/cmcurl/lib/curl_sspi.c index 11a7120a9..1d0de4ed3 100644 --- a/Utilities/cmcurl/lib/curl_sspi.c +++ b/Utilities/cmcurl/lib/curl_sspi.c @@ -90,8 +90,9 @@ CURLcode Curl_sspi_global_init(void) return CURLE_FAILED_INIT; /* Get address of the InitSecurityInterfaceA function from the SSPI dll */ - pInitSecurityInterface = (INITSECURITYINTERFACE_FN) - GetProcAddress(s_hSecDll, SECURITYENTRYPOINT); + pInitSecurityInterface = + CURLX_FUNCTION_CAST(INITSECURITYINTERFACE_FN, + (GetProcAddress(s_hSecDll, SECURITYENTRYPOINT))); if(!pInitSecurityInterface) return CURLE_FAILED_INIT; @@ -131,7 +132,7 @@ void Curl_sspi_global_cleanup(void) * Parameters: * * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * identity [in/out] - The identity structure. * * Returns CURLE_OK on success. diff --git a/Utilities/cmcurl/lib/curl_threads.c b/Utilities/cmcurl/lib/curl_threads.c index c1624a913..8e5937aa0 100644 --- a/Utilities/cmcurl/lib/curl_threads.c +++ b/Utilities/cmcurl/lib/curl_threads.c @@ -104,12 +104,21 @@ int Curl_thread_join(curl_thread_t *hnd) curl_thread_t Curl_thread_create(unsigned int (CURL_STDCALL *func) (void *), void *arg) { +#ifdef _WIN32_WCE + typedef HANDLE curl_win_thread_handle_t; +#elif defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) + typedef unsigned long curl_win_thread_handle_t; +#else + typedef uintptr_t curl_win_thread_handle_t; +#endif curl_thread_t t; + curl_win_thread_handle_t thread_handle; #ifdef _WIN32_WCE - t = CreateThread(NULL, 0, func, arg, 0, NULL); + thread_handle = CreateThread(NULL, 0, func, arg, 0, NULL); #else - t = (curl_thread_t)_beginthreadex(NULL, 0, func, arg, 0, NULL); + thread_handle = _beginthreadex(NULL, 0, func, arg, 0, NULL); #endif + t = (curl_thread_t)thread_handle; if((t == 0) || (t == LongToHandle(-1L))) { #ifdef _WIN32_WCE DWORD gle = GetLastError(); diff --git a/Utilities/cmcurl/lib/curl_threads.h b/Utilities/cmcurl/lib/curl_threads.h index 9e0d14a30..2a93644c5 100644 --- a/Utilities/cmcurl/lib/curl_threads.h +++ b/Utilities/cmcurl/lib/curl_threads.h @@ -38,7 +38,8 @@ # define curl_thread_t HANDLE # define curl_thread_t_null (HANDLE)0 # if !defined(_WIN32_WINNT) || !defined(_WIN32_WINNT_VISTA) || \ - (_WIN32_WINNT < _WIN32_WINNT_VISTA) + (_WIN32_WINNT < _WIN32_WINNT_VISTA) || \ + (defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR)) # define Curl_mutex_init(m) InitializeCriticalSection(m) # else # define Curl_mutex_init(m) InitializeCriticalSectionEx(m, 0, 1) diff --git a/Utilities/cmcurl/lib/curlx.h b/Utilities/cmcurl/lib/curlx.h index 6e418266b..4c77d4f2e 100644 --- a/Utilities/cmcurl/lib/curlx.h +++ b/Utilities/cmcurl/lib/curlx.h @@ -102,4 +102,3 @@ #endif /* ENABLE_CURLX_PRINTF */ #endif /* HEADER_CURL_CURLX_H */ - diff --git a/Utilities/cmcurl/lib/dict.c b/Utilities/cmcurl/lib/dict.c index 4fc85521d..78ef046d4 100644 --- a/Utilities/cmcurl/lib/dict.c +++ b/Utilities/cmcurl/lib/dict.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -95,17 +95,17 @@ static char *unescape_word(struct Curl_easy *data, const char *inputbuff) { char *newp = NULL; char *dictp; - char *ptr; size_t len; - char ch; - int olen = 0; CURLcode result = Curl_urldecode(data, inputbuff, 0, &newp, &len, FALSE); if(!newp || result) return NULL; - dictp = malloc(((size_t)len)*2 + 1); /* add one for terminating zero */ + dictp = malloc(len*2 + 1); /* add one for terminating zero */ if(dictp) { + char *ptr; + char ch; + int olen = 0; /* According to RFC2229 section 2.2, these letters need to be escaped with \[letter] */ for(ptr = newp; @@ -136,7 +136,7 @@ static CURLcode dict_do(struct connectdata *conn, bool *done) struct Curl_easy *data = conn->data; curl_socket_t sockfd = conn->sock[FIRSTSOCKET]; - char *path = data->state.path; + char *path = data->state.up.path; curl_off_t *bytecount = &data->req.bytecount; *done = TRUE; /* unconditionally */ diff --git a/Utilities/cmcurl/lib/doh.c b/Utilities/cmcurl/lib/doh.c new file mode 100644 index 000000000..ef6013db9 --- /dev/null +++ b/Utilities/cmcurl/lib/doh.c @@ -0,0 +1,920 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "curl_setup.h" + +#include "urldata.h" +#include "curl_addrinfo.h" +#include "doh.h" + +#ifdef USE_NGHTTP2 +#include "sendf.h" +#include "multiif.h" +#include "url.h" +#include "share.h" +#include "curl_base64.h" +#include "connect.h" +/* The last 3 #include files should be in this order */ +#include "curl_printf.h" +#include "curl_memory.h" +#include "memdebug.h" + +#define DNS_CLASS_IN 0x01 +#define DOH_MAX_RESPONSE_SIZE 3000 /* bytes */ + +#ifndef CURL_DISABLE_VERBOSE_STRINGS +static const char * const errors[]={ + "", + "Bad label", + "Out of range", + "Label loop", + "Too small", + "Out of memory", + "RDATA length", + "Malformat", + "Bad RCODE", + "Unexpected TYPE", + "Unexpected CLASS", + "No content", + "Bad ID" +}; + +static const char *doh_strerror(DOHcode code) +{ + if((code >= DOH_OK) && (code <= DOH_DNS_BAD_ID)) + return errors[code]; + return "bad error code"; +} +#endif + +#ifdef DEBUGBUILD +#define UNITTEST +#else +#define UNITTEST static +#endif + +UNITTEST DOHcode doh_encode(const char *host, + DNStype dnstype, + unsigned char *dnsp, /* buffer */ + size_t len, /* buffer size */ + size_t *olen) /* output length */ +{ + size_t hostlen = strlen(host); + unsigned char *orig = dnsp; + const char *hostp = host; + + if(len < (12 + hostlen + 4)) + return DOH_TOO_SMALL_BUFFER; + + *dnsp++ = 0; /* 16 bit id */ + *dnsp++ = 0; + *dnsp++ = 0x01; /* |QR| Opcode |AA|TC|RD| Set the RD bit */ + *dnsp++ = '\0'; /* |RA| Z | RCODE | */ + *dnsp++ = '\0'; + *dnsp++ = 1; /* QDCOUNT (number of entries in the question section) */ + *dnsp++ = '\0'; + *dnsp++ = '\0'; /* ANCOUNT */ + *dnsp++ = '\0'; + *dnsp++ = '\0'; /* NSCOUNT */ + *dnsp++ = '\0'; + *dnsp++ = '\0'; /* ARCOUNT */ + + /* store a QNAME */ + do { + char *dot = strchr(hostp, '.'); + size_t labellen; + bool found = false; + if(dot) { + found = true; + labellen = dot - hostp; + } + else + labellen = strlen(hostp); + if(labellen > 63) { + /* too long label, error out */ + *olen = 0; + return DOH_DNS_BAD_LABEL; + } + *dnsp++ = (unsigned char)labellen; + memcpy(dnsp, hostp, labellen); + dnsp += labellen; + hostp += labellen + 1; + if(!found) { + *dnsp++ = 0; /* terminating zero */ + break; + } + } while(1); + + *dnsp++ = '\0'; /* upper 8 bit TYPE */ + *dnsp++ = (unsigned char)dnstype; + *dnsp++ = '\0'; /* upper 8 bit CLASS */ + *dnsp++ = DNS_CLASS_IN; /* IN - "the Internet" */ + + *olen = dnsp - orig; + return DOH_OK; +} + +static size_t +doh_write_cb(void *contents, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + struct dohresponse *mem = (struct dohresponse *)userp; + + if((mem->size + realsize) > DOH_MAX_RESPONSE_SIZE) + /* suspiciously much for us */ + return 0; + + mem->memory = realloc(mem->memory, mem->size + realsize); + if(mem->memory == NULL) + /* out of memory! */ + return 0; + + memcpy(&(mem->memory[mem->size]), contents, realsize); + mem->size += realsize; + + return realsize; +} + +/* called from multi.c when this DOH transfer is complete */ +static int Curl_doh_done(struct Curl_easy *doh, CURLcode result) +{ + struct Curl_easy *data = doh->set.dohfor; + /* so one of the DOH request done for the 'data' transfer is now complete! */ + data->req.doh.pending--; + infof(data, "a DOH request is completed, %d to go\n", data->req.doh.pending); + if(result) + infof(data, "DOH request %s\n", curl_easy_strerror(result)); + + if(!data->req.doh.pending) { + /* DOH completed */ + curl_slist_free_all(data->req.doh.headers); + data->req.doh.headers = NULL; + Curl_expire(data, 0, EXPIRE_RUN_NOW); + } + return 0; +} + +#define ERROR_CHECK_SETOPT(x,y) result = curl_easy_setopt(doh, x, y); \ + if(result) goto error + +static CURLcode dohprobe(struct Curl_easy *data, + struct dnsprobe *p, DNStype dnstype, + const char *host, + const char *url, CURLM *multi, + struct curl_slist *headers) +{ + struct Curl_easy *doh = NULL; + char *nurl = NULL; + CURLcode result = CURLE_OK; + timediff_t timeout_ms; + DOHcode d = doh_encode(host, dnstype, p->dohbuffer, sizeof(p->dohbuffer), + &p->dohlen); + if(d) { + failf(data, "Failed to encode DOH packet [%d]\n", d); + return CURLE_OUT_OF_MEMORY; + } + + p->dnstype = dnstype; + p->serverdoh.memory = NULL; + /* the memory will be grown as needed by realloc in the doh_write_cb + function */ + p->serverdoh.size = 0; + + /* Note: this is code for sending the DoH request with GET but there's still + no logic that actually enables this. We should either add that ability or + yank out the GET code. Discuss! */ + if(data->set.doh_get) { + char *b64; + size_t b64len; + result = Curl_base64url_encode(data, (char *)p->dohbuffer, p->dohlen, + &b64, &b64len); + if(result) + goto error; + nurl = aprintf("%s?dns=%s", url, b64); + free(b64); + if(!nurl) { + result = CURLE_OUT_OF_MEMORY; + goto error; + } + url = nurl; + } + + timeout_ms = Curl_timeleft(data, NULL, TRUE); + + /* Curl_open() is the internal version of curl_easy_init() */ + result = Curl_open(&doh); + if(!result) { + /* pass in the struct pointer via a local variable to please coverity and + the gcc typecheck helpers */ + struct dohresponse *resp = &p->serverdoh; + ERROR_CHECK_SETOPT(CURLOPT_URL, url); + ERROR_CHECK_SETOPT(CURLOPT_WRITEFUNCTION, doh_write_cb); + ERROR_CHECK_SETOPT(CURLOPT_WRITEDATA, resp); + if(!data->set.doh_get) { + ERROR_CHECK_SETOPT(CURLOPT_POSTFIELDS, p->dohbuffer); + ERROR_CHECK_SETOPT(CURLOPT_POSTFIELDSIZE, (long)p->dohlen); + } + ERROR_CHECK_SETOPT(CURLOPT_HTTPHEADER, headers); + ERROR_CHECK_SETOPT(CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); +#ifndef CURLDEBUG + /* enforce HTTPS if not debug */ + ERROR_CHECK_SETOPT(CURLOPT_PROTOCOLS, CURLPROTO_HTTPS); +#endif + ERROR_CHECK_SETOPT(CURLOPT_TIMEOUT_MS, (long)timeout_ms); + ERROR_CHECK_SETOPT(CURLOPT_VERBOSE, 1L); + doh->set.fmultidone = Curl_doh_done; + doh->set.dohfor = data; /* identify for which transfer this is done */ + p->easy = doh; + + /* add this transfer to the multi handle */ + if(curl_multi_add_handle(multi, doh)) + goto error; + } + else + goto error; + free(nurl); + return CURLE_OK; + + error: + free(nurl); + Curl_close(doh); + return result; +} + +/* + * Curl_doh() resolves a name using DOH. It resolves a name and returns a + * 'Curl_addrinfo *' with the address information. + */ + +Curl_addrinfo *Curl_doh(struct connectdata *conn, + const char *hostname, + int port, + int *waitp) +{ + struct Curl_easy *data = conn->data; + CURLcode result = CURLE_OK; + *waitp = TRUE; /* this never returns synchronously */ + (void)conn; + (void)hostname; + (void)port; + + /* start clean, consider allocating this struct on demand */ + memset(&data->req.doh, 0, sizeof(struct dohdata)); + + data->req.doh.host = hostname; + data->req.doh.port = port; + data->req.doh.headers = + curl_slist_append(NULL, + "Content-Type: application/dns-message"); + if(!data->req.doh.headers) + goto error; + + if(conn->ip_version != CURL_IPRESOLVE_V6) { + /* create IPv4 DOH request */ + result = dohprobe(data, &data->req.doh.probe[0], DNS_TYPE_A, + hostname, data->set.str[STRING_DOH], + data->multi, data->req.doh.headers); + if(result) + goto error; + data->req.doh.pending++; + } + + if(conn->ip_version != CURL_IPRESOLVE_V4) { + /* create IPv6 DOH request */ + result = dohprobe(data, &data->req.doh.probe[1], DNS_TYPE_AAAA, + hostname, data->set.str[STRING_DOH], + data->multi, data->req.doh.headers); + if(result) + goto error; + data->req.doh.pending++; + } + return NULL; + + error: + curl_slist_free_all(data->req.doh.headers); + data->req.doh.headers = NULL; + curl_easy_cleanup(data->req.doh.probe[0].easy); + data->req.doh.probe[0].easy = NULL; + curl_easy_cleanup(data->req.doh.probe[1].easy); + data->req.doh.probe[1].easy = NULL; + return NULL; +} + +static DOHcode skipqname(unsigned char *doh, size_t dohlen, + unsigned int *indexp) +{ + unsigned char length; + do { + if(dohlen < (*indexp + 1)) + return DOH_DNS_OUT_OF_RANGE; + length = doh[*indexp]; + if((length & 0xc0) == 0xc0) { + /* name pointer, advance over it and be done */ + if(dohlen < (*indexp + 2)) + return DOH_DNS_OUT_OF_RANGE; + *indexp += 2; + break; + } + if(length & 0xc0) + return DOH_DNS_BAD_LABEL; + if(dohlen < (*indexp + 1 + length)) + return DOH_DNS_OUT_OF_RANGE; + *indexp += 1 + length; + } while(length); + return DOH_OK; +} + +static unsigned short get16bit(unsigned char *doh, int index) +{ + return (unsigned short)((doh[index] << 8) | doh[index + 1]); +} + +static unsigned int get32bit(unsigned char *doh, int index) +{ + return (doh[index] << 24) | (doh[index + 1] << 16) | + (doh[index + 2] << 8) | doh[index + 3]; +} + +static DOHcode store_a(unsigned char *doh, int index, struct dohentry *d) +{ + /* silently ignore addresses over the limit */ + if(d->numaddr < DOH_MAX_ADDR) { + struct dohaddr *a = &d->addr[d->numaddr]; + a->type = DNS_TYPE_A; + memcpy(&a->ip.v4, &doh[index], 4); + d->numaddr++; + } + return DOH_OK; +} + +static DOHcode store_aaaa(unsigned char *doh, int index, struct dohentry *d) +{ + /* silently ignore addresses over the limit */ + if(d->numaddr < DOH_MAX_ADDR) { + struct dohaddr *a = &d->addr[d->numaddr]; + a->type = DNS_TYPE_AAAA; + memcpy(&a->ip.v6, &doh[index], 16); + d->numaddr++; + } + return DOH_OK; +} + +static DOHcode cnameappend(struct cnamestore *c, + unsigned char *src, + size_t len) +{ + if(!c->alloc) { + c->allocsize = len + 1; + c->alloc = malloc(c->allocsize); + if(!c->alloc) + return DOH_OUT_OF_MEM; + } + else if(c->allocsize < (c->allocsize + len + 1)) { + char *ptr; + c->allocsize += len + 1; + ptr = realloc(c->alloc, c->allocsize); + if(!ptr) { + free(c->alloc); + return DOH_OUT_OF_MEM; + } + c->alloc = ptr; + } + memcpy(&c->alloc[c->len], src, len); + c->len += len; + c->alloc[c->len] = 0; /* keep it zero terminated */ + return DOH_OK; +} + +static DOHcode store_cname(unsigned char *doh, + size_t dohlen, + unsigned int index, + struct dohentry *d) +{ + struct cnamestore *c; + unsigned int loop = 128; /* a valid DNS name can never loop this much */ + unsigned char length; + + if(d->numcname == DOH_MAX_CNAME) + return DOH_OK; /* skip! */ + + c = &d->cname[d->numcname++]; + do { + if(index >= dohlen) + return DOH_DNS_OUT_OF_RANGE; + length = doh[index]; + if((length & 0xc0) == 0xc0) { + int newpos; + /* name pointer, get the new offset (14 bits) */ + if((index + 1) >= dohlen) + return DOH_DNS_OUT_OF_RANGE; + + /* move to the the new index */ + newpos = (length & 0x3f) << 8 | doh[index + 1]; + index = newpos; + continue; + } + else if(length & 0xc0) + return DOH_DNS_BAD_LABEL; /* bad input */ + else + index++; + + if(length) { + DOHcode rc; + if(c->len) { + rc = cnameappend(c, (unsigned char *)".", 1); + if(rc) + return rc; + } + if((index + length) > dohlen) + return DOH_DNS_BAD_LABEL; + + rc = cnameappend(c, &doh[index], length); + if(rc) + return rc; + index += length; + } + } while(length && --loop); + + if(!loop) + return DOH_DNS_LABEL_LOOP; + return DOH_OK; +} + +static DOHcode rdata(unsigned char *doh, + size_t dohlen, + unsigned short rdlength, + unsigned short type, + int index, + struct dohentry *d) +{ + /* RDATA + - A (TYPE 1): 4 bytes + - AAAA (TYPE 28): 16 bytes + - NS (TYPE 2): N bytes */ + DOHcode rc; + + switch(type) { + case DNS_TYPE_A: + if(rdlength != 4) + return DOH_DNS_RDATA_LEN; + rc = store_a(doh, index, d); + if(rc) + return rc; + break; + case DNS_TYPE_AAAA: + if(rdlength != 16) + return DOH_DNS_RDATA_LEN; + rc = store_aaaa(doh, index, d); + if(rc) + return rc; + break; + case DNS_TYPE_CNAME: + rc = store_cname(doh, dohlen, index, d); + if(rc) + return rc; + break; + default: + /* unsupported type, just skip it */ + break; + } + return DOH_OK; +} + +static void init_dohentry(struct dohentry *de) +{ + memset(de, 0, sizeof(*de)); + de->ttl = INT_MAX; +} + + +UNITTEST DOHcode doh_decode(unsigned char *doh, + size_t dohlen, + DNStype dnstype, + struct dohentry *d) +{ + unsigned char rcode; + unsigned short qdcount; + unsigned short ancount; + unsigned short type = 0; + unsigned short class; + unsigned short rdlength; + unsigned short nscount; + unsigned short arcount; + unsigned int index = 12; + DOHcode rc; + + if(dohlen < 12) + return DOH_TOO_SMALL_BUFFER; /* too small */ + if(doh[0] || doh[1]) + return DOH_DNS_BAD_ID; /* bad ID */ + rcode = doh[3] & 0x0f; + if(rcode) + return DOH_DNS_BAD_RCODE; /* bad rcode */ + + qdcount = get16bit(doh, 4); + while(qdcount) { + rc = skipqname(doh, dohlen, &index); + if(rc) + return rc; /* bad qname */ + if(dohlen < (index + 4)) + return DOH_DNS_OUT_OF_RANGE; + index += 4; /* skip question's type and class */ + qdcount--; + } + + ancount = get16bit(doh, 6); + while(ancount) { + unsigned int ttl; + + rc = skipqname(doh, dohlen, &index); + if(rc) + return rc; /* bad qname */ + + if(dohlen < (index + 2)) + return DOH_DNS_OUT_OF_RANGE; + + type = get16bit(doh, index); + if((type != DNS_TYPE_CNAME) && (type != dnstype)) + /* Not the same type as was asked for nor CNAME */ + return DOH_DNS_UNEXPECTED_TYPE; + index += 2; + + if(dohlen < (index + 2)) + return DOH_DNS_OUT_OF_RANGE; + class = get16bit(doh, index); + if(DNS_CLASS_IN != class) + return DOH_DNS_UNEXPECTED_CLASS; /* unsupported */ + index += 2; + + if(dohlen < (index + 4)) + return DOH_DNS_OUT_OF_RANGE; + + ttl = get32bit(doh, index); + if(ttl < d->ttl) + d->ttl = ttl; + index += 4; + + if(dohlen < (index + 2)) + return DOH_DNS_OUT_OF_RANGE; + + rdlength = get16bit(doh, index); + index += 2; + if(dohlen < (index + rdlength)) + return DOH_DNS_OUT_OF_RANGE; + + rc = rdata(doh, dohlen, rdlength, type, index, d); + if(rc) + return rc; /* bad rdata */ + index += rdlength; + ancount--; + } + + nscount = get16bit(doh, 8); + while(nscount) { + rc = skipqname(doh, dohlen, &index); + if(rc) + return rc; /* bad qname */ + + if(dohlen < (index + 8)) + return DOH_DNS_OUT_OF_RANGE; + + index += 2 + 2 + 4; /* type, class and ttl */ + + if(dohlen < (index + 2)) + return DOH_DNS_OUT_OF_RANGE; + + rdlength = get16bit(doh, index); + index += 2; + if(dohlen < (index + rdlength)) + return DOH_DNS_OUT_OF_RANGE; + index += rdlength; + nscount--; + } + + arcount = get16bit(doh, 10); + while(arcount) { + rc = skipqname(doh, dohlen, &index); + if(rc) + return rc; /* bad qname */ + + if(dohlen < (index + 8)) + return DOH_DNS_OUT_OF_RANGE; + + index += 2 + 2 + 4; /* type, class and ttl */ + + if(dohlen < (index + 2)) + return DOH_DNS_OUT_OF_RANGE; + + rdlength = get16bit(doh, index); + index += 2; + if(dohlen < (index + rdlength)) + return DOH_DNS_OUT_OF_RANGE; + index += rdlength; + arcount--; + } + + if(index != dohlen) + return DOH_DNS_MALFORMAT; /* something is wrong */ + + if((type != DNS_TYPE_NS) && !d->numcname && !d->numaddr) + /* nothing stored! */ + return DOH_NO_CONTENT; + + return DOH_OK; /* ok */ +} + +#ifndef CURL_DISABLE_VERBOSE_STRINGS +static void showdoh(struct Curl_easy *data, + struct dohentry *d) +{ + int i; + infof(data, "TTL: %u seconds\n", d->ttl); + for(i = 0; i < d->numaddr; i++) { + struct dohaddr *a = &d->addr[i]; + if(a->type == DNS_TYPE_A) { + infof(data, "DOH A: %u.%u.%u.%u\n", + a->ip.v4[0], a->ip.v4[1], + a->ip.v4[2], a->ip.v4[3]); + } + else if(a->type == DNS_TYPE_AAAA) { + int j; + char buffer[128]; + char *ptr; + size_t len; + snprintf(buffer, 128, "DOH AAAA: "); + ptr = &buffer[10]; + len = 118; + for(j = 0; j < 16; j += 2) { + size_t l; + snprintf(ptr, len, "%s%02x%02x", j?":":"", d->addr[i].ip.v6[j], + d->addr[i].ip.v6[j + 1]); + l = strlen(ptr); + len -= l; + ptr += l; + } + infof(data, "%s\n", buffer); + } + } + for(i = 0; i < d->numcname; i++) { + infof(data, "CNAME: %s\n", d->cname[i].alloc); + } +} +#else +#define showdoh(x,y) +#endif + +/* + * doh2ai() + * + * This function returns a pointer to the first element of a newly allocated + * Curl_addrinfo struct linked list filled with the data from a set of DOH + * lookups. Curl_addrinfo is meant to work like the addrinfo struct does for + * a IPv6 stack, but usable also for IPv4, all hosts and environments. + * + * The memory allocated by this function *MUST* be free'd later on calling + * Curl_freeaddrinfo(). For each successful call to this function there + * must be an associated call later to Curl_freeaddrinfo(). + */ + +static Curl_addrinfo * +doh2ai(const struct dohentry *de, const char *hostname, int port) +{ + Curl_addrinfo *ai; + Curl_addrinfo *prevai = NULL; + Curl_addrinfo *firstai = NULL; + struct sockaddr_in *addr; +#ifdef ENABLE_IPV6 + struct sockaddr_in6 *addr6; +#endif + CURLcode result = CURLE_OK; + int i; + + if(!de) + /* no input == no output! */ + return NULL; + + for(i = 0; i < de->numaddr; i++) { + size_t ss_size; + CURL_SA_FAMILY_T addrtype; + if(de->addr[i].type == DNS_TYPE_AAAA) { +#ifndef ENABLE_IPV6 + /* we can't handle IPv6 addresses */ + continue; +#else + ss_size = sizeof(struct sockaddr_in6); + addrtype = AF_INET6; +#endif + } + else { + ss_size = sizeof(struct sockaddr_in); + addrtype = AF_INET; + } + + ai = calloc(1, sizeof(Curl_addrinfo)); + if(!ai) { + result = CURLE_OUT_OF_MEMORY; + break; + } + ai->ai_canonname = strdup(hostname); + if(!ai->ai_canonname) { + result = CURLE_OUT_OF_MEMORY; + free(ai); + break; + } + ai->ai_addr = calloc(1, ss_size); + if(!ai->ai_addr) { + result = CURLE_OUT_OF_MEMORY; + free(ai->ai_canonname); + free(ai); + break; + } + + if(!firstai) + /* store the pointer we want to return from this function */ + firstai = ai; + + if(prevai) + /* make the previous entry point to this */ + prevai->ai_next = ai; + + ai->ai_family = addrtype; + + /* we return all names as STREAM, so when using this address for TFTP + the type must be ignored and conn->socktype be used instead! */ + ai->ai_socktype = SOCK_STREAM; + + ai->ai_addrlen = (curl_socklen_t)ss_size; + + /* leave the rest of the struct filled with zero */ + + switch(ai->ai_family) { + case AF_INET: + addr = (void *)ai->ai_addr; /* storage area for this info */ + DEBUGASSERT(sizeof(struct in_addr) == sizeof(de->addr[i].ip.v4)); + memcpy(&addr->sin_addr, &de->addr[i].ip.v4, sizeof(struct in_addr)); + addr->sin_family = (CURL_SA_FAMILY_T)addrtype; + addr->sin_port = htons((unsigned short)port); + break; + +#ifdef ENABLE_IPV6 + case AF_INET6: + addr6 = (void *)ai->ai_addr; /* storage area for this info */ + DEBUGASSERT(sizeof(struct in6_addr) == sizeof(de->addr[i].ip.v6)); + memcpy(&addr6->sin6_addr, &de->addr[i].ip.v6, sizeof(struct in6_addr)); + addr6->sin6_family = (CURL_SA_FAMILY_T)addrtype; + addr6->sin6_port = htons((unsigned short)port); + break; +#endif + } + + prevai = ai; + } + + if(result) { + Curl_freeaddrinfo(firstai); + firstai = NULL; + } + + return firstai; +} + +#ifndef CURL_DISABLE_VERBOSE_STRINGS +static const char *type2name(DNStype dnstype) +{ + return (dnstype == DNS_TYPE_A)?"A":"AAAA"; +} +#endif + +UNITTEST void de_cleanup(struct dohentry *d) +{ + int i = 0; + for(i = 0; i < d->numcname; i++) { + free(d->cname[i].alloc); + } +} + +CURLcode Curl_doh_is_resolved(struct connectdata *conn, + struct Curl_dns_entry **dnsp) +{ + struct Curl_easy *data = conn->data; + *dnsp = NULL; /* defaults to no response */ + + if(!data->req.doh.probe[0].easy && !data->req.doh.probe[1].easy) { + failf(data, "Could not DOH-resolve: %s", conn->async.hostname); + return conn->bits.proxy?CURLE_COULDNT_RESOLVE_PROXY: + CURLE_COULDNT_RESOLVE_HOST; + } + else if(!data->req.doh.pending) { + DOHcode rc; + DOHcode rc2; + struct dohentry de; + struct Curl_dns_entry *dns; + struct Curl_addrinfo *ai; + /* remove DOH handles from multi handle and close them */ + curl_multi_remove_handle(data->multi, data->req.doh.probe[0].easy); + Curl_close(data->req.doh.probe[0].easy); + curl_multi_remove_handle(data->multi, data->req.doh.probe[1].easy); + Curl_close(data->req.doh.probe[1].easy); + + /* parse the responses, create the struct and return it! */ + init_dohentry(&de); + rc = doh_decode(data->req.doh.probe[0].serverdoh.memory, + data->req.doh.probe[0].serverdoh.size, + data->req.doh.probe[0].dnstype, + &de); + free(data->req.doh.probe[0].serverdoh.memory); + if(rc) { + infof(data, "DOH: %s type %s for %s\n", doh_strerror(rc), + type2name(data->req.doh.probe[0].dnstype), + data->req.doh.host); + } + rc2 = doh_decode(data->req.doh.probe[1].serverdoh.memory, + data->req.doh.probe[1].serverdoh.size, + data->req.doh.probe[1].dnstype, + &de); + free(data->req.doh.probe[1].serverdoh.memory); + if(rc2) { + infof(data, "DOG: %s type %s for %s\n", doh_strerror(rc2), + type2name(data->req.doh.probe[1].dnstype), + data->req.doh.host); + } + if(!rc || !rc2) { + infof(data, "DOH Host name: %s\n", data->req.doh.host); + showdoh(data, &de); + + ai = doh2ai(&de, data->req.doh.host, data->req.doh.port); + if(!ai) { + de_cleanup(&de); + return CURLE_OUT_OF_MEMORY; + } + + if(data->share) + Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); + + /* we got a response, store it in the cache */ + dns = Curl_cache_addr(data, ai, data->req.doh.host, data->req.doh.port); + + if(data->share) + Curl_share_unlock(data, CURL_LOCK_DATA_DNS); + + de_cleanup(&de); + if(!dns) + /* returned failure, bail out nicely */ + Curl_freeaddrinfo(ai); + else { + conn->async.dns = dns; + *dnsp = dns; + return CURLE_OK; + } + } + de_cleanup(&de); + + return CURLE_COULDNT_RESOLVE_HOST; + } + + return CURLE_OK; +} + +#else /* !USE_NGHTTP2 */ +/* + */ +Curl_addrinfo *Curl_doh(struct connectdata *conn, + const char *hostname, + int port, + int *waitp) +{ + (void)conn; + (void)hostname; + (void)port; + (void)waitp; + return NULL; +} + +CURLcode Curl_doh_is_resolved(struct connectdata *conn, + struct Curl_dns_entry **dnsp) +{ + (void)conn; + (void)dnsp; + return CURLE_NOT_BUILT_IN; +} + +#endif /* USE_NGHTTP2 */ diff --git a/Utilities/cmcurl/lib/doh.h b/Utilities/cmcurl/lib/doh.h new file mode 100644 index 000000000..83c79bc5d --- /dev/null +++ b/Utilities/cmcurl/lib/doh.h @@ -0,0 +1,105 @@ +#ifndef HEADER_CURL_DOH_H +#define HEADER_CURL_DOH_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "urldata.h" +#include "curl_addrinfo.h" + +/* + * Curl_doh() resolve a name using DoH (DNS-over-HTTPS). It resolves a name + * and returns a 'Curl_addrinfo *' with the address information. + */ + +Curl_addrinfo *Curl_doh(struct connectdata *conn, + const char *hostname, + int port, + int *waitp); + +CURLcode Curl_doh_is_resolved(struct connectdata *conn, + struct Curl_dns_entry **dns); + +int Curl_doh_getsock(struct connectdata *conn, curl_socket_t *socks, + int numsocks); + +typedef enum { + DOH_OK, + DOH_DNS_BAD_LABEL, /* 1 */ + DOH_DNS_OUT_OF_RANGE, /* 2 */ + DOH_DNS_LABEL_LOOP, /* 3 */ + DOH_TOO_SMALL_BUFFER, /* 4 */ + DOH_OUT_OF_MEM, /* 5 */ + DOH_DNS_RDATA_LEN, /* 6 */ + DOH_DNS_MALFORMAT, /* 7 */ + DOH_DNS_BAD_RCODE, /* 8 - no such name */ + DOH_DNS_UNEXPECTED_TYPE, /* 9 */ + DOH_DNS_UNEXPECTED_CLASS, /* 10 */ + DOH_NO_CONTENT, /* 11 */ + DOH_DNS_BAD_ID /* 12 */ +} DOHcode; + +typedef enum { + DNS_TYPE_A = 1, + DNS_TYPE_NS = 2, + DNS_TYPE_CNAME = 5, + DNS_TYPE_AAAA = 28 +} DNStype; + +#define DOH_MAX_ADDR 24 +#define DOH_MAX_CNAME 4 + +struct cnamestore { + size_t len; /* length of cname */ + char *alloc; /* allocated pointer */ + size_t allocsize; /* allocated size */ +}; + +struct dohaddr { + int type; + union { + unsigned char v4[4]; /* network byte order */ + unsigned char v6[16]; + } ip; +}; + +struct dohentry { + unsigned int ttl; + int numaddr; + struct dohaddr addr[DOH_MAX_ADDR]; + int numcname; + struct cnamestore cname[DOH_MAX_CNAME]; +}; + + +#ifdef DEBUGBUILD +DOHcode doh_encode(const char *host, + DNStype dnstype, + unsigned char *dnsp, /* buffer */ + size_t len, /* buffer size */ + size_t *olen); /* output length */ +DOHcode doh_decode(unsigned char *doh, + size_t dohlen, + DNStype dnstype, + struct dohentry *d); +void de_cleanup(struct dohentry *d); +#endif +#endif /* HEADER_CURL_DOH_H */ diff --git a/Utilities/cmcurl/lib/dotdot.c b/Utilities/cmcurl/lib/dotdot.c index cbb308d78..2c6177aea 100644 --- a/Utilities/cmcurl/lib/dotdot.c +++ b/Utilities/cmcurl/lib/dotdot.c @@ -62,6 +62,8 @@ char *Curl_dedotdotify(const char *input) if(!out) return NULL; /* out of memory */ + *out = 0; /* zero terminates, for inputs like "./" */ + /* get a cloned copy of the input */ clone = strdup(input); if(!clone) { diff --git a/Utilities/cmcurl/lib/dotdot.h b/Utilities/cmcurl/lib/dotdot.h index fac8e6f2a..125af4367 100644 --- a/Utilities/cmcurl/lib/dotdot.h +++ b/Utilities/cmcurl/lib/dotdot.h @@ -22,4 +22,4 @@ * ***************************************************************************/ char *Curl_dedotdotify(const char *input); -#endif +#endif /* HEADER_CURL_DOTDOT_H */ diff --git a/Utilities/cmcurl/lib/easy.c b/Utilities/cmcurl/lib/easy.c index 6b914353d..4de4e6522 100644 --- a/Utilities/cmcurl/lib/easy.c +++ b/Utilities/cmcurl/lib/easy.c @@ -113,7 +113,7 @@ static CURLcode win32_init(void) res = WSAStartup(wVersionRequested, &wsaData); if(res != 0) - /* Tell the user that we couldn't find a useable */ + /* Tell the user that we couldn't find a usable */ /* winsock.dll. */ return CURLE_FAILED_INIT; @@ -125,7 +125,7 @@ static CURLcode win32_init(void) if(LOBYTE(wsaData.wVersion) != LOBYTE(wVersionRequested) || HIBYTE(wsaData.wVersion) != HIBYTE(wVersionRequested) ) { - /* Tell the user that we couldn't find a useable */ + /* Tell the user that we couldn't find a usable */ /* winsock.dll. */ WSACleanup(); @@ -661,38 +661,27 @@ static CURLcode easy_transfer(struct Curl_multi *multi) bool done = FALSE; CURLMcode mcode = CURLM_OK; CURLcode result = CURLE_OK; - struct curltime before; - int without_fds = 0; /* count number of consecutive returns from - curl_multi_wait() without any filedescriptors */ while(!done && !mcode) { int still_running = 0; int rc; - before = Curl_now(); mcode = curl_multi_wait(multi, NULL, 0, 1000, &rc); if(!mcode) { if(!rc) { - struct curltime after = Curl_now(); + long sleep_ms; /* If it returns without any filedescriptor instantly, we need to avoid busy-looping during periods where it has nothing particular to wait for */ - if(Curl_timediff(after, before) <= 10) { - without_fds++; - if(without_fds > 2) { - int sleep_ms = without_fds < 10 ? (1 << (without_fds - 1)) : 1000; - Curl_wait_ms(sleep_ms); - } + curl_multi_timeout(multi, &sleep_ms); + if(sleep_ms) { + if(sleep_ms > 1000) + sleep_ms = 1000; + Curl_wait_ms((int)sleep_ms); } - else - /* it wasn't "instant", restart counter */ - without_fds = 0; } - else - /* got file descriptor, restart counter */ - without_fds = 0; mcode = curl_multi_perform(multi, &still_running); } @@ -969,6 +958,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->change.referer_alloc = TRUE; } + /* Reinitialize an SSL engine for the new handle + * note: the engine name has already been copied by dupset */ + if(outcurl->set.str[STRING_SSL_ENGINE]) { + if(Curl_ssl_set_engine(outcurl, outcurl->set.str[STRING_SSL_ENGINE])) + goto fail; + } + /* Clone the resolver handle, if present, for the new handle */ if(Curl_resolver_duphandle(&outcurl->state.resolver, data->state.resolver)) @@ -1006,10 +1002,6 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) */ void curl_easy_reset(struct Curl_easy *data) { - Curl_safefree(data->state.pathbuffer); - - data->state.path = NULL; - Curl_free_request_state(data); /* zero out UserDefined data: */ @@ -1201,3 +1193,22 @@ CURLcode curl_easy_send(struct Curl_easy *data, const void *buffer, return result; } + +/* + * Performs connection upkeep for the given session handle. + */ +CURLcode curl_easy_upkeep(struct Curl_easy *data) +{ + /* Verify that we got an easy handle we can work with. */ + if(!GOOD_EASY_HANDLE(data)) + return CURLE_BAD_FUNCTION_ARGUMENT; + + if(data->multi_easy) { + /* Use the common function to keep connections alive. */ + return Curl_upkeep(&data->multi_easy->conn_cache, data); + } + else { + /* No connections, so just return success */ + return CURLE_OK; + } +} diff --git a/Utilities/cmcurl/lib/easyif.h b/Utilities/cmcurl/lib/easyif.h index f6132cc70..6ba7e549d 100644 --- a/Utilities/cmcurl/lib/easyif.h +++ b/Utilities/cmcurl/lib/easyif.h @@ -30,4 +30,3 @@ CURL_EXTERN CURLcode curl_easy_perform_ev(struct Curl_easy *easy); #endif #endif /* HEADER_CURL_EASYIF_H */ - diff --git a/Utilities/cmcurl/lib/escape.c b/Utilities/cmcurl/lib/escape.c index b7e2d32a6..afd3899f9 100644 --- a/Utilities/cmcurl/lib/escape.c +++ b/Utilities/cmcurl/lib/escape.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -41,7 +41,7 @@ its behavior is altered by the current locale. See https://tools.ietf.org/html/rfc3986#section-2.3 */ -static bool Curl_isunreserved(unsigned char in) +bool Curl_isunreserved(unsigned char in) { switch(in) { case '0': case '1': case '2': case '3': case '4': @@ -82,7 +82,6 @@ char *curl_easy_escape(struct Curl_easy *data, const char *string, size_t alloc; char *ns; char *testing_ptr = NULL; - unsigned char in; /* we need to treat the characters unsigned */ size_t newlen; size_t strindex = 0; size_t length; @@ -100,7 +99,7 @@ char *curl_easy_escape(struct Curl_easy *data, const char *string, length = alloc-1; while(length--) { - in = *string; + unsigned char in = *string; /* we need to treat the characters unsigned */ if(Curl_isunreserved(in)) /* just copy this */ @@ -142,6 +141,8 @@ char *curl_easy_escape(struct Curl_easy *data, const char *string, * Returns a pointer to a malloced string in *ostring with length given in * *olen. If length == 0, the length is assumed to be strlen(string). * + * 'data' can be set to NULL but then this function can't convert network + * data to host for non-ascii. */ CURLcode Curl_urldecode(struct Curl_easy *data, const char *string, size_t length, @@ -150,16 +151,15 @@ CURLcode Curl_urldecode(struct Curl_easy *data, { size_t alloc = (length?length:strlen(string)) + 1; char *ns = malloc(alloc); - unsigned char in; size_t strindex = 0; unsigned long hex; - CURLcode result; + CURLcode result = CURLE_OK; if(!ns) return CURLE_OUT_OF_MEMORY; while(--alloc > 0) { - in = *string; + unsigned char in = *string; if(('%' == in) && (alloc > 2) && ISXDIGIT(string[1]) && ISXDIGIT(string[2])) { /* this is two hexadecimal digits following a '%' */ @@ -173,11 +173,13 @@ CURLcode Curl_urldecode(struct Curl_easy *data, in = curlx_ultouc(hex); /* this long is never bigger than 255 anyway */ - result = Curl_convert_from_network(data, (char *)&in, 1); - if(result) { - /* Curl_convert_from_network calls failf if unsuccessful */ - free(ns); - return result; + if(data) { + result = Curl_convert_from_network(data, (char *)&in, 1); + if(result) { + /* Curl_convert_from_network calls failf if unsuccessful */ + free(ns); + return result; + } } string += 2; diff --git a/Utilities/cmcurl/lib/escape.h b/Utilities/cmcurl/lib/escape.h index 638666f03..d8bbe5cb0 100644 --- a/Utilities/cmcurl/lib/escape.h +++ b/Utilities/cmcurl/lib/escape.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -24,10 +24,10 @@ /* Escape and unescape URL encoding in strings. The functions return a new * allocated string or NULL if an error occurred. */ +bool Curl_isunreserved(unsigned char in); CURLcode Curl_urldecode(struct Curl_easy *data, const char *string, size_t length, char **ostring, size_t *olen, bool reject_crlf); #endif /* HEADER_CURL_ESCAPE_H */ - diff --git a/Utilities/cmcurl/lib/file.c b/Utilities/cmcurl/lib/file.c index db04cc2da..722b55e9d 100644 --- a/Utilities/cmcurl/lib/file.c +++ b/Utilities/cmcurl/lib/file.c @@ -143,7 +143,7 @@ static CURLcode file_connect(struct connectdata *conn, bool *done) #endif size_t real_path_len; - CURLcode result = Curl_urldecode(data, data->state.path, 0, &real_path, + CURLcode result = Curl_urldecode(data, data->state.up.path, 0, &real_path, &real_path_len, FALSE); if(result) return result; @@ -197,7 +197,7 @@ static CURLcode file_connect(struct connectdata *conn, bool *done) file->fd = fd; if(!data->set.upload && (fd == -1)) { - failf(data, "Couldn't open file %s", data->state.path); + failf(data, "Couldn't open file %s", data->state.up.path); file_done(conn, CURLE_FILE_COULDNT_READ_FILE, FALSE); return CURLE_FILE_COULDNT_READ_FILE; } @@ -256,8 +256,6 @@ static CURLcode file_upload(struct connectdata *conn) CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; char *buf = data->state.buffer; - size_t nread; - size_t nwrite; curl_off_t bytecount = 0; struct_stat file_stat; const char *buf2; @@ -306,15 +304,17 @@ static CURLcode file_upload(struct connectdata *conn) } while(!result) { - int readcount; - result = Curl_fillreadbuffer(conn, (int)data->set.buffer_size, &readcount); + size_t nread; + size_t nwrite; + size_t readcount; + result = Curl_fillreadbuffer(conn, data->set.buffer_size, &readcount); if(result) break; if(readcount <= 0) /* fix questionable compare error. curlvms */ break; - nread = (size_t)readcount; + nread = readcount; /*skip bytes before resume point*/ if(data->state.resume_from) { @@ -378,7 +378,6 @@ static CURLcode file_do(struct connectdata *conn, bool *done) curl_off_t expected_size = 0; bool size_known; bool fstated = FALSE; - ssize_t nread; struct Curl_easy *data = conn->data; char *buf = data->state.buffer; curl_off_t bytecount = 0; @@ -387,7 +386,6 @@ static CURLcode file_do(struct connectdata *conn, bool *done) *done = TRUE; /* unconditionally */ - Curl_initinfo(data); Curl_pgrsStartNow(data); if(data->set.upload) @@ -414,21 +412,18 @@ static CURLcode file_do(struct connectdata *conn, bool *done) } } - /* If we have selected NOBODY and HEADER, it means that we only want file - information. Which for FILE can't be much more than the file size and - date. */ - if(data->set.opt_no_body && data->set.include_header && fstated) { + if(fstated) { time_t filetime; struct tm buffer; const struct tm *tm = &buffer; char header[80]; snprintf(header, sizeof(header), "Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", expected_size); - result = Curl_client_write(conn, CLIENTWRITE_BOTH, header, 0); + result = Curl_client_write(conn, CLIENTWRITE_HEADER, header, 0); if(result) return result; - result = Curl_client_write(conn, CLIENTWRITE_BOTH, + result = Curl_client_write(conn, CLIENTWRITE_HEADER, (char *)"Accept-ranges: bytes\r\n", 0); if(result) return result; @@ -440,19 +435,22 @@ static CURLcode file_do(struct connectdata *conn, bool *done) /* format: "Tue, 15 Nov 1994 12:45:26 GMT" */ snprintf(header, sizeof(header), - "Last-Modified: %s, %02d %s %4d %02d:%02d:%02d GMT\r\n", + "Last-Modified: %s, %02d %s %4d %02d:%02d:%02d GMT\r\n%s", Curl_wkday[tm->tm_wday?tm->tm_wday-1:6], tm->tm_mday, Curl_month[tm->tm_mon], tm->tm_year + 1900, tm->tm_hour, tm->tm_min, - tm->tm_sec); - result = Curl_client_write(conn, CLIENTWRITE_BOTH, header, 0); - if(!result) - /* set the file size to make it available post transfer */ - Curl_pgrsSetDownloadSize(data, expected_size); - return result; + tm->tm_sec, + data->set.opt_no_body ? "": "\r\n"); + result = Curl_client_write(conn, CLIENTWRITE_HEADER, header, 0); + if(result) + return result; + /* set the file size to make it available post transfer */ + Curl_pgrsSetDownloadSize(data, expected_size); + if(data->set.opt_no_body) + return result; } /* Check whether file range has been specified */ @@ -461,7 +459,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done) return result; /* Adjust the start offset in case we want to get the N last bytes - * of the stream iff the filesize could be determined */ + * of the stream if the filesize could be determined */ if(data->state.resume_from < 0) { if(!fstated) { failf(data, "Can't get the size of file."); @@ -502,6 +500,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done) Curl_pgrsTime(data, TIMER_STARTTRANSFER); while(!result) { + ssize_t nread; /* Don't fill a whole buffer if we want less than all data */ size_t bytestoread; diff --git a/Utilities/cmcurl/lib/file.h b/Utilities/cmcurl/lib/file.h index c12ae0e09..20828ad4a 100644 --- a/Utilities/cmcurl/lib/file.h +++ b/Utilities/cmcurl/lib/file.h @@ -38,4 +38,3 @@ extern const struct Curl_handler Curl_handler_file; #endif #endif /* HEADER_CURL_FILE_H */ - diff --git a/Utilities/cmcurl/lib/formdata.c b/Utilities/cmcurl/lib/formdata.c index f91241052..202d930c7 100644 --- a/Utilities/cmcurl/lib/formdata.c +++ b/Utilities/cmcurl/lib/formdata.c @@ -39,16 +39,13 @@ #include "sendf.h" #include "strdup.h" #include "rand.h" +#include "warnless.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" #include "curl_memory.h" #include "memdebug.h" -/* What kind of Content-Type to use on un-specified files with unrecognized - extensions. */ -#define HTTPPOST_CONTENTTYPE_DEFAULT "application/octet-stream" - #define HTTPPOST_PTRNAME CURL_HTTPPOST_PTRNAME #define HTTPPOST_FILENAME CURL_HTTPPOST_FILENAME #define HTTPPOST_PTRCONTENTS CURL_HTTPPOST_PTRCONTENTS @@ -305,7 +302,8 @@ CURLFORMcode FormAdd(struct curl_httppost **httppost, * Set the contents property. */ case CURLFORM_PTRCONTENTS: - current_form->flags |= HTTPPOST_PTRCONTENTS; /* fall through */ + current_form->flags |= HTTPPOST_PTRCONTENTS; + /* FALLTHROUGH */ case CURLFORM_COPYCONTENTS: if(current_form->value) return_value = CURL_FORMADD_OPTION_TWICE; @@ -725,7 +723,7 @@ int curl_formget(struct curl_httppost *form, void *arg, while(!result) { char buffer[8192]; - size_t nread = Curl_mime_read(buffer, 1, sizeof buffer, &toppart); + size_t nread = Curl_mime_read(buffer, 1, sizeof(buffer), &toppart); if(!nread) break; @@ -812,7 +810,6 @@ CURLcode Curl_getformdata(struct Curl_easy *data, { CURLcode result = CURLE_OK; curl_mime *form = NULL; - curl_mime *multipart; curl_mimepart *part; struct curl_httppost *file; @@ -831,7 +828,7 @@ CURLcode Curl_getformdata(struct Curl_easy *data, /* Process each top part. */ for(; !result && post; post = post->next) { /* If we have more than a file here, create a mime subpart and fill it. */ - multipart = form; + curl_mime *multipart = form; if(post->more) { part = curl_mime_addpart(form); if(!part) @@ -883,7 +880,8 @@ CURLcode Curl_getformdata(struct Curl_easy *data, compatibility: use of "-" pseudo file name should be avoided. */ result = curl_mime_data_cb(part, (curl_off_t) -1, (curl_read_callback) fread, - (curl_seek_callback) fseek, + CURLX_FUNCTION_CAST(curl_seek_callback, + fseek), NULL, (void *) stdin); } else diff --git a/Utilities/cmcurl/lib/ftp.c b/Utilities/cmcurl/lib/ftp.c index a42c0c7c6..ce889ab29 100644 --- a/Utilities/cmcurl/lib/ftp.c +++ b/Utilities/cmcurl/lib/ftp.c @@ -239,8 +239,8 @@ static void close_secondarysocket(struct connectdata *conn) static void freedirs(struct ftp_conn *ftpc) { - int i; if(ftpc->dirs) { + int i; for(i = 0; i < ftpc->dirdepth; i++) { free(ftpc->dirs[i]); ftpc->dirs[i] = NULL; @@ -637,8 +637,6 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */ * line in a response or continue reading. */ curl_socket_t sockfd = conn->sock[FIRSTSOCKET]; - time_t timeout; /* timeout in milliseconds */ - time_t interval_ms; struct Curl_easy *data = conn->data; CURLcode result = CURLE_OK; struct ftp_conn *ftpc = &conn->proto.ftpc; @@ -657,7 +655,8 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */ while(!*ftpcode && !result) { /* check and reset timeout value every lap */ - timeout = Curl_pp_state_timeout(pp); + time_t timeout = Curl_pp_state_timeout(pp); /* timeout in milliseconds */ + time_t interval_ms; if(timeout <= 0) { failf(data, "FTP response timeout"); @@ -1445,6 +1444,7 @@ static CURLcode ftp_state_list(struct connectdata *conn) { CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; + struct FTP *ftp = data->req.protop; /* If this output is to be machine-parsed, the NLST command might be better to use, since the LIST command output is not specified or standard in any @@ -1461,7 +1461,7 @@ static CURLcode ftp_state_list(struct connectdata *conn) then just do LIST (in that case: nothing to do here) */ char *cmd, *lstArg, *slashPos; - const char *inpath = data->state.path; + const char *inpath = ftp->path; lstArg = NULL; if((data->set.ftp_filemethod == FTPFILE_NOCWD) && @@ -1589,7 +1589,6 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn, struct FTP *ftp = conn->data->req.protop; struct Curl_easy *data = conn->data; struct ftp_conn *ftpc = &conn->proto.ftpc; - int seekerr = CURL_SEEKFUNC_OK; if((data->state.resume_from && !sizechecked) || ((data->state.resume_from > 0) && sizechecked)) { @@ -1605,6 +1604,7 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn, /* 3. pass file-size number of bytes in the source file */ /* 4. lower the infilesize counter */ /* => transfer as usual */ + int seekerr = CURL_SEEKFUNC_OK; if(data->state.resume_from < 0) { /* Got no given size to start from, figure it out */ @@ -2782,7 +2782,6 @@ static CURLcode ftp_statemach_act(struct connectdata *conn) char *ptr = &data->state.buffer[4]; /* start on the first letter */ const size_t buf_size = data->set.buffer_size; char *dir; - char *store; bool entry_extracted = FALSE; dir = malloc(nread + 1); @@ -2805,6 +2804,7 @@ static CURLcode ftp_statemach_act(struct connectdata *conn) if('\"' == *ptr) { /* it started good */ + char *store; ptr++; for(store = dir; *ptr;) { if('\"' == *ptr) { @@ -3142,7 +3142,6 @@ static CURLcode ftp_done(struct connectdata *conn, CURLcode status, int ftpcode; CURLcode result = CURLE_OK; char *path = NULL; - const char *path_to_use = data->state.path; if(!ftp) return CURLE_OK; @@ -3194,7 +3193,7 @@ static CURLcode ftp_done(struct connectdata *conn, CURLcode status, if(!result) /* get the "raw" path */ - result = Curl_urldecode(data, path_to_use, 0, &path, NULL, TRUE); + result = Curl_urldecode(data, ftp->path, 0, &path, NULL, TRUE); if(result) { /* We can limp along anyway (and should try to since we may already be in * the error path) */ @@ -3214,9 +3213,11 @@ static CURLcode ftp_done(struct connectdata *conn, CURLcode status, ftpc->prevpath[dlen] = 0; /* terminate */ } else { + free(path); /* we never changed dir */ ftpc->prevpath = strdup(""); - free(path); + if(!ftpc->prevpath) + return CURLE_OUT_OF_MEMORY; } if(ftpc->prevpath) infof(data, "Remembering we are in dir \"%s\"\n", ftpc->prevpath); @@ -3347,7 +3348,7 @@ static CURLcode ftp_done(struct connectdata *conn, CURLcode status, /* Send any post-transfer QUOTE strings? */ if(!status && !result && !premature && data->set.postquote) result = ftp_sendquote(conn, data->set.postquote); - + Curl_safefree(ftp->pathalloc); return result; } @@ -3697,12 +3698,13 @@ static void wc_data_dtor(void *ptr) static CURLcode init_wc_data(struct connectdata *conn) { char *last_slash; - char *path = conn->data->state.path; + struct FTP *ftp = conn->data->req.protop; + char *path = ftp->path; struct WildcardData *wildcard = &(conn->data->wildcard); CURLcode result = CURLE_OK; struct ftp_wc *ftpwc = NULL; - last_slash = strrchr(conn->data->state.path, '/'); + last_slash = strrchr(ftp->path, '/'); if(last_slash) { last_slash++; if(last_slash[0] == '\0') { @@ -3759,7 +3761,7 @@ static CURLcode init_wc_data(struct connectdata *conn) goto fail; } - wildcard->path = strdup(conn->data->state.path); + wildcard->path = strdup(ftp->path); if(!wildcard->path) { result = CURLE_OUT_OF_MEMORY; goto fail; @@ -3830,16 +3832,15 @@ static CURLcode wc_statemach(struct connectdata *conn) /* filelist has at least one file, lets get first one */ struct ftp_conn *ftpc = &conn->proto.ftpc; struct curl_fileinfo *finfo = wildcard->filelist.head->ptr; + struct FTP *ftp = conn->data->req.protop; char *tmp_path = aprintf("%s%s", wildcard->path, finfo->filename); if(!tmp_path) return CURLE_OUT_OF_MEMORY; - /* switch default "state.pathbuffer" and tmp_path, good to see - ftp_parse_url_path function to understand this trick */ - Curl_safefree(conn->data->state.pathbuffer); - conn->data->state.pathbuffer = tmp_path; - conn->data->state.path = tmp_path; + /* switch default ftp->path and tmp_path */ + free(ftp->pathalloc); + ftp->pathalloc = ftp->path = tmp_path; infof(conn->data, "Wildcard - START of \"%s\"\n", finfo->filename); if(conn->data->set.chunk_bgn) { @@ -3965,10 +3966,14 @@ CURLcode Curl_ftpsend(struct connectdata *conn, const char *cmd) enum protection_level data_sec = conn->data_prot; #endif + if(!cmd) + return CURLE_BAD_FUNCTION_ARGUMENT; + write_len = strlen(cmd); - if(write_len > (sizeof(s) -3)) + if(!write_len || write_len > (sizeof(s) -3)) return CURLE_BAD_FUNCTION_ARGUMENT; + memcpy(&s, cmd, write_len); strcpy(&s[write_len], "\r\n"); /* append a trailing CRLF */ write_len += 2; bytes_written = 0; @@ -3993,8 +3998,7 @@ CURLcode Curl_ftpsend(struct connectdata *conn, const char *cmd) break; if(conn->data->set.verbose) - Curl_debug(conn->data, CURLINFO_HEADER_OUT, - sptr, (size_t)bytes_written, conn); + Curl_debug(conn->data, CURLINFO_HEADER_OUT, sptr, (size_t)bytes_written); if(bytes_written != (ssize_t)write_len) { write_len -= bytes_written; @@ -4104,7 +4108,7 @@ CURLcode ftp_parse_url_path(struct connectdata *conn) struct FTP *ftp = data->req.protop; struct ftp_conn *ftpc = &conn->proto.ftpc; const char *slash_pos; /* position of the first '/' char in curpos */ - const char *path_to_use = data->state.path; + const char *path_to_use = ftp->path; const char *cur_pos; const char *filename = NULL; @@ -4190,7 +4194,7 @@ CURLcode ftp_parse_url_path(struct connectdata *conn) /* parse the URL path into separate path components */ while((slash_pos = strchr(cur_pos, '/')) != NULL) { /* 1 or 0 pointer offset to indicate absolute directory */ - ssize_t absolute_dir = ((cur_pos - data->state.path > 0) && + ssize_t absolute_dir = ((cur_pos - ftp->path > 0) && (ftpc->dirdepth == 0))?1:0; /* seek out the next path component */ @@ -4267,7 +4271,7 @@ CURLcode ftp_parse_url_path(struct connectdata *conn) size_t dlen; char *path; CURLcode result = - Curl_urldecode(conn->data, data->state.path, 0, &path, &dlen, TRUE); + Curl_urldecode(conn->data, ftp->path, 0, &path, &dlen, TRUE); if(result) { freedirs(ftpc); return result; @@ -4385,24 +4389,24 @@ static CURLcode ftp_setup_connection(struct connectdata *conn) { struct Curl_easy *data = conn->data; char *type; - char command; struct FTP *ftp; - conn->data->req.protop = ftp = malloc(sizeof(struct FTP)); + conn->data->req.protop = ftp = calloc(sizeof(struct FTP), 1); if(NULL == ftp) return CURLE_OUT_OF_MEMORY; - data->state.path++; /* don't include the initial slash */ + ftp->path = &data->state.up.path[1]; /* don't include the initial slash */ data->state.slash_removed = TRUE; /* we've skipped the slash */ /* FTP URLs support an extension like ";type=<typecode>" that * we'll try to get now! */ - type = strstr(data->state.path, ";type="); + type = strstr(ftp->path, ";type="); if(!type) type = strstr(conn->host.rawalloc, ";type="); if(type) { + char command; *type = 0; /* it was in the middle of the hostname */ command = Curl_raw_toupper(type[6]); conn->bits.type_set = TRUE; diff --git a/Utilities/cmcurl/lib/ftp.h b/Utilities/cmcurl/lib/ftp.h index 7ec339118..38d03223c 100644 --- a/Utilities/cmcurl/lib/ftp.h +++ b/Utilities/cmcurl/lib/ftp.h @@ -105,6 +105,8 @@ struct FTP { curl_off_t *bytecountp; char *user; /* user name string */ char *passwd; /* password string */ + char *path; /* points to the urlpieces struct field */ + char *pathalloc; /* if non-NULL a pointer to an allocated path */ /* transfer a file/body or not, done as a typedefed enum just to make debuggers display the full symbol and not just the numerical value */ diff --git a/Utilities/cmcurl/lib/getinfo.c b/Utilities/cmcurl/lib/getinfo.c index d280eebfa..54c2c2f1c 100644 --- a/Utilities/cmcurl/lib/getinfo.c +++ b/Utilities/cmcurl/lib/getinfo.c @@ -85,7 +85,6 @@ CURLcode Curl_initinfo(struct Curl_easy *data) #ifdef USE_SSL Curl_ssl_free_certinfo(data); #endif - return CURLE_OK; } @@ -281,6 +280,28 @@ static CURLcode getinfo_offt(struct Curl_easy *data, CURLINFO info, *param_offt = (data->progress.flags & PGRS_UL_SIZE_KNOWN)? data->progress.size_ul:-1; break; + case CURLINFO_TOTAL_TIME_T: + *param_offt = data->progress.timespent; + break; + case CURLINFO_NAMELOOKUP_TIME_T: + *param_offt = data->progress.t_nslookup; + break; + case CURLINFO_CONNECT_TIME_T: + *param_offt = data->progress.t_connect; + break; + case CURLINFO_APPCONNECT_TIME_T: + *param_offt = data->progress.t_appconnect; + break; + case CURLINFO_PRETRANSFER_TIME_T: + *param_offt = data->progress.t_pretransfer; + break; + case CURLINFO_STARTTRANSFER_TIME_T: + *param_offt = data->progress.t_starttransfer; + break; + case CURLINFO_REDIRECT_TIME_T: + *param_offt = data->progress.t_redirect; + break; + default: return CURLE_UNKNOWN_OPTION; } diff --git a/Utilities/cmcurl/lib/gopher.c b/Utilities/cmcurl/lib/gopher.c index b7c31b695..b441a641d 100644 --- a/Utilities/cmcurl/lib/gopher.c +++ b/Utilities/cmcurl/lib/gopher.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -78,7 +78,7 @@ static CURLcode gopher_do(struct connectdata *conn, bool *done) curl_socket_t sockfd = conn->sock[FIRSTSOCKET]; curl_off_t *bytecount = &data->req.bytecount; - char *path = data->state.path; + char *path = data->state.up.path; char *sel = NULL; char *sel_org = NULL; ssize_t amount, k; @@ -89,22 +89,15 @@ static CURLcode gopher_do(struct connectdata *conn, bool *done) /* Create selector. Degenerate cases: / and /1 => convert to "" */ if(strlen(path) <= 2) { sel = (char *)""; - len = (int)strlen(sel); + len = strlen(sel); } else { char *newp; - size_t j, i; /* Otherwise, drop / and the first character (i.e., item type) ... */ newp = path; newp += 2; - /* ... then turn ? into TAB for search servers, Veronica, etc. ... */ - j = strlen(newp); - for(i = 0; i<j; i++) - if(newp[i] == '?') - newp[i] = '\x09'; - /* ... and finally unescape */ result = Curl_urldecode(data, newp, 0, &sel, &len, FALSE); if(result) diff --git a/Utilities/cmcurl/lib/hash.c b/Utilities/cmcurl/lib/hash.c index 15a128fec..421d68f76 100644 --- a/Utilities/cmcurl/lib/hash.c +++ b/Utilities/cmcurl/lib/hash.c @@ -60,8 +60,6 @@ Curl_hash_init(struct curl_hash *h, comp_function comparator, curl_hash_dtor dtor) { - int i; - if(!slots || !hfunc || !comparator ||!dtor) { return 1; /* failure */ } @@ -74,6 +72,7 @@ Curl_hash_init(struct curl_hash *h, h->table = malloc(slots * sizeof(struct curl_llist)); if(h->table) { + int i; for(i = 0; i < slots; ++i) Curl_llist_init(&h->table[i], (curl_llist_dtor) hash_element_dtor); return 0; /* fine */ @@ -140,11 +139,10 @@ Curl_hash_add(struct curl_hash *h, void *key, size_t key_len, void *p) int Curl_hash_delete(struct curl_hash *h, void *key, size_t key_len) { struct curl_llist_element *le; - struct curl_hash_element *he; struct curl_llist *l = FETCH_LIST(h, key, key_len); for(le = l->head; le; le = le->next) { - he = le->ptr; + struct curl_hash_element *he = le->ptr; if(h->comp_func(he->key, he->key_len, key, key_len)) { Curl_llist_remove(l, le, (void *) h); --h->size; @@ -162,13 +160,12 @@ void * Curl_hash_pick(struct curl_hash *h, void *key, size_t key_len) { struct curl_llist_element *le; - struct curl_hash_element *he; struct curl_llist *l; if(h) { l = FETCH_LIST(h, key, key_len); for(le = l->head; le; le = le->next) { - he = le->ptr; + struct curl_hash_element *he = le->ptr; if(h->comp_func(he->key, he->key_len, key, key_len)) { return he->ptr; } @@ -291,7 +288,6 @@ void Curl_hash_start_iterate(struct curl_hash *hash, struct curl_hash_element * Curl_hash_next_element(struct curl_hash_iterator *iter) { - int i; struct curl_hash *h = iter->hash; /* Get the next element in the current list, if any */ @@ -300,6 +296,7 @@ Curl_hash_next_element(struct curl_hash_iterator *iter) /* If we have reached the end of the list, find the next one */ if(!iter->current_element) { + int i; for(i = iter->slot_index; i < h->slots; i++) { if(h->table[i].head) { iter->current_element = h->table[i].head; diff --git a/Utilities/cmcurl/lib/hmac.c b/Utilities/cmcurl/lib/hmac.c index dae95054b..bf49ebec5 100644 --- a/Utilities/cmcurl/lib/hmac.c +++ b/Utilities/cmcurl/lib/hmac.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -58,7 +58,7 @@ Curl_HMAC_init(const HMAC_params * hashparams, unsigned char b; /* Create HMAC context. */ - i = sizeof *ctxt + 2 * hashparams->hmac_ctxtsize + + i = sizeof(*ctxt) + 2 * hashparams->hmac_ctxtsize + hashparams->hmac_resultlen; ctxt = malloc(i); diff --git a/Utilities/cmcurl/lib/hostasyn.c b/Utilities/cmcurl/lib/hostasyn.c index 7b6e8568a..6ff60ba61 100644 --- a/Utilities/cmcurl/lib/hostasyn.c +++ b/Utilities/cmcurl/lib/hostasyn.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -111,31 +111,6 @@ CURLcode Curl_addrinfo_callback(struct connectdata *conn, return result; } -/* Call this function after Curl_connect() has returned async=TRUE and - then a successful name resolve has been received. - - Note: this function disconnects and frees the conn data in case of - resolve failure */ -CURLcode Curl_async_resolved(struct connectdata *conn, - bool *protocol_done) -{ - CURLcode result; - - if(conn->async.dns) { - conn->dns_entry = conn->async.dns; - conn->async.dns = NULL; - } - - result = Curl_setup_conn(conn, protocol_done); - - if(result) - /* We're not allowed to return failure with memory left allocated - in the connectdata struct, free those here */ - Curl_disconnect(conn, FALSE); /* close the connection */ - - return result; -} - /* * Curl_getaddrinfo() is the generic low-level name resolve API within this * source file. There are several versions of this function - for different diff --git a/Utilities/cmcurl/lib/hostcheck.h b/Utilities/cmcurl/lib/hostcheck.h index 86e3b96a9..f562df9ae 100644 --- a/Utilities/cmcurl/lib/hostcheck.h +++ b/Utilities/cmcurl/lib/hostcheck.h @@ -29,4 +29,3 @@ int Curl_cert_hostcheck(const char *match_pattern, const char *hostname); #endif /* HEADER_CURL_HOSTCHECK_H */ - diff --git a/Utilities/cmcurl/lib/hostip.c b/Utilities/cmcurl/lib/hostip.c index c2f9defd9..f589a0b2c 100644 --- a/Utilities/cmcurl/lib/hostip.c +++ b/Utilities/cmcurl/lib/hostip.c @@ -60,6 +60,7 @@ #include "url.h" #include "inet_ntop.h" #include "multiif.h" +#include "doh.h" #include "warnless.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" @@ -454,7 +455,7 @@ Curl_cache_addr(struct Curl_easy *data, /* shuffle addresses if requested */ if(data->set.dns_shuffle_addresses) { CURLcode result = Curl_shuffle_addr(data, &addr); - if(!result) + if(result) return NULL; } @@ -565,23 +566,27 @@ int Curl_resolv(struct connectdata *conn, return CURLRESOLV_ERROR; } - /* If Curl_getaddrinfo() returns NULL, 'respwait' might be set to a - non-zero value indicating that we need to wait for the response to the - resolve call */ - addr = Curl_getaddrinfo(conn, + if(data->set.doh) { + addr = Curl_doh(conn, hostname, port, &respwait); + } + else { + /* If Curl_getaddrinfo() returns NULL, 'respwait' might be set to a + non-zero value indicating that we need to wait for the response to the + resolve call */ + addr = Curl_getaddrinfo(conn, #ifdef DEBUGBUILD - (data->set.str[STRING_DEVICE] - && !strcmp(data->set.str[STRING_DEVICE], - "LocalHost"))?"localhost": + (data->set.str[STRING_DEVICE] + && !strcmp(data->set.str[STRING_DEVICE], + "LocalHost"))?"localhost": #endif - hostname, port, &respwait); - + hostname, port, &respwait); + } if(!addr) { if(respwait) { /* the response to our resolve call will come asynchronously at a later time, good or bad */ /* First, check that we haven't received the info by now */ - result = Curl_resolver_is_resolved(conn, &dns); + result = Curl_resolv_check(conn, &dns); if(result) /* error detected */ return CURLRESOLV_ERROR; if(dns) @@ -907,7 +912,9 @@ CURLcode Curl_loadhostpairs(struct Curl_easy *data) char *entry_id; size_t entry_len; char address[64]; +#if !defined(CURL_DISABLE_VERBOSE_STRINGS) char *addresses = NULL; +#endif char *addr_begin; char *addr_end; char *port_ptr; @@ -930,7 +937,9 @@ CURLcode Curl_loadhostpairs(struct Curl_easy *data) goto err; port = (int)tmp_port; +#if !defined(CURL_DISABLE_VERBOSE_STRINGS) addresses = end_ptr + 1; +#endif while(*end_ptr) { size_t alen; @@ -1010,24 +1019,28 @@ CURLcode Curl_loadhostpairs(struct Curl_easy *data) /* See if its already in our dns cache */ dns = Curl_hash_pick(data->dns.hostcache, entry_id, entry_len + 1); + if(dns) { + infof(data, "RESOLVE %s:%d is - old addresses discarded!\n", + hostname, port); + /* delete old entry entry, there are two reasons for this + 1. old entry may have different addresses. + 2. even if entry with correct addresses is already in the cache, + but if it is close to expire, then by the time next http + request is made, it can get expired and pruned because old + entry is not necessarily marked as added by CURLOPT_RESOLVE. */ + + Curl_hash_delete(data->dns.hostcache, entry_id, entry_len + 1); + } /* free the allocated entry_id again */ free(entry_id); - if(!dns) { - /* if not in the cache already, put this host in the cache */ - dns = Curl_cache_addr(data, head, hostname, port); - if(dns) { - dns->timestamp = 0; /* mark as added by CURLOPT_RESOLVE */ - /* release the returned reference; the cache itself will keep the - * entry alive: */ - dns->inuse--; - } - } - else { - /* this is a duplicate, free it again */ - infof(data, "RESOLVE %s:%d is already cached, %s not stored!\n", - hostname, port, addresses); - Curl_freeaddrinfo(head); + /* put this new host in the cache */ + dns = Curl_cache_addr(data, head, hostname, port); + if(dns) { + dns->timestamp = 0; /* mark as added by CURLOPT_RESOLVE */ + /* release the returned reference; the cache itself will keep the + * entry alive: */ + dns->inuse--; } if(data->share) @@ -1045,3 +1058,54 @@ CURLcode Curl_loadhostpairs(struct Curl_easy *data) return CURLE_OK; } + +CURLcode Curl_resolv_check(struct connectdata *conn, + struct Curl_dns_entry **dns) +{ + if(conn->data->set.doh) + return Curl_doh_is_resolved(conn, dns); + return Curl_resolver_is_resolved(conn, dns); +} + +int Curl_resolv_getsock(struct connectdata *conn, + curl_socket_t *socks, + int numsocks) +{ +#ifdef CURLRES_ASYNCH + if(conn->data->set.doh) + /* nothing to wait for during DOH resolve, those handles have their own + sockets */ + return GETSOCK_BLANK; + return Curl_resolver_getsock(conn, socks, numsocks); +#else + (void)conn; + (void)socks; + (void)numsocks; + return GETSOCK_BLANK; +#endif +} + +/* Call this function after Curl_connect() has returned async=TRUE and + then a successful name resolve has been received. + + Note: this function disconnects and frees the conn data in case of + resolve failure */ +CURLcode Curl_once_resolved(struct connectdata *conn, + bool *protocol_done) +{ + CURLcode result; + + if(conn->async.dns) { + conn->dns_entry = conn->async.dns; + conn->async.dns = NULL; + } + + result = Curl_setup_conn(conn, protocol_done); + + if(result) + /* We're not allowed to return failure with memory left allocated + in the connectdata struct, free those here */ + Curl_disconnect(conn->data, conn, TRUE); /* close the connection */ + + return result; +} diff --git a/Utilities/cmcurl/lib/hostip.h b/Utilities/cmcurl/lib/hostip.h index 1de4bee8d..29fd1ef7c 100644 --- a/Utilities/cmcurl/lib/hostip.h +++ b/Utilities/cmcurl/lib/hostip.h @@ -145,12 +145,7 @@ int curl_dogetnameinfo(GETNAMEINFO_QUAL_ARG1 GETNAMEINFO_TYPE_ARG1 sa, /* IPv4 threadsafe resolve function used for synch and asynch builds */ Curl_addrinfo *Curl_ipv4_resolve_r(const char *hostname, int port); -CURLcode Curl_async_resolved(struct connectdata *conn, - bool *protocol_connect); - -#ifndef CURLRES_ASYNCH -#define Curl_async_resolved(x,y) CURLE_OK -#endif +CURLcode Curl_once_resolved(struct connectdata *conn, bool *protocol_connect); /* * Curl_addrinfo_callback() is used when we build with any asynch specialty. @@ -258,4 +253,10 @@ void Curl_hostcache_destroy(struct Curl_easy *data); */ CURLcode Curl_loadhostpairs(struct Curl_easy *data); +CURLcode Curl_resolv_check(struct connectdata *conn, + struct Curl_dns_entry **dns); +int Curl_resolv_getsock(struct connectdata *conn, + curl_socket_t *socks, + int numsocks); + #endif /* HEADER_CURL_HOSTIP_H */ diff --git a/Utilities/cmcurl/lib/hostip6.c b/Utilities/cmcurl/lib/hostip6.c index 7c9988f41..3bf47b467 100644 --- a/Utilities/cmcurl/lib/hostip6.c +++ b/Utilities/cmcurl/lib/hostip6.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -59,39 +59,6 @@ #include "curl_memory.h" #include "memdebug.h" -#if defined(CURLDEBUG) && defined(HAVE_GETNAMEINFO) -/* These are strictly for memory tracing and are using the same style as the - * family otherwise present in memdebug.c. I put these ones here since they - * require a bunch of structs I didn't want to include in memdebug.c - */ - -/* - * For CURLRES_ARS, this should be written using ares_gethostbyaddr() - * (ignoring the fact c-ares doesn't return 'serv'). - */ - -int curl_dogetnameinfo(GETNAMEINFO_QUAL_ARG1 GETNAMEINFO_TYPE_ARG1 sa, - GETNAMEINFO_TYPE_ARG2 salen, - char *host, GETNAMEINFO_TYPE_ARG46 hostlen, - char *serv, GETNAMEINFO_TYPE_ARG46 servlen, - GETNAMEINFO_TYPE_ARG7 flags, - int line, const char *source) -{ - int res = (getnameinfo)(sa, salen, - host, hostlen, - serv, servlen, - flags); - if(0 == res) - /* success */ - curl_memlog("GETNAME %s:%d getnameinfo()\n", - source, line); - else - curl_memlog("GETNAME %s:%d getnameinfo() failed = %d\n", - source, line, res); - return res; -} -#endif /* defined(CURLDEBUG) && defined(HAVE_GETNAMEINFO) */ - /* * Curl_ipv6works() returns TRUE if IPv6 seems to work. */ diff --git a/Utilities/cmcurl/lib/http.c b/Utilities/cmcurl/lib/http.c index ff1d6813a..46ac15a6e 100644 --- a/Utilities/cmcurl/lib/http.c +++ b/Utilities/cmcurl/lib/http.c @@ -158,18 +158,20 @@ CURLcode Curl_http_setup_conn(struct connectdata *conn) /* allocate the HTTP-specific struct for the Curl_easy, only to survive during this request */ struct HTTP *http; - DEBUGASSERT(conn->data->req.protop == NULL); + struct Curl_easy *data = conn->data; + DEBUGASSERT(data->req.protop == NULL); http = calloc(1, sizeof(struct HTTP)); if(!http) return CURLE_OUT_OF_MEMORY; Curl_mime_initpart(&http->form, conn->data); - conn->data->req.protop = http; - - Curl_http2_setup_conn(conn); - Curl_http2_setup_req(conn->data); + data->req.protop = http; + if(!CONN_INUSE(conn)) + /* if not already multi-using, setup connection details */ + Curl_http2_setup_conn(conn); + Curl_http2_setup_req(data); return CURLE_OK; } @@ -310,22 +312,49 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy) return result; } +/* + * http_output_bearer() sets up an Authorization: header + * for HTTP Bearer authentication. + * + * Returns CURLcode. + */ +static CURLcode http_output_bearer(struct connectdata *conn) +{ + char **userp; + CURLcode result = CURLE_OK; + + userp = &conn->allocptr.userpwd; + free(*userp); + *userp = aprintf("Authorization: Bearer %s\r\n", + conn->oauth_bearer); + + if(!*userp) { + result = CURLE_OUT_OF_MEMORY; + goto fail; + } + + fail: + return result; +} + /* pickoneauth() selects the most favourable authentication method from the * ones available and the ones we want. * * return TRUE if one was picked */ -static bool pickoneauth(struct auth *pick) +static bool pickoneauth(struct auth *pick, unsigned long mask) { bool picked; /* only deal with authentication we want */ - unsigned long avail = pick->avail & pick->want; + unsigned long avail = pick->avail & pick->want & mask; picked = TRUE; /* The order of these checks is highly relevant, as this will be the order of preference in case of the existence of multiple accepted types. */ if(avail & CURLAUTH_NEGOTIATE) pick->picked = CURLAUTH_NEGOTIATE; + else if(avail & CURLAUTH_BEARER) + pick->picked = CURLAUTH_BEARER; else if(avail & CURLAUTH_DIGEST) pick->picked = CURLAUTH_DIGEST; else if(avail & CURLAUTH_NTLM) @@ -479,6 +508,10 @@ CURLcode Curl_http_auth_act(struct connectdata *conn) bool pickhost = FALSE; bool pickproxy = FALSE; CURLcode result = CURLE_OK; + unsigned long authmask = ~0ul; + + if(!conn->oauth_bearer) + authmask &= (unsigned long)~CURLAUTH_BEARER; if(100 <= data->req.httpcode && 199 >= data->req.httpcode) /* this is a transient response code, ignore */ @@ -487,30 +520,23 @@ CURLcode Curl_http_auth_act(struct connectdata *conn) if(data->state.authproblem) return data->set.http_fail_on_error?CURLE_HTTP_RETURNED_ERROR:CURLE_OK; - if(conn->bits.user_passwd && + if((conn->bits.user_passwd || conn->oauth_bearer) && ((data->req.httpcode == 401) || (conn->bits.authneg && data->req.httpcode < 300))) { - pickhost = pickoneauth(&data->state.authhost); + pickhost = pickoneauth(&data->state.authhost, authmask); if(!pickhost) data->state.authproblem = TRUE; } if(conn->bits.proxy_user_passwd && ((data->req.httpcode == 407) || (conn->bits.authneg && data->req.httpcode < 300))) { - pickproxy = pickoneauth(&data->state.authproxy); + pickproxy = pickoneauth(&data->state.authproxy, + authmask & ~CURLAUTH_BEARER); if(!pickproxy) data->state.authproblem = TRUE; } if(pickhost || pickproxy) { - /* In case this is GSS auth, the newurl field is already allocated so - we must make sure to free it before allocating a new one. As figured - out in bug #2284386 */ - Curl_safefree(data->req.newurl); - data->req.newurl = strdup(data->change.url); /* clone URL */ - if(!data->req.newurl) - return CURLE_OUT_OF_MEMORY; - if((data->set.httpreq != HTTPREQ_GET) && (data->set.httpreq != HTTPREQ_HEAD) && !conn->bits.rewindaftersend) { @@ -518,6 +544,13 @@ CURLcode Curl_http_auth_act(struct connectdata *conn) if(result) return result; } + /* In case this is GSS auth, the newurl field is already allocated so + we must make sure to free it before allocating a new one. As figured + out in bug #2284386 */ + Curl_safefree(data->req.newurl); + data->req.newurl = strdup(data->change.url); /* clone URL */ + if(!data->req.newurl) + return CURLE_OUT_OF_MEMORY; } else if((data->req.httpcode < 300) && (!data->state.authhost.done) && @@ -628,6 +661,20 @@ output_auth_headers(struct connectdata *conn, functions work that way */ authstatus->done = TRUE; } + if(authstatus->picked == CURLAUTH_BEARER) { + /* Bearer */ + if((!proxy && conn->oauth_bearer && + !Curl_checkheaders(conn, "Authorization:"))) { + auth = "Bearer"; + result = http_output_bearer(conn); + if(result) + return result; + } + + /* NOTE: this function should set 'done' TRUE, as the other auth + functions work that way */ + authstatus->done = TRUE; + } if(auth) { infof(data, "%s auth using %s with user '%s'\n", @@ -674,7 +721,7 @@ Curl_http_output_auth(struct connectdata *conn, authproxy = &data->state.authproxy; if((conn->bits.httpproxy && conn->bits.proxy_user_passwd) || - conn->bits.user_passwd) + conn->bits.user_passwd || conn->oauth_bearer) /* continue please */; else { authhost->done = TRUE; @@ -883,6 +930,18 @@ CURLcode Curl_http_input_auth(struct connectdata *conn, bool proxy, data->state.authproblem = TRUE; } } + else + if(checkprefix("Bearer", auth)) { + *availp |= CURLAUTH_BEARER; + authp->avail |= CURLAUTH_BEARER; + if(authp->picked == CURLAUTH_BEARER) { + /* We asked for Bearer authentication but got a 40X back + anyway, which basically means our token isn't valid. */ + authp->avail = CURLAUTH_NONE; + infof(data, "Authentication problem. Ignoring this.\n"); + data->state.authproblem = TRUE; + } + } /* there may be multiple methods on one line, so keep reading */ while(*auth && *auth != ',') /* read up to the next comma */ @@ -1034,11 +1093,13 @@ Curl_send_buffer *Curl_add_buffer_init(void) /* * Curl_add_buffer_free() frees all associated resources. */ -void Curl_add_buffer_free(Curl_send_buffer *buff) +void Curl_add_buffer_free(Curl_send_buffer **inp) { - if(buff) /* deal with NULL input */ - free(buff->buffer); - free(buff); + Curl_send_buffer *in = *inp; + if(in) /* deal with NULL input */ + free(in->buffer); + free(in); + *inp = NULL; } /* @@ -1047,7 +1108,7 @@ void Curl_add_buffer_free(Curl_send_buffer *buff) * * Returns CURLcode */ -CURLcode Curl_add_buffer_send(Curl_send_buffer *in, +CURLcode Curl_add_buffer_send(Curl_send_buffer **inp, struct connectdata *conn, /* add the number of sent bytes to this @@ -1063,10 +1124,12 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, CURLcode result; char *ptr; size_t size; - struct HTTP *http = conn->data->req.protop; + struct Curl_easy *data = conn->data; + struct HTTP *http = data->req.protop; size_t sendsize; curl_socket_t sockfd; size_t headersize; + Curl_send_buffer *in = *inp; DEBUGASSERT(socketindex <= SECONDARYSOCKET); @@ -1083,11 +1146,11 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, DEBUGASSERT(size > included_body_bytes); - result = Curl_convert_to_network(conn->data, ptr, headersize); + result = Curl_convert_to_network(data, ptr, headersize); /* Curl_convert_to_network calls failf if unsuccessful */ if(result) { /* conversion failed, free memory and return to the caller */ - Curl_add_buffer_free(in); + Curl_add_buffer_free(inp); return result; } @@ -1108,8 +1171,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, must copy the data to the uploadbuffer first, since that is the buffer we will be using if this send is retried later. */ - memcpy(conn->data->state.uploadbuffer, ptr, sendsize); - ptr = conn->data->state.uploadbuffer; + result = Curl_get_upload_buffer(data); + if(result) { + /* malloc failed, free memory and return to the caller */ + Curl_add_buffer_free(&in); + return result; + } + memcpy(data->state.ulbuf, ptr, sendsize); + ptr = data->state.ulbuf; } else sendsize = size; @@ -1126,14 +1195,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, size_t headlen = (size_t)amount>headersize ? headersize : (size_t)amount; size_t bodylen = amount - headlen; - if(conn->data->set.verbose) { + if(data->set.verbose) { /* this data _may_ contain binary stuff */ - Curl_debug(conn->data, CURLINFO_HEADER_OUT, ptr, headlen, conn); + Curl_debug(data, CURLINFO_HEADER_OUT, ptr, headlen); if(bodylen) { /* there was body data sent beyond the initial header part, pass that on to the debug callback too */ - Curl_debug(conn->data, CURLINFO_DATA_OUT, - ptr + headlen, bodylen, conn); + Curl_debug(data, CURLINFO_DATA_OUT, + ptr + headlen, bodylen); } } @@ -1157,14 +1226,14 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, ptr = in->buffer + amount; /* backup the currently set pointers */ - http->backup.fread_func = conn->data->state.fread_func; - http->backup.fread_in = conn->data->state.in; + http->backup.fread_func = data->state.fread_func; + http->backup.fread_in = data->state.in; http->backup.postdata = http->postdata; http->backup.postsize = http->postsize; /* set the new pointers for the request-sending */ - conn->data->state.fread_func = (curl_read_callback)readmoredata; - conn->data->state.in = (void *)conn; + data->state.fread_func = (curl_read_callback)readmoredata; + data->state.in = (void *)conn; http->postdata = ptr; http->postsize = (curl_off_t)size; @@ -1189,7 +1258,7 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, Curl_pipeline_leave_write(conn); } } - Curl_add_buffer_free(in); + Curl_add_buffer_free(&in); return result; } @@ -1198,32 +1267,35 @@ CURLcode Curl_add_buffer_send(Curl_send_buffer *in, /* * add_bufferf() add the formatted input to the buffer. */ -CURLcode Curl_add_bufferf(Curl_send_buffer *in, const char *fmt, ...) +CURLcode Curl_add_bufferf(Curl_send_buffer **inp, const char *fmt, ...) { char *s; va_list ap; + Curl_send_buffer *in = *inp; va_start(ap, fmt); s = vaprintf(fmt, ap); /* this allocs a new string to append */ va_end(ap); if(s) { - CURLcode result = Curl_add_buffer(in, s, strlen(s)); + CURLcode result = Curl_add_buffer(inp, s, strlen(s)); free(s); return result; } /* If we failed, we cleanup the whole buffer and return error */ free(in->buffer); free(in); + *inp = NULL; return CURLE_OUT_OF_MEMORY; } /* - * add_buffer() appends a memory chunk to the existing buffer + * Curl_add_buffer() appends a memory chunk to the existing buffer */ -CURLcode Curl_add_buffer(Curl_send_buffer *in, const void *inptr, size_t size) +CURLcode Curl_add_buffer(Curl_send_buffer **inp, const void *inptr, + size_t size) { char *new_rb; - size_t new_size; + Curl_send_buffer *in = *inp; if(~size < in->size_used) { /* If resulting used size of send buffer would wrap size_t, cleanup @@ -1231,15 +1303,16 @@ CURLcode Curl_add_buffer(Curl_send_buffer *in, const void *inptr, size_t size) size will fit into a single allocatable memory chunk */ Curl_safefree(in->buffer); free(in); + *inp = NULL; return CURLE_OUT_OF_MEMORY; } if(!in->buffer || ((in->size_used + size) > (in->size_max - 1))) { - /* If current buffer size isn't enough to hold the result, use a buffer size that doubles the required size. If this new size would wrap size_t, then just use the largest possible one */ + size_t new_size; if((size > (size_t)-1 / 2) || (in->size_used > (size_t)-1 / 2) || (~(size * 2) < (in->size_used * 2))) @@ -1257,6 +1330,7 @@ CURLcode Curl_add_buffer(Curl_send_buffer *in, const void *inptr, size_t size) if(!new_rb) { /* If we failed, we cleanup the whole buffer and return error */ free(in); + *inp = NULL; return CURLE_OUT_OF_MEMORY; } @@ -1406,7 +1480,7 @@ static CURLcode add_haproxy_protocol_header(struct connectdata *conn) } snprintf(proxy_header, - sizeof proxy_header, + sizeof(proxy_header), "PROXY %s %s %s %li %li\r\n", tcp_version, conn->data->info.conn_local_ip, @@ -1418,11 +1492,11 @@ static CURLcode add_haproxy_protocol_header(struct connectdata *conn) if(!req_buffer) return CURLE_OUT_OF_MEMORY; - result = Curl_add_bufferf(req_buffer, proxy_header); + result = Curl_add_bufferf(&req_buffer, proxy_header); if(result) return result; - result = Curl_add_buffer_send(req_buffer, + result = Curl_add_buffer_send(&req_buffer, conn, &conn->data->info.request_size, 0, @@ -1495,8 +1569,7 @@ CURLcode Curl_http_done(struct connectdata *conn, return CURLE_OK; if(http->send_buffer) { - Curl_add_buffer_free(http->send_buffer); - http->send_buffer = NULL; /* clear the pointer */ + Curl_add_buffer_free(&http->send_buffer); } Curl_http2_done(conn, premature); @@ -1574,7 +1647,6 @@ static CURLcode expect100(struct Curl_easy *data, Curl_send_buffer *req_buffer) { CURLcode result = CURLE_OK; - const char *ptr; data->state.expect100header = FALSE; /* default to false unless it is set to TRUE below */ if(use_http_1_1plus(data, conn) && @@ -1582,14 +1654,14 @@ static CURLcode expect100(struct Curl_easy *data, /* if not doing HTTP 1.0 or version 2, or disabled explicitly, we add an Expect: 100-continue to the headers which actually speeds up post operations (as there is one packet coming back from the web server) */ - ptr = Curl_checkheaders(conn, "Expect"); + const char *ptr = Curl_checkheaders(conn, "Expect"); if(ptr) { data->state.expect100header = Curl_compareheader(ptr, "Expect:", "100-continue"); } else { - result = Curl_add_bufferf(req_buffer, - "Expect: 100-continue\r\n"); + result = Curl_add_bufferf(&req_buffer, + "Expect: 100-continue\r\n"); if(!result) data->state.expect100header = TRUE; } @@ -1720,7 +1792,7 @@ CURLcode Curl_add_custom_headers(struct connectdata *conn, !strcasecompare(data->state.first_host, conn->host.name))) ; else { - result = Curl_add_bufferf(req_buffer, "%s\r\n", headers->data); + result = Curl_add_bufferf(&req_buffer, "%s\r\n", headers->data); } if(semicolonp) *semicolonp = ';'; /* put back the semicolon */ @@ -1789,7 +1861,7 @@ CURLcode Curl_add_timecondition(struct Curl_easy *data, tm->tm_min, tm->tm_sec); - result = Curl_add_buffer(req_buffer, datestr, strlen(datestr)); + result = Curl_add_buffer(&req_buffer, datestr, strlen(datestr)); return result; } @@ -1804,7 +1876,8 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) struct Curl_easy *data = conn->data; CURLcode result = CURLE_OK; struct HTTP *http; - const char *ppath = data->state.path; + const char *path = data->state.up.path; + const char *query = data->state.up.query; bool paste_ftp_userpwd = FALSE; char ftp_typecode[sizeof("/;type=?")] = ""; const char *host = conn->host.name; @@ -1819,7 +1892,6 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) const char *httpstring; Curl_send_buffer *req_buffer; curl_off_t postsize = 0; /* curl_off_t to handle large file sizes */ - int seekerr = CURL_SEEKFUNC_CANTSEEK; /* Always consider the DO phase done after this function call, even if there may be parts of the request that is not yet sent, since we can deal with @@ -1863,6 +1935,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) } http = data->req.protop; + DEBUGASSERT(http); if(!data->state.this_is_a_follow) { /* Free to avoid leaking memory on multiple requests*/ @@ -1922,7 +1995,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) } /* setup the authentication headers */ - result = Curl_http_output_auth(conn, request, ppath, FALSE); + result = Curl_http_output_auth(conn, request, path, FALSE); if(result) return result; @@ -2088,7 +2161,6 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) else { /* If the host begins with '[', we start searching for the port after the bracket has been closed */ - int startsearch = 0; if(*cookiehost == '[') { char *closingbracket; /* since the 'cookiehost' is an allocated memory area that will be @@ -2099,6 +2171,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) *closingbracket = 0; } else { + int startsearch = 0; char *colon = strchr(cookiehost + startsearch, ':'); if(colon) *colon = 0; /* The host must not include an embedded port number */ @@ -2150,47 +2223,59 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) /* The path sent to the proxy is in fact the entire URL. But if the remote host is a IDN-name, we must make sure that the request we produce only uses the encoded host name! */ + + /* and no fragment part */ + CURLUcode uc; + char *url; + CURLU *h = curl_url_dup(data->state.uh); + if(!h) + return CURLE_OUT_OF_MEMORY; + if(conn->host.dispname != conn->host.name) { - char *url = data->change.url; - ptr = strstr(url, conn->host.dispname); - if(ptr) { - /* This is where the display name starts in the URL, now replace this - part with the encoded name. TODO: This method of replacing the host - name is rather crude as I believe there's a slight risk that the - user has entered a user name or password that contain the host name - string. */ - size_t currlen = strlen(conn->host.dispname); - size_t newlen = strlen(conn->host.name); - size_t urllen = strlen(url); - - char *newurl; - - newurl = malloc(urllen + newlen - currlen + 1); - if(newurl) { - /* copy the part before the host name */ - memcpy(newurl, url, ptr - url); - /* append the new host name instead of the old */ - memcpy(newurl + (ptr - url), conn->host.name, newlen); - /* append the piece after the host name */ - memcpy(newurl + newlen + (ptr - url), - ptr + currlen, /* copy the trailing zero byte too */ - urllen - (ptr-url) - currlen + 1); - if(data->change.url_alloc) { - Curl_safefree(data->change.url); - data->change.url_alloc = FALSE; - } - data->change.url = newurl; - data->change.url_alloc = TRUE; - } - else - return CURLE_OUT_OF_MEMORY; + uc = curl_url_set(h, CURLUPART_HOST, conn->host.name, 0); + if(uc) { + curl_url_cleanup(h); + return CURLE_OUT_OF_MEMORY; + } + } + uc = curl_url_set(h, CURLUPART_FRAGMENT, NULL, 0); + if(uc) { + curl_url_cleanup(h); + return CURLE_OUT_OF_MEMORY; + } + + if(strcasecompare("http", data->state.up.scheme)) { + /* when getting HTTP, we don't want the userinfo the URL */ + uc = curl_url_set(h, CURLUPART_USER, NULL, 0); + if(uc) { + curl_url_cleanup(h); + return CURLE_OUT_OF_MEMORY; } + uc = curl_url_set(h, CURLUPART_PASSWORD, NULL, 0); + if(uc) { + curl_url_cleanup(h); + return CURLE_OUT_OF_MEMORY; + } + } + /* now extract the new version of the URL */ + uc = curl_url_get(h, CURLUPART_URL, &url, 0); + if(uc) { + curl_url_cleanup(h); + return CURLE_OUT_OF_MEMORY; } - ppath = data->change.url; - if(checkprefix("ftp://", ppath)) { + + if(data->change.url_alloc) + free(data->change.url); + + data->change.url = url; + data->change.url_alloc = TRUE; + + curl_url_cleanup(h); + + if(strcasecompare("ftp", data->state.up.scheme)) { if(data->set.proxy_transfer_mode) { /* when doing ftp, append ;type=<a|i> if not present */ - char *type = strstr(ppath, ";type="); + char *type = strstr(path, ";type="); if(type && type[6] && type[7] == 0) { switch(Curl_raw_toupper(type[6])) { case 'A': @@ -2205,7 +2290,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) char *p = ftp_typecode; /* avoid sending invalid URLs like ftp://example.com;type=i if the * user specified ftp://example.com without the slash */ - if(!*data->state.path && ppath[strlen(ppath) - 1] != '/') { + if(!*data->state.up.path && path[strlen(path) - 1] != '/') { *p++ = '/'; } snprintf(p, sizeof(ftp_typecode) - 1, ";type=%c", @@ -2244,6 +2329,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) /* Now, let's read off the proper amount of bytes from the input. */ + int seekerr = CURL_SEEKFUNC_CANTSEEK; if(conn->seek_func) { Curl_set_in_callback(data, true); seekerr = conn->seek_func(conn->seek_client, data->state.resume_from, @@ -2353,25 +2439,36 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) /* add the main request stuff */ /* GET/HEAD/POST/PUT */ - result = Curl_add_bufferf(req_buffer, "%s ", request); + result = Curl_add_bufferf(&req_buffer, "%s ", request); if(result) return result; - if(data->set.str[STRING_TARGET]) - ppath = data->set.str[STRING_TARGET]; + if(data->set.str[STRING_TARGET]) { + path = data->set.str[STRING_TARGET]; + query = NULL; + } /* url */ - if(paste_ftp_userpwd) - result = Curl_add_bufferf(req_buffer, "ftp://%s:%s@%s", + if(conn->bits.httpproxy && !conn->bits.tunnel_proxy) { + char *url = data->change.url; + result = Curl_add_buffer(&req_buffer, url, strlen(url)); + } + else if(paste_ftp_userpwd) + result = Curl_add_bufferf(&req_buffer, "ftp://%s:%s@%s", conn->user, conn->passwd, - ppath + sizeof("ftp://") - 1); - else - result = Curl_add_buffer(req_buffer, ppath, strlen(ppath)); + path + sizeof("ftp://") - 1); + else { + result = Curl_add_buffer(&req_buffer, path, strlen(path)); + if(result) + return result; + if(query) + result = Curl_add_bufferf(&req_buffer, "?%s", query); + } if(result) return result; result = - Curl_add_bufferf(req_buffer, + Curl_add_bufferf(&req_buffer, "%s" /* ftp typecode (;type=x) */ " HTTP/%s\r\n" /* HTTP version */ "%s" /* host */ @@ -2441,7 +2538,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) co = Curl_cookie_getlist(data->cookies, conn->allocptr.cookiehost? conn->allocptr.cookiehost:host, - data->state.path, + data->state.up.path, (conn->handler->protocol&CURLPROTO_HTTPS)? TRUE:FALSE); Curl_share_unlock(data, CURL_LOCK_DATA_COOKIE); @@ -2452,11 +2549,11 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) while(co) { if(co->value) { if(0 == count) { - result = Curl_add_bufferf(req_buffer, "Cookie: "); + result = Curl_add_bufferf(&req_buffer, "Cookie: "); if(result) break; } - result = Curl_add_bufferf(req_buffer, + result = Curl_add_bufferf(&req_buffer, "%s%s=%s", count?"; ":"", co->name, co->value); if(result) @@ -2469,15 +2566,15 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) } if(addcookies && !result) { if(!count) - result = Curl_add_bufferf(req_buffer, "Cookie: "); + result = Curl_add_bufferf(&req_buffer, "Cookie: "); if(!result) { - result = Curl_add_bufferf(req_buffer, "%s%s", count?"; ":"", + result = Curl_add_bufferf(&req_buffer, "%s%s", count?"; ":"", addcookies); count++; } } if(count && !result) - result = Curl_add_buffer(req_buffer, "\r\n", 2); + result = Curl_add_buffer(&req_buffer, "\r\n", 2); if(result) return result; @@ -2511,7 +2608,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) if((postsize != -1) && !data->req.upload_chunky && (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length"))) { /* only add Content-Length if not uploading chunked */ - result = Curl_add_bufferf(req_buffer, + result = Curl_add_bufferf(&req_buffer, "Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", postsize); if(result) @@ -2524,7 +2621,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) return result; } - result = Curl_add_buffer(req_buffer, "\r\n", 2); /* end of headers */ + result = Curl_add_buffer(&req_buffer, "\r\n", 2); /* end of headers */ if(result) return result; @@ -2532,7 +2629,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) Curl_pgrsSetUploadSize(data, postsize); /* this sends the buffer and frees all the buffer resources */ - result = Curl_add_buffer_send(req_buffer, conn, + result = Curl_add_buffer_send(&req_buffer, conn, &data->info.request_size, 0, FIRSTSOCKET); if(result) failf(data, "Failed sending PUT request"); @@ -2550,11 +2647,11 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) /* This is form posting using mime data. */ if(conn->bits.authneg) { /* nothing to post! */ - result = Curl_add_bufferf(req_buffer, "Content-Length: 0\r\n\r\n"); + result = Curl_add_bufferf(&req_buffer, "Content-Length: 0\r\n\r\n"); if(result) return result; - result = Curl_add_buffer_send(req_buffer, conn, + result = Curl_add_buffer_send(&req_buffer, conn, &data->info.request_size, 0, FIRSTSOCKET); if(result) failf(data, "Failed sending POST request"); @@ -2574,7 +2671,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length"))) { /* we allow replacing this header if not during auth negotiation, although it isn't very wise to actually set your own */ - result = Curl_add_bufferf(req_buffer, + result = Curl_add_bufferf(&req_buffer, "Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", postsize); if(result) @@ -2586,7 +2683,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) struct curl_slist *hdr; for(hdr = http->sendit->curlheaders; hdr; hdr = hdr->next) { - result = Curl_add_bufferf(req_buffer, "%s\r\n", hdr->data); + result = Curl_add_bufferf(&req_buffer, "%s\r\n", hdr->data); if(result) return result; } @@ -2610,7 +2707,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) data->state.expect100header = FALSE; /* make the request end in a true CRLF */ - result = Curl_add_buffer(req_buffer, "\r\n", 2); + result = Curl_add_buffer(&req_buffer, "\r\n", 2); if(result) return result; @@ -2623,7 +2720,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) http->sending = HTTPSEND_BODY; /* this sends the buffer and frees all the buffer resources */ - result = Curl_add_buffer_send(req_buffer, conn, + result = Curl_add_buffer_send(&req_buffer, conn, &data->info.request_size, 0, FIRSTSOCKET); if(result) failf(data, "Failed sending POST request"); @@ -2653,7 +2750,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) (conn->bits.authneg || !Curl_checkheaders(conn, "Content-Length"))) { /* we allow replacing this header if not during auth negotiation, although it isn't very wise to actually set your own */ - result = Curl_add_bufferf(req_buffer, + result = Curl_add_bufferf(&req_buffer, "Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", postsize); if(result) @@ -2661,7 +2758,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) } if(!Curl_checkheaders(conn, "Content-Type")) { - result = Curl_add_bufferf(req_buffer, + result = Curl_add_bufferf(&req_buffer, "Content-Type: application/" "x-www-form-urlencoded\r\n"); if(result) @@ -2699,31 +2796,31 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) is no magic limit but only set to prevent really huge POSTs to get the data duplicated with malloc() and family. */ - result = Curl_add_buffer(req_buffer, "\r\n", 2); /* end of headers! */ + result = Curl_add_buffer(&req_buffer, "\r\n", 2); /* end of headers! */ if(result) return result; if(!data->req.upload_chunky) { /* We're not sending it 'chunked', append it to the request already now to reduce the number if send() calls */ - result = Curl_add_buffer(req_buffer, data->set.postfields, + result = Curl_add_buffer(&req_buffer, data->set.postfields, (size_t)postsize); included_body = postsize; } else { if(postsize) { /* Append the POST data chunky-style */ - result = Curl_add_bufferf(req_buffer, "%x\r\n", (int)postsize); + result = Curl_add_bufferf(&req_buffer, "%x\r\n", (int)postsize); if(!result) { - result = Curl_add_buffer(req_buffer, data->set.postfields, + result = Curl_add_buffer(&req_buffer, data->set.postfields, (size_t)postsize); if(!result) - result = Curl_add_buffer(req_buffer, "\r\n", 2); + result = Curl_add_buffer(&req_buffer, "\r\n", 2); included_body = postsize + 2; } } if(!result) - result = Curl_add_buffer(req_buffer, "\x30\x0d\x0a\x0d\x0a", 5); + result = Curl_add_buffer(&req_buffer, "\x30\x0d\x0a\x0d\x0a", 5); /* 0 CR LF CR LF */ included_body += 5; } @@ -2745,20 +2842,20 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) /* set the upload size to the progress meter */ Curl_pgrsSetUploadSize(data, http->postsize); - result = Curl_add_buffer(req_buffer, "\r\n", 2); /* end of headers! */ + result = Curl_add_buffer(&req_buffer, "\r\n", 2); /* end of headers! */ if(result) return result; } } else { - result = Curl_add_buffer(req_buffer, "\r\n", 2); /* end of headers! */ + result = Curl_add_buffer(&req_buffer, "\r\n", 2); /* end of headers! */ if(result) return result; if(data->req.upload_chunky && conn->bits.authneg) { /* Chunky upload is selected and we're negotiating auth still, send end-of-data only */ - result = Curl_add_buffer(req_buffer, + result = Curl_add_buffer(&req_buffer, "\x30\x0d\x0a\x0d\x0a", 5); /* 0 CR LF CR LF */ if(result) @@ -2779,7 +2876,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) } } /* issue the request */ - result = Curl_add_buffer_send(req_buffer, conn, &data->info.request_size, + result = Curl_add_buffer_send(&req_buffer, conn, &data->info.request_size, (size_t)included_body, FIRSTSOCKET); if(result) @@ -2791,12 +2888,12 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) break; default: - result = Curl_add_buffer(req_buffer, "\r\n", 2); + result = Curl_add_buffer(&req_buffer, "\r\n", 2); if(result) return result; /* issue the request */ - result = Curl_add_buffer_send(req_buffer, conn, + result = Curl_add_buffer_send(&req_buffer, conn, &data->info.request_size, 0, FIRSTSOCKET); if(result) @@ -2828,6 +2925,8 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) data->req.exp100 = EXP100_SEND_DATA; /* already sent */ Curl_expire_done(data, EXPIRE_100_TIMEOUT); } + else + data->req.writebytecount = http->writebytecount; } if((conn->httpversion == 20) && data->req.upload_chunky) @@ -2838,17 +2937,32 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) return result; } +typedef enum { + STATUS_UNKNOWN, /* not enough data to tell yet */ + STATUS_DONE, /* a status line was read */ + STATUS_BAD /* not a status line */ +} statusline; + + +/* Check a string for a prefix. Check no more than 'len' bytes */ +static bool checkprefixmax(const char *prefix, const char *buffer, size_t len) +{ + size_t ch = CURLMIN(strlen(prefix), len); + return curl_strnequal(prefix, buffer, ch); +} + /* * checkhttpprefix() * * Returns TRUE if member of the list matches prefix of string */ -static bool +static statusline checkhttpprefix(struct Curl_easy *data, - const char *s) + const char *s, size_t len) { struct curl_slist *head = data->set.http200aliases; - bool rc = FALSE; + statusline rc = STATUS_BAD; + statusline onmatch = len >= 5? STATUS_DONE : STATUS_UNKNOWN; #ifdef CURL_DOES_CONVERSIONS /* convert from the network encoding using a scratch area */ char *scratch = strdup(s); @@ -2865,15 +2979,15 @@ checkhttpprefix(struct Curl_easy *data, #endif /* CURL_DOES_CONVERSIONS */ while(head) { - if(checkprefix(head->data, s)) { - rc = TRUE; + if(checkprefixmax(head->data, s, len)) { + rc = onmatch; break; } head = head->next; } - if(!rc && (checkprefix("HTTP/", s))) - rc = TRUE; + if((rc != STATUS_DONE) && (checkprefixmax("HTTP/", s, len))) + rc = onmatch; #ifdef CURL_DOES_CONVERSIONS free(scratch); @@ -2882,11 +2996,12 @@ checkhttpprefix(struct Curl_easy *data, } #ifndef CURL_DISABLE_RTSP -static bool +static statusline checkrtspprefix(struct Curl_easy *data, - const char *s) + const char *s, size_t len) { - bool result = FALSE; + statusline result = STATUS_BAD; + statusline onmatch = len >= 5? STATUS_DONE : STATUS_UNKNOWN; #ifdef CURL_DOES_CONVERSIONS /* convert from the network encoding using a scratch area */ @@ -2899,30 +3014,31 @@ checkrtspprefix(struct Curl_easy *data, /* Curl_convert_from_network calls failf if unsuccessful */ result = FALSE; /* can't return CURLE_foobar so return FALSE */ } - else - result = checkprefix("RTSP/", scratch)? TRUE: FALSE; + else if(checkprefixmax("RTSP/", scratch, len)) + result = onmatch; free(scratch); #else (void)data; /* unused */ - result = checkprefix("RTSP/", s)? TRUE: FALSE; + if(checkprefixmax("RTSP/", s, len)) + result = onmatch; #endif /* CURL_DOES_CONVERSIONS */ return result; } #endif /* CURL_DISABLE_RTSP */ -static bool +static statusline checkprotoprefix(struct Curl_easy *data, struct connectdata *conn, - const char *s) + const char *s, size_t len) { #ifndef CURL_DISABLE_RTSP if(conn->handler->protocol & CURLPROTO_RTSP) - return checkrtspprefix(data, s); + return checkrtspprefix(data, s, len); #else (void)conn; #endif /* CURL_DISABLE_RTSP */ - return checkhttpprefix(data, s); + return checkhttpprefix(data, s, len); } /* @@ -2939,7 +3055,7 @@ static CURLcode header_append(struct Curl_easy *data, /* The reason to have a max limit for this is to avoid the risk of a bad server feeding libcurl with a never-ending header that will cause reallocs infinitely */ - failf(data, "Rejected %zd bytes header (max is %d)!", newsize, + failf(data, "Rejected %zu bytes header (max is %d)!", newsize, CURL_MAX_HTTP_HEADER); return CURLE_OUT_OF_MEMORY; } @@ -3036,12 +3152,15 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, if(result) return result; - if(!k->headerline && (k->hbuflen>5)) { - /* make a first check that this looks like a protocol header */ - if(!checkprotoprefix(data, conn, data->state.headerbuff)) { + if(!k->headerline) { + /* check if this looks like a protocol header */ + statusline st = checkprotoprefix(data, conn, data->state.headerbuff, + k->hbuflen); + if(st == STATUS_BAD) { /* this is not the beginning of a protocol first header line */ k->header = FALSE; k->badheader = HEADER_ALLBAD; + streamclose(conn, "bad HTTP: No end-of-message indicator"); break; } } @@ -3070,8 +3189,10 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, if(!k->headerline) { /* the first read header */ - if((k->hbuflen>5) && - !checkprotoprefix(data, conn, data->state.headerbuff)) { + statusline st = checkprotoprefix(data, conn, data->state.headerbuff, + k->hbuflen); + if(st == STATUS_BAD) { + streamclose(conn, "bad HTTP: No end-of-message indicator"); /* this is not the beginning of a protocol first header line */ k->header = FALSE; if(*nread) @@ -3354,7 +3475,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, if(data->set.verbose) Curl_debug(data, CURLINFO_HEADER_IN, - k->str_start, headerlen, conn); + k->str_start, headerlen); break; /* exit header line loop */ } @@ -3440,7 +3561,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, compare header line against list of aliases */ if(!nc) { - if(checkhttpprefix(data, k->p)) { + if(checkhttpprefix(data, k->p, k->hbuflen) == STATUS_DONE) { nc = 1; k->httpcode = 200; conn->httpversion = 10; @@ -3487,21 +3608,18 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, * depending on how authentication is working. Other codes * are definitely errors, so give up here. */ - if(data->set.http_fail_on_error && (k->httpcode >= 400) && + if(data->state.resume_from && data->set.httpreq == HTTPREQ_GET && + k->httpcode == 416) { + /* "Requested Range Not Satisfiable", just proceed and + pretend this is no error */ + k->ignorebody = TRUE; /* Avoid appending error msg to good data. */ + } + else if(data->set.http_fail_on_error && (k->httpcode >= 400) && ((k->httpcode != 401) || !conn->bits.user_passwd) && ((k->httpcode != 407) || !conn->bits.proxy_user_passwd) ) { - - if(data->state.resume_from && - (data->set.httpreq == HTTPREQ_GET) && - (k->httpcode == 416)) { - /* "Requested Range Not Satisfiable", just proceed and - pretend this is no error */ - } - else { - /* serious error, go home! */ - print_http_error(data); - return CURLE_HTTP_RETURNED_ERROR; - } + /* serious error, go home! */ + print_http_error(data); + return CURLE_HTTP_RETURNED_ERROR; } if(conn->httpversion == 10) { @@ -3741,7 +3859,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, here, or else use real peer host name. */ conn->allocptr.cookiehost? conn->allocptr.cookiehost:conn->host.name, - data->state.path); + data->state.up.path); Curl_share_unlock(data, CURL_LOCK_DATA_COOKIE); } #endif @@ -3812,8 +3930,7 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data, writetype |= CLIENTWRITE_BODY; if(data->set.verbose) - Curl_debug(data, CURLINFO_HEADER_IN, - k->p, (size_t)k->hbuflen, conn); + Curl_debug(data, CURLINFO_HEADER_IN, k->p, (size_t)k->hbuflen); result = Curl_client_write(conn, writetype, k->p, k->hbuflen); if(result) diff --git a/Utilities/cmcurl/lib/http.h b/Utilities/cmcurl/lib/http.h index 1d373e8f4..21fa701ab 100644 --- a/Utilities/cmcurl/lib/http.h +++ b/Utilities/cmcurl/lib/http.h @@ -58,10 +58,12 @@ struct Curl_send_buffer { typedef struct Curl_send_buffer Curl_send_buffer; Curl_send_buffer *Curl_add_buffer_init(void); -void Curl_add_buffer_free(Curl_send_buffer *buff); -CURLcode Curl_add_bufferf(Curl_send_buffer *in, const char *fmt, ...); -CURLcode Curl_add_buffer(Curl_send_buffer *in, const void *inptr, size_t size); -CURLcode Curl_add_buffer_send(Curl_send_buffer *in, +void Curl_add_buffer_free(Curl_send_buffer **inp); +CURLcode Curl_add_bufferf(Curl_send_buffer **inp, const char *fmt, ...) + WARN_UNUSED_RESULT; +CURLcode Curl_add_buffer(Curl_send_buffer **inp, const void *inptr, + size_t size) WARN_UNUSED_RESULT; +CURLcode Curl_add_buffer_send(Curl_send_buffer **inp, struct connectdata *conn, long *bytes_written, size_t included_body_bytes, @@ -154,9 +156,11 @@ struct HTTP { HTTPSEND_LAST /* never use this */ } sending; - void *send_buffer; /* used if the request couldn't be sent in one chunk, - points to an allocated send_buffer struct */ - +#ifndef CURL_DISABLE_HTTP + Curl_send_buffer *send_buffer; /* used if the request couldn't be sent in + one chunk, points to an allocated + send_buffer struct */ +#endif #ifdef USE_NGHTTP2 /*********** for HTTP/2 we store stream-local data here *************/ int32_t stream_id; /* stream we are interested in */ @@ -253,4 +257,3 @@ Curl_http_output_auth(struct connectdata *conn, up the proxy tunnel */ #endif /* HEADER_CURL_HTTP_H */ - diff --git a/Utilities/cmcurl/lib/http2.c b/Utilities/cmcurl/lib/http2.c index da001dfd0..0c5f6db0b 100644 --- a/Utilities/cmcurl/lib/http2.c +++ b/Utilities/cmcurl/lib/http2.c @@ -42,7 +42,6 @@ #include "memdebug.h" #define H2_BUFSIZE 32768 -#define MIN(x,y) ((x)<(y)?(x):(y)) #if (NGHTTP2_VERSION_NUM < 0x010000) #error too old nghttp2 version, upgrade! @@ -142,10 +141,8 @@ static int http2_getsock(struct connectdata *conn, static void http2_stream_free(struct HTTP *http) { if(http) { - Curl_add_buffer_free(http->header_recvbuf); - http->header_recvbuf = NULL; /* clear the pointer */ - Curl_add_buffer_free(http->trailer_recvbuf); - http->trailer_recvbuf = NULL; /* clear the pointer */ + Curl_add_buffer_free(&http->header_recvbuf); + Curl_add_buffer_free(&http->trailer_recvbuf); for(; http->push_headers_used > 0; --http->push_headers_used) { free(http->push_headers[http->push_headers_used - 1]); } @@ -154,6 +151,11 @@ static void http2_stream_free(struct HTTP *http) } } +/* + * Disconnects *a* connection used for HTTP/2. It might be an old one from the + * connection cache and not the "main" one. Don't touch the easy handle! + */ + static CURLcode http2_disconnect(struct connectdata *conn, bool dead_connection) { @@ -164,8 +166,6 @@ static CURLcode http2_disconnect(struct connectdata *conn, nghttp2_session_del(c->h2); Curl_safefree(c->inbuf); - http2_stream_free(conn->data->req.protop); - conn->data->state.drain = 0; H2BUGF(infof(conn->data, "HTTP/2 DISCONNECT done\n")); @@ -201,7 +201,7 @@ static bool http2_connisdead(struct connectdata *conn) dead = !Curl_connalive(conn); if(!dead) { /* This happens before we've sent off a request and the connection is - not in use by any other thransfer, there shouldn't be any data here, + not in use by any other transfer, there shouldn't be any data here, only "protocol frames" */ CURLcode result; struct http_conn *httpc = &conn->proto.httpc; @@ -231,12 +231,43 @@ static unsigned int http2_conncheck(struct connectdata *check, unsigned int checks_to_perform) { unsigned int ret_val = CONNRESULT_NONE; + struct http_conn *c = &check->proto.httpc; + int rc; + bool send_frames = false; if(checks_to_perform & CONNCHECK_ISDEAD) { if(http2_connisdead(check)) ret_val |= CONNRESULT_DEAD; } + if(checks_to_perform & CONNCHECK_KEEPALIVE) { + struct curltime now = Curl_now(); + time_t elapsed = Curl_timediff(now, check->keepalive); + + if(elapsed > check->upkeep_interval_ms) { + /* Perform an HTTP/2 PING */ + rc = nghttp2_submit_ping(c->h2, 0, ZERO_NULL); + if(!rc) { + /* Successfully added a PING frame to the session. Need to flag this + so the frame is sent. */ + send_frames = true; + } + else { + failf(check->data, "nghttp2_submit_ping() failed: %s(%d)", + nghttp2_strerror(rc), rc); + } + + check->keepalive = now; + } + } + + if(send_frames) { + rc = nghttp2_session_send(c->h2); + if(rc) + failf(check->data, "nghttp2_session_send() failed: %s(%d)", + nghttp2_strerror(rc), rc); + } + return ret_val; } @@ -345,7 +376,7 @@ const char *Curl_http2_strerror(uint32_t err) "INADEQUATE_SECURITY", /* 0xC */ "HTTP_1_1_REQUIRED" /* 0xD */ }; - return (err < sizeof str / sizeof str[0]) ? str[err] : "unknown"; + return (err < sizeof(str) / sizeof(str[0])) ? str[err] : "unknown"; #else return nghttp2_http2_strerror(err); #endif @@ -368,6 +399,10 @@ static ssize_t send_callback(nghttp2_session *h2, (void)h2; (void)flags; + if(!c->send_underlying) + /* called before setup properly! */ + return NGHTTP2_ERR_CALLBACK_FAILURE; + written = ((Curl_send*)c->send_underlying)(conn, FIRSTSOCKET, data, length, &result); @@ -441,6 +476,28 @@ char *curl_pushheader_byname(struct curl_pushheaders *h, const char *header) return NULL; } +/* + * This specific transfer on this connection has been "drained". + */ +static void drained_transfer(struct Curl_easy *data, + struct http_conn *httpc) +{ + DEBUGASSERT(httpc->drain_total >= data->state.drain); + httpc->drain_total -= data->state.drain; + data->state.drain = 0; +} + +/* + * Mark this transfer to get "drained". + */ +static void drain_this(struct Curl_easy *data, + struct http_conn *httpc) +{ + data->state.drain++; + httpc->drain_total++; + DEBUGASSERT(httpc->drain_total >= data->state.drain); +} + static struct Curl_easy *duphandle(struct Curl_easy *data) { struct Curl_easy *second = curl_easy_duphandle(data); @@ -520,6 +577,7 @@ static int push_promise(struct Curl_easy *data, if(rv) { /* denied, kill off the new handle again */ http2_stream_free(newhandle->req.protop); + newhandle->req.protop = NULL; (void)Curl_close(newhandle); goto fail; } @@ -535,14 +593,22 @@ static int push_promise(struct Curl_easy *data, if(rc) { infof(data, "failed to add handle to multi\n"); http2_stream_free(newhandle->req.protop); + newhandle->req.protop = NULL; Curl_close(newhandle); rv = 1; goto fail; } httpc = &conn->proto.httpc; - nghttp2_session_set_stream_user_data(httpc->h2, - frame->promised_stream_id, newhandle); + rv = nghttp2_session_set_stream_user_data(httpc->h2, + frame->promised_stream_id, + newhandle); + if(rv) { + infof(data, "failed to set user_data for stream %d\n", + frame->promised_stream_id); + DEBUGASSERT(0); + goto fail; + } } else { H2BUGF(infof(data, "Got PUSH_PROMISE, ignore it!\n")); @@ -562,6 +628,7 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, int rv; size_t left, ncopy; int32_t stream_id = frame->hd.stream_id; + CURLcode result; if(!stream_id) { /* stream ID zero is for connection-oriented stuff */ @@ -581,7 +648,7 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, if(max_conn != httpc->settings.max_concurrent_streams) { /* only signal change if the value actually changed */ infof(conn->data, - "Connection state changed (MAX_CONCURRENT_STREAMS == %d)!\n", + "Connection state changed (MAX_CONCURRENT_STREAMS == %u)!\n", httpc->settings.max_concurrent_streams); Curl_multi_connchanged(conn->data->multi); } @@ -637,10 +704,12 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, stream->status_code = -1; } - Curl_add_buffer(stream->header_recvbuf, "\r\n", 2); + result = Curl_add_buffer(&stream->header_recvbuf, "\r\n", 2); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; left = stream->header_recvbuf->size_used - stream->nread_header_recvbuf; - ncopy = MIN(stream->len, left); + ncopy = CURLMIN(stream->len, left); memcpy(&stream->mem[stream->memlen], stream->header_recvbuf->buffer + stream->nread_header_recvbuf, @@ -653,8 +722,7 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, stream->len -= ncopy; stream->memlen += ncopy; - data_s->state.drain++; - httpc->drain_total++; + drain_this(data_s, httpc); { /* get the pointer from userp again since it was re-assigned above */ struct connectdata *conn_s = (struct connectdata *)userp; @@ -683,25 +751,6 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame, return 0; } -static int on_invalid_frame_recv(nghttp2_session *session, - const nghttp2_frame *frame, - int lib_error_code, void *userp) -{ - struct Curl_easy *data_s = NULL; - (void)userp; -#if !defined(DEBUG_HTTP2) || defined(CURL_DISABLE_VERBOSE_STRINGS) - (void)lib_error_code; -#endif - - data_s = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); - if(data_s) { - H2BUGF(infof(data_s, - "on_invalid_frame_recv() was called, error=%d:%s\n", - lib_error_code, nghttp2_strerror(lib_error_code))); - } - return 0; -} - static int on_data_chunk_recv(nghttp2_session *session, uint8_t flags, int32_t stream_id, const uint8_t *data, size_t len, void *userp) @@ -727,14 +776,13 @@ static int on_data_chunk_recv(nghttp2_session *session, uint8_t flags, if(!stream) return NGHTTP2_ERR_CALLBACK_FAILURE; - nread = MIN(stream->len, len); + nread = CURLMIN(stream->len, len); memcpy(&stream->mem[stream->memlen], data, nread); stream->len -= nread; stream->memlen += nread; - data_s->state.drain++; - conn->proto.httpc.drain_total++; + drain_this(data_s, &conn->proto.httpc); /* if we receive data for another handle, wake that up */ if(conn->data != data_s) @@ -768,58 +816,13 @@ static int on_data_chunk_recv(nghttp2_session *session, uint8_t flags, return 0; } -static int before_frame_send(nghttp2_session *session, - const nghttp2_frame *frame, - void *userp) -{ - struct Curl_easy *data_s; - (void)userp; - - data_s = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); - if(data_s) { - H2BUGF(infof(data_s, "before_frame_send() was called\n")); - } - - return 0; -} -static int on_frame_send(nghttp2_session *session, - const nghttp2_frame *frame, - void *userp) -{ - struct Curl_easy *data_s; - (void)userp; - - data_s = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); - if(data_s) { - H2BUGF(infof(data_s, "on_frame_send() was called, length = %zd\n", - frame->hd.length)); - } - return 0; -} -static int on_frame_not_send(nghttp2_session *session, - const nghttp2_frame *frame, - int lib_error_code, void *userp) -{ - struct Curl_easy *data_s; - (void)userp; -#if !defined(DEBUG_HTTP2) || defined(CURL_DISABLE_VERBOSE_STRINGS) - (void)lib_error_code; -#endif - - data_s = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); - if(data_s) { - H2BUGF(infof(data_s, - "on_frame_not_send() was called, lib_error_code = %d\n", - lib_error_code)); - } - return 0; -} static int on_stream_close(nghttp2_session *session, int32_t stream_id, uint32_t error_code, void *userp) { struct Curl_easy *data_s; struct HTTP *stream; struct connectdata *conn = (struct connectdata *)userp; + int rv; (void)session; (void)stream_id; @@ -840,14 +843,19 @@ static int on_stream_close(nghttp2_session *session, int32_t stream_id, return NGHTTP2_ERR_CALLBACK_FAILURE; stream->closed = TRUE; - data_s->state.drain++; httpc = &conn->proto.httpc; - httpc->drain_total++; + drain_this(data_s, httpc); httpc->error_code = error_code; /* remove the entry from the hash as the stream is now gone */ - nghttp2_session_set_stream_user_data(session, stream_id, 0); + rv = nghttp2_session_set_stream_user_data(session, stream_id, 0); + if(rv) { + infof(data_s, "http/2: failed to clear user_data for stream %d!\n", + stream_id); + DEBUGASSERT(0); + } H2BUGF(infof(data_s, "Removed stream %u hash!\n", stream_id)); + stream->stream_id = 0; /* cleared */ } return 0; } @@ -922,6 +930,7 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame, struct Curl_easy *data_s; int32_t stream_id = frame->hd.stream_id; struct connectdata *conn = (struct connectdata *)userp; + CURLcode result; (void)flags; DEBUGASSERT(stream_id); /* should never be a zero stream ID here */ @@ -948,6 +957,8 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame, stream->push_headers_alloc = 10; stream->push_headers = malloc(stream->push_headers_alloc * sizeof(char *)); + if(!stream->push_headers) + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; stream->push_headers_used = 0; } else if(stream->push_headers_used == @@ -976,11 +987,21 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame, H2BUGF(infof(data_s, "h2 trailer: %.*s: %.*s\n", namelen, name, valuelen, value)); - Curl_add_buffer(stream->trailer_recvbuf, &n, sizeof(n)); - Curl_add_buffer(stream->trailer_recvbuf, name, namelen); - Curl_add_buffer(stream->trailer_recvbuf, ": ", 2); - Curl_add_buffer(stream->trailer_recvbuf, value, valuelen); - Curl_add_buffer(stream->trailer_recvbuf, "\r\n\0", 3); + result = Curl_add_buffer(&stream->trailer_recvbuf, &n, sizeof(n)); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; + result = Curl_add_buffer(&stream->trailer_recvbuf, name, namelen); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; + result = Curl_add_buffer(&stream->trailer_recvbuf, ": ", 2); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; + result = Curl_add_buffer(&stream->trailer_recvbuf, value, valuelen); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; + result = Curl_add_buffer(&stream->trailer_recvbuf, "\r\n\0", 3); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; return 0; } @@ -993,10 +1014,16 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame, stream->status_code = decode_status_code(value, valuelen); DEBUGASSERT(stream->status_code != -1); - Curl_add_buffer(stream->header_recvbuf, "HTTP/2 ", 7); - Curl_add_buffer(stream->header_recvbuf, value, valuelen); + result = Curl_add_buffer(&stream->header_recvbuf, "HTTP/2 ", 7); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; + result = Curl_add_buffer(&stream->header_recvbuf, value, valuelen); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; /* the space character after the status code is mandatory */ - Curl_add_buffer(stream->header_recvbuf, " \r\n", 3); + result = Curl_add_buffer(&stream->header_recvbuf, " \r\n", 3); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; /* if we receive data for another handle, wake that up */ if(conn->data != data_s) Curl_expire(data_s, 0, EXPIRE_RUN_NOW); @@ -1009,10 +1036,18 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame, /* nghttp2 guarantees that namelen > 0, and :status was already received, and this is not pseudo-header field . */ /* convert to a HTTP1-style header */ - Curl_add_buffer(stream->header_recvbuf, name, namelen); - Curl_add_buffer(stream->header_recvbuf, ": ", 2); - Curl_add_buffer(stream->header_recvbuf, value, valuelen); - Curl_add_buffer(stream->header_recvbuf, "\r\n", 2); + result = Curl_add_buffer(&stream->header_recvbuf, name, namelen); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; + result = Curl_add_buffer(&stream->header_recvbuf, ": ", 2); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; + result = Curl_add_buffer(&stream->header_recvbuf, value, valuelen); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; + result = Curl_add_buffer(&stream->header_recvbuf, "\r\n", 2); + if(result) + return NGHTTP2_ERR_CALLBACK_FAILURE; /* if we receive data for another handle, wake that up */ if(conn->data != data_s) Curl_expire(data_s, 0, EXPIRE_RUN_NOW); @@ -1052,7 +1087,7 @@ static ssize_t data_source_read_callback(nghttp2_session *session, else return NGHTTP2_ERR_INVALID_ARGUMENT; - nread = MIN(stream->upload_len, length); + nread = CURLMIN(stream->upload_len, length); if(nread > 0) { memcpy(buf, stream->upload_mem, nread); stream->upload_mem += nread; @@ -1073,7 +1108,8 @@ static ssize_t data_source_read_callback(nghttp2_session *session, return nread; } -#ifdef NGHTTP2_HAS_ERROR_CALLBACK +#if defined(NGHTTP2_HAS_ERROR_CALLBACK) && \ + !defined(CURL_DISABLE_VERBOSE_STRINGS) static int error_callback(nghttp2_session *session, const char *msg, size_t len, @@ -1109,11 +1145,11 @@ void Curl_http2_done(struct connectdata *conn, bool premature) struct HTTP *http = data->req.protop; struct http_conn *httpc = &conn->proto.httpc; + /* there might be allocated resources done before this got the 'h2' pointer + setup */ if(http->header_recvbuf) { - Curl_add_buffer_free(http->header_recvbuf); - http->header_recvbuf = NULL; /* clear the pointer */ - Curl_add_buffer_free(http->trailer_recvbuf); - http->trailer_recvbuf = NULL; /* clear the pointer */ + Curl_add_buffer_free(&http->header_recvbuf); + Curl_add_buffer_free(&http->trailer_recvbuf); if(http->push_headers) { /* if they weren't used and then freed before */ for(; http->push_headers_used > 0; --http->push_headers_used) { @@ -1124,17 +1160,32 @@ void Curl_http2_done(struct connectdata *conn, bool premature) } } + if(!httpc->h2) /* not HTTP/2 ? */ + return; + + if(data->state.drain) + drained_transfer(data, httpc); + if(premature) { /* RST_STREAM */ - nghttp2_submit_rst_stream(httpc->h2, NGHTTP2_FLAG_NONE, http->stream_id, - NGHTTP2_STREAM_CLOSED); + if(!nghttp2_submit_rst_stream(httpc->h2, NGHTTP2_FLAG_NONE, + http->stream_id, NGHTTP2_STREAM_CLOSED)) + (void)nghttp2_session_send(httpc->h2); + if(http->stream_id == httpc->pause_stream_id) { infof(data, "stopped the pause stream!\n"); httpc->pause_stream_id = 0; } } - if(http->stream_id) { - nghttp2_session_set_stream_user_data(httpc->h2, http->stream_id, 0); + /* -1 means unassigned and 0 means cleared */ + if(http->stream_id > 0) { + int rv = nghttp2_session_set_stream_user_data(httpc->h2, + http->stream_id, 0); + if(rv) { + infof(data, "http/2: failed to clear user_data for stream %d!\n", + http->stream_id); + DEBUGASSERT(0); + } http->stream_id = 0; } } @@ -1164,21 +1215,9 @@ CURLcode Curl_http2_init(struct connectdata *conn) /* nghttp2_on_frame_recv_callback */ nghttp2_session_callbacks_set_on_frame_recv_callback (callbacks, on_frame_recv); - /* nghttp2_on_invalid_frame_recv_callback */ - nghttp2_session_callbacks_set_on_invalid_frame_recv_callback - (callbacks, on_invalid_frame_recv); /* nghttp2_on_data_chunk_recv_callback */ nghttp2_session_callbacks_set_on_data_chunk_recv_callback (callbacks, on_data_chunk_recv); - /* nghttp2_before_frame_send_callback */ - nghttp2_session_callbacks_set_before_frame_send_callback - (callbacks, before_frame_send); - /* nghttp2_on_frame_send_callback */ - nghttp2_session_callbacks_set_on_frame_send_callback - (callbacks, on_frame_send); - /* nghttp2_on_frame_not_send_callback */ - nghttp2_session_callbacks_set_on_frame_not_send_callback - (callbacks, on_frame_not_send); /* nghttp2_on_stream_close_callback */ nghttp2_session_callbacks_set_on_stream_close_callback (callbacks, on_stream_close); @@ -1188,7 +1227,9 @@ CURLcode Curl_http2_init(struct connectdata *conn) /* nghttp2_on_header_callback */ nghttp2_session_callbacks_set_on_header_callback(callbacks, on_header); +#ifndef CURL_DISABLE_VERBOSE_STRINGS nghttp2_session_callbacks_set_error_callback(callbacks, error_callback); +#endif /* The nghttp2 session is not yet setup, do it */ rc = nghttp2_session_client_new(&conn->proto.httpc.h2, callbacks, conn); @@ -1225,7 +1266,7 @@ CURLcode Curl_http2_request_upgrade(Curl_send_buffer *req, httpc->local_settings_num); if(!binlen) { failf(conn->data, "nghttp2 unexpectedly failed on pack_settings_payload"); - Curl_add_buffer_free(req); + Curl_add_buffer_free(&req); return CURLE_FAILED_INIT; } conn->proto.httpc.binlen = binlen; @@ -1233,11 +1274,11 @@ CURLcode Curl_http2_request_upgrade(Curl_send_buffer *req, result = Curl_base64url_encode(conn->data, (const char *)binsettings, binlen, &base64, &blen); if(result) { - Curl_add_buffer_free(req); + Curl_add_buffer_free(&req); return result; } - result = Curl_add_bufferf(req, + result = Curl_add_bufferf(&req, "Connection: Upgrade, HTTP2-Settings\r\n" "Upgrade: %s\r\n" "HTTP2-Settings: %s\r\n", @@ -1280,7 +1321,7 @@ static int h2_process_pending_input(struct connectdata *conn, if(rv < 0) { failf(data, "h2_process_pending_input: nghttp2_session_mem_recv() returned " - "%d:%s\n", rv, nghttp2_strerror((int)rv)); + "%zd:%s\n", rv, nghttp2_strerror((int)rv)); *err = CURLE_RECV_ERROR; return -1; } @@ -1352,7 +1393,6 @@ CURLcode Curl_http2_done_sending(struct connectdata *conn) return result; } - static ssize_t http2_handle_stream_close(struct connectdata *conn, struct Curl_easy *data, struct HTTP *stream, CURLcode *err) @@ -1365,9 +1405,7 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn, httpc->pause_stream_id = 0; } - DEBUGASSERT(httpc->drain_total >= data->state.drain); - httpc->drain_total -= data->state.drain; - data->state.drain = 0; + drained_transfer(data, httpc); if(httpc->pause_stream_id == 0) { if(h2_process_pending_input(conn, httpc, err) != 0) { @@ -1388,7 +1426,7 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn, return -1; } else if(httpc->error_code != NGHTTP2_NO_ERROR) { - failf(data, "HTTP/2 stream %u was not closed cleanly: %s (err %d)", + failf(data, "HTTP/2 stream %d was not closed cleanly: %s (err %u)", stream->stream_id, Curl_http2_strerror(httpc->error_code), httpc->error_code); *err = CURLE_HTTP2_STREAM; @@ -1396,7 +1434,7 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn, } if(!stream->bodystarted) { - failf(data, "HTTP/2 stream %u was closed cleanly, but before getting " + failf(data, "HTTP/2 stream %d was closed cleanly, but before getting " " all response header fields, treated as error", stream->stream_id); *err = CURLE_HTTP2_STREAM; @@ -1510,7 +1548,7 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex, /* If there is body data pending for this stream to return, do that */ size_t left = stream->header_recvbuf->size_used - stream->nread_header_recvbuf; - size_t ncopy = MIN(len, left); + size_t ncopy = CURLMIN(len, left); memcpy(mem, stream->header_recvbuf->buffer + stream->nread_header_recvbuf, ncopy); stream->nread_header_recvbuf += ncopy; @@ -1546,13 +1584,13 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex, } else if(stream->pausedata) { DEBUGASSERT(httpc->pause_stream_id == stream->stream_id); - nread = MIN(len, stream->pauselen); + nread = CURLMIN(len, stream->pauselen); memcpy(mem, stream->pausedata, nread); stream->pausedata += nread; stream->pauselen -= nread; - infof(data, "%zu data bytes written\n", nread); + infof(data, "%zd data bytes written\n", nread); if(stream->pauselen == 0) { H2BUGF(infof(data, "Unpaused by stream %u\n", stream->stream_id)); DEBUGASSERT(httpc->pause_stream_id == stream->stream_id); @@ -1635,7 +1673,7 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex, rv = nghttp2_session_mem_recv(httpc->h2, (const uint8_t *)inbuf, nread); if(nghttp2_is_fatal((int)rv)) { - failf(data, "nghttp2_session_mem_recv() returned %d:%s\n", + failf(data, "nghttp2_session_mem_recv() returned %zd:%s\n", rv, nghttp2_strerror((int)rv)); *err = CURLE_RECV_ERROR; return -1; @@ -1678,9 +1716,7 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex, stream->stream_id)); } else if(!stream->closed) { - DEBUGASSERT(httpc->drain_total >= data->state.drain); - httpc->drain_total -= data->state.drain; - data->state.drain = 0; /* this stream is hereby drained */ + drained_transfer(data, httpc); } return retlen; @@ -2040,7 +2076,7 @@ static ssize_t http2_send(struct connectdata *conn, int sockindex, } infof(conn->data, "Using Stream ID: %x (easy handle %p)\n", - stream_id, conn->data); + stream_id, (void *)conn->data); stream->stream_id = stream_id; /* this does not call h2_session_send() since there can not have been any @@ -2086,8 +2122,11 @@ CURLcode Curl_http2_setup(struct connectdata *conn) stream->stream_id = -1; - if(!stream->header_recvbuf) + if(!stream->header_recvbuf) { stream->header_recvbuf = Curl_add_buffer_init(); + if(!stream->header_recvbuf) + return CURLE_OUT_OF_MEMORY; + } if((conn->handler == &Curl_handler_http2_ssl) || (conn->handler == &Curl_handler_http2)) @@ -2099,8 +2138,10 @@ CURLcode Curl_http2_setup(struct connectdata *conn) conn->handler = &Curl_handler_http2; result = Curl_http2_init(conn); - if(result) + if(result) { + Curl_add_buffer_free(&stream->header_recvbuf); return result; + } infof(conn->data, "Using HTTP2, server supports multi-use\n"); stream->upload_left = 0; @@ -2154,9 +2195,14 @@ CURLcode Curl_http2_switched(struct connectdata *conn, return CURLE_HTTP2; } - nghttp2_session_set_stream_user_data(httpc->h2, - stream->stream_id, - conn->data); + rv = nghttp2_session_set_stream_user_data(httpc->h2, + stream->stream_id, + data); + if(rv) { + infof(data, "http/2: failed to set user_data for stream %d!\n", + stream->stream_id); + DEBUGASSERT(0); + } } else { populate_settings(conn, httpc); diff --git a/Utilities/cmcurl/lib/http2.h b/Utilities/cmcurl/lib/http2.h index f597c805e..4492ec211 100644 --- a/Utilities/cmcurl/lib/http2.h +++ b/Utilities/cmcurl/lib/http2.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -65,7 +65,7 @@ void Curl_http2_cleanup_dependencies(struct Curl_easy *data); #define Curl_http2_request_upgrade(x,y) CURLE_UNSUPPORTED_PROTOCOL #define Curl_http2_setup(x) CURLE_UNSUPPORTED_PROTOCOL #define Curl_http2_switched(x,y,z) CURLE_UNSUPPORTED_PROTOCOL -#define Curl_http2_setup_conn(x) +#define Curl_http2_setup_conn(x) Curl_nop_stmt #define Curl_http2_setup_req(x) #define Curl_http2_init_state(x) #define Curl_http2_init_userset(x) @@ -77,4 +77,3 @@ void Curl_http2_cleanup_dependencies(struct Curl_easy *data); #endif #endif /* HEADER_CURL_HTTP2_H */ - diff --git a/Utilities/cmcurl/lib/http_chunks.h b/Utilities/cmcurl/lib/http_chunks.h index 3a8b4ddf3..b969c5590 100644 --- a/Utilities/cmcurl/lib/http_chunks.h +++ b/Utilities/cmcurl/lib/http_chunks.h @@ -88,4 +88,3 @@ struct Curl_chunker { }; #endif /* HEADER_CURL_HTTP_CHUNKS_H */ - diff --git a/Utilities/cmcurl/lib/http_ntlm.c b/Utilities/cmcurl/lib/http_ntlm.c index fd5540b5d..a9b33f98e 100644 --- a/Utilities/cmcurl/lib/http_ntlm.c +++ b/Utilities/cmcurl/lib/http_ntlm.c @@ -228,7 +228,7 @@ CURLcode Curl_output_ntlm(struct connectdata *conn, bool proxy) /* connection is already authenticated, * don't send a header in future requests */ ntlm->state = NTLMSTATE_LAST; - /* fall-through */ + /* FALLTHROUGH */ case NTLMSTATE_LAST: Curl_safefree(*allocuserpwd); authp->done = TRUE; diff --git a/Utilities/cmcurl/lib/http_proxy.c b/Utilities/cmcurl/lib/http_proxy.c index e10a48829..2e0d92edd 100644 --- a/Utilities/cmcurl/lib/http_proxy.c +++ b/Utilities/cmcurl/lib/http_proxy.c @@ -188,12 +188,10 @@ static CURLcode CONNECT(struct connectdata *conn, struct SingleRequest *k = &data->req; CURLcode result; curl_socket_t tunnelsocket = conn->sock[sockindex]; - timediff_t check; struct http_connect_state *s = conn->connect_state; #define SELECT_OK 0 #define SELECT_ERROR 1 -#define SELECT_TIMEOUT 2 if(Curl_connect_complete(conn)) return CURLE_OK; /* CONNECT is already completed */ @@ -201,12 +199,13 @@ static CURLcode CONNECT(struct connectdata *conn, conn->bits.proxy_connect_closed = FALSE; do { + timediff_t check; if(TUNNEL_INIT == s->tunnel_state) { /* BEGIN CONNECT PHASE */ char *host_port; Curl_send_buffer *req_buffer; - infof(data, "Establish HTTP proxy tunnel to %s:%hu\n", + infof(data, "Establish HTTP proxy tunnel to %s:%d\n", hostname, remote_port); /* This only happens if we've looped here due to authentication @@ -223,7 +222,7 @@ static CURLcode CONNECT(struct connectdata *conn, host_port = aprintf("%s:%d", hostname, remote_port); if(!host_port) { - Curl_add_buffer_free(req_buffer); + Curl_add_buffer_free(&req_buffer); return CURLE_OUT_OF_MEMORY; } @@ -248,7 +247,7 @@ static CURLcode CONNECT(struct connectdata *conn, aprintf("%s%s%s:%d", ipv6_ip?"[":"", hostname, ipv6_ip?"]":"", remote_port); if(!hostheader) { - Curl_add_buffer_free(req_buffer); + Curl_add_buffer_free(&req_buffer); return CURLE_OUT_OF_MEMORY; } @@ -256,7 +255,7 @@ static CURLcode CONNECT(struct connectdata *conn, host = aprintf("Host: %s\r\n", hostheader); if(!host) { free(hostheader); - Curl_add_buffer_free(req_buffer); + Curl_add_buffer_free(&req_buffer); return CURLE_OUT_OF_MEMORY; } } @@ -268,7 +267,7 @@ static CURLcode CONNECT(struct connectdata *conn, useragent = conn->allocptr.uagent; result = - Curl_add_bufferf(req_buffer, + Curl_add_bufferf(&req_buffer, "CONNECT %s HTTP/%s\r\n" "%s" /* Host: */ "%s" /* Proxy-Authorization */ @@ -291,13 +290,13 @@ static CURLcode CONNECT(struct connectdata *conn, if(!result) /* CRLF terminate the request */ - result = Curl_add_bufferf(req_buffer, "\r\n"); + result = Curl_add_bufferf(&req_buffer, "\r\n"); if(!result) { /* Send the connect request to the proxy */ /* BLOCKING */ result = - Curl_add_buffer_send(req_buffer, conn, + Curl_add_buffer_send(&req_buffer, conn, &data->info.request_size, 0, sockindex); } req_buffer = NULL; @@ -305,7 +304,7 @@ static CURLcode CONNECT(struct connectdata *conn, failf(data, "Failed sending CONNECT to proxy"); } - Curl_add_buffer_free(req_buffer); + Curl_add_buffer_free(&req_buffer); if(result) return result; @@ -419,7 +418,7 @@ static CURLcode CONNECT(struct connectdata *conn, /* output debug if that is requested */ if(data->set.verbose) Curl_debug(data, CURLINFO_HEADER_IN, - s->line_start, (size_t)s->perline, conn); + s->line_start, (size_t)s->perline); if(!data->set.suppress_connect_headers) { /* send the header to the callback */ diff --git a/Utilities/cmcurl/lib/imap.c b/Utilities/cmcurl/lib/imap.c index cf278a22b..3ef89097f 100644 --- a/Utilities/cmcurl/lib/imap.c +++ b/Utilities/cmcurl/lib/imap.c @@ -159,7 +159,8 @@ const struct Curl_handler Curl_handler_imaps = { ZERO_NULL, /* connection_check */ PORT_IMAPS, /* defport */ CURLPROTO_IMAPS, /* protocol */ - PROTOPT_CLOSEACTION | PROTOPT_SSL /* flags */ + PROTOPT_CLOSEACTION | PROTOPT_SSL | /* flags */ + PROTOPT_URLOPTIONS }; #endif @@ -421,7 +422,6 @@ static CURLcode imap_perform_capability(struct connectdata *conn) { CURLcode result = CURLE_OK; struct imap_conn *imapc = &conn->proto.imapc; - imapc->sasl.authmechs = SASL_AUTH_NONE; /* No known auth. mechanisms yet */ imapc->sasl.authused = SASL_AUTH_NONE; /* Clear the auth. mechanism used */ imapc->tls_supported = FALSE; /* Clear the TLS capability */ @@ -609,7 +609,6 @@ static CURLcode imap_perform_list(struct connectdata *conn) CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; struct IMAP *imap = data->req.protop; - char *mailbox; if(imap->custom) /* Send the custom request */ @@ -617,7 +616,8 @@ static CURLcode imap_perform_list(struct connectdata *conn) imap->custom_params ? imap->custom_params : ""); else { /* Make sure the mailbox is in the correct atom format if necessary */ - mailbox = imap->mailbox ? imap_atom(imap->mailbox, true) : strdup(""); + char *mailbox = imap->mailbox ? imap_atom(imap->mailbox, true) + : strdup(""); if(!mailbox) return CURLE_OUT_OF_MEMORY; @@ -683,24 +683,37 @@ static CURLcode imap_perform_fetch(struct connectdata *conn) { CURLcode result = CURLE_OK; struct IMAP *imap = conn->data->req.protop; - /* Check we have a UID */ - if(!imap->uid) { - failf(conn->data, "Cannot FETCH without a UID."); - return CURLE_URL_MALFORMAT; + if(imap->uid) { + + /* Send the FETCH command */ + if(imap->partial) + result = imap_sendf(conn, "UID FETCH %s BODY[%s]<%s>", + imap->uid, + imap->section ? imap->section : "", + imap->partial); + else + result = imap_sendf(conn, "UID FETCH %s BODY[%s]", + imap->uid, + imap->section ? imap->section : ""); + } + else if(imap->mindex) { + + /* Send the FETCH command */ + if(imap->partial) + result = imap_sendf(conn, "FETCH %s BODY[%s]<%s>", + imap->mindex, + imap->section ? imap->section : "", + imap->partial); + else + result = imap_sendf(conn, "FETCH %s BODY[%s]", + imap->mindex, + imap->section ? imap->section : ""); + } + else { + failf(conn->data, "Cannot FETCH without a UID."); + return CURLE_URL_MALFORMAT; } - - /* Send the FETCH command */ - if(imap->partial) - result = imap_sendf(conn, "FETCH %s BODY[%s]<%s>", - imap->uid, - imap->section ? imap->section : "", - imap->partial); - else - result = imap_sendf(conn, "FETCH %s BODY[%s]", - imap->uid, - imap->section ? imap->section : ""); - if(!result) state(conn, IMAP_FETCH); @@ -854,7 +867,6 @@ static CURLcode imap_state_capability_resp(struct connectdata *conn, struct Curl_easy *data = conn->data; struct imap_conn *imapc = &conn->proto.imapc; const char *line = data->state.buffer; - size_t wordlen; (void)instate; /* no use for this yet */ @@ -864,6 +876,7 @@ static CURLcode imap_state_capability_resp(struct connectdata *conn, /* Loop through the data line */ for(;;) { + size_t wordlen; while(*line && (*line == ' ' || *line == '\t' || *line == '\r' || *line == '\n')) { @@ -1046,12 +1059,12 @@ static CURLcode imap_state_select_resp(struct connectdata *conn, int imapcode, struct IMAP *imap = conn->data->req.protop; struct imap_conn *imapc = &conn->proto.imapc; const char *line = data->state.buffer; - char tmp[20]; (void)instate; /* no use for this yet */ if(imapcode == '*') { /* See if this is an UIDVALIDITY response */ + char tmp[20]; if(sscanf(line + 2, "OK [UIDVALIDITY %19[0123456789]]", tmp) == 1) { Curl_safefree(imapc->mailbox_uidvalidity); imapc->mailbox_uidvalidity = strdup(tmp); @@ -1119,7 +1132,7 @@ static CURLcode imap_state_fetch_resp(struct connectdata *conn, int imapcode, } if(parsed) { - infof(data, "Found %" CURL_FORMAT_CURL_OFF_TU " bytes to download\n", + infof(data, "Found %" CURL_FORMAT_CURL_OFF_T " bytes to download\n", size); Curl_pgrsSetDownloadSize(data, size); @@ -1144,10 +1157,8 @@ static CURLcode imap_state_fetch_resp(struct connectdata *conn, int imapcode, data->req.bytecount += chunk; - infof(data, "Written %" CURL_FORMAT_CURL_OFF_TU - " bytes, %" CURL_FORMAT_CURL_OFF_TU - " bytes are left for transfer\n", (curl_off_t)chunk, - size - chunk); + infof(data, "Written %zu bytes, %" CURL_FORMAT_CURL_OFF_TU + " bytes are left for transfer\n", chunk, size - chunk); /* Have we used the entire cache or just part of it?*/ if(pp->cache_size > chunk) { @@ -1466,9 +1477,10 @@ static CURLcode imap_done(struct connectdata *conn, CURLcode status, result = status; /* use the already set error code */ } else if(!data->set.connect_only && !imap->custom && - (imap->uid || data->set.upload || + (imap->uid || imap->mindex || data->set.upload || data->set.mimepost.kind != MIMEKIND_NONE)) { /* Handle responses after FETCH or APPEND transfer has finished */ + if(!data->set.upload && data->set.mimepost.kind == MIMEKIND_NONE) state(conn, IMAP_FETCH_FINAL); else { @@ -1492,6 +1504,7 @@ static CURLcode imap_done(struct connectdata *conn, CURLcode status, Curl_safefree(imap->mailbox); Curl_safefree(imap->uidvalidity); Curl_safefree(imap->uid); + Curl_safefree(imap->mindex); Curl_safefree(imap->section); Curl_safefree(imap->partial); Curl_safefree(imap->query); @@ -1545,14 +1558,14 @@ static CURLcode imap_perform(struct connectdata *conn, bool *connected, else if(imap->custom && (selected || !imap->mailbox)) /* Custom command using the same mailbox or no mailbox */ result = imap_perform_list(conn); - else if(!imap->custom && selected && imap->uid) + else if(!imap->custom && selected && (imap->uid || imap->mindex)) /* FETCH from the same mailbox */ result = imap_perform_fetch(conn); else if(!imap->custom && selected && imap->query) /* SEARCH the current mailbox */ result = imap_perform_search(conn); else if(imap->mailbox && !selected && - (imap->custom || imap->uid || imap->query)) + (imap->custom || imap->uid || imap->mindex || imap->query)) /* SELECT the mailbox */ result = imap_perform_select(conn); else @@ -1704,8 +1717,6 @@ static CURLcode imap_regular_transfer(struct connectdata *conn, static CURLcode imap_setup_connection(struct connectdata *conn) { - struct Curl_easy *data = conn->data; - /* Initialise the IMAP layer */ CURLcode result = imap_init(conn); if(result) @@ -1713,7 +1724,6 @@ static CURLcode imap_setup_connection(struct connectdata *conn) /* Clear the TLS upgraded flag */ conn->tls_upgraded = FALSE; - data->state.path++; /* don't include the initial slash */ return CURLE_OK; } @@ -1946,7 +1956,7 @@ static CURLcode imap_parse_url_path(struct connectdata *conn) CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; struct IMAP *imap = data->req.protop; - const char *begin = data->state.path; + const char *begin = &data->state.up.path[1]; /* skip leading slash */ const char *ptr = begin; /* See how much of the URL is a valid path and decode it */ @@ -2018,6 +2028,13 @@ static CURLcode imap_parse_url_path(struct connectdata *conn) imap->uid = value; value = NULL; } + else if(strcasecompare(name, "MAILINDEX") && !imap->mindex) { + if(valuelen > 0 && value[valuelen - 1] == '/') + value[valuelen - 1] = '\0'; + + imap->mindex = value; + value = NULL; + } else if(strcasecompare(name, "SECTION") && !imap->section) { if(valuelen > 0 && value[valuelen - 1] == '/') value[valuelen - 1] = '\0'; @@ -2045,17 +2062,10 @@ static CURLcode imap_parse_url_path(struct connectdata *conn) /* Does the URL contain a query parameter? Only valid when we have a mailbox and no UID as per RFC-5092 */ - if(imap->mailbox && !imap->uid && *ptr == '?') { - /* Find the length of the query parameter */ - begin = ++ptr; - while(imap_is_bchar(*ptr)) - ptr++; - - /* Decode the query parameter */ - result = Curl_urldecode(data, begin, ptr - begin, &imap->query, NULL, - TRUE); - if(result) - return result; + if(imap->mailbox && !imap->uid && !imap->mindex) { + /* Get the query parameter, URL decoded */ + (void)curl_url_get(data->state.uh, CURLUPART_QUERY, &imap->query, + CURLU_URLDECODE); } /* Any extra stuff at the end of the URL is an error */ diff --git a/Utilities/cmcurl/lib/imap.h b/Utilities/cmcurl/lib/imap.h index 9fc4ff5a3..0efcfd293 100644 --- a/Utilities/cmcurl/lib/imap.h +++ b/Utilities/cmcurl/lib/imap.h @@ -58,6 +58,7 @@ struct IMAP { char *mailbox; /* Mailbox to select */ char *uidvalidity; /* UIDVALIDITY to check in select */ char *uid; /* Message UID to fetch */ + char *mindex; /* Index in mail box of mail to fetch */ char *section; /* Message SECTION to fetch */ char *partial; /* Message PARTIAL to fetch */ char *query; /* Query to search for */ diff --git a/Utilities/cmcurl/lib/inet_ntop.c b/Utilities/cmcurl/lib/inet_ntop.c index fb91a505d..ac5d2d4d6 100644 --- a/Utilities/cmcurl/lib/inet_ntop.c +++ b/Utilities/cmcurl/lib/inet_ntop.c @@ -49,7 +49,7 @@ */ static char *inet_ntop4 (const unsigned char *src, char *dst, size_t size) { - char tmp[sizeof "255.255.255.255"]; + char tmp[sizeof("255.255.255.255")]; size_t len; DEBUGASSERT(size >= 16); diff --git a/Utilities/cmcurl/lib/inet_ntop.h b/Utilities/cmcurl/lib/inet_ntop.h index 9f4461271..d150bb693 100644 --- a/Utilities/cmcurl/lib/inet_ntop.h +++ b/Utilities/cmcurl/lib/inet_ntop.h @@ -35,4 +35,3 @@ char *Curl_inet_ntop(int af, const void *addr, char *buf, size_t size); #endif #endif /* HEADER_CURL_INET_NTOP_H */ - diff --git a/Utilities/cmcurl/lib/inet_pton.h b/Utilities/cmcurl/lib/inet_pton.h index e216f4efa..0209b9b7b 100644 --- a/Utilities/cmcurl/lib/inet_pton.h +++ b/Utilities/cmcurl/lib/inet_pton.h @@ -37,4 +37,3 @@ int Curl_inet_pton(int, const char *, void *); #endif #endif /* HEADER_CURL_INET_PTON_H */ - diff --git a/Utilities/cmcurl/lib/krb5.c b/Utilities/cmcurl/lib/krb5.c index 59c0d71a8..147ab024e 100644 --- a/Utilities/cmcurl/lib/krb5.c +++ b/Utilities/cmcurl/lib/krb5.c @@ -206,7 +206,7 @@ krb5_auth(void *app_data, struct connectdata *conn) if(maj != GSS_S_COMPLETE) { gss_release_name(&min, &gssname); if(service == srv_host) { - Curl_failf(data, "Error importing service name %s@%s", service, host); + failf(data, "Error importing service name %s@%s", service, host); return AUTH_ERROR; } service = srv_host; @@ -265,6 +265,7 @@ krb5_auth(void *app_data, struct connectdata *conn) result = CURLE_OUT_OF_MEMORY; free(p); + free(cmd); if(result) { ret = -2; @@ -290,8 +291,7 @@ krb5_auth(void *app_data, struct connectdata *conn) (unsigned char **)&_gssresp.value, &_gssresp.length); if(result) { - Curl_failf(data, "base64-decoding: %s", - curl_easy_strerror(result)); + failf(data, "base64-decoding: %s", curl_easy_strerror(result)); ret = AUTH_CONTINUE; break; } diff --git a/Utilities/cmcurl/lib/ldap.c b/Utilities/cmcurl/lib/ldap.c index 89047bcbd..ceaa71d08 100644 --- a/Utilities/cmcurl/lib/ldap.c +++ b/Utilities/cmcurl/lib/ldap.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -54,15 +54,6 @@ # endif /* HAVE_LDAP_SSL && HAVE_LDAP_SSL_H */ #endif -/* These are macros in both <wincrypt.h> (in above <winldap.h>) and typedefs - * in BoringSSL's <openssl/x509.h> - */ -#ifdef HAVE_BORINGSSL -# undef X509_NAME -# undef X509_CERT_PAIR -# undef X509_EXTENSIONS -#endif - #include "urldata.h" #include <curl/curl.h> #include "sendf.h" @@ -483,7 +474,13 @@ static CURLcode Curl_ldap(struct connectdata *conn, bool *done) #endif } if(rc != 0) { - failf(data, "LDAP local: ldap_simple_bind_s %s", ldap_err2string(rc)); +#ifdef USE_WIN32_LDAP + failf(data, "LDAP local: bind via ldap_win_bind %s", + ldap_err2string(rc)); +#else + failf(data, "LDAP local: bind via ldap_simple_bind_s %s", + ldap_err2string(rc)); +#endif result = CURLE_LDAP_CANNOT_BIND; goto quit; } @@ -847,9 +844,9 @@ static int _ldap_url_parse2(const struct connectdata *conn, LDAPURLDesc *ludp) size_t i; if(!conn->data || - !conn->data->state.path || - conn->data->state.path[0] != '/' || - !checkprefix("LDAP", conn->data->change.url)) + !conn->data->state.up.path || + conn->data->state.up.path[0] != '/' || + !strcasecompare("LDAP", conn->data->state.up.scheme)) return LDAP_INVALID_SYNTAX; ludp->lud_scope = LDAP_SCOPE_BASE; @@ -857,7 +854,7 @@ static int _ldap_url_parse2(const struct connectdata *conn, LDAPURLDesc *ludp) ludp->lud_host = conn->host.name; /* Duplicate the path */ - p = path = strdup(conn->data->state.path + 1); + p = path = strdup(conn->data->state.up.path + 1); if(!path) return LDAP_NO_MEMORY; diff --git a/Utilities/cmcurl/lib/llist.h b/Utilities/cmcurl/lib/llist.h index 6b644b99c..b9d4c89a9 100644 --- a/Utilities/cmcurl/lib/llist.h +++ b/Utilities/cmcurl/lib/llist.h @@ -51,4 +51,3 @@ void Curl_llist_move(struct curl_llist *, struct curl_llist_element *, struct curl_llist *, struct curl_llist_element *); #endif /* HEADER_CURL_LLIST_H */ - diff --git a/Utilities/cmcurl/lib/md4.c b/Utilities/cmcurl/lib/md4.c index 2bb7dcc25..d35060216 100644 --- a/Utilities/cmcurl/lib/md4.c +++ b/Utilities/cmcurl/lib/md4.c @@ -3,7 +3,7 @@ * MD4 Message-Digest Algorithm (RFC 1320). * * Homepage: - http://openwall.info/wiki/people/solar/software/public-domain-source-code/md4 + https://openwall.info/wiki/people/solar/software/public-domain-source-code/md4 * * Author: * Alexander Peslyak, better known as Solar Designer <solar at openwall.com> diff --git a/Utilities/cmcurl/lib/md5.c b/Utilities/cmcurl/lib/md5.c index 3096602b7..45f45bbd9 100644 --- a/Utilities/cmcurl/lib/md5.c +++ b/Utilities/cmcurl/lib/md5.c @@ -177,7 +177,7 @@ static void MD5_Final(unsigned char digest[16], MD5_CTX *ctx) * MD5 Message-Digest Algorithm (RFC 1321). * * Homepage: - http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 + https://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 * * Author: * Alexander Peslyak, better known as Solar Designer <solar at openwall.com> @@ -484,29 +484,35 @@ static void MD5_Final(unsigned char *result, MD5_CTX *ctx) #endif /* CRYPTO LIBS */ -/* Disable this picky gcc-8 compiler warning */ -#if defined(__GNUC__) && (__GNUC__ >= 8) -#pragma GCC diagnostic ignored "-Wcast-function-type" -#endif - const HMAC_params Curl_HMAC_MD5[] = { { - (HMAC_hinit_func) MD5_Init, /* Hash initialization function. */ - (HMAC_hupdate_func) MD5_Update, /* Hash update function. */ - (HMAC_hfinal_func) MD5_Final, /* Hash computation end function. */ - sizeof(MD5_CTX), /* Size of hash context structure. */ - 64, /* Maximum key length. */ - 16 /* Result size. */ + /* Hash initialization function. */ + CURLX_FUNCTION_CAST(HMAC_hinit_func, MD5_Init), + /* Hash update function. */ + CURLX_FUNCTION_CAST(HMAC_hupdate_func, MD5_Update), + /* Hash computation end function. */ + CURLX_FUNCTION_CAST(HMAC_hfinal_func, MD5_Final), + /* Size of hash context structure. */ + sizeof(MD5_CTX), + /* Maximum key length. */ + 64, + /* Result size. */ + 16 } }; const MD5_params Curl_DIGEST_MD5[] = { { - (Curl_MD5_init_func) MD5_Init, /* Digest initialization function */ - (Curl_MD5_update_func) MD5_Update, /* Digest update function */ - (Curl_MD5_final_func) MD5_Final, /* Digest computation end function */ - sizeof(MD5_CTX), /* Size of digest context struct */ - 16 /* Result size */ + /* Digest initialization function */ + CURLX_FUNCTION_CAST(Curl_MD5_init_func, MD5_Init), + /* Digest update function */ + CURLX_FUNCTION_CAST(Curl_MD5_update_func, MD5_Update), + /* Digest computation end function */ + CURLX_FUNCTION_CAST(Curl_MD5_final_func, MD5_Final), + /* Size of digest context struct */ + sizeof(MD5_CTX), + /* Result size */ + 16 } }; @@ -527,7 +533,7 @@ MD5_context *Curl_MD5_init(const MD5_params *md5params) MD5_context *ctxt; /* Create MD5 context */ - ctxt = malloc(sizeof *ctxt); + ctxt = malloc(sizeof(*ctxt)); if(!ctxt) return ctxt; diff --git a/Utilities/cmcurl/lib/memdebug.h b/Utilities/cmcurl/lib/memdebug.h index 6fb8b6851..233de65a4 100644 --- a/Utilities/cmcurl/lib/memdebug.h +++ b/Utilities/cmcurl/lib/memdebug.h @@ -8,7 +8,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -137,13 +137,6 @@ CURL_EXTERN int curl_fclose(FILE *file, int line, const char *source); #endif #endif /* HAVE_GETADDRINFO */ -#ifdef HAVE_GETNAMEINFO -#undef getnameinfo -#define getnameinfo(sa,salen,host,hostlen,serv,servlen,flags) \ - curl_dogetnameinfo(sa, salen, host, hostlen, serv, servlen, flags, \ - __LINE__, __FILE__) -#endif /* HAVE_GETNAMEINFO */ - #ifdef HAVE_FREEADDRINFO #undef freeaddrinfo #define freeaddrinfo(data) \ diff --git a/Utilities/cmcurl/lib/mime.c b/Utilities/cmcurl/lib/mime.c index 4c0d2eeba..ca492d11a 100644 --- a/Utilities/cmcurl/lib/mime.c +++ b/Utilities/cmcurl/lib/mime.c @@ -505,9 +505,6 @@ static size_t encoder_qp_read(char *buffer, size_t size, bool ateof, mime_encoder_state *st = &part->encstate; char *ptr = buffer; size_t cursize = 0; - int i; - size_t len; - size_t consumed; int softlinebreak; char buf[4]; @@ -516,9 +513,9 @@ static size_t encoder_qp_read(char *buffer, size_t size, bool ateof, character constants that can be interpreted as non-ascii on some platforms. Preserve ASCII encoding on output too. */ while(st->bufbeg < st->bufend) { - len = 1; - consumed = 1; - i = st->buf[st->bufbeg]; + size_t len = 1; + size_t consumed = 1; + int i = st->buf[st->bufbeg]; buf[0] = (char) i; buf[1] = aschex[(i >> 4) & 0xF]; buf[2] = aschex[i & 0xF]; @@ -787,10 +784,10 @@ static size_t read_encoded_part_content(curl_mimepart *part, st->bufbeg = 0; st->bufend = len; } - if(st->bufend >= sizeof st->buf) + if(st->bufend >= sizeof(st->buf)) return cursize? cursize: READ_ERROR; /* Buffer full. */ sz = read_part_content(part, st->buf + st->bufend, - sizeof st->buf - st->bufend); + sizeof(st->buf) - st->bufend); switch(sz) { case 0: ateof = TRUE; @@ -813,8 +810,6 @@ static size_t readback_part(curl_mimepart *part, char *buffer, size_t bufsize) { size_t cursize = 0; - size_t sz; - struct curl_slist *hdr; #ifdef CURL_DOES_CONVERSIONS char *convbuf = buffer; #endif @@ -822,8 +817,8 @@ static size_t readback_part(curl_mimepart *part, /* Readback from part. */ while(bufsize) { - sz = 0; - hdr = (struct curl_slist *) part->state.ptr; + size_t sz = 0; + struct curl_slist *hdr = (struct curl_slist *) part->state.ptr; switch(part->state.state) { case MIMESTATE_BEGIN: mimesetstate(&part->state, part->flags & MIME_BODY_ONLY? MIMESTATE_BODY: @@ -918,8 +913,6 @@ static size_t mime_subparts_read(char *buffer, size_t size, size_t nitems, { curl_mime *mime = (curl_mime *) instream; size_t cursize = 0; - size_t sz; - curl_mimepart *part; #ifdef CURL_DOES_CONVERSIONS char *convbuf = buffer; #endif @@ -927,8 +920,8 @@ static size_t mime_subparts_read(char *buffer, size_t size, size_t nitems, (void) size; /* Always 1. */ while(nitems) { - sz = 0; - part = mime->state.ptr; + size_t sz = 0; + curl_mimepart *part = mime->state.ptr; switch(mime->state.state) { case MIMESTATE_BEGIN: case MIMESTATE_BODY: @@ -1044,7 +1037,6 @@ static int mime_subparts_seek(void *instream, curl_off_t offset, int whence) curl_mime *mime = (curl_mime *) instream; curl_mimepart *part; int result = CURL_SEEKFUNC_OK; - int res; if(whence != SEEK_SET || offset) return CURL_SEEKFUNC_CANTSEEK; /* Only support full rewind. */ @@ -1053,7 +1045,7 @@ static int mime_subparts_seek(void *instream, curl_off_t offset, int whence) return CURL_SEEKFUNC_OK; /* Already rewound. */ for(part = mime->firstpart; part; part = part->nextpart) { - res = mime_part_rewind(part); + int res = mime_part_rewind(part); if(res != CURL_SEEKFUNC_OK) result = res; } @@ -1220,7 +1212,7 @@ curl_mime *curl_mime_init(struct Curl_easy *easy) { curl_mime *mime; - mime = (curl_mime *) malloc(sizeof *mime); + mime = (curl_mime *) malloc(sizeof(*mime)); if(mime) { mime->easy = easy; @@ -1236,8 +1228,13 @@ curl_mime *curl_mime_init(struct Curl_easy *easy) } memset(mime->boundary, '-', 24); - Curl_rand_hex(easy, (unsigned char *) mime->boundary + 24, - MIME_RAND_BOUNDARY_CHARS + 1); + if(Curl_rand_hex(easy, (unsigned char *) mime->boundary + 24, + MIME_RAND_BOUNDARY_CHARS + 1)) { + /* failed to get random separator, bail out */ + free(mime->boundary); + free(mime); + return NULL; + } mimesetstate(&mime->state, MIMESTATE_BEGIN, NULL); } @@ -1247,7 +1244,7 @@ curl_mime *curl_mime_init(struct Curl_easy *easy) /* Initialize a mime part. */ void Curl_mime_initpart(curl_mimepart *part, struct Curl_easy *easy) { - memset((char *) part, 0, sizeof *part); + memset((char *) part, 0, sizeof(*part)); part->easy = easy; mimesetstate(&part->state, MIMESTATE_BEGIN, NULL); } @@ -1260,7 +1257,7 @@ curl_mimepart *curl_mime_addpart(curl_mime *mime) if(!mime) return NULL; - part = (curl_mimepart *) malloc(sizeof *part); + part = (curl_mimepart *) malloc(sizeof(*part)); if(part) { Curl_mime_initpart(part, mime->easy); @@ -1349,7 +1346,6 @@ CURLcode curl_mime_data(curl_mimepart *part, CURLcode curl_mime_filedata(curl_mimepart *part, const char *filename) { CURLcode result = CURLE_OK; - char *base; if(!part) return CURLE_BAD_FUNCTION_ARGUMENT; @@ -1357,6 +1353,7 @@ CURLcode curl_mime_filedata(curl_mimepart *part, const char *filename) cleanup_part_content(part); if(filename) { + char *base; struct_stat sbuf; if(stat(filename, &sbuf) || access(filename, R_OK)) @@ -1564,7 +1561,6 @@ static size_t slist_size(struct curl_slist *s, static curl_off_t multipart_size(curl_mime *mime) { curl_off_t size; - curl_off_t sz; size_t boundarysize; curl_mimepart *part; @@ -1575,7 +1571,7 @@ static curl_off_t multipart_size(curl_mime *mime) size = boundarysize; /* Final boundary - CRLF after headers. */ for(part = mime->firstpart; part; part = part->nextpart) { - sz = Curl_mime_size(part); + curl_off_t sz = Curl_mime_size(part); if(sz < 0) size = sz; @@ -1643,8 +1639,6 @@ static CURLcode add_content_type(struct curl_slist **slp, const char *Curl_mime_contenttype(const char *filename) { - unsigned int i; - /* * If no content type was specified, we scan through a few well-known * extensions and pick the first we match! @@ -1669,8 +1663,9 @@ const char *Curl_mime_contenttype(const char *filename) if(filename) { size_t len1 = strlen(filename); const char *nameend = filename + len1; + unsigned int i; - for(i = 0; i < sizeof ctts / sizeof ctts[0]; i++) { + for(i = 0; i < sizeof(ctts) / sizeof(ctts[0]); i++) { size_t len2 = strlen(ctts[i].extension); if(len1 >= len2 && strcasecompare(nameend - len2, ctts[i].extension)) diff --git a/Utilities/cmcurl/lib/multi.c b/Utilities/cmcurl/lib/multi.c index f85284695..0db2a9730 100644 --- a/Utilities/cmcurl/lib/multi.c +++ b/Utilities/cmcurl/lib/multi.c @@ -31,6 +31,7 @@ #include "progress.h" #include "easyif.h" #include "share.h" +#include "psl.h" #include "multiif.h" #include "sendf.h" #include "timeval.h" @@ -68,8 +69,8 @@ #define GOOD_MULTI_HANDLE(x) \ ((x) && (x)->type == CURL_MULTI_HANDLE) -static void singlesocket(struct Curl_multi *multi, - struct Curl_easy *data); +static CURLMcode singlesocket(struct Curl_multi *multi, + struct Curl_easy *data); static int update_timer(struct Curl_multi *multi); static CURLMcode add_next_timeout(struct curltime now, @@ -106,6 +107,16 @@ static const char * const statename[]={ /* function pointer called once when switching TO a state */ typedef void (*init_multistate_func)(struct Curl_easy *data); +static void Curl_init_completed(struct Curl_easy *data) +{ + /* this is a completed transfer */ + + /* Important: reset the conn pointer so that we don't point to memory + that could be freed anytime */ + data->easy_conn = NULL; + Curl_expire_clear(data); /* stop all timers */ +} + /* always use this function to change state, to make debugging easier */ static void mstate(struct Curl_easy *data, CURLMstate state #ifdef DEBUGBUILD @@ -115,17 +126,25 @@ static void mstate(struct Curl_easy *data, CURLMstate state { CURLMstate oldstate = data->mstate; static const init_multistate_func finit[CURLM_STATE_LAST] = { - NULL, - NULL, + NULL, /* INIT */ + NULL, /* CONNECT_PEND */ Curl_init_CONNECT, /* CONNECT */ - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - Curl_connect_free /* DO */ - /* the rest is NULL too */ + NULL, /* WAITRESOLVE */ + NULL, /* WAITCONNECT */ + NULL, /* WAITPROXYCONNECT */ + NULL, /* SENDPROTOCONNECT */ + NULL, /* PROTOCONNECT */ + NULL, /* WAITDO */ + Curl_connect_free, /* DO */ + NULL, /* DOING */ + NULL, /* DO_MORE */ + NULL, /* DO_DONE */ + NULL, /* WAITPERFORM */ + NULL, /* PERFORM */ + NULL, /* TOOFAST */ + NULL, /* DONE */ + Curl_init_completed, /* COMPLETED */ + NULL /* MSGSENT */ }; #if defined(DEBUGBUILD) && defined(CURL_DISABLE_VERBOSE_STRINGS) @@ -328,6 +347,7 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */ Curl_llist_init(&multi->pending, multi_freeamsg); multi->max_pipeline_length = 5; + multi->pipelining = CURLPIPE_MULTIPLEX; /* -1 means it not set by user, use the default value */ multi->maxconnects = -1; @@ -409,6 +429,14 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi, else data->state.conn_cache = &multi->conn_cache; +#ifdef USE_LIBPSL + /* Do the same for PSL. */ + if(data->share && (data->share->specifier & (1 << CURL_LOCK_DATA_PSL))) + data->psl = &data->share->psl; + else + data->psl = &multi->psl; +#endif + /* This adds the new entry at the 'end' of the doubly-linked circular list of Curl_easy structs to try and maintain a FIFO queue so the pipelined requests are in order. */ @@ -449,7 +477,7 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi, happens if the lastcall time is set to the same time when the handle is removed as when the next handle is added, as then the check in update_timer() that prevents calling the application multiple times with - the same timer infor will not trigger and then the new handle's timeout + the same timer info will not trigger and then the new handle's timeout will not be notified to the app. The work-around is thus simply to clear the 'lastcall' variable to force @@ -464,6 +492,8 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi, data->state.conn_cache->closure_handle->set.timeout = data->set.timeout; data->state.conn_cache->closure_handle->set.server_response_timeout = data->set.server_response_timeout; + data->state.conn_cache->closure_handle->set.no_signal = + data->set.no_signal; update_timer(multi); return CURLM_OK; @@ -506,13 +536,16 @@ static CURLcode multi_done(struct connectdata **connp, /* Stop if multi_done() has already been called */ return CURLE_OK; + if(data->mstate == CURLM_STATE_WAITRESOLVE) { + /* still waiting for the resolve to complete */ + (void)Curl_resolver_wait_resolv(conn, NULL); + } + Curl_getoff_all_pipelines(data, conn); /* Cleanup possible redirect junk */ - free(data->req.newurl); - data->req.newurl = NULL; - free(data->req.location); - data->req.location = NULL; + Curl_safefree(data->req.newurl); + Curl_safefree(data->req.location); switch(status) { case CURLE_ABORTED_BY_CALLBACK: @@ -546,7 +579,7 @@ static CURLcode multi_done(struct connectdata **connp, if(conn->send_pipe.size || conn->recv_pipe.size) { /* Stop if pipeline is not empty . */ data->easy_conn = NULL; - DEBUGF(infof(data, "Connection still in use %d/%d, " + DEBUGF(infof(data, "Connection still in use %zu/%zu, " "no more multi_done now!\n", conn->send_pipe.size, conn->recv_pipe.size)); return CURLE_OK; @@ -560,6 +593,7 @@ static CURLcode multi_done(struct connectdata **connp, conn->dns_entry = NULL; } Curl_hostcache_prune(data); + Curl_safefree(data->state.ulbuf); /* if the transfer was completed in a paused state there can be buffered data left to free */ @@ -590,7 +624,7 @@ static CURLcode multi_done(struct connectdata **connp, #endif ) || conn->bits.close || (premature && !(conn->handler->flags & PROTOPT_STREAM))) { - CURLcode res2 = Curl_disconnect(conn, premature); /* close connection */ + CURLcode res2 = Curl_disconnect(data, conn, premature); /* If we had an error already, make sure we return that one. But if we got a new error, return that. */ @@ -608,7 +642,7 @@ static CURLcode multi_done(struct connectdata **connp, conn->bits.conn_to_host ? conn->conn_to_host.dispname : conn->host.dispname); - /* the connection is no longer in use */ + /* the connection is no longer in use by this transfer */ if(Curl_conncache_return_conn(conn)) { /* remember the most recently used connection */ data->state.lastconnect = conn; @@ -623,7 +657,6 @@ static CURLcode multi_done(struct connectdata **connp, cache here, and therefore cannot be used from this point on */ Curl_free_request_state(data); - return result; } @@ -698,6 +731,11 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi, Curl_getoff_all_pipelines(data, data->easy_conn); } + if(data->connect_queue.ptr) + /* the handle was in the pending list waiting for an available connection, + so go ahead and remove it */ + Curl_llist_remove(&multi->pending, &data->connect_queue, NULL); + if(data->dns.hostcachetype == HCACHE_MULTI) { /* stop using the multi handle's DNS cache, *after* the possible multi_done() call above */ @@ -727,6 +765,12 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi, data->easy_conn = NULL; } +#ifdef USE_LIBPSL + /* Remove the PSL association. */ + if(data->psl == &multi->psl) + data->psl = NULL; +#endif + data->multi = NULL; /* clear the association to this multi handle */ /* make sure there's no pending message in the queue sent from this easy @@ -831,12 +875,10 @@ static int multi_getsock(struct Curl_easy *data, of sockets */ int numsocks) { - /* If the pipe broke, or if there's no connection left for this easy handle, - then we MUST bail out now with no bitmask set. The no connection case can - happen when this is called from curl_multi_remove_handle() => - singlesocket() => multi_getsock(). + /* The no connection case can happen when this is called from + curl_multi_remove_handle() => singlesocket() => multi_getsock(). */ - if(data->state.pipe_broke || !data->easy_conn) + if(!data->easy_conn) return 0; if(data->mstate > CURLM_STATE_CONNECT && @@ -863,7 +905,7 @@ static int multi_getsock(struct Curl_easy *data, return 0; case CURLM_STATE_WAITRESOLVE: - return Curl_resolver_getsock(data->easy_conn, socks, numsocks); + return Curl_resolv_getsock(data->easy_conn, socks, numsocks); case CURLM_STATE_PROTOCONNECT: case CURLM_STATE_SENDPROTOCONNECT: @@ -902,7 +944,6 @@ CURLMcode curl_multi_fdset(struct Curl_multi *multi, struct Curl_easy *data; int this_max_fd = -1; curl_socket_t sockbunch[MAX_SOCKSPEREASYHANDLE]; - int bitmap; int i; (void)exc_fd_set; /* not used */ @@ -914,7 +955,7 @@ CURLMcode curl_multi_fdset(struct Curl_multi *multi, data = multi->easyp; while(data) { - bitmap = multi_getsock(data, sockbunch, MAX_SOCKSPEREASYHANDLE); + int bitmap = multi_getsock(data, sockbunch, MAX_SOCKSPEREASYHANDLE); for(i = 0; i< MAX_SOCKSPEREASYHANDLE; i++) { curl_socket_t s = CURL_SOCKET_BAD; @@ -968,13 +1009,6 @@ CURLMcode curl_multi_wait(struct Curl_multi *multi, if(multi->in_callback) return CURLM_RECURSIVE_API_CALL; - /* If the internally desired timeout is actually shorter than requested from - the outside, then use the shorter time! But only if the internal timer - is actually larger than -1! */ - (void)multi_timeout(multi, &timeout_internal); - if((timeout_internal >= 0) && (timeout_internal < (long)timeout_ms)) - timeout_ms = (int)timeout_internal; - /* Count up how many fds we have from the multi handle */ data = multi->easyp; while(data) { @@ -999,6 +1033,13 @@ CURLMcode curl_multi_wait(struct Curl_multi *multi, data = data->next; /* check next handle */ } + /* If the internally desired timeout is actually shorter than requested from + the outside, then use the shorter time! But only if the internal timer + is actually larger than -1! */ + (void)multi_timeout(multi, &timeout_internal); + if((timeout_internal >= 0) && (timeout_internal < (long)timeout_ms)) + timeout_ms = (int)timeout_internal; + curlfds = nfds; /* number of internal file descriptors */ nfds += extra_nfds; /* add the externally provided ones */ @@ -1068,8 +1109,6 @@ CURLMcode curl_multi_wait(struct Curl_multi *multi, int pollrc; /* wait... */ pollrc = Curl_poll(ufds, nfds, timeout_ms); - DEBUGF(infof(data, "Curl_poll(%d ds, %d ms) == %d\n", - nfds, timeout_ms, pollrc)); if(pollrc > 0) { retcode = pollrc; @@ -1196,7 +1235,7 @@ static CURLcode multi_reconnect_request(struct connectdata **connp) return result; /* Resolved, continue with the connection */ - result = Curl_async_resolved(conn, &protocol_done); + result = Curl_once_resolved(conn, &protocol_done); if(result) return result; } @@ -1312,24 +1351,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, bool stream_error = FALSE; rc = CURLM_OK; - /* Handle the case when the pipe breaks, i.e., the connection - we're using gets cleaned up and we're left with nothing. */ - if(data->state.pipe_broke) { - infof(data, "Pipe broke: handle %p, url = %s\n", - (void *)data, data->state.path); - - if(data->mstate < CURLM_STATE_COMPLETED) { - /* Head back to the CONNECT state */ - multistate(data, CURLM_STATE_CONNECT); - rc = CURLM_CALL_MULTI_PERFORM; - result = CURLE_OK; - } - - data->state.pipe_broke = FALSE; - data->easy_conn = NULL; - continue; - } - if(!data->easy_conn && data->mstate > CURLM_STATE_CONNECT && data->mstate < CURLM_STATE_DONE) { @@ -1490,7 +1511,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, } if(!dns) - result = Curl_resolver_is_resolved(data->easy_conn, &dns); + result = Curl_resolv_check(data->easy_conn, &dns); /* Update sockets here, because the socket(s) may have been closed and the application thus needs to be told, even if it @@ -1503,10 +1524,10 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, if(dns) { /* Perform the next step in the connection phase, and then move on to the WAITCONNECT state */ - result = Curl_async_resolved(data->easy_conn, &protocol_connect); + result = Curl_once_resolved(data->easy_conn, &protocol_connect); if(result) - /* if Curl_async_resolved() returns failure, the connection struct + /* if Curl_once_resolved() returns failure, the connection struct is already freed and gone */ data->easy_conn = NULL; /* no more connection */ else { @@ -1555,6 +1576,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, multistate(data, CURLM_STATE_SENDPROTOCONNECT); } } + else if(result) + stream_error = TRUE; break; #endif @@ -1585,7 +1608,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, case CURLM_STATE_SENDPROTOCONNECT: result = Curl_protocol_connect(data->easy_conn, &protocol_connect); - if(!protocol_connect) + if(!result && !protocol_connect) /* switch to waiting state */ multistate(data, CURLM_STATE_PROTOCONNECT); else if(!result) { @@ -1684,7 +1707,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, char *newurl = NULL; followtype follow = FOLLOW_NONE; CURLcode drc; - bool retry = FALSE; drc = Curl_retry_request(data->easy_conn, &newurl); if(drc) { @@ -1692,15 +1714,13 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, result = drc; stream_error = TRUE; } - else - retry = (newurl)?TRUE:FALSE; Curl_posttransfer(data); drc = multi_done(&data->easy_conn, result, FALSE); /* When set to retry the connection, we must to go back to * the CONNECT state */ - if(retry) { + if(newurl) { if(!drc || (drc == CURLE_SEND_ERROR)) { follow = FOLLOW_RETRY; drc = Curl_follow(data, newurl, follow); @@ -1909,6 +1929,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, CURLcode ret = Curl_retry_request(data->easy_conn, &newurl); if(!ret) retry = (newurl)?TRUE:FALSE; + else if(!result) + result = ret; if(retry) { /* if we are to retry, set the result to OK and consider the @@ -1968,6 +1990,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, rc = CURLM_CALL_MULTI_PERFORM; } } + free(newurl); } else { /* after the transfer is done, go DONE */ @@ -1979,18 +2002,21 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, newurl = data->req.location; data->req.location = NULL; result = Curl_follow(data, newurl, FOLLOW_FAKE); - if(result) + free(newurl); + if(result) { stream_error = TRUE; + result = multi_done(&data->easy_conn, result, TRUE); + } } - multistate(data, CURLM_STATE_DONE); - rc = CURLM_CALL_MULTI_PERFORM; + if(!result) { + multistate(data, CURLM_STATE_DONE); + rc = CURLM_CALL_MULTI_PERFORM; + } } } else if(comeback) rc = CURLM_CALL_MULTI_PERFORM; - - free(newurl); break; } @@ -2040,16 +2066,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, break; case CURLM_STATE_COMPLETED: - /* this is a completed transfer, it is likely to still be connected */ - - /* This node should be delinked from the list now and we should post - an information message that we are complete. */ - - /* Important: reset the conn pointer so that we don't point to memory - that could be freed anytime */ - data->easy_conn = NULL; - - Curl_expire_clear(data); /* stop all timers */ break; case CURLM_STATE_MSGSENT: @@ -2071,8 +2087,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, /* NOTE: no attempt to disconnect connections must be made in the case blocks above - cleanup happens only here */ - data->state.pipe_broke = FALSE; - /* Check if we can move pending requests to send pipe */ process_pending_handles(multi); /* connection */ @@ -2087,7 +2101,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, /* Don't attempt to send data over a connection that timed out */ bool dead_connection = result == CURLE_OPERATION_TIMEDOUT; /* disconnect properly */ - Curl_disconnect(data->easy_conn, dead_connection); + Curl_disconnect(data, data->easy_conn, dead_connection); /* This is where we make sure that the easy_conn pointer is reset. We don't have to do this in every case block above where a @@ -2101,6 +2115,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, } multistate(data, CURLM_STATE_COMPLETED); + rc = CURLM_CALL_MULTI_PERFORM; } /* if there's still a connection to use, call the progress function */ else if(data->easy_conn && Curl_pgrsUpdate(data->easy_conn)) { @@ -2117,22 +2132,26 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, } if(CURLM_STATE_COMPLETED == data->mstate) { - /* now fill in the Curl_message with this info */ - msg = &data->msg; - - msg->extmsg.msg = CURLMSG_DONE; - msg->extmsg.easy_handle = data; - msg->extmsg.data.result = result; + if(data->set.fmultidone) { + /* signal via callback instead */ + data->set.fmultidone(data, result); + } + else { + /* now fill in the Curl_message with this info */ + msg = &data->msg; - rc = multi_addmsg(multi, msg); + msg->extmsg.msg = CURLMSG_DONE; + msg->extmsg.easy_handle = data; + msg->extmsg.data.result = result; + rc = multi_addmsg(multi, msg); + DEBUGASSERT(!data->easy_conn); + } multistate(data, CURLM_STATE_MSGSENT); } } while((rc == CURLM_CALL_MULTI_PERFORM) || multi_ischanged(multi, FALSE)); data->result = result; - - return rc; } @@ -2220,6 +2239,11 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi) data->state.conn_cache = NULL; data->multi = NULL; /* clear the association */ +#ifdef USE_LIBPSL + if(data->psl == &multi->psl) + data->psl = NULL; +#endif + data = nextdata; } @@ -2232,6 +2256,7 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi) Curl_llist_destroy(&multi->pending, NULL); Curl_hash_destroy(&multi->hostcache); + Curl_psl_destroy(&multi->psl); /* Free the blacklists by setting them to NULL */ Curl_pipeline_set_site_blacklist(NULL, &multi->pipelining_site_bl); @@ -2286,8 +2311,8 @@ CURLMsg *curl_multi_info_read(struct Curl_multi *multi, int *msgs_in_queue) * and if we have a different state in any of those sockets from last time we * call the callback accordingly. */ -static void singlesocket(struct Curl_multi *multi, - struct Curl_easy *data) +static CURLMcode singlesocket(struct Curl_multi *multi, + struct Curl_easy *data) { curl_socket_t socks[MAX_SOCKSPEREASYHANDLE]; int i; @@ -2334,7 +2359,7 @@ static void singlesocket(struct Curl_multi *multi, entry = sh_addentry(&multi->sockhash, s, data); if(!entry) /* fatal */ - return; + return CURLM_OUT_OF_MEMORY; } /* we know (entry != NULL) at this point, see the logic above */ @@ -2422,6 +2447,7 @@ static void singlesocket(struct Curl_multi *multi, memcpy(data->sockets, socks, num*sizeof(curl_socket_t)); data->numsocks = num; + return CURLM_OK; } void Curl_updatesocket(struct Curl_easy *data) @@ -2442,20 +2468,23 @@ void Curl_updatesocket(struct Curl_easy *data) void Curl_multi_closed(struct connectdata *conn, curl_socket_t s) { - struct Curl_multi *multi = conn->data->multi; - if(multi) { - /* this is set if this connection is part of a handle that is added to - a multi handle, and only then this is necessary */ - struct Curl_sh_entry *entry = sh_getentry(&multi->sockhash, s); - - if(entry) { - if(multi->socket_cb) - multi->socket_cb(conn->data, s, CURL_POLL_REMOVE, - multi->socket_userp, - entry->socketp); + if(conn->data) { + /* if there's still an easy handle associated with this connection */ + struct Curl_multi *multi = conn->data->multi; + if(multi) { + /* this is set if this connection is part of a handle that is added to + a multi handle, and only then this is necessary */ + struct Curl_sh_entry *entry = sh_getentry(&multi->sockhash, s); + + if(entry) { + if(multi->socket_cb) + multi->socket_cb(conn->data, s, CURL_POLL_REMOVE, + multi->socket_userp, + entry->socketp); - /* now remove it from the socket hash */ - sh_delentry(&multi->sockhash, s); + /* now remove it from the socket hash */ + sh_delentry(&multi->sockhash, s); + } } } } @@ -2535,8 +2564,8 @@ static CURLMcode multi_socket(struct Curl_multi *multi, and callbacks */ if(result != CURLM_BAD_HANDLE) { data = multi->easyp; - while(data) { - singlesocket(multi, data); + while(data && !result) { + result = singlesocket(multi, data); data = data->next; } } @@ -2590,10 +2619,13 @@ static CURLMcode multi_socket(struct Curl_multi *multi, /* clear the bitmask only if not locked */ data->easy_conn->cselect_bits = 0; - if(CURLM_OK >= result) + if(CURLM_OK >= result) { /* get the socket(s) and check if the state has been changed since last */ - singlesocket(multi, data); + result = singlesocket(multi, data); + if(result) + return result; + } /* Now we fall-through and do the timer-based stuff, since we don't want to force the user to have to deal with timeouts as long as at least @@ -2627,10 +2659,13 @@ static CURLMcode multi_socket(struct Curl_multi *multi, result = multi_runsingle(multi, now, data); sigpipe_restore(&pipe_st); - if(CURLM_OK >= result) + if(CURLM_OK >= result) { /* get the socket(s) and check if the state has been changed since last */ - singlesocket(multi, data); + result = singlesocket(multi, data); + if(result) + return result; + } } /* Check if there's one (more) expired timer to deal with! This function @@ -2677,7 +2712,7 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, multi->push_userp = va_arg(param, void *); break; case CURLMOPT_PIPELINING: - multi->pipelining = va_arg(param, long); + multi->pipelining = va_arg(param, long) & CURLPIPE_MULTIPLEX; break; case CURLMOPT_TIMERFUNCTION: multi->timer_cb = va_arg(param, curl_multi_timer_callback); @@ -2924,7 +2959,6 @@ void Curl_expire(struct Curl_easy *data, time_t milli, expire_id id) { struct Curl_multi *multi = data->multi; struct curltime *nowp = &data->state.expiretime; - int rc; struct curltime set; /* this is only interesting while there is still an associated multi struct @@ -2955,6 +2989,7 @@ void Curl_expire(struct Curl_easy *data, time_t milli, expire_id id) Compare if the new time is earlier, and only remove-old/add-new if it is. */ timediff_t diff = Curl_timediff(set, *nowp); + int rc; if(diff > 0) { /* The current splay tree entry is sooner than this new expiry time. @@ -3000,7 +3035,6 @@ void Curl_expire_clear(struct Curl_easy *data) { struct Curl_multi *multi = data->multi; struct curltime *nowp = &data->state.expiretime; - int rc; /* this is only interesting while there is still an associated multi struct remaining! */ @@ -3011,6 +3045,7 @@ void Curl_expire_clear(struct Curl_easy *data) /* Since this is an cleared time, we must remove the previous entry from the splay tree */ struct curl_llist *list = &data->state.timeoutlist; + int rc; rc = Curl_splayremovebyaddr(multi->timetree, &data->state.timenode, @@ -3100,12 +3135,15 @@ static void process_pending_handles(struct Curl_multi *multi) } } -void Curl_set_in_callback(struct Curl_easy *easy, bool value) +void Curl_set_in_callback(struct Curl_easy *data, bool value) { - if(easy->multi_easy) - easy->multi_easy->in_callback = value; - else if(easy->multi) - easy->multi->in_callback = value; + /* might get called when there is no data pointer! */ + if(data) { + if(data->multi_easy) + data->multi_easy->in_callback = value; + else if(data->multi) + data->multi->in_callback = value; + } } bool Curl_is_in_callback(struct Curl_easy *easy) diff --git a/Utilities/cmcurl/lib/multihandle.h b/Utilities/cmcurl/lib/multihandle.h index 1a5017f4a..ea2bf352d 100644 --- a/Utilities/cmcurl/lib/multihandle.h +++ b/Utilities/cmcurl/lib/multihandle.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -23,6 +23,7 @@ ***************************************************************************/ #include "conncache.h" +#include "psl.h" struct Curl_message { struct curl_llist_element list; @@ -97,6 +98,11 @@ struct Curl_multi { /* Hostname cache */ struct curl_hash hostcache; +#ifdef USE_LIBPSL + /* PSL cache. */ + struct PslCache psl; +#endif + /* timetree points to the splay-tree of time nodes to figure out expire times of all currently set timers */ struct Curl_tree *timetree; diff --git a/Utilities/cmcurl/lib/netrc.c b/Utilities/cmcurl/lib/netrc.c index dbcc59ac9..1724b35b0 100644 --- a/Utilities/cmcurl/lib/netrc.c +++ b/Utilities/cmcurl/lib/netrc.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -57,7 +57,11 @@ int Curl_parsenetrc(const char *host, { FILE *file; int retcode = 1; - int specific_login = (*loginp && **loginp != 0); + char *login = *loginp; + char *password = *passwordp; + bool specific_login = (login && *login != 0); + bool login_alloc = FALSE; + bool password_alloc = FALSE; bool netrc_alloc = FALSE; enum host_lookup_state state = NOTHING; @@ -115,7 +119,7 @@ int Curl_parsenetrc(const char *host, char *tok; char *tok_buf; bool done = FALSE; - char netrcbuffer[256]; + char netrcbuffer[4096]; int netrcbuffsize = (int)sizeof(netrcbuffer); while(!done && fgets(netrcbuffer, netrcbuffsize, file)) { @@ -125,7 +129,7 @@ int Curl_parsenetrc(const char *host, continue; while(!done && tok) { - if((*loginp && **loginp) && (*passwordp && **passwordp)) { + if((login && *login) && (password && *password)) { done = TRUE; break; } @@ -158,26 +162,34 @@ int Curl_parsenetrc(const char *host, /* we are now parsing sub-keywords concerning "our" host */ if(state_login) { if(specific_login) { - state_our_login = strcasecompare(*loginp, tok); + state_our_login = strcasecompare(login, tok); } else { - free(*loginp); - *loginp = strdup(tok); - if(!*loginp) { + if(login_alloc) { + free(login); + login_alloc = FALSE; + } + login = strdup(tok); + if(!login) { retcode = -1; /* allocation failed */ goto out; } + login_alloc = TRUE; } state_login = 0; } else if(state_password) { if(state_our_login || !specific_login) { - free(*passwordp); - *passwordp = strdup(tok); - if(!*passwordp) { + if(password_alloc) { + free(password); + password_alloc = FALSE; + } + password = strdup(tok); + if(!password) { retcode = -1; /* allocation failed */ goto out; } + password_alloc = TRUE; } state_password = 0; } @@ -198,6 +210,24 @@ int Curl_parsenetrc(const char *host, } /* while fgets() */ out: + if(!retcode) { + if(login_alloc) { + if(*loginp) + free(*loginp); + *loginp = login; + } + if(password_alloc) { + if(*passwordp) + free(*passwordp); + *passwordp = password; + } + } + else { + if(login_alloc) + free(login); + if(password_alloc) + free(password); + } fclose(file); } diff --git a/Utilities/cmcurl/lib/nonblock.c b/Utilities/cmcurl/lib/nonblock.c index 5959281e5..4d105c1fe 100644 --- a/Utilities/cmcurl/lib/nonblock.c +++ b/Utilities/cmcurl/lib/nonblock.c @@ -48,7 +48,8 @@ int curlx_nonblock(curl_socket_t sockfd, /* operate on this */ int nonblock /* TRUE or FALSE */) { #if defined(USE_BLOCKING_SOCKETS) - + (void)sockfd; + (void)nonblock; return 0; /* returns success */ #elif defined(HAVE_FCNTL_O_NONBLOCK) diff --git a/Utilities/cmcurl/lib/nonblock.h b/Utilities/cmcurl/lib/nonblock.h index 98cdc25ab..eb18ea1c3 100644 --- a/Utilities/cmcurl/lib/nonblock.h +++ b/Utilities/cmcurl/lib/nonblock.h @@ -28,4 +28,3 @@ int curlx_nonblock(curl_socket_t sockfd, /* operate on this */ int nonblock /* TRUE or FALSE */); #endif /* HEADER_CURL_NONBLOCK_H */ - diff --git a/Utilities/cmcurl/lib/nwlib.c b/Utilities/cmcurl/lib/nwlib.c index 215d933ac..7bf5f51c3 100644 --- a/Utilities/cmcurl/lib/nwlib.c +++ b/Utilities/cmcurl/lib/nwlib.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -195,7 +195,7 @@ int GetOrSetUpData(int id, libdata_t **appData, if(!app_data->tenbytes || !app_data->lock) { if(app_data->lock) NXMutexFree(app_data->lock); - + free(app_data->tenbytes); free(app_data); app_data = (libdata_t *) NULL; err = ENOMEM; @@ -213,6 +213,9 @@ int GetOrSetUpData(int id, libdata_t **appData, err = set_app_data(gLibId, app_data); if(err) { + if(app_data->lock) + NXMutexFree(app_data->lock); + free(app_data->tenbytes); free(app_data); app_data = (libdata_t *) NULL; err = ENOMEM; diff --git a/Utilities/cmcurl/lib/parsedate.h b/Utilities/cmcurl/lib/parsedate.h index 2e59eb17c..8dc3b90ec 100644 --- a/Utilities/cmcurl/lib/parsedate.h +++ b/Utilities/cmcurl/lib/parsedate.h @@ -28,4 +28,3 @@ extern const char * const Curl_month[12]; CURLcode Curl_gmtime(time_t intime, struct tm *store); #endif /* HEADER_CURL_PARSEDATE_H */ - diff --git a/Utilities/cmcurl/lib/pingpong.c b/Utilities/cmcurl/lib/pingpong.c index ad370ee82..2e93d201f 100644 --- a/Utilities/cmcurl/lib/pingpong.c +++ b/Utilities/cmcurl/lib/pingpong.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -49,7 +49,6 @@ time_t Curl_pp_state_timeout(struct pingpong *pp) struct connectdata *conn = pp->conn; struct Curl_easy *data = conn->data; time_t timeout_ms; /* in milliseconds */ - time_t timeout2_ms; /* in milliseconds */ long response_time = (data->set.server_response_timeout)? data->set.server_response_timeout: pp->response_time; @@ -65,7 +64,7 @@ time_t Curl_pp_state_timeout(struct pingpong *pp) if(data->set.timeout) { /* if timeout is requested, find out how much remaining time we have */ - timeout2_ms = data->set.timeout - /* timeout time */ + time_t timeout2_ms = data->set.timeout - /* timeout time */ Curl_timediff(Curl_now(), conn->now); /* spent time */ /* pick the lowest number */ @@ -222,8 +221,7 @@ CURLcode Curl_pp_vsendf(struct pingpong *pp, } if(conn->data->set.verbose) - Curl_debug(conn->data, CURLINFO_HEADER_OUT, - s, (size_t)bytes_written, conn); + Curl_debug(conn->data, CURLINFO_HEADER_OUT, s, (size_t)bytes_written); if(bytes_written != (ssize_t)write_len) { /* the whole chunk was not sent, keep it around and adjust sizes */ @@ -371,7 +369,7 @@ CURLcode Curl_pp_readresp(curl_socket_t sockfd, #endif if(data->set.verbose) Curl_debug(data, CURLINFO_HEADER_IN, - pp->linestart_resp, (size_t)perline, conn); + pp->linestart_resp, (size_t)perline); /* * We pass all response-lines to the callback function registered diff --git a/Utilities/cmcurl/lib/pipeline.c b/Utilities/cmcurl/lib/pipeline.c index 4d41b0413..8de3babd7 100644 --- a/Utilities/cmcurl/lib/pipeline.c +++ b/Utilities/cmcurl/lib/pipeline.c @@ -6,7 +6,7 @@ * \___|\___/|_| \_\_____| * * Copyright (C) 2013, Linus Nielsen Feltzing, <linus@haxx.se> - * Copyright (C) 2013 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 2013 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -82,7 +82,8 @@ bool Curl_pipeline_penalized(struct Curl_easy *data, penalized = TRUE; infof(data, "Conn: %ld (%p) Receive pipe weight: (%" - CURL_FORMAT_CURL_OFF_T "/%zu), penalized: %s\n", + CURL_FORMAT_CURL_OFF_T "/%" CURL_FORMAT_CURL_OFF_T + "), penalized: %s\n", conn->connection_id, (void *)conn, recv_size, conn->chunk.datasize, penalized?"TRUE":"FALSE"); return penalized; @@ -109,8 +110,8 @@ CURLcode Curl_add_handle_to_pipeline(struct Curl_easy *handle, pipeline = &conn->send_pipe; result = addHandleToPipeline(handle, pipeline); - - if(pipeline == &conn->send_pipe && sendhead != conn->send_pipe.head) { + if((conn->bundle->multiuse == BUNDLE_PIPELINING) && + (pipeline == &conn->send_pipe && sendhead != conn->send_pipe.head)) { /* this is a new one as head, expire it */ Curl_pipeline_leave_write(conn); /* not in use yet */ Curl_expire(conn->send_pipe.head->ptr, 0, EXPIRE_RUN_NOW); diff --git a/Utilities/cmcurl/lib/pop3.c b/Utilities/cmcurl/lib/pop3.c index 78f6afef1..5e0fd2299 100644 --- a/Utilities/cmcurl/lib/pop3.c +++ b/Utilities/cmcurl/lib/pop3.c @@ -613,7 +613,6 @@ static CURLcode pop3_state_servergreet_resp(struct connectdata *conn, struct pop3_conn *pop3c = &conn->proto.pop3c; const char *line = data->state.buffer; size_t len = strlen(line); - size_t i; (void)instate; /* no use for this yet */ @@ -625,6 +624,7 @@ static CURLcode pop3_state_servergreet_resp(struct connectdata *conn, /* Does the server support APOP authentication? */ if(len >= 4 && line[len - 2] == '>') { /* Look for the APOP timestamp */ + size_t i; for(i = 3; i < len - 2; ++i) { if(line[i] == '<') { /* Calculate the length of the timestamp */ @@ -664,7 +664,6 @@ static CURLcode pop3_state_capa_resp(struct connectdata *conn, int pop3code, struct pop3_conn *pop3c = &conn->proto.pop3c; const char *line = data->state.buffer; size_t len = strlen(line); - size_t wordlen; (void)instate; /* no use for this yet */ @@ -689,6 +688,7 @@ static CURLcode pop3_state_capa_resp(struct connectdata *conn, int pop3code, /* Loop through the data line */ for(;;) { size_t llen; + size_t wordlen; unsigned int mechbit; while(len && @@ -1303,8 +1303,6 @@ static CURLcode pop3_regular_transfer(struct connectdata *conn, static CURLcode pop3_setup_connection(struct connectdata *conn) { - struct Curl_easy *data = conn->data; - /* Initialise the POP3 layer */ CURLcode result = pop3_init(conn); if(result) @@ -1312,7 +1310,6 @@ static CURLcode pop3_setup_connection(struct connectdata *conn) /* Clear the TLS upgraded flag */ conn->tls_upgraded = FALSE; - data->state.path++; /* don't include the initial slash */ return CURLE_OK; } @@ -1387,7 +1384,7 @@ static CURLcode pop3_parse_url_path(struct connectdata *conn) /* The POP3 struct is already initialised in pop3_connect() */ struct Curl_easy *data = conn->data; struct POP3 *pop3 = data->req.protop; - const char *path = data->state.path; + const char *path = &data->state.up.path[1]; /* skip leading path */ /* URL decode the path for the message ID */ return Curl_urldecode(data, path, 0, &pop3->id, NULL, TRUE); diff --git a/Utilities/cmcurl/lib/progress.c b/Utilities/cmcurl/lib/progress.c index f59faa3d3..a94668dc2 100644 --- a/Utilities/cmcurl/lib/progress.c +++ b/Utilities/cmcurl/lib/progress.c @@ -35,22 +35,22 @@ byte) */ static void time2str(char *r, curl_off_t seconds) { - curl_off_t d, h, m, s; + curl_off_t h; if(seconds <= 0) { strcpy(r, "--:--:--"); return; } h = seconds / CURL_OFF_T_C(3600); if(h <= CURL_OFF_T_C(99)) { - m = (seconds - (h*CURL_OFF_T_C(3600))) / CURL_OFF_T_C(60); - s = (seconds - (h*CURL_OFF_T_C(3600))) - (m*CURL_OFF_T_C(60)); + curl_off_t m = (seconds - (h*CURL_OFF_T_C(3600))) / CURL_OFF_T_C(60); + curl_off_t s = (seconds - (h*CURL_OFF_T_C(3600))) - (m*CURL_OFF_T_C(60)); snprintf(r, 9, "%2" CURL_FORMAT_CURL_OFF_T ":%02" CURL_FORMAT_CURL_OFF_T ":%02" CURL_FORMAT_CURL_OFF_T, h, m, s); } else { /* this equals to more than 99 hours, switch to a more suitable output format to fit within the limits. */ - d = seconds / CURL_OFF_T_C(86400); + curl_off_t d = seconds / CURL_OFF_T_C(86400); h = (seconds - (d*CURL_OFF_T_C(86400))) / CURL_OFF_T_C(3600); if(d <= CURL_OFF_T_C(999)) snprintf(r, 9, "%3" CURL_FORMAT_CURL_OFF_T @@ -369,25 +369,10 @@ void Curl_pgrsSetUploadSize(struct Curl_easy *data, curl_off_t size) int Curl_pgrsUpdate(struct connectdata *conn) { struct curltime now; - int result; - char max5[6][10]; - curl_off_t dlpercen = 0; - curl_off_t ulpercen = 0; - curl_off_t total_percen = 0; - curl_off_t total_transfer; - curl_off_t total_expected_transfer; curl_off_t timespent; curl_off_t timespent_ms; /* milliseconds */ struct Curl_easy *data = conn->data; int nowindex = data->progress.speeder_c% CURR_TIME; - int checkindex; - int countindex; /* amount of seconds stored in the speeder array */ - char time_left[10]; - char time_total[10]; - char time_spent[10]; - curl_off_t ulestimate = 0; - curl_off_t dlestimate = 0; - curl_off_t total_estimate; bool shownow = FALSE; curl_off_t dl = data->progress.downloaded; curl_off_t ul = data->progress.uploaded; @@ -413,6 +398,7 @@ int Curl_pgrsUpdate(struct connectdata *conn) /* Calculations done at most once a second, unless end is reached */ if(data->progress.lastshow != now.tv_sec) { + int countindex; /* amount of seconds stored in the speeder array */ shownow = TRUE; data->progress.lastshow = now.tv_sec; @@ -438,6 +424,7 @@ int Curl_pgrsUpdate(struct connectdata *conn) /* first of all, we don't do this if there's no counted seconds yet */ if(countindex) { + int checkindex; timediff_t span_ms; /* Get the index position to compare with the 'nowindex' position. @@ -477,8 +464,21 @@ int Curl_pgrsUpdate(struct connectdata *conn) if(!(data->progress.flags & PGRS_HIDE)) { /* progress meter has not been shut off */ + char max5[6][10]; + curl_off_t dlpercen = 0; + curl_off_t ulpercen = 0; + curl_off_t total_percen = 0; + curl_off_t total_transfer; + curl_off_t total_expected_transfer; + char time_left[10]; + char time_total[10]; + char time_spent[10]; + curl_off_t ulestimate = 0; + curl_off_t dlestimate = 0; + curl_off_t total_estimate; if(data->set.fxferinfo) { + int result; /* There's a callback set, call that */ Curl_set_in_callback(data, true); result = data->set.fxferinfo(data->set.progress_client, @@ -492,6 +492,7 @@ int Curl_pgrsUpdate(struct connectdata *conn) return result; } if(data->set.fprogress) { + int result; /* The older deprecated callback is set, call that */ Curl_set_in_callback(data, true); result = data->set.fprogress(data->set.progress_client, diff --git a/Utilities/cmcurl/lib/progress.h b/Utilities/cmcurl/lib/progress.h index 2baa925db..3515ac6d5 100644 --- a/Utilities/cmcurl/lib/progress.h +++ b/Utilities/cmcurl/lib/progress.h @@ -56,23 +56,9 @@ timediff_t Curl_pgrsLimitWaitTime(curl_off_t cursize, struct curltime start, struct curltime now); -/* Don't show progress for sizes smaller than: */ -#define LEAST_SIZE_PROGRESS BUFSIZE - -#define PROGRESS_DOWNLOAD (1<<0) -#define PROGRESS_UPLOAD (1<<1) -#define PROGRESS_DOWN_AND_UP (PROGRESS_UPLOAD | PROGRESS_DOWNLOAD) - -#define PGRS_SHOW_DL (1<<0) -#define PGRS_SHOW_UL (1<<1) -#define PGRS_DONE_DL (1<<2) -#define PGRS_DONE_UL (1<<3) #define PGRS_HIDE (1<<4) #define PGRS_UL_SIZE_KNOWN (1<<5) #define PGRS_DL_SIZE_KNOWN (1<<6) - #define PGRS_HEADERS_OUT (1<<7) /* set when the headers have been written */ - #endif /* HEADER_CURL_PROGRESS_H */ - diff --git a/Utilities/cmcurl/lib/psl.c b/Utilities/cmcurl/lib/psl.c new file mode 100644 index 000000000..568baff03 --- /dev/null +++ b/Utilities/cmcurl/lib/psl.c @@ -0,0 +1,111 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "curl_setup.h" + +#include <curl/curl.h> + +#ifdef USE_LIBPSL + +#include "psl.h" +#include "share.h" + +/* The last 3 #include files should be in this order */ +#include "curl_printf.h" +#include "curl_memory.h" +#include "memdebug.h" + +void Curl_psl_destroy(struct PslCache *pslcache) +{ + if(pslcache->psl) { + if(pslcache->dynamic) + psl_free((psl_ctx_t *) pslcache->psl); + pslcache->psl = NULL; + pslcache->dynamic = FALSE; + } +} + +static time_t now_seconds(void) +{ + struct curltime now = Curl_now(); + + return now.tv_sec; +} + +const psl_ctx_t *Curl_psl_use(struct Curl_easy *easy) +{ + struct PslCache *pslcache = easy->psl; + const psl_ctx_t *psl; + time_t now; + + if(!pslcache) + return NULL; + + Curl_share_lock(easy, CURL_LOCK_DATA_PSL, CURL_LOCK_ACCESS_SHARED); + now = now_seconds(); + if(!pslcache->psl || pslcache->expires <= now) { + /* Let a chance to other threads to do the job: avoids deadlock. */ + Curl_share_unlock(easy, CURL_LOCK_DATA_PSL); + + /* Update cache: this needs an exclusive lock. */ + Curl_share_lock(easy, CURL_LOCK_DATA_PSL, CURL_LOCK_ACCESS_SINGLE); + + /* Recheck in case another thread did the job. */ + now = now_seconds(); + if(!pslcache->psl || pslcache->expires <= now) { + bool dynamic = FALSE; + time_t expires = TIME_T_MAX; + +#if defined(PSL_VERSION_NUMBER) && PSL_VERSION_NUMBER >= 0x001000 + psl = psl_latest(NULL); + dynamic = psl != NULL; + /* Take care of possible time computation overflow. */ + expires = now < TIME_T_MAX - PSL_TTL? now + PSL_TTL: TIME_T_MAX; + + /* Only get the built-in PSL if we do not already have the "latest". */ + if(!psl && !pslcache->dynamic) +#endif + + psl = psl_builtin(); + + if(psl) { + Curl_psl_destroy(pslcache); + pslcache->psl = psl; + pslcache->dynamic = dynamic; + pslcache->expires = expires; + } + } + Curl_share_unlock(easy, CURL_LOCK_DATA_PSL); /* Release exclusive lock. */ + Curl_share_lock(easy, CURL_LOCK_DATA_PSL, CURL_LOCK_ACCESS_SHARED); + } + psl = pslcache->psl; + if(!psl) + Curl_share_unlock(easy, CURL_LOCK_DATA_PSL); + return psl; +} + +void Curl_psl_release(struct Curl_easy *easy) +{ + Curl_share_unlock(easy, CURL_LOCK_DATA_PSL); +} + +#endif /* USE_LIBPSL */ diff --git a/Utilities/cmcurl/lib/psl.h b/Utilities/cmcurl/lib/psl.h new file mode 100644 index 000000000..e9f99d03e --- /dev/null +++ b/Utilities/cmcurl/lib/psl.h @@ -0,0 +1,47 @@ +#ifndef HEADER_PSL_H +#define HEADER_PSL_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#ifdef USE_LIBPSL +#include <libpsl.h> + +#define PSL_TTL (72 * 3600) /* PSL time to live before a refresh. */ + +struct PslCache { + const psl_ctx_t *psl; /* The PSL. */ + time_t expires; /* Time this PSL life expires. */ + bool dynamic; /* PSL should be released when no longer needed. */ +}; + +const psl_ctx_t *Curl_psl_use(struct Curl_easy *easy); +void Curl_psl_release(struct Curl_easy *easy); +void Curl_psl_destroy(struct PslCache *pslcache); + +#else + +#define Curl_psl_use(easy) NULL +#define Curl_psl_release(easy) +#define Curl_psl_destroy(pslcache) + +#endif /* USE_LIBPSL */ +#endif /* HEADER_PSL_H */ diff --git a/Utilities/cmcurl/lib/rand.c b/Utilities/cmcurl/lib/rand.c index 0769ed151..6ee45feb1 100644 --- a/Utilities/cmcurl/lib/rand.c +++ b/Utilities/cmcurl/lib/rand.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -158,7 +158,7 @@ CURLcode Curl_rand_hex(struct Curl_easy *data, unsigned char *rnd, DEBUGASSERT(num > 1); #ifdef __clang_analyzer__ - /* This silences a scan-build warning about accesssing this buffer with + /* This silences a scan-build warning about accessing this buffer with uninitialized memory. */ memset(buffer, 0, sizeof(buffer)); #endif @@ -174,6 +174,8 @@ CURLcode Curl_rand_hex(struct Curl_easy *data, unsigned char *rnd, return result; while(num) { + /* clang-tidy warns on this line without this comment: */ + /* NOLINTNEXTLINE(clang-analyzer-core.UndefinedBinaryOperatorResult) */ *rnd++ = hex[(*bufp & 0xF0)>>4]; *rnd++ = hex[*bufp & 0x0F]; bufp++; diff --git a/Utilities/cmcurl/lib/rtsp.c b/Utilities/cmcurl/lib/rtsp.c index 41f300013..01dfce640 100644 --- a/Utilities/cmcurl/lib/rtsp.c +++ b/Utilities/cmcurl/lib/rtsp.c @@ -218,8 +218,6 @@ static CURLcode rtsp_done(struct connectdata *conn, struct Curl_easy *data = conn->data; struct RTSP *rtsp = data->req.protop; CURLcode httpStatus; - long CSeq_sent; - long CSeq_recv; /* Bypass HTTP empty-reply checks on receive */ if(data->set.rtspreq == RTSPREQ_RECEIVE) @@ -229,8 +227,8 @@ static CURLcode rtsp_done(struct connectdata *conn, if(rtsp) { /* Check the sequence numbers */ - CSeq_sent = rtsp->CSeq_sent; - CSeq_recv = rtsp->CSeq_recv; + long CSeq_sent = rtsp->CSeq_sent; + long CSeq_recv = rtsp->CSeq_recv; if((data->set.rtspreq != RTSPREQ_RECEIVE) && (CSeq_sent != CSeq_recv)) { failf(data, "The CSeq of this request %ld did not match the response %ld", @@ -464,7 +462,7 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done) return CURLE_OUT_OF_MEMORY; result = - Curl_add_bufferf(req_buffer, + Curl_add_bufferf(&req_buffer, "%s %s RTSP/1.0\r\n" /* Request Stream-URI RTSP/1.0 */ "CSeq: %ld\r\n", /* CSeq */ p_request, p_stream_uri, rtsp->CSeq_sent); @@ -476,7 +474,7 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done) * to make comparison easier */ if(p_session_id) { - result = Curl_add_bufferf(req_buffer, "Session: %s\r\n", p_session_id); + result = Curl_add_bufferf(&req_buffer, "Session: %s\r\n", p_session_id); if(result) return result; } @@ -484,7 +482,7 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done) /* * Shared HTTP-like options */ - result = Curl_add_bufferf(req_buffer, + result = Curl_add_bufferf(&req_buffer, "%s" /* transport */ "%s" /* accept */ "%s" /* accept-encoding */ @@ -543,9 +541,10 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done) /* As stated in the http comments, it is probably not wise to * actually set a custom Content-Length in the headers */ if(!Curl_checkheaders(conn, "Content-Length")) { - result = Curl_add_bufferf(req_buffer, - "Content-Length: %" CURL_FORMAT_CURL_OFF_T"\r\n", - (data->set.upload ? putsize : postsize)); + result = + Curl_add_bufferf(&req_buffer, + "Content-Length: %" CURL_FORMAT_CURL_OFF_T"\r\n", + (data->set.upload ? putsize : postsize)); if(result) return result; } @@ -553,8 +552,8 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done) if(rtspreq == RTSPREQ_SET_PARAMETER || rtspreq == RTSPREQ_GET_PARAMETER) { if(!Curl_checkheaders(conn, "Content-Type")) { - result = Curl_add_bufferf(req_buffer, - "Content-Type: text/parameters\r\n"); + result = Curl_add_bufferf(&req_buffer, + "Content-Type: text/parameters\r\n"); if(result) return result; } @@ -562,8 +561,8 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done) if(rtspreq == RTSPREQ_ANNOUNCE) { if(!Curl_checkheaders(conn, "Content-Type")) { - result = Curl_add_bufferf(req_buffer, - "Content-Type: application/sdp\r\n"); + result = Curl_add_bufferf(&req_buffer, + "Content-Type: application/sdp\r\n"); if(result) return result; } @@ -581,19 +580,19 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done) /* RTSP never allows chunked transfer */ data->req.forbidchunk = TRUE; /* Finish the request buffer */ - result = Curl_add_buffer(req_buffer, "\r\n", 2); + result = Curl_add_buffer(&req_buffer, "\r\n", 2); if(result) return result; if(postsize > 0) { - result = Curl_add_buffer(req_buffer, data->set.postfields, + result = Curl_add_buffer(&req_buffer, data->set.postfields, (size_t)postsize); if(result) return result; } /* issue the request */ - result = Curl_add_buffer_send(req_buffer, conn, + result = Curl_add_buffer_send(&req_buffer, conn, &data->info.request_size, 0, FIRSTSOCKET); if(result) { failf(data, "Failed sending RTSP request"); diff --git a/Utilities/cmcurl/lib/rtsp.h b/Utilities/cmcurl/lib/rtsp.h index 8375a5317..2f9cc32c8 100644 --- a/Utilities/cmcurl/lib/rtsp.h +++ b/Utilities/cmcurl/lib/rtsp.h @@ -64,4 +64,3 @@ struct RTSP { #endif /* HEADER_CURL_RTSP_H */ - diff --git a/Utilities/cmcurl/lib/security.c b/Utilities/cmcurl/lib/security.c index d17198547..c278406ba 100644 --- a/Utilities/cmcurl/lib/security.c +++ b/Utilities/cmcurl/lib/security.c @@ -61,7 +61,9 @@ #include "strcase.h" #include "warnless.h" #include "strdup.h" -/* The last #include file should be: */ +/* The last 3 #include files should be in this order */ +#include "curl_printf.h" +#include "curl_memory.h" #include "memdebug.h" static const struct { @@ -390,7 +392,7 @@ int Curl_sec_read_msg(struct connectdata *conn, char *buffer, if(conn->data->set.verbose) { buf[decoded_len] = '\n'; - Curl_debug(conn->data, CURLINFO_HEADER_IN, buf, decoded_len + 1, conn); + Curl_debug(conn->data, CURLINFO_HEADER_IN, buf, decoded_len + 1); } buf[decoded_len] = '\0'; @@ -422,7 +424,7 @@ static int sec_set_protection_level(struct connectdata *conn) if(!conn->sec_complete) { infof(conn->data, "Trying to change the protection level after the" - "completion of the data exchange.\n"); + " completion of the data exchange.\n"); return -1; } @@ -488,7 +490,7 @@ static CURLcode choose_mech(struct connectdata *conn) tmp_allocation = realloc(conn->app_data, mech->size); if(tmp_allocation == NULL) { - failf(data, "Failed realloc of size %u", mech->size); + failf(data, "Failed realloc of size %zu", mech->size); mech = NULL; return CURLE_OUT_OF_MEMORY; } diff --git a/Utilities/cmcurl/lib/select.c b/Utilities/cmcurl/lib/select.c index 0406dd26a..6e7389015 100644 --- a/Utilities/cmcurl/lib/select.c +++ b/Utilities/cmcurl/lib/select.c @@ -80,7 +80,6 @@ int Curl_wait_ms(int timeout_ms) #endif struct curltime initial_tv; int pending_ms; - int error; #endif int r = 0; @@ -98,6 +97,7 @@ int Curl_wait_ms(int timeout_ms) pending_ms = timeout_ms; initial_tv = Curl_now(); do { + int error; #if defined(HAVE_POLL_FINE) r = poll(NULL, 0, pending_ms); #else @@ -160,7 +160,6 @@ int Curl_socket_check(curl_socket_t readfd0, /* two sockets to read from */ #endif struct curltime initial_tv = {0, 0}; int pending_ms = 0; - int error; int r; int ret; @@ -210,6 +209,7 @@ int Curl_socket_check(curl_socket_t readfd0, /* two sockets to read from */ } do { + int error; if(timeout_ms < 0) pending_ms = -1; else if(!timeout_ms) @@ -291,6 +291,7 @@ int Curl_socket_check(curl_socket_t readfd0, /* two sockets to read from */ ptimeout = (timeout_ms < 0) ? NULL : &pending_tv; do { + int error; if(timeout_ms > 0) { pending_tv.tv_sec = pending_ms / 1000; pending_tv.tv_usec = (pending_ms % 1000) * 1000; @@ -402,7 +403,6 @@ int Curl_poll(struct pollfd ufds[], unsigned int nfds, int timeout_ms) bool fds_none = TRUE; unsigned int i; int pending_ms = 0; - int error; int r; if(ufds) { @@ -431,6 +431,7 @@ int Curl_poll(struct pollfd ufds[], unsigned int nfds, int timeout_ms) #ifdef HAVE_POLL_FINE do { + int error; if(timeout_ms < 0) pending_ms = -1; else if(!timeout_ms) @@ -502,6 +503,7 @@ int Curl_poll(struct pollfd ufds[], unsigned int nfds, int timeout_ms) ptimeout = (timeout_ms < 0) ? NULL : &pending_tv; do { + int error; if(timeout_ms > 0) { pending_tv.tv_sec = pending_ms / 1000; pending_tv.tv_usec = (pending_ms % 1000) * 1000; diff --git a/Utilities/cmcurl/lib/select.h b/Utilities/cmcurl/lib/select.h index 4351786c3..9a1ba45a7 100644 --- a/Utilities/cmcurl/lib/select.h +++ b/Utilities/cmcurl/lib/select.h @@ -113,4 +113,3 @@ int tpf_select_libcurl(int maxfds, fd_set* reads, fd_set* writes, #endif #endif /* HEADER_CURL_SELECT_H */ - diff --git a/Utilities/cmcurl/lib/sendf.c b/Utilities/cmcurl/lib/sendf.c index 27c0ccc73..d3c10b369 100644 --- a/Utilities/cmcurl/lib/sendf.c +++ b/Utilities/cmcurl/lib/sendf.c @@ -240,7 +240,7 @@ void Curl_infof(struct Curl_easy *data, const char *fmt, ...) vsnprintf(print_buffer, sizeof(print_buffer), fmt, ap); va_end(ap); len = strlen(print_buffer); - Curl_debug(data, CURLINFO_TEXT, print_buffer, len, NULL); + Curl_debug(data, CURLINFO_TEXT, print_buffer, len); } } @@ -265,7 +265,7 @@ void Curl_failf(struct Curl_easy *data, const char *fmt, ...) if(data->set.verbose) { error[len] = '\n'; error[++len] = '\0'; - Curl_debug(data, CURLINFO_TEXT, error, len, NULL); + Curl_debug(data, CURLINFO_TEXT, error, len); } va_end(ap); } @@ -300,7 +300,7 @@ CURLcode Curl_sendf(curl_socket_t sockfd, struct connectdata *conn, break; if(data->set.verbose) - Curl_debug(data, CURLINFO_DATA_OUT, sptr, (size_t)bytes_written, conn); + Curl_debug(data, CURLINFO_DATA_OUT, sptr, (size_t)bytes_written); if((size_t)bytes_written != write_len) { /* if not all was written at once, we must advance the pointer, decrease @@ -762,8 +762,8 @@ CURLcode Curl_read(struct connectdata *conn, /* connection data */ } /* return 0 on success */ -static int showit(struct Curl_easy *data, curl_infotype type, - char *ptr, size_t size) +int Curl_debug(struct Curl_easy *data, curl_infotype type, + char *ptr, size_t size) { static const char s_infotype[CURLINFO_END][3] = { "* ", "< ", "> ", "{ ", "} ", "{ ", "} " }; @@ -834,41 +834,3 @@ static int showit(struct Curl_easy *data, curl_infotype type, #endif return rc; } - -int Curl_debug(struct Curl_easy *data, curl_infotype type, - char *ptr, size_t size, - struct connectdata *conn) -{ - int rc; - if(data->set.printhost && conn && conn->host.dispname) { - char buffer[160]; - const char *t = NULL; - const char *w = "Data"; - switch(type) { - case CURLINFO_HEADER_IN: - w = "Header"; - /* FALLTHROUGH */ - case CURLINFO_DATA_IN: - t = "from"; - break; - case CURLINFO_HEADER_OUT: - w = "Header"; - /* FALLTHROUGH */ - case CURLINFO_DATA_OUT: - t = "to"; - break; - default: - break; - } - - if(t) { - snprintf(buffer, sizeof(buffer), "[%s %s %s]", w, t, - conn->host.dispname); - rc = showit(data, CURLINFO_TEXT, buffer, strlen(buffer)); - if(rc) - return rc; - } - } - rc = showit(data, type, ptr, size); - return rc; -} diff --git a/Utilities/cmcurl/lib/sendf.h b/Utilities/cmcurl/lib/sendf.h index 7c9134dec..c68b017da 100644 --- a/Utilities/cmcurl/lib/sendf.h +++ b/Utilities/cmcurl/lib/sendf.h @@ -36,7 +36,7 @@ void Curl_failf(struct Curl_easy *, const char *fmt, ...); #elif defined(HAVE_VARIADIC_MACROS_GCC) #define infof(x...) Curl_nop_stmt #else -#define infof (void) +#error "missing VARIADIC macro define, fix and rebuild!" #endif #else /* CURL_DISABLE_VERBOSE_STRINGS */ @@ -84,9 +84,8 @@ CURLcode Curl_write_plain(struct connectdata *conn, ssize_t *written); /* the function used to output verbose information */ -int Curl_debug(struct Curl_easy *handle, curl_infotype type, - char *data, size_t size, - struct connectdata *conn); +int Curl_debug(struct Curl_easy *data, curl_infotype type, + char *ptr, size_t size); #endif /* HEADER_CURL_SENDF_H */ diff --git a/Utilities/cmcurl/lib/setopt.c b/Utilities/cmcurl/lib/setopt.c index af53ee3ef..22956a20f 100644 --- a/Utilities/cmcurl/lib/setopt.c +++ b/Utilities/cmcurl/lib/setopt.c @@ -127,9 +127,11 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, data->set.dns_cache_timeout = arg; break; case CURLOPT_DNS_USE_GLOBAL_CACHE: +#if 0 /* deprecated */ /* remember we want this enabled */ arg = va_arg(param, long); data->set.global_dns_cache = (0 != arg) ? TRUE : FALSE; +#endif break; case CURLOPT_SSL_CIPHER_LIST: /* set a list of cipher we want to use in the SSL connection */ @@ -142,6 +144,25 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_arg(param, char *)); break; + case CURLOPT_TLS13_CIPHERS: + if(Curl_ssl_tls13_ciphersuites()) { + /* set preferred list of TLS 1.3 cipher suites */ + result = Curl_setstropt(&data->set.str[STRING_SSL_CIPHER13_LIST_ORIG], + va_arg(param, char *)); + } + else + return CURLE_NOT_BUILT_IN; + break; + case CURLOPT_PROXY_TLS13_CIPHERS: + if(Curl_ssl_tls13_ciphersuites()) { + /* set preferred list of TLS 1.3 cipher suites for proxy */ + result = Curl_setstropt(&data->set.str[STRING_SSL_CIPHER13_LIST_PROXY], + va_arg(param, char *)); + } + else + return CURLE_NOT_BUILT_IN; + break; + case CURLOPT_RANDOM_FILE: /* * This is the path name to a file that contains random data to seed @@ -822,6 +843,8 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, #else if(arg > CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE) return CURLE_UNSUPPORTED_PROTOCOL; + if(arg == CURL_HTTP_VERSION_NONE) + arg = CURL_HTTP_VERSION_2TLS; #endif data->set.httpversion = arg; break; @@ -1584,14 +1607,19 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, * String that holds the SSL crypto engine. */ argptr = va_arg(param, char *); - if(argptr && argptr[0]) - result = Curl_ssl_set_engine(data, argptr); + if(argptr && argptr[0]) { + result = Curl_setstropt(&data->set.str[STRING_SSL_ENGINE], argptr); + if(!result) { + result = Curl_ssl_set_engine(data, argptr); + } + } break; case CURLOPT_SSLENGINE_DEFAULT: /* * flag to set engine as default. */ + Curl_setstropt(&data->set.str[STRING_SSL_ENGINE], NULL); result = Curl_ssl_set_engine_default(data); break; case CURLOPT_CRLF: @@ -1912,6 +1940,22 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, break; + case CURLOPT_UPLOAD_BUFFERSIZE: + /* + * The application kindly asks for a differently sized upload buffer. + * Cap it to sensible. + */ + arg = va_arg(param, long); + + if(arg > UPLOADBUFFER_MAX) + arg = UPLOADBUFFER_MAX; + else if(arg < UPLOADBUFFER_MIN) + arg = UPLOADBUFFER_MIN; + + data->set.upload_buffer_size = arg; + Curl_safefree(data->state.ulbuf); /* force a realloc next opportunity */ + break; + case CURLOPT_NOSIGNAL: /* * The application asks not to set any signal() or alarm() handlers, @@ -1942,6 +1986,11 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, if(data->share->sslsession == data->state.session) data->state.session = NULL; +#ifdef USE_LIBPSL + if(data->psl == &data->share->psl) + data->psl = data->multi? &data->multi->psl: NULL; +#endif + data->share->dirty--; Curl_share_unlock(data, CURL_LOCK_DATA_SHARE); @@ -1973,8 +2022,12 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, data->set.general_ssl.max_ssl_sessions = data->share->max_ssl_sessions; data->state.session = data->share->sslsession; } - Curl_share_unlock(data, CURL_LOCK_DATA_SHARE); +#ifdef USE_LIBPSL + if(data->share->specifier & (1 << CURL_LOCK_DATA_PSL)) + data->psl = &data->share->psl; +#endif + Curl_share_unlock(data, CURL_LOCK_DATA_SHARE); } /* check for host cache not needed, * it will be done by curl_easy_perform */ @@ -2562,6 +2615,21 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, case CURLOPT_DNS_SHUFFLE_ADDRESSES: data->set.dns_shuffle_addresses = (0 != va_arg(param, long)) ? TRUE:FALSE; break; + case CURLOPT_DISALLOW_USERNAME_IN_URL: + data->set.disallow_username_in_url = + (0 != va_arg(param, long)) ? TRUE : FALSE; + break; + case CURLOPT_DOH_URL: + result = Curl_setstropt(&data->set.str[STRING_DOH], + va_arg(param, char *)); + data->set.doh = data->set.str[STRING_DOH]?TRUE:FALSE; + break; + case CURLOPT_UPKEEP_INTERVAL_MS: + arg = va_arg(param, long); + if(arg < 0) + return CURLE_BAD_FUNCTION_ARGUMENT; + data->set.upkeep_interval_ms = arg; + break; default: /* unknown tag and its companion, just ignore: */ result = CURLE_UNKNOWN_OPTION; diff --git a/Utilities/cmcurl/lib/sha256.c b/Utilities/cmcurl/lib/sha256.c index 55716c63b..f9287af23 100644 --- a/Utilities/cmcurl/lib/sha256.c +++ b/Utilities/cmcurl/lib/sha256.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Florin Petriuc, <petriuc.florin@gmail.com> + * Copyright (C) 1998 - 2018, Florin Petriuc, <petriuc.florin@gmail.com> * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -123,15 +123,11 @@ static const unsigned long K[64] = { #define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) #define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) #define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) -#ifndef MIN -#define MIN(x, y) (((x) < (y)) ? (x) : (y)) -#endif /* compress 512-bits */ static int sha256_compress(struct sha256_state *md, unsigned char *buf) { - unsigned long S[8], W[64], t0, t1; - unsigned long t; + unsigned long S[8], W[64]; int i; /* copy state into S */ for(i = 0; i < 8; i++) { @@ -146,12 +142,13 @@ static int sha256_compress(struct sha256_state *md, W[i - 16]; } /* Compress */ -#define RND(a,b,c,d,e,f,g,h,i) \ - t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ - t1 = Sigma0(a) + Maj(a, b, c); \ - d += t0; \ +#define RND(a,b,c,d,e,f,g,h,i) \ + unsigned long t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + unsigned long t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ h = t0 + t1; for(i = 0; i < 64; ++i) { + unsigned long t; RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], i); t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; @@ -200,7 +197,7 @@ static int SHA256_Update(struct sha256_state *md, inlen -= block_size; } else { - n = MIN(inlen, (block_size - md->curlen)); + n = CURLMIN(inlen, (block_size - md->curlen)); memcpy(md->buf + md->curlen, in, n); md->curlen += n; in += n; diff --git a/Utilities/cmcurl/lib/share.c b/Utilities/cmcurl/lib/share.c index 870b191fc..3d5108610 100644 --- a/Utilities/cmcurl/lib/share.c +++ b/Utilities/cmcurl/lib/share.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -25,6 +25,7 @@ #include <curl/curl.h> #include "urldata.h" #include "share.h" +#include "psl.h" #include "vtls/vtls.h" #include "curl_memory.h" @@ -106,6 +107,12 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) res = CURLSHE_NOMEM; break; + case CURL_LOCK_DATA_PSL: +#ifndef USE_LIBPSL + res = CURLSHE_NOT_BUILT_IN; +#endif + break; + default: res = CURLSHE_BAD_OPTION; } @@ -205,6 +212,8 @@ curl_share_cleanup(struct Curl_share *share) } #endif + Curl_psl_destroy(&share->psl); + if(share->unlockfunc) share->unlockfunc(NULL, CURL_LOCK_DATA_SHARE, share->clientdata); free(share); diff --git a/Utilities/cmcurl/lib/share.h b/Utilities/cmcurl/lib/share.h index 4b13406d9..a7dea41ad 100644 --- a/Utilities/cmcurl/lib/share.h +++ b/Utilities/cmcurl/lib/share.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -25,6 +25,7 @@ #include "curl_setup.h" #include <curl/curl.h> #include "cookie.h" +#include "psl.h" #include "urldata.h" #include "conncache.h" @@ -49,6 +50,9 @@ struct Curl_share { #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES) struct CookieInfo *cookies; #endif +#ifdef USE_LIBPSL + struct PslCache psl; +#endif struct curl_ssl_session *sslsession; size_t max_ssl_sessions; diff --git a/Utilities/cmcurl/lib/slist.c b/Utilities/cmcurl/lib/slist.c index e5adc0e71..392b84d13 100644 --- a/Utilities/cmcurl/lib/slist.c +++ b/Utilities/cmcurl/lib/slist.c @@ -142,4 +142,3 @@ void curl_slist_free_all(struct curl_slist *list) item = next; } while(next); } - diff --git a/Utilities/cmcurl/lib/slist.h b/Utilities/cmcurl/lib/slist.h index b3f498c35..d73dbf672 100644 --- a/Utilities/cmcurl/lib/slist.h +++ b/Utilities/cmcurl/lib/slist.h @@ -37,4 +37,3 @@ struct curl_slist *Curl_slist_append_nodup(struct curl_slist *list, char *data); #endif /* HEADER_CURL_SLIST_H */ - diff --git a/Utilities/cmcurl/lib/smb.c b/Utilities/cmcurl/lib/smb.c index 9ac61505c..e4f266e19 100644 --- a/Utilities/cmcurl/lib/smb.c +++ b/Utilities/cmcurl/lib/smb.c @@ -59,6 +59,7 @@ static CURLcode smb_setup_connection(struct connectdata *conn); static CURLcode smb_connect(struct connectdata *conn, bool *done); static CURLcode smb_connection_state(struct connectdata *conn, bool *done); +static CURLcode smb_do(struct connectdata *conn, bool *done); static CURLcode smb_request_state(struct connectdata *conn, bool *done); static CURLcode smb_done(struct connectdata *conn, CURLcode status, bool premature); @@ -73,7 +74,7 @@ static CURLcode smb_parse_url_path(struct connectdata *conn); const struct Curl_handler Curl_handler_smb = { "SMB", /* scheme */ smb_setup_connection, /* setup_connection */ - ZERO_NULL, /* do_it */ + smb_do, /* do_it */ smb_done, /* done */ ZERO_NULL, /* do_more */ smb_connect, /* connect_it */ @@ -98,7 +99,7 @@ const struct Curl_handler Curl_handler_smb = { const struct Curl_handler Curl_handler_smbs = { "SMBS", /* scheme */ smb_setup_connection, /* setup_connection */ - ZERO_NULL, /* do_it */ + smb_do, /* do_it */ smb_done, /* done */ ZERO_NULL, /* do_more */ smb_connect, /* connect_it */ @@ -173,7 +174,6 @@ enum smb_req_state { /* SMB request data */ struct smb_request { enum smb_req_state state; - char *share; char *path; unsigned short tid; /* Even if we connect to the same tree as another */ unsigned short fid; /* request, the tid will be different */ @@ -182,7 +182,7 @@ struct smb_request { static void conn_state(struct connectdata *conn, enum smb_conn_state newstate) { - struct smb_conn *smb = &conn->proto.smbc; + struct smb_conn *smbc = &conn->proto.smbc; #if defined(DEBUGBUILD) && !defined(CURL_DISABLE_VERBOSE_STRINGS) /* For debug purposes */ static const char * const names[] = { @@ -194,12 +194,12 @@ static void conn_state(struct connectdata *conn, enum smb_conn_state newstate) /* LAST */ }; - if(smb->state != newstate) + if(smbc->state != newstate) infof(conn->data, "SMB conn %p state change from %s to %s\n", - (void *)smb, names[smb->state], names[newstate]); + (void *)smbc, names[smbc->state], names[newstate]); #endif - smb->state = newstate; + smbc->state = newstate; } static void request_state(struct connectdata *conn, @@ -228,6 +228,8 @@ static void request_state(struct connectdata *conn, req->state = newstate; } +/* this should setup things in the connection, not in the easy + handle */ static CURLcode smb_setup_connection(struct connectdata *conn) { struct smb_request *req; @@ -253,7 +255,6 @@ static CURLcode smb_connect(struct connectdata *conn, bool *done) return CURLE_LOGIN_DENIED; /* Initialize the connection state */ - memset(smbc, 0, sizeof(*smbc)); smbc->state = SMB_CONNECTING; smbc->recv_buf = malloc(MAX_MESSAGE_SIZE); if(!smbc->recv_buf) @@ -366,7 +367,7 @@ static CURLcode smb_send(struct connectdata *conn, ssize_t len, ssize_t bytes_written; CURLcode result; - result = Curl_write(conn, FIRSTSOCKET, conn->data->state.uploadbuffer, + result = Curl_write(conn, FIRSTSOCKET, conn->data->state.ulbuf, len, &bytes_written); if(result) return result; @@ -392,7 +393,7 @@ static CURLcode smb_flush(struct connectdata *conn) return CURLE_OK; result = Curl_write(conn, FIRSTSOCKET, - conn->data->state.uploadbuffer + smbc->sent, + conn->data->state.ulbuf + smbc->sent, len, &bytes_written); if(result) return result; @@ -408,9 +409,12 @@ static CURLcode smb_flush(struct connectdata *conn) static CURLcode smb_send_message(struct connectdata *conn, unsigned char cmd, const void *msg, size_t msg_len) { - smb_format_message(conn, (struct smb_header *)conn->data->state.uploadbuffer, + CURLcode result = Curl_get_upload_buffer(conn->data); + if(result) + return result; + smb_format_message(conn, (struct smb_header *)conn->data->state.ulbuf, cmd, msg_len); - memcpy(conn->data->state.uploadbuffer + sizeof(struct smb_header), + memcpy(conn->data->state.ulbuf + sizeof(struct smb_header), msg, msg_len); return smb_send(conn, sizeof(struct smb_header) + msg_len, 0); @@ -475,11 +479,11 @@ static CURLcode smb_send_setup(struct connectdata *conn) static CURLcode smb_send_tree_connect(struct connectdata *conn) { - struct smb_request *req = conn->data->req.protop; struct smb_tree_connect msg; + struct smb_conn *smbc = &conn->proto.smbc; char *p = msg.bytes; - size_t byte_count = strlen(conn->host.name) + strlen(req->share); + size_t byte_count = strlen(conn->host.name) + strlen(smbc->share); byte_count += strlen(SERVICENAME) + 5; /* 2 nulls and 3 backslashes */ if(byte_count > sizeof(msg.bytes)) return CURLE_FILESIZE_EXCEEDED; @@ -491,7 +495,7 @@ static CURLcode smb_send_tree_connect(struct connectdata *conn) MSGCAT("\\\\"); MSGCAT(conn->host.name); MSGCAT("\\"); - MSGCATNULL(req->share); + MSGCATNULL(smbc->share); MSGCATNULL(SERVICENAME); /* Match any type of service */ byte_count = p - msg.bytes; msg.byte_count = smb_swap16((unsigned short)byte_count); @@ -571,11 +575,15 @@ static CURLcode smb_send_read(struct connectdata *conn) static CURLcode smb_send_write(struct connectdata *conn) { - struct smb_write *msg = (struct smb_write *)conn->data->state.uploadbuffer; + struct smb_write *msg; struct smb_request *req = conn->data->req.protop; curl_off_t offset = conn->data->req.offset; - curl_off_t upload_size = conn->data->req.size - conn->data->req.bytecount; + CURLcode result = Curl_get_upload_buffer(conn->data); + if(result) + return result; + msg = (struct smb_write *)conn->data->state.ulbuf; + if(upload_size >= MAX_PAYLOAD_SIZE - 1) /* There is one byte of padding */ upload_size = MAX_PAYLOAD_SIZE - 1; @@ -602,9 +610,10 @@ static CURLcode smb_send_and_recv(struct connectdata *conn, void **msg) /* Check if there is data in the transfer buffer */ if(!smbc->send_size && smbc->upload_size) { - int nread = smbc->upload_size > UPLOAD_BUFSIZE ? UPLOAD_BUFSIZE : - (int) smbc->upload_size; - conn->data->req.upload_fromhere = conn->data->state.uploadbuffer; + size_t nread = smbc->upload_size > conn->data->set.upload_buffer_size ? + conn->data->set.upload_buffer_size : + smbc->upload_size; + conn->data->req.upload_fromhere = conn->data->state.ulbuf; result = Curl_fillreadbuffer(conn, nread, &nread); if(result && result != CURLE_AGAIN) return result; @@ -910,93 +919,84 @@ static CURLcode smb_request_state(struct connectdata *conn, bool *done) static CURLcode smb_done(struct connectdata *conn, CURLcode status, bool premature) { - struct smb_request *req = conn->data->req.protop; - (void) premature; - - Curl_safefree(req->share); Curl_safefree(conn->data->req.protop); - return status; } static CURLcode smb_disconnect(struct connectdata *conn, bool dead) { struct smb_conn *smbc = &conn->proto.smbc; - struct smb_request *req = conn->data->req.protop; - (void) dead; - + Curl_safefree(smbc->share); Curl_safefree(smbc->domain); Curl_safefree(smbc->recv_buf); - - /* smb_done is not always called, so cleanup the request */ - if(req) { - Curl_safefree(req->share); - } - return CURLE_OK; } static int smb_getsock(struct connectdata *conn, curl_socket_t *socks, int numsocks) { - struct smb_conn *smbc = &conn->proto.smbc; - if(!numsocks) return GETSOCK_BLANK; socks[0] = conn->sock[FIRSTSOCKET]; + return GETSOCK_READSOCK(0) | GETSOCK_WRITESOCK(0); +} - if(smbc->send_size || smbc->upload_size) - return GETSOCK_WRITESOCK(0); +static CURLcode smb_do(struct connectdata *conn, bool *done) +{ + struct smb_conn *smbc = &conn->proto.smbc; + struct smb_request *req = conn->data->req.protop; - return GETSOCK_READSOCK(0); + *done = FALSE; + if(smbc->share) { + req->path = strchr(smbc->share, '\0'); + if(req->path) { + req->path++; + return CURLE_OK; + } + } + return CURLE_URL_MALFORMAT; } static CURLcode smb_parse_url_path(struct connectdata *conn) { CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; - struct smb_request *req = data->req.protop; + struct smb_conn *smbc = &conn->proto.smbc; char *path; char *slash; /* URL decode the path */ - result = Curl_urldecode(data, data->state.path, 0, &path, NULL, TRUE); + result = Curl_urldecode(data, data->state.up.path, 0, &path, NULL, TRUE); if(result) return result; /* Parse the path for the share */ - req->share = strdup((*path == '/' || *path == '\\') ? path + 1 : path); - if(!req->share) { - free(path); - + smbc->share = strdup((*path == '/' || *path == '\\') ? path + 1 : path); + free(path); + if(!smbc->share) return CURLE_OUT_OF_MEMORY; - } - slash = strchr(req->share, '/'); + slash = strchr(smbc->share, '/'); if(!slash) - slash = strchr(req->share, '\\'); + slash = strchr(smbc->share, '\\'); /* The share must be present */ if(!slash) { - free(path); - + Curl_safefree(smbc->share); return CURLE_URL_MALFORMAT; } /* Parse the path for the file path converting any forward slashes into backslashes */ *slash++ = 0; - req->path = slash; + for(; *slash; slash++) { if(*slash == '/') *slash = '\\'; } - - free(path); - return CURLE_OK; } diff --git a/Utilities/cmcurl/lib/smb.h b/Utilities/cmcurl/lib/smb.h index c3ee7ae03..9ce6b5615 100644 --- a/Utilities/cmcurl/lib/smb.h +++ b/Utilities/cmcurl/lib/smb.h @@ -35,6 +35,7 @@ struct smb_conn { enum smb_conn_state state; char *user; char *domain; + char *share; unsigned char challenge[8]; unsigned int session_key; unsigned short uid; diff --git a/Utilities/cmcurl/lib/smtp.c b/Utilities/cmcurl/lib/smtp.c index 3f3b45a95..587562306 100644 --- a/Utilities/cmcurl/lib/smtp.c +++ b/Utilities/cmcurl/lib/smtp.c @@ -704,7 +704,6 @@ static CURLcode smtp_state_ehlo_resp(struct connectdata *conn, int smtpcode, struct smtp_conn *smtpc = &conn->proto.smtpc; const char *line = data->state.buffer; size_t len = strlen(line); - size_t wordlen; (void)instate; /* no use for this yet */ @@ -739,6 +738,7 @@ static CURLcode smtp_state_ehlo_resp(struct connectdata *conn, int smtpcode, /* Loop through the data line */ for(;;) { size_t llen; + size_t wordlen; unsigned int mechbit; while(len && @@ -1441,7 +1441,6 @@ static CURLcode smtp_regular_transfer(struct connectdata *conn, static CURLcode smtp_setup_connection(struct connectdata *conn) { - struct Curl_easy *data = conn->data; CURLcode result; /* Clear the TLS upgraded flag */ @@ -1452,8 +1451,6 @@ static CURLcode smtp_setup_connection(struct connectdata *conn) if(result) return result; - data->state.path++; /* don't include the initial slash */ - return CURLE_OK; } @@ -1507,7 +1504,7 @@ static CURLcode smtp_parse_url_path(struct connectdata *conn) /* The SMTP struct is already initialised in smtp_connect() */ struct Curl_easy *data = conn->data; struct smtp_conn *smtpc = &conn->proto.smtpc; - const char *path = data->state.path; + const char *path = &data->state.up.path[1]; /* skip leading path */ char localhost[HOSTNAME_MAX + 1]; /* Calculate the path if necessary */ @@ -1563,13 +1560,14 @@ CURLcode Curl_smtp_escape_eob(struct connectdata *conn, const ssize_t nread) if(!scratch || data->set.crlf) { oldscratch = scratch; - scratch = newscratch = malloc(2 * data->set.buffer_size); + scratch = newscratch = malloc(2 * data->set.upload_buffer_size); if(!newscratch) { failf(data, "Failed to alloc scratch buffer!"); return CURLE_OUT_OF_MEMORY; } } + DEBUGASSERT(data->set.upload_buffer_size >= (size_t)nread); /* Have we already sent part of the EOB? */ eob_sent = smtp->eob; diff --git a/Utilities/cmcurl/lib/sockaddr.h b/Utilities/cmcurl/lib/sockaddr.h index 95ba4c3c9..db146803a 100644 --- a/Utilities/cmcurl/lib/sockaddr.h +++ b/Utilities/cmcurl/lib/sockaddr.h @@ -40,4 +40,3 @@ struct Curl_sockaddr_storage { }; #endif /* HEADER_CURL_SOCKADDR_H */ - diff --git a/Utilities/cmcurl/lib/socks.c b/Utilities/cmcurl/lib/socks.c index ac4270eac..d2209ad89 100644 --- a/Utilities/cmcurl/lib/socks.c +++ b/Utilities/cmcurl/lib/socks.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -57,10 +57,9 @@ int Curl_blockread_all(struct connectdata *conn, /* connection data */ ssize_t nread; ssize_t allread = 0; int result; - timediff_t timeleft; *n = 0; for(;;) { - timeleft = Curl_timeleft(conn->data, NULL, TRUE); + timediff_t timeleft = Curl_timeleft(conn->data, NULL, TRUE); if(timeleft < 0) { /* we already got the timeout */ result = CURLE_OPERATION_TIMEDOUT; @@ -99,7 +98,7 @@ int Curl_blockread_all(struct connectdata *conn, /* connection data */ * destination server. * * Reference : -* http://socks.permeo.com/protocol/socks4.protocol +* https://www.openssh.com/txt/socks4.protocol * * Note : * Set protocol4a=true for "SOCKS 4A (Simple Extension to SOCKS 4 Protocol)" @@ -116,7 +115,6 @@ CURLcode Curl_SOCKS4(const char *proxy_user, #define SOCKS4REQLEN 262 unsigned char socksreq[SOCKS4REQLEN]; /* room for SOCKS4 request incl. user id */ - int result; CURLcode code; curl_socket_t sock = conn->sock[sockindex]; struct Curl_easy *data = conn->data; @@ -220,11 +218,12 @@ CURLcode Curl_SOCKS4(const char *proxy_user, * Make connection */ { + int result; ssize_t actualread; ssize_t written; ssize_t hostnamelen = 0; - int packetsize = 9 + - (int)strlen((char *)socksreq + 8); /* size including NUL */ + ssize_t packetsize = 9 + + strlen((char *)socksreq + 8); /* size including NUL */ /* If SOCKS4a, set special invalid IP address 0.0.0.x */ if(protocol4a) { @@ -617,11 +616,11 @@ CURLcode Curl_SOCKS5(const char *proxy_user, if(dns) hp = dns->addr; if(hp) { - int i; char buf[64]; Curl_printable_address(hp, buf, sizeof(buf)); if(hp->ai_family == AF_INET) { + int i; struct sockaddr_in *saddr_in; socksreq[len++] = 1; /* ATYP: IPv4 = 1 */ @@ -634,6 +633,7 @@ CURLcode Curl_SOCKS5(const char *proxy_user, } #ifdef ENABLE_IPV6 else if(hp->ai_family == AF_INET6) { + int i; struct sockaddr_in6 *saddr_in6; socksreq[len++] = 4; /* ATYP: IPv6 = 4 */ @@ -789,4 +789,3 @@ CURLcode Curl_SOCKS5(const char *proxy_user, } #endif /* CURL_DISABLE_PROXY */ - diff --git a/Utilities/cmcurl/lib/socks.h b/Utilities/cmcurl/lib/socks.h index 348707e74..daa07c127 100644 --- a/Utilities/cmcurl/lib/socks.h +++ b/Utilities/cmcurl/lib/socks.h @@ -73,4 +73,3 @@ CURLcode Curl_SOCKS5_gssapi_negotiate(int sockindex, #endif /* CURL_DISABLE_PROXY */ #endif /* HEADER_CURL_SOCKS_H */ - diff --git a/Utilities/cmcurl/lib/splay.c b/Utilities/cmcurl/lib/splay.c index 69af446eb..baf07e00d 100644 --- a/Utilities/cmcurl/lib/splay.c +++ b/Utilities/cmcurl/lib/splay.c @@ -41,7 +41,6 @@ struct Curl_tree *Curl_splay(struct curltime i, struct Curl_tree *t) { struct Curl_tree N, *l, *r, *y; - long comp; if(t == NULL) return t; @@ -49,7 +48,7 @@ struct Curl_tree *Curl_splay(struct curltime i, l = r = &N; for(;;) { - comp = compare(i, t->key); + long comp = compare(i, t->key); if(comp < 0) { if(t->smaller == NULL) break; @@ -275,4 +274,3 @@ int Curl_splayremovebyaddr(struct Curl_tree *t, return 0; } - diff --git a/Utilities/cmcurl/lib/ssh-libssh.c b/Utilities/cmcurl/lib/ssh-libssh.c index 34ef490c4..7d590891c 100644 --- a/Utilities/cmcurl/lib/ssh-libssh.c +++ b/Utilities/cmcurl/lib/ssh-libssh.c @@ -204,11 +204,21 @@ static CURLcode sftp_error_to_CURLE(int err) return CURLE_SSH; } +#ifndef DEBUGBUILD +#define state(x,y) mystate(x,y) +#else +#define state(x,y) mystate(x,y, __LINE__) +#endif + /* * SSH State machine related code */ /* This is the ONLY way to change SSH state! */ -static void state(struct connectdata *conn, sshstate nowstate) +static void mystate(struct connectdata *conn, sshstate nowstate +#ifdef DEBUGBUILD + , int lineno +#endif + ) { struct ssh_conn *sshc = &conn->proto.sshc; #if defined(DEBUGBUILD) && !defined(CURL_DISABLE_VERBOSE_STRINGS) @@ -278,8 +288,9 @@ static void state(struct connectdata *conn, sshstate nowstate) if(sshc->state != nowstate) { - infof(conn->data, "SSH %p state change from %s to %s\n", - (void *) sshc, names[sshc->state], names[nowstate]); + infof(conn->data, "SSH %p state change from %s to %s (line %d)\n", + (void *) sshc, names[sshc->state], names[nowstate], + lineno); } #endif @@ -418,7 +429,7 @@ cleanup: } #define MOVE_TO_ERROR_STATE(_r) { \ - state(conn, SSH_SESSION_FREE); \ + state(conn, SSH_SESSION_DISCONNECT); \ sshc->actualcode = _r; \ rc = SSH_ERROR; \ break; \ @@ -486,7 +497,7 @@ restart: if(rc < 0) return SSH_ERROR; - /* fallthrough */ + /* FALLTHROUGH */ case 1: sshc->kbd_state = 1; @@ -561,7 +572,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) ssh_set_blocking(sshc->ssh_session, 0); state(conn, SSH_S_STARTUP); - /* fall-through */ + /* FALLTHROUGH */ case SSH_S_STARTUP: rc = ssh_connect(sshc->ssh_session); @@ -575,7 +586,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) state(conn, SSH_HOSTKEY); - /* fall-through */ + /* FALLTHROUGH */ case SSH_HOSTKEY: rc = myssh_is_known(conn); @@ -584,7 +595,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) } state(conn, SSH_AUTHLIST); - /* fall through */ + /* FALLTHROUGH */ case SSH_AUTHLIST:{ sshc->authed = FALSE; @@ -607,6 +618,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) sshc->auth_methods = ssh_userauth_list(sshc->ssh_session, NULL); if(sshc->auth_methods & SSH_AUTH_METHOD_PUBLICKEY) { state(conn, SSH_AUTH_PKEY_INIT); + infof(data, "Authentication using SSH public key file\n"); } else if(sshc->auth_methods & SSH_AUTH_METHOD_GSSAPI_MIC) { state(conn, SSH_AUTH_GSSAPI); @@ -651,6 +663,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) if(rc != SSH_OK) { failf(data, "Could not load private key file %s", data->set.str[STRING_SSH_PRIVATE_KEY]); + MOVE_TO_ERROR_STATE(CURLE_LOGIN_DENIED); break; } @@ -659,8 +672,6 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) } else { - infof(data, "Authentication using SSH public key file\n"); - rc = ssh_userauth_publickey_auto(sshc->ssh_session, NULL, data->set.ssl.key_passwd); if(rc == SSH_AUTH_AGAIN) { @@ -748,7 +759,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) MOVE_TO_ERROR_STATE(CURLE_LOGIN_DENIED); } state(conn, SSH_AUTH_PASS); - /* fall through */ + /* FALLTHROUGH */ case SSH_AUTH_PASS: rc = ssh_userauth_password(sshc->ssh_session, NULL, conn->passwd); @@ -812,7 +823,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) break; } state(conn, SSH_SFTP_REALPATH); - /* fall through */ + /* FALLTHROUGH */ case SSH_SFTP_REALPATH: /* * Get the "home" directory @@ -1279,7 +1290,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) if(sshc->readdir_attrs) { sshc->readdir_filename = sshc->readdir_attrs->name; sshc->readdir_longentry = sshc->readdir_attrs->longname; - sshc->readdir_len = (int)strlen(sshc->readdir_filename); + sshc->readdir_len = strlen(sshc->readdir_filename); if(data->set.ftp_list_only) { char *tmpLine; @@ -1306,11 +1317,11 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) if(data->set.verbose) { Curl_debug(data, CURLINFO_DATA_OUT, (char *)sshc->readdir_filename, - sshc->readdir_len, conn); + sshc->readdir_len); } } else { - sshc->readdir_currLen = (int)strlen(sshc->readdir_longentry); + sshc->readdir_currLen = strlen(sshc->readdir_longentry); sshc->readdir_totalLen = 80 + sshc->readdir_currLen; sshc->readdir_line = calloc(sshc->readdir_totalLen, 1); if(!sshc->readdir_line) { @@ -1371,12 +1382,12 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) if(sshc->readdir_filename == NULL) sshc->readdir_len = 0; else - sshc->readdir_len = (int)strlen(sshc->readdir_tmp); + sshc->readdir_len = strlen(sshc->readdir_tmp); sshc->readdir_longentry = NULL; sshc->readdir_filename = sshc->readdir_tmp; } else { - sshc->readdir_len = (int)strlen(sshc->readdir_link_attrs->name); + sshc->readdir_len = strlen(sshc->readdir_link_attrs->name); sshc->readdir_filename = sshc->readdir_link_attrs->name; sshc->readdir_longentry = sshc->readdir_link_attrs->longname; } @@ -1408,7 +1419,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) sshc->readdir_longentry = NULL; state(conn, SSH_SFTP_READDIR_BOTTOM); - /* fall through */ + /* FALLTHROUGH */ case SSH_SFTP_READDIR_BOTTOM: sshc->readdir_currLen += snprintf(sshc->readdir_line + sshc->readdir_currLen, @@ -1423,7 +1434,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) /* output debug output if that is requested */ if(data->set.verbose) { Curl_debug(data, CURLINFO_DATA_OUT, sshc->readdir_line, - sshc->readdir_currLen, conn); + sshc->readdir_currLen); } data->req.bytecount += sshc->readdir_currLen; } @@ -1740,7 +1751,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) MOVE_TO_ERROR_STATE(CURLE_COULDNT_CONNECT); } state(conn, SSH_SCP_DOWNLOAD); - /* fall through */ + /* FALLTHROUGH */ case SSH_SCP_DOWNLOAD:{ curl_off_t bytecount; @@ -1805,7 +1816,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) ssh_set_blocking(sshc->ssh_session, 0); state(conn, SSH_SESSION_DISCONNECT); - /* fall through */ + /* FALLTHROUGH */ case SSH_SESSION_DISCONNECT: /* during weird times when we've been prematurely aborted, the channel @@ -1822,7 +1833,7 @@ static CURLcode myssh_statemach_act(struct connectdata *conn, bool *block) conn->data->state.most_recent_ftp_entrypath = NULL; state(conn, SSH_SESSION_FREE); - /* fall through */ + /* FALLTHROUGH */ case SSH_SESSION_FREE: if(sshc->ssh_session) { ssh_free(sshc->ssh_session); @@ -2379,7 +2390,8 @@ static CURLcode sftp_done(struct connectdata *conn, CURLcode status, /* Post quote commands are executed after the SFTP_CLOSE state to avoid errors that could happen due to open file handles during POSTQUOTE operation */ - if(!status && !premature && conn->data->set.postquote) { + if(!status && !premature && conn->data->set.postquote && + !conn->bits.retry) { sshc->nextstate = SSH_SFTP_POSTQUOTE_INIT; state(conn, SSH_SFTP_CLOSE); } @@ -2437,7 +2449,7 @@ static ssize_t sftp_recv(struct connectdata *conn, int sockindex, return -1; } - /* fall-through */ + /* FALLTHROUGH */ case 1: conn->proto.sshc.sftp_recv_state = 1; @@ -2500,8 +2512,8 @@ static void sftp_quote(struct connectdata *conn) return; } if(data->set.verbose) { - Curl_debug(data, CURLINFO_HEADER_OUT, (char *) "PWD\n", 4, conn); - Curl_debug(data, CURLINFO_HEADER_IN, tmp, strlen(tmp), conn); + Curl_debug(data, CURLINFO_HEADER_OUT, (char *) "PWD\n", 4); + Curl_debug(data, CURLINFO_HEADER_IN, tmp, strlen(tmp)); } /* this sends an FTP-like "header" to the header callback so that the current directory can be read very similar to how it is read when diff --git a/Utilities/cmcurl/lib/ssh.c b/Utilities/cmcurl/lib/ssh.c index bf7bd54f9..da896196f 100644 --- a/Utilities/cmcurl/lib/ssh.c +++ b/Utilities/cmcurl/lib/ssh.c @@ -659,7 +659,7 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) libssh2_session_set_blocking(sshc->ssh_session, 0); state(conn, SSH_S_STARTUP); - /* fall-through */ + /* FALLTHROUGH */ case SSH_S_STARTUP: rc = libssh2_session_startup(sshc->ssh_session, (int)sock); @@ -675,7 +675,7 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) state(conn, SSH_HOSTKEY); - /* fall-through */ + /* FALLTHROUGH */ case SSH_HOSTKEY: /* * Before we authenticate we should check the hostkey's fingerprint @@ -1175,8 +1175,8 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) break; } if(data->set.verbose) { - Curl_debug(data, CURLINFO_HEADER_OUT, (char *)"PWD\n", 4, conn); - Curl_debug(data, CURLINFO_HEADER_IN, tmp, strlen(tmp), conn); + Curl_debug(data, CURLINFO_HEADER_OUT, (char *)"PWD\n", 4); + Curl_debug(data, CURLINFO_HEADER_IN, tmp, strlen(tmp)); } /* this sends an FTP-like "header" to the header callback so that the current directory can be read very similar to how it is read when @@ -1933,17 +1933,17 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) break; case SSH_SFTP_READDIR: - sshc->readdir_len = libssh2_sftp_readdir_ex(sshc->sftp_handle, - sshc->readdir_filename, - PATH_MAX, - sshc->readdir_longentry, - PATH_MAX, - &sshc->readdir_attrs); - if(sshc->readdir_len == LIBSSH2_ERROR_EAGAIN) { - rc = LIBSSH2_ERROR_EAGAIN; + rc = libssh2_sftp_readdir_ex(sshc->sftp_handle, + sshc->readdir_filename, + PATH_MAX, + sshc->readdir_longentry, + PATH_MAX, + &sshc->readdir_attrs); + if(rc == LIBSSH2_ERROR_EAGAIN) { break; } - if(sshc->readdir_len > 0) { + if(rc > 0) { + sshc->readdir_len = (size_t) rc; sshc->readdir_filename[sshc->readdir_len] = '\0'; if(data->set.ftp_list_only) { @@ -1970,11 +1970,11 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) /* output debug output if that is requested */ if(data->set.verbose) { Curl_debug(data, CURLINFO_DATA_OUT, sshc->readdir_filename, - sshc->readdir_len, conn); + sshc->readdir_len); } } else { - sshc->readdir_currLen = (int)strlen(sshc->readdir_longentry); + sshc->readdir_currLen = strlen(sshc->readdir_longentry); sshc->readdir_totalLen = 80 + sshc->readdir_currLen; sshc->readdir_line = calloc(sshc->readdir_totalLen, 1); if(!sshc->readdir_line) { @@ -2008,13 +2008,13 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) break; } } - else if(sshc->readdir_len == 0) { + else if(rc == 0) { Curl_safefree(sshc->readdir_filename); Curl_safefree(sshc->readdir_longentry); state(conn, SSH_SFTP_READDIR_DONE); break; } - else if(sshc->readdir_len <= 0) { + else if(rc < 0) { err = sftp_libssh2_last_error(sshc->sftp_session); result = sftp_libssh2_error_to_CURLE(err); sshc->actualcode = result?result:CURLE_SSH; @@ -2029,16 +2029,16 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) break; case SSH_SFTP_READDIR_LINK: - sshc->readdir_len = + rc = libssh2_sftp_symlink_ex(sshc->sftp_session, sshc->readdir_linkPath, curlx_uztoui(strlen(sshc->readdir_linkPath)), sshc->readdir_filename, PATH_MAX, LIBSSH2_SFTP_READLINK); - if(sshc->readdir_len == LIBSSH2_ERROR_EAGAIN) { - rc = LIBSSH2_ERROR_EAGAIN; + if(rc == LIBSSH2_ERROR_EAGAIN) { break; } + sshc->readdir_len = (size_t) rc; Curl_safefree(sshc->readdir_linkPath); /* get room for the filename and extra output */ @@ -2079,7 +2079,7 @@ static CURLcode ssh_statemach_act(struct connectdata *conn, bool *block) /* output debug output if that is requested */ if(data->set.verbose) { Curl_debug(data, CURLINFO_DATA_OUT, sshc->readdir_line, - sshc->readdir_currLen, conn); + sshc->readdir_currLen); } data->req.bytecount += sshc->readdir_currLen; } @@ -2926,7 +2926,7 @@ static CURLcode ssh_connect(struct connectdata *conn, bool *done) int rc; ssh->kh = libssh2_knownhost_init(ssh->ssh_session); if(!ssh->kh) { - /* eeek. TODO: free the ssh_session! */ + libssh2_session_free(ssh->ssh_session); return CURLE_FAILED_INIT; } @@ -3219,7 +3219,8 @@ static CURLcode sftp_done(struct connectdata *conn, CURLcode status, /* Post quote commands are executed after the SFTP_CLOSE state to avoid errors that could happen due to open file handles during POSTQUOTE operation */ - if(!status && !premature && conn->data->set.postquote) { + if(!status && !premature && conn->data->set.postquote && + !conn->bits.retry) { sshc->nextstate = SSH_SFTP_POSTQUOTE_INIT; state(conn, SSH_SFTP_CLOSE); } diff --git a/Utilities/cmcurl/lib/ssh.h b/Utilities/cmcurl/lib/ssh.h index 1c1355077..0620aac32 100644 --- a/Utilities/cmcurl/lib/ssh.h +++ b/Utilities/cmcurl/lib/ssh.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -131,7 +131,7 @@ struct ssh_conn { quote command fails) */ char *homedir; /* when doing SFTP we figure out home dir in the connect phase */ - int readdir_len, readdir_totalLen, readdir_currLen; + size_t readdir_len, readdir_totalLen, readdir_currLen; char *readdir_line; char *readdir_linkPath; /* end of READDIR stuff */ diff --git a/Utilities/cmcurl/lib/strcase.h b/Utilities/cmcurl/lib/strcase.h index ea2abc8b6..6fee3840e 100644 --- a/Utilities/cmcurl/lib/strcase.h +++ b/Utilities/cmcurl/lib/strcase.h @@ -46,6 +46,5 @@ char Curl_raw_toupper(char in); #define checkprefix(a,b) curl_strnequal(a,b,strlen(a)) void Curl_strntoupper(char *dest, const char *src, size_t n); -char Curl_raw_toupper(char in); #endif /* HEADER_CURL_STRCASE_H */ diff --git a/Utilities/cmcurl/lib/strdup.c b/Utilities/cmcurl/lib/strdup.c index 19cb04416..51e7978b7 100644 --- a/Utilities/cmcurl/lib/strdup.c +++ b/Utilities/cmcurl/lib/strdup.c @@ -81,7 +81,7 @@ void *Curl_memdup(const void *src, size_t length) * Curl_saferealloc(ptr, size) * * Does a normal realloc(), but will free the data pointer if the realloc - * fails. If 'size' is zero, it will free the data and return a failure. + * fails. If 'size' is non-zero, it will free the data and return a failure. * * This convenience function is provided and used to help us avoid a common * mistake pattern when we could pass in a zero, catch the NULL return and end diff --git a/Utilities/cmcurl/lib/strerror.c b/Utilities/cmcurl/lib/strerror.c index 0295d6c27..47ef44a66 100644 --- a/Utilities/cmcurl/lib/strerror.c +++ b/Utilities/cmcurl/lib/strerror.c @@ -191,9 +191,6 @@ curl_easy_strerror(CURLcode error) case CURLE_TELNET_OPTION_SYNTAX : return "Malformed telnet option"; - case CURLE_PEER_FAILED_VERIFICATION: - return "SSL peer certificate or SSH remote key was not OK"; - case CURLE_GOT_NOTHING: return "Server returned nothing (no headers, no data)"; @@ -218,9 +215,8 @@ curl_easy_strerror(CURLcode error) case CURLE_SSL_CIPHER: return "Couldn't use specified SSL cipher"; - case CURLE_SSL_CACERT: - return "Peer certificate cannot be authenticated with given CA " - "certificates"; + case CURLE_PEER_FAILED_VERIFICATION: + return "SSL peer certificate or SSH remote key was not OK"; case CURLE_SSL_CACERT_BADFILE: return "Problem with the SSL CA cert (path? access rights?)"; @@ -324,6 +320,7 @@ curl_easy_strerror(CURLcode error) case CURLE_OBSOLETE44: case CURLE_OBSOLETE46: case CURLE_OBSOLETE50: + case CURLE_OBSOLETE51: case CURLE_OBSOLETE57: case CURL_LAST: break; diff --git a/Utilities/cmcurl/lib/system_win32.c b/Utilities/cmcurl/lib/system_win32.c index cfbbf3279..6b8004e5b 100644 --- a/Utilities/cmcurl/lib/system_win32.c +++ b/Utilities/cmcurl/lib/system_win32.c @@ -26,6 +26,7 @@ #include <curl/curl.h> #include "system_win32.h" +#include "warnless.h" /* The last #include files should be: */ #include "curl_memory.h" @@ -134,8 +135,9 @@ bool Curl_verify_windows_version(const unsigned int majorVersion, break; case VERSION_LESS_THAN_EQUAL: - if(osver.dwMajorVersion <= majorVersion && - osver.dwMinorVersion <= minorVersion) + if(osver.dwMajorVersion < majorVersion || + (osver.dwMajorVersion == majorVersion && + osver.dwMinorVersion <= minorVersion)) matched = TRUE; break; @@ -146,8 +148,9 @@ bool Curl_verify_windows_version(const unsigned int majorVersion, break; case VERSION_GREATER_THAN_EQUAL: - if(osver.dwMajorVersion >= majorVersion && - osver.dwMinorVersion >= minorVersion) + if(osver.dwMajorVersion > majorVersion || + (osver.dwMajorVersion == majorVersion && + osver.dwMinorVersion >= minorVersion)) matched = TRUE; break; @@ -278,7 +281,9 @@ HMODULE Curl_load_library(LPCTSTR filename) /* Attempt to find LoadLibraryEx() which is only available on Windows 2000 and above */ - pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX); + pLoadLibraryEx = + CURLX_FUNCTION_CAST(LOADLIBRARYEX_FN, + (GetProcAddress(hKernel32, LOADLIBARYEX))); /* Detect if there's already a path in the filename and load the library if there is. Note: Both back slashes and forward slashes have been supported diff --git a/Utilities/cmcurl/lib/telnet.c b/Utilities/cmcurl/lib/telnet.c index d71c8e067..05fe744db 100644 --- a/Utilities/cmcurl/lib/telnet.c +++ b/Utilities/cmcurl/lib/telnet.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -52,10 +52,6 @@ #include "connect.h" #include "progress.h" #include "system_win32.h" - -#define TELOPTS -#define TELCMDS - #include "arpa_telnet.h" #include "select.h" #include "strcase.h" @@ -74,10 +70,10 @@ x->subend = x->subpointer; \ CURL_SB_CLEAR(x); \ } WHILE_FALSE -#define CURL_SB_ACCUM(x,c) \ - do { \ - if(x->subpointer < (x->subbuffer + sizeof x->subbuffer)) \ - *x->subpointer++ = (c); \ +#define CURL_SB_ACCUM(x,c) \ + do { \ + if(x->subpointer < (x->subbuffer + sizeof(x->subbuffer))) \ + *x->subpointer++ = (c); \ } WHILE_FALSE #define CURL_SB_GET(x) ((*x->subpointer++)&0xff) @@ -92,6 +88,7 @@ #endif #ifdef USE_WINSOCK +typedef WSAEVENT (WINAPI *WSOCK2_EVENT)(void); typedef FARPROC WSOCK2_FUNC; static CURLcode check_wsock2(struct Curl_easy *data); #endif @@ -109,8 +106,10 @@ static void printoption(struct Curl_easy *data, static void negotiate(struct connectdata *); static void send_negotiation(struct connectdata *, int cmd, int option); -static void set_local_option(struct connectdata *, int cmd, int option); -static void set_remote_option(struct connectdata *, int cmd, int option); +static void set_local_option(struct connectdata *conn, + int option, int newstate); +static void set_remote_option(struct connectdata *conn, + int option, int newstate); static void printsub(struct Curl_easy *data, int direction, unsigned char *pointer, @@ -311,9 +310,6 @@ static void negotiate(struct connectdata *conn) static void printoption(struct Curl_easy *data, const char *direction, int cmd, int option) { - const char *fmt; - const char *opt; - if(data->set.verbose) { if(cmd == CURL_IAC) { if(CURL_TELCMD_OK(option)) @@ -322,9 +318,12 @@ static void printoption(struct Curl_easy *data, infof(data, "%s IAC %d\n", direction, option); } else { - fmt = (cmd == CURL_WILL) ? "WILL" : (cmd == CURL_WONT) ? "WONT" : - (cmd == CURL_DO) ? "DO" : (cmd == CURL_DONT) ? "DONT" : 0; + const char *fmt = (cmd == CURL_WILL) ? "WILL" : + (cmd == CURL_WONT) ? "WONT" : + (cmd == CURL_DO) ? "DO" : + (cmd == CURL_DONT) ? "DONT" : 0; if(fmt) { + const char *opt; if(CURL_TELOPT_OK(option)) opt = CURL_TELOPT(option); else if(option == CURL_TELOPT_EXOPL) @@ -348,7 +347,6 @@ static void send_negotiation(struct connectdata *conn, int cmd, int option) { unsigned char buf[3]; ssize_t bytes_written; - int err; struct Curl_easy *data = conn->data; buf[0] = CURL_IAC; @@ -357,7 +355,7 @@ static void send_negotiation(struct connectdata *conn, int cmd, int option) bytes_written = swrite(conn->sock[FIRSTSOCKET], buf, 3); if(bytes_written < 0) { - err = SOCKERRNO; + int err = SOCKERRNO; failf(data,"Sending data failed (%d)",err); } @@ -710,9 +708,8 @@ static void printsub(struct Curl_easy *data, unsigned char *pointer, /* where suboption data is */ size_t length) /* length of suboption data */ { - unsigned int i = 0; - if(data->set.verbose) { + unsigned int i = 0; if(direction) { infof(data, "%s IAC SB ", (direction == '<')? "RCVD":"SENT"); if(length >= 3) { @@ -764,7 +761,7 @@ static void printsub(struct Curl_easy *data, switch(pointer[0]) { case CURL_TELOPT_NAWS: if(length > 4) - infof(data, "Width: %hu ; Height: %hu", (pointer[1]<<8) | pointer[2], + infof(data, "Width: %d ; Height: %d", (pointer[1]<<8) | pointer[2], (pointer[3]<<8) | pointer[4]); break; default: @@ -928,7 +925,6 @@ static void suboption(struct connectdata *conn) unsigned char temp[2048]; ssize_t bytes_written; size_t len; - size_t tmplen; int err; char varname[128] = ""; char varval[128] = ""; @@ -968,7 +964,7 @@ static void suboption(struct connectdata *conn) len = 4; for(v = tn->telnet_vars; v; v = v->next) { - tmplen = (strlen(v->data) + 1); + size_t tmplen = (strlen(v->data) + 1); /* Add the variable only if it fits */ if(len + tmplen < (int)sizeof(temp)-6) { if(sscanf(v->data, "%127[^,],%127s", varname, varval)) { @@ -1017,7 +1013,7 @@ static void sendsuboption(struct connectdata *conn, int option) CURL_SB_ACCUM(tn, CURL_IAC); CURL_SB_ACCUM(tn, CURL_SB); CURL_SB_ACCUM(tn, CURL_TELOPT_NAWS); - /* We must deal either with litte or big endian processors */ + /* We must deal either with little or big endian processors */ /* Window size must be sent according to the 'network order' */ x = htons(tn->subopt_wsx); y = htons(tn->subopt_wsy); @@ -1223,7 +1219,7 @@ CURLcode telrcv(struct connectdata *conn, static CURLcode send_telnet_data(struct connectdata *conn, char *buffer, ssize_t nread) { - ssize_t escapes, i, j, outlen; + ssize_t escapes, i, outlen; unsigned char *outbuf = NULL; CURLcode result = CURLE_OK; ssize_t bytes_written, total_written; @@ -1238,6 +1234,7 @@ static CURLcode send_telnet_data(struct connectdata *conn, if(outlen == nread) outbuf = (unsigned char *)buffer; else { + ssize_t j; outbuf = malloc(nread + escapes + 1); if(!outbuf) return CURLE_OUT_OF_MEMORY; @@ -1306,7 +1303,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) #ifdef USE_WINSOCK HMODULE wsock2; WSOCK2_FUNC close_event_func; - WSOCK2_FUNC create_event_func; + WSOCK2_EVENT create_event_func; WSOCK2_FUNC event_select_func; WSOCK2_FUNC enum_netevents_func; WSAEVENT event_handle; @@ -1315,7 +1312,6 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) HANDLE objs[2]; DWORD obj_count; DWORD wait_timeout; - DWORD waitret; DWORD readfile_read; int err; #else @@ -1361,7 +1357,9 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) } /* Grab a pointer to WSACreateEvent */ - create_event_func = GetProcAddress(wsock2, "WSACreateEvent"); + create_event_func = + CURLX_FUNCTION_CAST(WSOCK2_EVENT, + (GetProcAddress(wsock2, "WSACreateEvent"))); if(create_event_func == NULL) { failf(data, "failed to find WSACreateEvent function (%u)", GetLastError()); FreeLibrary(wsock2); @@ -1438,7 +1436,8 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) /* Keep on listening and act on events */ while(keepon) { const DWORD buf_size = (DWORD)data->set.buffer_size; - waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout); + DWORD waitret = WaitForMultipleObjects(obj_count, objs, + FALSE, wait_timeout); switch(waitret) { case WAIT_TIMEOUT: { @@ -1607,7 +1606,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) case 0: /* timeout */ pfd[0].revents = 0; pfd[1].revents = 0; - /* fall through */ + /* FALLTHROUGH */ default: /* read! */ if(pfd[0].revents & POLLIN) { /* read data from network */ diff --git a/Utilities/cmcurl/lib/telnet.h b/Utilities/cmcurl/lib/telnet.h index 419a399b7..668a78a13 100644 --- a/Utilities/cmcurl/lib/telnet.h +++ b/Utilities/cmcurl/lib/telnet.h @@ -26,4 +26,3 @@ extern const struct Curl_handler Curl_handler_telnet; #endif #endif /* HEADER_CURL_TELNET_H */ - diff --git a/Utilities/cmcurl/lib/tftp.c b/Utilities/cmcurl/lib/tftp.c index b32960f82..5b74e8e08 100644 --- a/Utilities/cmcurl/lib/tftp.c +++ b/Utilities/cmcurl/lib/tftp.c @@ -451,7 +451,6 @@ static CURLcode tftp_send_first(tftp_state_data_t *state, tftp_event_t event) ssize_t senddata; const char *mode = "octet"; char *filename; - char buf[64]; struct Curl_easy *data = state->conn->data; CURLcode result = CURLE_OK; @@ -486,7 +485,7 @@ static CURLcode tftp_send_first(tftp_state_data_t *state, tftp_event_t event) /* As RFC3617 describes the separator slash is not actually part of the file name so we skip the always-present first letter of the path string. */ - result = Curl_urldecode(data, &state->conn->data->state.path[1], 0, + result = Curl_urldecode(data, &state->conn->data->state.up.path[1], 0, &filename, NULL, FALSE); if(result) return result; @@ -504,6 +503,7 @@ static CURLcode tftp_send_first(tftp_state_data_t *state, tftp_event_t event) /* optional addition of TFTP options */ if(!data->set.tftp_no_options) { + char buf[64]; /* add tsize option */ if(data->set.upload && (data->state.infilesize != -1)) snprintf(buf, sizeof(buf), "%" CURL_FORMAT_CURL_OFF_T, @@ -710,10 +710,9 @@ static CURLcode tftp_tx(tftp_state_data_t *state, tftp_event_t event) { struct Curl_easy *data = state->conn->data; ssize_t sbytes; - int rblock; CURLcode result = CURLE_OK; struct SingleRequest *k = &data->req; - int cb; /* Bytes currently read */ + size_t cb; /* Bytes currently read */ switch(event) { @@ -721,7 +720,7 @@ static CURLcode tftp_tx(tftp_state_data_t *state, tftp_event_t event) case TFTP_EVENT_OACK: if(event == TFTP_EVENT_ACK) { /* Ack the packet */ - rblock = getrpacketblock(&state->rpacket); + int rblock = getrpacketblock(&state->rpacket); if(rblock != state->block && /* There's a bug in tftpd-hpa that causes it to send us an ack for @@ -766,7 +765,7 @@ static CURLcode tftp_tx(tftp_state_data_t *state, tftp_event_t event) state->retries = 0; setpacketevent(&state->spacket, TFTP_EVENT_DATA); setpacketblock(&state->spacket, state->block); - if(state->block > 1 && state->sbytes < (int)state->blksize) { + if(state->block > 1 && state->sbytes < state->blksize) { state->state = TFTP_STATE_FIN; return CURLE_OK; } @@ -782,7 +781,7 @@ static CURLcode tftp_tx(tftp_state_data_t *state, tftp_event_t event) &cb); if(result) return result; - state->sbytes += cb; + state->sbytes += (int)cb; state->conn->data->req.upload_fromhere += cb; } while(state->sbytes < state->blksize && cb != 0); @@ -969,7 +968,7 @@ static CURLcode tftp_disconnect(struct connectdata *conn, bool dead_connection) static CURLcode tftp_connect(struct connectdata *conn, bool *done) { tftp_state_data_t *state; - int blksize, rc; + int blksize; blksize = TFTP_BLKSIZE_DEFAULT; @@ -1028,8 +1027,8 @@ static CURLcode tftp_connect(struct connectdata *conn, bool *done) * assume uses the same IP version and thus hopefully this works for both * IPv4 and IPv6... */ - rc = bind(state->sockfd, (struct sockaddr *)&state->local_addr, - conn->ip_addr->ai_addrlen); + int rc = bind(state->sockfd, (struct sockaddr *)&state->local_addr, + conn->ip_addr->ai_addrlen); if(rc) { failf(conn->data, "bind() failed; %s", Curl_strerror(conn, SOCKERRNO)); @@ -1148,8 +1147,11 @@ static CURLcode tftp_receive_packet(struct connectdata *conn) case TFTP_EVENT_ERROR: { unsigned short error = getrpacketblock(&state->rpacket); + char *str = (char *)state->rpacket.data + 4; + size_t strn = state->rbytes - 4; state->error = (tftp_error_t)error; - infof(data, "%s\n", (const char *)state->rpacket.data + 4); + if(Curl_strnlen(str, strn) < strn) + infof(data, "TFTP error: %s\n", str); break; } case TFTP_EVENT_ACK: @@ -1221,7 +1223,6 @@ static long tftp_state_timeout(struct connectdata *conn, tftp_event_t *event) **********************************************************/ static CURLcode tftp_multi_statemach(struct connectdata *conn, bool *done) { - int rc; tftp_event_t event; CURLcode result = CURLE_OK; struct Curl_easy *data = conn->data; @@ -1245,7 +1246,7 @@ static CURLcode tftp_multi_statemach(struct connectdata *conn, bool *done) } else { /* no timeouts to handle, check our socket */ - rc = SOCKET_READABLE(state->sockfd, 0); + int rc = SOCKET_READABLE(state->sockfd, 0); if(rc == -1) { /* bail out */ @@ -1368,18 +1369,18 @@ static CURLcode tftp_setup_connection(struct connectdata * conn) { struct Curl_easy *data = conn->data; char *type; - char command; conn->socktype = SOCK_DGRAM; /* UDP datagram based */ /* TFTP URLs support an extension like ";mode=<typecode>" that * we'll try to get now! */ - type = strstr(data->state.path, ";mode="); + type = strstr(data->state.up.path, ";mode="); if(!type) type = strstr(conn->host.rawalloc, ";mode="); if(type) { + char command; *type = 0; /* it was in the middle of the hostname */ command = Curl_raw_toupper(type[6]); diff --git a/Utilities/cmcurl/lib/tftp.h b/Utilities/cmcurl/lib/tftp.h index c2325b232..1335f64bd 100644 --- a/Utilities/cmcurl/lib/tftp.h +++ b/Utilities/cmcurl/lib/tftp.h @@ -26,4 +26,3 @@ extern const struct Curl_handler Curl_handler_tftp; #endif #endif /* HEADER_CURL_TFTP_H */ - diff --git a/Utilities/cmcurl/lib/timeval.c b/Utilities/cmcurl/lib/timeval.c index f4bf83531..dce1a761e 100644 --- a/Utilities/cmcurl/lib/timeval.c +++ b/Utilities/cmcurl/lib/timeval.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -33,7 +33,8 @@ struct curltime Curl_now(void) */ struct curltime now; #if !defined(_WIN32_WINNT) || !defined(_WIN32_WINNT_VISTA) || \ - (_WIN32_WINNT < _WIN32_WINNT_VISTA) + (_WIN32_WINNT < _WIN32_WINNT_VISTA) || \ + (defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR)) DWORD milliseconds = GetTickCount(); now.tv_sec = milliseconds / 1000; now.tv_usec = (milliseconds % 1000) * 1000; @@ -60,7 +61,23 @@ struct curltime Curl_now(void) struct timeval now; struct curltime cnow; struct timespec tsnow; - if(0 == clock_gettime(CLOCK_MONOTONIC, &tsnow)) { + + /* + ** clock_gettime() may be defined by Apple's SDK as weak symbol thus + ** code compiles but fails during run-time if clock_gettime() is + ** called on unsupported OS version. + */ +#if defined(__APPLE__) && (HAVE_BUILTIN_AVAILABLE == 1) + bool have_clock_gettime = FALSE; + if(__builtin_available(macOS 10.12, iOS 10, tvOS 10, watchOS 3, *)) + have_clock_gettime = TRUE; +#endif + + if( +#if defined(__APPLE__) && (HAVE_BUILTIN_AVAILABLE == 1) + have_clock_gettime && +#endif + (0 == clock_gettime(CLOCK_MONOTONIC, &tsnow))) { cnow.tv_sec = tsnow.tv_sec; cnow.tv_usec = (unsigned int)(tsnow.tv_nsec / 1000); } diff --git a/Utilities/cmcurl/lib/transfer.c b/Utilities/cmcurl/lib/transfer.c index 131f2dc7c..05ba862c2 100644 --- a/Utilities/cmcurl/lib/transfer.c +++ b/Utilities/cmcurl/lib/transfer.c @@ -75,6 +75,7 @@ #include "http2.h" #include "mime.h" #include "strcase.h" +#include "urlapi-int.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" @@ -106,15 +107,26 @@ char *Curl_checkheaders(const struct connectdata *conn, } #endif +CURLcode Curl_get_upload_buffer(struct Curl_easy *data) +{ + if(!data->state.ulbuf) { + data->state.ulbuf = malloc(data->set.upload_buffer_size); + if(!data->state.ulbuf) + return CURLE_OUT_OF_MEMORY; + } + return CURLE_OK; +} + /* * This function will call the read callback to fill our buffer with data * to upload. */ -CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) +CURLcode Curl_fillreadbuffer(struct connectdata *conn, size_t bytes, + size_t *nreadp) { struct Curl_easy *data = conn->data; - size_t buffersize = (size_t)bytes; - int nread; + size_t buffersize = bytes; + size_t nread; #ifdef CURL_DOES_CONVERSIONS bool sending_http_headers = FALSE; @@ -134,11 +146,9 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */ } - /* this function returns a size_t, so we typecast to int to prevent warnings - with picky compilers */ Curl_set_in_callback(data, true); - nread = (int)data->state.fread_func(data->req.upload_fromhere, 1, - buffersize, data->state.in); + nread = data->state.fread_func(data->req.upload_fromhere, 1, + buffersize, data->state.in); Curl_set_in_callback(data, false); if(nread == CURL_READFUNC_ABORT) { @@ -167,7 +177,7 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) return CURLE_OK; /* nothing was read */ } - else if((size_t)nread > buffersize) { + else if(nread > buffersize) { /* the read function returned a too large value */ *nreadp = 0; failf(data, "read function returned funny value"); @@ -226,13 +236,13 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) #ifdef CURL_DOES_CONVERSIONS { CURLcode result; - int length; + size_t length; if(data->set.prefer_ascii) /* translate the protocol and data */ length = nread; else /* just translate the protocol portion */ - length = (int)strlen(hexbuffer); + length = strlen(hexbuffer); result = Curl_convert_to_network(data, data->req.upload_fromhere, length); /* Curl_convert_to_network calls failf if unsuccessful */ @@ -247,7 +257,7 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) infof(data, "Signaling end of chunked upload via terminating chunk.\n"); } - nread += (int)strlen(endofline_native); /* for the added end of line */ + nread += strlen(endofline_native); /* for the added end of line */ } #ifdef CURL_DOES_CONVERSIONS else if((data->set.prefer_ascii) && (!sending_http_headers)) { @@ -353,7 +363,7 @@ static int data_pending(const struct connectdata *conn) return conn->handler->protocol&(CURLPROTO_SCP|CURLPROTO_SFTP) || #if defined(USE_NGHTTP2) Curl_ssl_data_pending(conn, FIRSTSOCKET) || - /* For HTTP/2, we may read up everything including responde body + /* For HTTP/2, we may read up everything including response body with header fields in Curl_http_readwrite_headers. If no content-length is provided, curl waits for the connection close, which we emulate it using conn->proto.httpc.closed = @@ -444,7 +454,6 @@ static CURLcode readwrite_data(struct Curl_easy *data, CURLcode result = CURLE_OK; ssize_t nread; /* number of bytes read */ size_t excess = 0; /* excess bytes read */ - bool is_empty_data = FALSE; bool readmore = FALSE; /* used by RTP to signal for more data */ int maxloops = 100; @@ -454,6 +463,7 @@ static CURLcode readwrite_data(struct Curl_easy *data, /* This is where we loop until we have read everything there is to read or we get a CURLE_AGAIN */ do { + bool is_empty_data = FALSE; size_t buffersize = data->set.buffer_size; size_t bytestoread = buffersize; @@ -557,7 +567,7 @@ static CURLcode readwrite_data(struct Curl_easy *data, infof(data, "Rewinding stream by : %zd" " bytes on url %s (zero-length body)\n", - nread, data->state.path); + nread, data->state.up.path); read_rewind(conn, (size_t)nread); } else { @@ -565,7 +575,7 @@ static CURLcode readwrite_data(struct Curl_easy *data, "Excess found in a non pipelined read:" " excess = %zd" " url = %s (zero-length body)\n", - nread, data->state.path); + nread, data->state.up.path); } } @@ -660,14 +670,14 @@ static CURLcode readwrite_data(struct Curl_easy *data, if(data->set.verbose) { if(k->badheader) { Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff, - (size_t)k->hbuflen, conn); + (size_t)k->hbuflen); if(k->badheader == HEADER_PARTHEADER) Curl_debug(data, CURLINFO_DATA_IN, - k->str, (size_t)nread, conn); + k->str, (size_t)nread); } else Curl_debug(data, CURLINFO_DATA_IN, - k->str, (size_t)nread, conn); + k->str, (size_t)nread); } #ifndef CURL_DISABLE_HTTP @@ -734,7 +744,7 @@ static CURLcode readwrite_data(struct Curl_easy *data, " bytes on url %s (size = %" CURL_FORMAT_CURL_OFF_T ", maxdownload = %" CURL_FORMAT_CURL_OFF_T ", bytecount = %" CURL_FORMAT_CURL_OFF_T ", nread = %zd)\n", - excess, data->state.path, + excess, data->state.up.path, k->size, k->maxdownload, k->bytecount, nread); read_rewind(conn, excess); } @@ -797,7 +807,7 @@ static CURLcode readwrite_data(struct Curl_easy *data, nread); } } - else + else if(!k->ignorebody) result = Curl_unencode_write(conn, k->writer_stack, k->str, nread); } k->badheader = HEADER_NORMAL; /* taken care of now */ @@ -869,6 +879,26 @@ static CURLcode done_sending(struct connectdata *conn, return CURLE_OK; } +#if defined(WIN32) && !defined(USE_LWIPSOCK) +#ifndef SIO_IDEAL_SEND_BACKLOG_QUERY +#define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747B +#endif + +static void win_update_buffer_size(curl_socket_t sockfd) +{ + int result; + ULONG ideal; + DWORD ideallen; + result = WSAIoctl(sockfd, SIO_IDEAL_SEND_BACKLOG_QUERY, 0, 0, + &ideal, sizeof(ideal), &ideallen, 0, 0); + if(result == 0) { + setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, + (const char *)&ideal, sizeof(ideal)); + } +} +#else +#define win_update_buffer_size(x) +#endif /* * Send data to upload to the server, when the socket is writable. @@ -894,13 +924,16 @@ static CURLcode readwrite_upload(struct Curl_easy *data, /* only read more data if there's no upload data already present in the upload buffer */ if(0 == k->upload_present) { + result = Curl_get_upload_buffer(data); + if(result) + return result; /* init the "upload from here" pointer */ - k->upload_fromhere = data->state.uploadbuffer; + k->upload_fromhere = data->state.ulbuf; if(!k->upload_done) { /* HTTP pollution, this should be written nicer to become more protocol agnostic. */ - int fillcount; + size_t fillcount; struct HTTP *http = k->protop; if((k->exp100 == EXP100_SENDING_REQUEST) && @@ -927,11 +960,12 @@ static CURLcode readwrite_upload(struct Curl_easy *data, sending_http_headers = FALSE; } - result = Curl_fillreadbuffer(conn, UPLOAD_BUFSIZE, &fillcount); + result = Curl_fillreadbuffer(conn, data->set.upload_buffer_size, + &fillcount); if(result) return result; - nread = (ssize_t)fillcount; + nread = fillcount; } else nread = 0; /* we're done uploading/reading */ @@ -959,7 +993,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data, (data->set.crlf))) { /* Do we need to allocate a scratch buffer? */ if(!data->state.scratch) { - data->state.scratch = malloc(2 * data->set.buffer_size); + data->state.scratch = malloc(2 * data->set.upload_buffer_size); if(!data->state.scratch) { failf(data, "Failed to alloc scratch buffer!"); @@ -1020,14 +1054,15 @@ static CURLcode readwrite_upload(struct Curl_easy *data, k->upload_fromhere, /* buffer pointer */ k->upload_present, /* buffer size */ &bytes_written); /* actually sent */ - if(result) return result; + win_update_buffer_size(conn->writesockfd); + if(data->set.verbose) /* show the data before we change the pointer upload_fromhere */ Curl_debug(data, CURLINFO_DATA_OUT, k->upload_fromhere, - (size_t)bytes_written, conn); + (size_t)bytes_written); k->writebytecount += bytes_written; @@ -1050,7 +1085,10 @@ static CURLcode readwrite_upload(struct Curl_easy *data, } else { /* we've uploaded that buffer now */ - k->upload_fromhere = data->state.uploadbuffer; + result = Curl_get_upload_buffer(data); + if(result) + return result; + k->upload_fromhere = data->state.ulbuf; k->upload_present = 0; /* no more bytes left */ if(k->upload_done) { @@ -1417,314 +1455,11 @@ CURLcode Curl_posttransfer(struct Curl_easy *data) return CURLE_OK; } -#ifndef CURL_DISABLE_HTTP -/* - * Find the separator at the end of the host name, or the '?' in cases like - * http://www.url.com?id=2380 - */ -static const char *find_host_sep(const char *url) -{ - const char *sep; - const char *query; - - /* Find the start of the hostname */ - sep = strstr(url, "//"); - if(!sep) - sep = url; - else - sep += 2; - - query = strchr(sep, '?'); - sep = strchr(sep, '/'); - - if(!sep) - sep = url + strlen(url); - - if(!query) - query = url + strlen(url); - - return sep < query ? sep : query; -} - -/* - * Decide in an encoding-independent manner whether a character in an - * URL must be escaped. The same criterion must be used in strlen_url() - * and strcpy_url(). - */ -static bool urlchar_needs_escaping(int c) -{ - return !(ISCNTRL(c) || ISSPACE(c) || ISGRAPH(c)); -} - -/* - * strlen_url() returns the length of the given URL if the spaces within the - * URL were properly URL encoded. - * URL encoding should be skipped for host names, otherwise IDN resolution - * will fail. - */ -static size_t strlen_url(const char *url, bool relative) -{ - const unsigned char *ptr; - size_t newlen = 0; - bool left = TRUE; /* left side of the ? */ - const unsigned char *host_sep = (const unsigned char *) url; - - if(!relative) - host_sep = (const unsigned char *) find_host_sep(url); - - for(ptr = (unsigned char *)url; *ptr; ptr++) { - - if(ptr < host_sep) { - ++newlen; - continue; - } - - switch(*ptr) { - case '?': - left = FALSE; - /* fall through */ - default: - if(urlchar_needs_escaping(*ptr)) - newlen += 2; - newlen++; - break; - case ' ': - if(left) - newlen += 3; - else - newlen++; - break; - } - } - return newlen; -} - -/* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in - * the source URL accordingly. - * URL encoding should be skipped for host names, otherwise IDN resolution - * will fail. - */ -static void strcpy_url(char *output, const char *url, bool relative) -{ - /* we must add this with whitespace-replacing */ - bool left = TRUE; - const unsigned char *iptr; - char *optr = output; - const unsigned char *host_sep = (const unsigned char *) url; - - if(!relative) - host_sep = (const unsigned char *) find_host_sep(url); - - for(iptr = (unsigned char *)url; /* read from here */ - *iptr; /* until zero byte */ - iptr++) { - - if(iptr < host_sep) { - *optr++ = *iptr; - continue; - } - - switch(*iptr) { - case '?': - left = FALSE; - /* fall through */ - default: - if(urlchar_needs_escaping(*iptr)) { - snprintf(optr, 4, "%%%02x", *iptr); - optr += 3; - } - else - *optr++=*iptr; - break; - case ' ': - if(left) { - *optr++='%'; /* add a '%' */ - *optr++='2'; /* add a '2' */ - *optr++='0'; /* add a '0' */ - } - else - *optr++='+'; /* add a '+' here */ - break; - } - } - *optr = 0; /* zero terminate output buffer */ - -} - -/* - * Returns true if the given URL is absolute (as opposed to relative) - */ -static bool is_absolute_url(const char *url) -{ - char prot[16]; /* URL protocol string storage */ - char letter; /* used for a silly sscanf */ - - return (2 == sscanf(url, "%15[^?&/:]://%c", prot, &letter)) ? TRUE : FALSE; -} - -/* - * Concatenate a relative URL to a base URL making it absolute. - * URL-encodes any spaces. - * The returned pointer must be freed by the caller unless NULL - * (returns NULL on out of memory). - */ -static char *concat_url(const char *base, const char *relurl) -{ - /*** - TRY to append this new path to the old URL - to the right of the host part. Oh crap, this is doomed to cause - problems in the future... - */ - char *newest; - char *protsep; - char *pathsep; - size_t newlen; - bool host_changed = FALSE; - - const char *useurl = relurl; - size_t urllen; - - /* we must make our own copy of the URL to play with, as it may - point to read-only data */ - char *url_clone = strdup(base); - - if(!url_clone) - return NULL; /* skip out of this NOW */ - - /* protsep points to the start of the host name */ - protsep = strstr(url_clone, "//"); - if(!protsep) - protsep = url_clone; - else - protsep += 2; /* pass the slashes */ - - if('/' != relurl[0]) { - int level = 0; - - /* First we need to find out if there's a ?-letter in the URL, - and cut it and the right-side of that off */ - pathsep = strchr(protsep, '?'); - if(pathsep) - *pathsep = 0; - - /* we have a relative path to append to the last slash if there's one - available, or if the new URL is just a query string (starts with a - '?') we append the new one at the end of the entire currently worked - out URL */ - if(useurl[0] != '?') { - pathsep = strrchr(protsep, '/'); - if(pathsep) - *pathsep = 0; - } - - /* Check if there's any slash after the host name, and if so, remember - that position instead */ - pathsep = strchr(protsep, '/'); - if(pathsep) - protsep = pathsep + 1; - else - protsep = NULL; - - /* now deal with one "./" or any amount of "../" in the newurl - and act accordingly */ - - if((useurl[0] == '.') && (useurl[1] == '/')) - useurl += 2; /* just skip the "./" */ - - while((useurl[0] == '.') && - (useurl[1] == '.') && - (useurl[2] == '/')) { - level++; - useurl += 3; /* pass the "../" */ - } - - if(protsep) { - while(level--) { - /* cut off one more level from the right of the original URL */ - pathsep = strrchr(protsep, '/'); - if(pathsep) - *pathsep = 0; - else { - *protsep = 0; - break; - } - } - } - } - else { - /* We got a new absolute path for this server */ - - if((relurl[0] == '/') && (relurl[1] == '/')) { - /* the new URL starts with //, just keep the protocol part from the - original one */ - *protsep = 0; - useurl = &relurl[2]; /* we keep the slashes from the original, so we - skip the new ones */ - host_changed = TRUE; - } - else { - /* cut off the original URL from the first slash, or deal with URLs - without slash */ - pathsep = strchr(protsep, '/'); - if(pathsep) { - /* When people use badly formatted URLs, such as - "http://www.url.com?dir=/home/daniel" we must not use the first - slash, if there's a ?-letter before it! */ - char *sep = strchr(protsep, '?'); - if(sep && (sep < pathsep)) - pathsep = sep; - *pathsep = 0; - } - else { - /* There was no slash. Now, since we might be operating on a badly - formatted URL, such as "http://www.url.com?id=2380" which doesn't - use a slash separator as it is supposed to, we need to check for a - ?-letter as well! */ - pathsep = strchr(protsep, '?'); - if(pathsep) - *pathsep = 0; - } - } - } - - /* If the new part contains a space, this is a mighty stupid redirect - but we still make an effort to do "right". To the left of a '?' - letter we replace each space with %20 while it is replaced with '+' - on the right side of the '?' letter. - */ - newlen = strlen_url(useurl, !host_changed); - - urllen = strlen(url_clone); - - newest = malloc(urllen + 1 + /* possible slash */ - newlen + 1 /* zero byte */); - - if(!newest) { - free(url_clone); /* don't leak this */ - return NULL; - } - - /* copy over the root url part */ - memcpy(newest, url_clone, urllen); - - /* check if we need to append a slash */ - if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0])) - ; - else - newest[urllen++]='/'; - - /* then append the new piece on the right side */ - strcpy_url(&newest[urllen], useurl, !host_changed); - - free(url_clone); - - return newest; -} -#endif /* CURL_DISABLE_HTTP */ - /* * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string * as given by the remote server and set up the new URL to request. + * + * This function DOES NOT FREE the given url. */ CURLcode Curl_follow(struct Curl_easy *data, char *newurl, /* the Location: string */ @@ -1741,6 +1476,7 @@ CURLcode Curl_follow(struct Curl_easy *data, /* Location: redirect */ bool disallowport = FALSE; bool reachedmax = FALSE; + CURLUcode uc; if(type == FOLLOW_REDIR) { if((data->set.maxredirs != -1) && @@ -1773,33 +1509,19 @@ CURLcode Curl_follow(struct Curl_easy *data, } } - if(!is_absolute_url(newurl)) { - /*** - *DANG* this is an RFC 2068 violation. The URL is supposed - to be absolute and this doesn't seem to be that! - */ - char *absolute = concat_url(data->change.url, newurl); - if(!absolute) - return CURLE_OUT_OF_MEMORY; - newurl = absolute; - } - else { - /* The new URL MAY contain space or high byte values, that means a mighty - stupid redirect URL but we still make an effort to do "right". */ - char *newest; - size_t newlen = strlen_url(newurl, FALSE); - + if(Curl_is_absolute_url(newurl, NULL, MAX_SCHEME_LEN)) /* This is an absolute URL, don't allow the custom port number */ disallowport = TRUE; - newest = malloc(newlen + 1); /* get memory for this */ - if(!newest) - return CURLE_OUT_OF_MEMORY; + DEBUGASSERT(data->state.uh); + uc = curl_url_set(data->state.uh, CURLUPART_URL, newurl, + (type == FOLLOW_FAKE) ? CURLU_NON_SUPPORT_SCHEME : 0); + if(uc) + return Curl_uc_to_curlcode(uc); - strcpy_url(newest, newurl, FALSE); /* create a space-free URL */ - newurl = newest; /* use this instead now */ - - } + uc = curl_url_get(data->state.uh, CURLUPART_URL, &newurl, 0); + if(uc) + return Curl_uc_to_curlcode(uc); if(type == FOLLOW_FAKE) { /* we're only figuring out the new url if we would've followed locations @@ -1816,10 +1538,8 @@ CURLcode Curl_follow(struct Curl_easy *data, if(disallowport) data->state.allow_port = FALSE; - if(data->change.url_alloc) { + if(data->change.url_alloc) Curl_safefree(data->change.url); - data->change.url_alloc = FALSE; - } data->change.url = newurl; data->change.url_alloc = TRUE; @@ -1985,8 +1705,13 @@ CURLcode Curl_retry_request(struct connectdata *conn, if(conn->handler->protocol&PROTO_FAMILY_HTTP) { struct HTTP *http = data->req.protop; - if(http->writebytecount) - return Curl_readrewind(conn); + if(http->writebytecount) { + CURLcode result = Curl_readrewind(conn); + if(result) { + Curl_safefree(*url); + return result; + } + } } } return CURLE_OK; diff --git a/Utilities/cmcurl/lib/transfer.h b/Utilities/cmcurl/lib/transfer.h index 9ba398d27..9742455ae 100644 --- a/Utilities/cmcurl/lib/transfer.h +++ b/Utilities/cmcurl/lib/transfer.h @@ -51,9 +51,11 @@ int Curl_single_getsock(const struct connectdata *conn, curl_socket_t *socks, int numsocks); CURLcode Curl_readrewind(struct connectdata *conn); -CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp); +CURLcode Curl_fillreadbuffer(struct connectdata *conn, size_t bytes, + size_t *nreadp); CURLcode Curl_retry_request(struct connectdata *conn, char **url); bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc); +CURLcode Curl_get_upload_buffer(struct Curl_easy *data); /* This sets up a forthcoming transfer */ void @@ -69,4 +71,3 @@ Curl_setup_transfer (struct connectdata *data, ); #endif /* HEADER_CURL_TRANSFER_H */ - diff --git a/Utilities/cmcurl/lib/url.c b/Utilities/cmcurl/lib/url.c index 701f83ab3..0d5a13f99 100644 --- a/Utilities/cmcurl/lib/url.c +++ b/Utilities/cmcurl/lib/url.c @@ -92,6 +92,7 @@ bool curl_win32_idn_to_ascii(const char *in, char **out); #include "non-ascii.h" #include "inet_pton.h" #include "getinfo.h" +#include "urlapi-int.h" /* And now for the protocols */ #include "ftp.h" @@ -127,11 +128,6 @@ bool curl_win32_idn_to_ascii(const char *in, char **out); static void conn_free(struct connectdata *conn); static void free_fixed_hostname(struct hostname *host); -static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke); -static CURLcode parse_url_login(struct Curl_easy *data, - struct connectdata *conn, - char **userptr, char **passwdptr, - char **optionsptr); static unsigned int get_protocol_family(unsigned int protocol); /* Some parts of the code (e.g. chunked encoding) assume this buffer has at @@ -295,6 +291,22 @@ void Curl_freeset(struct Curl_easy *data) Curl_mime_cleanpart(&data->set.mimepost); } +/* free the URL pieces */ +void Curl_up_free(struct Curl_easy *data) +{ + struct urlpieces *up = &data->state.up; + Curl_safefree(up->scheme); + Curl_safefree(up->hostname); + Curl_safefree(up->port); + Curl_safefree(up->user); + Curl_safefree(up->password); + Curl_safefree(up->options); + Curl_safefree(up->path); + Curl_safefree(up->query); + curl_url_cleanup(data->state.uh); + data->state.uh = NULL; +} + /* * This is the internal function curl_easy_cleanup() calls. This should * cleanup and free all resources associated with this sessionhandle. @@ -314,16 +326,17 @@ CURLcode Curl_close(struct Curl_easy *data) Curl_expire_clear(data); /* shut off timers */ m = data->multi; - if(m) /* This handle is still part of a multi handle, take care of this first and detach this handle from there. */ curl_multi_remove_handle(data->multi, data); - if(data->multi_easy) + if(data->multi_easy) { /* when curl_easy_perform() is used, it creates its own multi handle to use and this is the one */ curl_multi_cleanup(data->multi_easy); + data->multi_easy = NULL; + } /* Destroy the timeout list that is held in the easy handle. It is /normally/ done by curl_multi_remove_handle() but this is "just in @@ -337,10 +350,6 @@ CURLcode Curl_close(struct Curl_easy *data) if(data->state.rangestringalloc) free(data->state.range); - /* Free the pathbuffer */ - Curl_safefree(data->state.pathbuffer); - data->state.path = NULL; - /* freed here just in case DONE wasn't called */ Curl_free_request_state(data); @@ -360,19 +369,12 @@ CURLcode Curl_close(struct Curl_easy *data) } data->change.referer = NULL; - if(data->change.url_alloc) { - Curl_safefree(data->change.url); - data->change.url_alloc = FALSE; - } - data->change.url = NULL; - + Curl_up_free(data); Curl_safefree(data->state.buffer); Curl_safefree(data->state.headerbuff); - + Curl_safefree(data->state.ulbuf); Curl_flush_cookies(data, 1); - Curl_digest_cleanup(data); - Curl_safefree(data->info.contenttype); Curl_safefree(data->info.wouldredirect); @@ -519,24 +521,28 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data) set->wildcard_enabled = FALSE; set->chunk_bgn = ZERO_NULL; set->chunk_end = ZERO_NULL; - - /* tcp keepalives are disabled by default, but provide reasonable values for - * the interval and idle times. - */ set->tcp_keepalive = FALSE; set->tcp_keepintvl = 60; set->tcp_keepidle = 60; set->tcp_fastopen = FALSE; set->tcp_nodelay = TRUE; - set->ssl_enable_npn = TRUE; set->ssl_enable_alpn = TRUE; - set->expect_100_timeout = 1000L; /* Wait for a second by default. */ set->sep_headers = TRUE; /* separated header lists by default */ set->buffer_size = READBUFFER_SIZE; + set->upload_buffer_size = UPLOADBUFFER_DEFAULT; set->happy_eyeballs_timeout = CURL_HET_DEFAULT; - + set->fnmatch = ZERO_NULL; + set->upkeep_interval_ms = CURL_UPKEEP_INTERVAL_DEFAULT; + set->maxconnects = DEFAULT_CONNCACHE_SIZE; /* for easy handles */ + set->httpversion = +#ifdef USE_NGHTTP2 + CURL_HTTP_VERSION_2TLS +#else + CURL_HTTP_VERSION_1_1 +#endif + ; Curl_http2_init_userset(set); return result; } @@ -596,8 +602,6 @@ CURLcode Curl_open(struct Curl_easy **curl) data->progress.flags |= PGRS_HIDE; data->state.current_speed = -1; /* init to negative == impossible */ - data->set.fnmatch = ZERO_NULL; - data->set.maxconnects = DEFAULT_CONNCACHE_SIZE; /* for easy handles */ Curl_http2_init_state(&data->state); } @@ -734,20 +738,23 @@ static void conn_free(struct connectdata *conn) * primary connection, like when freeing room in the connection cache or * killing of a dead old connection. * + * A connection needs an easy handle when closing down. We support this passed + * in separately since the connection to get closed here is often already + * disassociated from an easy handle. + * * This function MUST NOT reset state in the Curl_easy struct if that * isn't strictly bound to the life-time of *this* particular connection. * */ -CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) +CURLcode Curl_disconnect(struct Curl_easy *data, + struct connectdata *conn, bool dead_connection) { - struct Curl_easy *data; if(!conn) return CURLE_OK; /* this is closed and fine already */ - data = conn->data; if(!data) { - DEBUGF(fprintf(stderr, "DISCONNECT without easy handle, ignoring\n")); + DEBUGF(infof(data, "DISCONNECT without easy handle, ignoring\n")); return CURLE_OK; } @@ -755,13 +762,12 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) * If this connection isn't marked to force-close, leave it open if there * are other users of it */ - if(!conn->bits.close && - (conn->send_pipe.size + conn->recv_pipe.size)) { - DEBUGF(infof(data, "Curl_disconnect, usecounter: %d\n", - conn->send_pipe.size + conn->recv_pipe.size)); + if(CONN_INUSE(conn) && !dead_connection) { + DEBUGF(infof(data, "Curl_disconnect when inuse: %zu\n", CONN_INUSE(conn))); return CURLE_OK; } + conn->data = data; if(conn->dns_entry != NULL) { Curl_resolv_unlock(data, conn->dns_entry); conn->dns_entry = NULL; @@ -787,16 +793,12 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) free_fixed_hostname(&conn->http_proxy.host); free_fixed_hostname(&conn->socks_proxy.host); + DEBUGASSERT(conn->data == data); + /* this assumes that the pointer is still there after the connection was + detected from the cache */ Curl_ssl_close(conn, FIRSTSOCKET); - /* Indicate to all handles on the pipe that we're dead */ - if(Curl_pipeline_wanted(data->multi, CURLPIPE_ANY)) { - signalPipeClose(&conn->send_pipe, TRUE); - signalPipeClose(&conn->recv_pipe, TRUE); - } - conn_free(conn); - return CURLE_OK; } @@ -848,6 +850,7 @@ static int IsPipeliningPossible(const struct Curl_easy *handle, return avail; } +/* Returns non-zero if a handle was removed */ int Curl_removeHandleFromPipeline(struct Curl_easy *handle, struct curl_llist *pipeline) { @@ -884,6 +887,16 @@ static void Curl_printPipeline(struct curl_llist *pipeline) static struct Curl_easy* gethandleathead(struct curl_llist *pipeline) { struct curl_llist_element *curr = pipeline->head; +#ifdef DEBUGBUILD + { + struct curl_llist_element *p = pipeline->head; + while(p) { + struct Curl_easy *e = p->ptr; + DEBUGASSERT(GOOD_EASY_HANDLE(e)); + p = p->next; + } + } +#endif if(curr) { return (struct Curl_easy *) curr->ptr; } @@ -896,41 +909,22 @@ static struct Curl_easy* gethandleathead(struct curl_llist *pipeline) void Curl_getoff_all_pipelines(struct Curl_easy *data, struct connectdata *conn) { - bool recv_head = (conn->readchannel_inuse && - Curl_recvpipe_head(data, conn)); - bool send_head = (conn->writechannel_inuse && - Curl_sendpipe_head(data, conn)); - - if(Curl_removeHandleFromPipeline(data, &conn->recv_pipe) && recv_head) - Curl_pipeline_leave_read(conn); - if(Curl_removeHandleFromPipeline(data, &conn->send_pipe) && send_head) - Curl_pipeline_leave_write(conn); -} - -static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke) -{ - struct curl_llist_element *curr; - - if(!pipeline) + if(!conn->bundle) return; + if(conn->bundle->multiuse == BUNDLE_PIPELINING) { + bool recv_head = (conn->readchannel_inuse && + Curl_recvpipe_head(data, conn)); + bool send_head = (conn->writechannel_inuse && + Curl_sendpipe_head(data, conn)); - curr = pipeline->head; - while(curr) { - struct curl_llist_element *next = curr->next; - struct Curl_easy *data = (struct Curl_easy *) curr->ptr; - -#ifdef DEBUGBUILD /* debug-only code */ - if(data->magic != CURLEASY_MAGIC_NUMBER) { - /* MAJOR BADNESS */ - infof(data, "signalPipeClose() found BAAD easy handle\n"); - } -#endif - - if(pipe_broke) - data->state.pipe_broke = TRUE; - Curl_multi_handlePipeBreak(data); - Curl_llist_remove(pipeline, curr, NULL); - curr = next; + if(Curl_removeHandleFromPipeline(data, &conn->recv_pipe) && recv_head) + Curl_pipeline_leave_read(conn); + if(Curl_removeHandleFromPipeline(data, &conn->send_pipe) && send_head) + Curl_pipeline_leave_write(conn); + } + else { + (void)Curl_removeHandleFromPipeline(data, &conn->recv_pipe); + (void)Curl_removeHandleFromPipeline(data, &conn->send_pipe); } } @@ -959,12 +953,13 @@ static bool extract_if_dead(struct connectdata *conn, struct Curl_easy *data) { size_t pipeLen = conn->send_pipe.size + conn->recv_pipe.size; - if(!pipeLen && !conn->inuse) { + if(!pipeLen && !CONN_INUSE(conn)) { /* The check for a dead socket makes sense only if there are no handles in pipeline and the connection isn't already marked in use */ bool dead; + conn->data = data; if(conn->handler->connection_check) { /* The protocol has a special method for checking the state of the connection. Use it to check if the connection is dead. */ @@ -979,9 +974,9 @@ static bool extract_if_dead(struct connectdata *conn, } if(dead) { - conn->data = data; infof(data, "Connection %ld seems to be dead!\n", conn->connection_id); Curl_conncache_remove_conn(conn, FALSE); + conn->data = NULL; /* detach */ return TRUE; } } @@ -1025,7 +1020,7 @@ static void prune_dead_connections(struct Curl_easy *data) while(Curl_conncache_foreach(data, data->state.conn_cache, &prune, call_extract_if_dead)) { /* disconnect it */ - (void)Curl_disconnect(prune.extracted, /* dead_connection */TRUE); + (void)Curl_disconnect(data, prune.extracted, /* dead_connection */TRUE); } data->state.conn_cache->last_cleanup = now; } @@ -1139,7 +1134,7 @@ ConnectionExists(struct Curl_easy *data, if(extract_if_dead(check, data)) { /* disconnect it */ - (void)Curl_disconnect(check, /* dead_connection */TRUE); + (void)Curl_disconnect(data, check, /* dead_connection */TRUE); continue; } @@ -1267,12 +1262,12 @@ ConnectionExists(struct Curl_easy *data, } } - if(!canpipe && check->inuse) + if(!canpipe && CONN_INUSE(check)) /* this request can't be pipelined but the checked connection is already in use so we skip it */ continue; - if((check->inuse) && (check->data->multi != needle->data->multi)) + if(CONN_INUSE(check) && (check->data->multi != needle->data->multi)) /* this could be subject for pipeline/multiplex use, but only if they belong to the same multi handle */ continue; @@ -1464,7 +1459,7 @@ ConnectionExists(struct Curl_easy *data, if(chosen) { /* mark it as used before releasing the lock */ - chosen->inuse = TRUE; + chosen->data = data; /* own it! */ Curl_conncache_unlock(needle); *usethis = chosen; return TRUE; /* yes, we found one to use! */ @@ -1554,7 +1549,11 @@ int Curl_protocol_getsock(struct connectdata *conn, { if(conn->handler->proto_getsock) return conn->handler->proto_getsock(conn, socks, numsocks); - return GETSOCK_BLANK; + /* Backup getsock logic. Since there is a live socket in use, we must wait + for it or it will be removed from watching when the multi_socket API is + used. */ + socks[0] = conn->sock[FIRSTSOCKET]; + return GETSOCK_READSOCK(0) | GETSOCK_WRITESOCK(0); } int Curl_doing_getsock(struct connectdata *conn, @@ -1838,6 +1837,12 @@ static struct connectdata *allocate_conn(struct Curl_easy *data) /* Store creation time to help future close decision making */ conn->created = Curl_now(); + /* Store current time to give a baseline to keepalive connection times. */ + conn->keepalive = Curl_now(); + + /* Store off the configured connection upkeep time. */ + conn->upkeep_interval_ms = data->set.upkeep_interval_ms; + conn->data = data; /* Setup the association between this connection and the Curl_easy */ @@ -1944,30 +1949,37 @@ static struct connectdata *allocate_conn(struct Curl_easy *data) return NULL; } -static CURLcode findprotocol(struct Curl_easy *data, - struct connectdata *conn, - const char *protostr) +/* returns the handler if the given scheme is built-in */ +const struct Curl_handler *Curl_builtin_scheme(const char *scheme) { const struct Curl_handler * const *pp; const struct Curl_handler *p; - - /* Scan protocol handler table and match against 'protostr' to set a few - variables based on the URL. Now that the handler may be changed later - when the protocol specific setup function is called. */ - for(pp = protocols; (p = *pp) != NULL; pp++) { - if(strcasecompare(p->scheme, protostr)) { + /* Scan protocol handler table and match against 'scheme'. The handler may + be changed later when the protocol specific setup function is called. */ + for(pp = protocols; (p = *pp) != NULL; pp++) + if(strcasecompare(p->scheme, scheme)) /* Protocol found in table. Check if allowed */ - if(!(data->set.allowed_protocols & p->protocol)) - /* nope, get out */ - break; + return p; + return NULL; /* not found */ +} + + +static CURLcode findprotocol(struct Curl_easy *data, + struct connectdata *conn, + const char *protostr) +{ + const struct Curl_handler *p = Curl_builtin_scheme(protostr); - /* it is allowed for "normal" request, now do an extra check if this is - the result of a redirect */ - if(data->state.this_is_a_follow && - !(data->set.redir_protocols & p->protocol)) - /* nope, get out */ - break; + if(p && /* Protocol found in table. Check if allowed */ + (data->set.allowed_protocols & p->protocol)) { + /* it is allowed for "normal" request, now do an extra check if this is + the result of a redirect */ + if(data->state.this_is_a_follow && + !(data->set.redir_protocols & p->protocol)) + /* nope, get out */ + ; + else { /* Perform setup complement if some. */ conn->handler = conn->given = p; @@ -1976,7 +1988,6 @@ static CURLcode findprotocol(struct Curl_easy *data, } } - /* The protocol was not found in the table, but we don't have to assign it to anything since it is already assigned to a dummy-struct in the create_conn() function when the connectdata struct is allocated. */ @@ -1986,379 +1997,134 @@ static CURLcode findprotocol(struct Curl_easy *data, return CURLE_UNSUPPORTED_PROTOCOL; } + +CURLcode Curl_uc_to_curlcode(CURLUcode uc) +{ + switch(uc) { + default: + return CURLE_URL_MALFORMAT; + case CURLUE_UNSUPPORTED_SCHEME: + return CURLE_UNSUPPORTED_PROTOCOL; + case CURLUE_OUT_OF_MEMORY: + return CURLE_OUT_OF_MEMORY; + case CURLUE_USER_NOT_ALLOWED: + return CURLE_LOGIN_DENIED; + } +} + /* * Parse URL and fill in the relevant members of the connection struct. */ static CURLcode parseurlandfillconn(struct Curl_easy *data, - struct connectdata *conn, - bool *prot_missing, - char **userp, char **passwdp, - char **optionsp) + struct connectdata *conn) { - char *at; - char *fragment; - char *path = data->state.path; - char *query; - int i; - int rc; - const char *protop = ""; CURLcode result; - bool rebuild_url = FALSE; - bool url_has_scheme = FALSE; - char protobuf[16]; - - *prot_missing = FALSE; + CURLU *uh; + CURLUcode uc; + char *hostname; - /* We might pass the entire URL into the request so we need to make sure - * there are no bad characters in there.*/ - if(strpbrk(data->change.url, "\r\n")) { - failf(data, "Illegal characters found in URL"); - return CURLE_URL_MALFORMAT; - } + Curl_up_free(data); /* cleanup previous leftovers first */ - /************************************************************* - * Parse the URL. - * - * We need to parse the url even when using the proxy, because we will need - * the hostname and port in case we are trying to SSL connect through the - * proxy -- and we don't know if we will need to use SSL until we parse the - * url ... - ************************************************************/ - if(data->change.url[0] == ':') { - failf(data, "Bad URL, colon is first character"); - return CURLE_URL_MALFORMAT; - } + /* parse the URL */ + uh = data->state.uh = curl_url(); + if(!uh) + return CURLE_OUT_OF_MEMORY; - /* MSDOS/Windows style drive prefix, eg c: in c:foo */ -#define STARTS_WITH_DRIVE_PREFIX(str) \ - ((('a' <= str[0] && str[0] <= 'z') || \ - ('A' <= str[0] && str[0] <= 'Z')) && \ - (str[1] == ':')) - - /* MSDOS/Windows style drive prefix, optionally with - * a '|' instead of ':', followed by a slash or NUL */ -#define STARTS_WITH_URL_DRIVE_PREFIX(str) \ - ((('a' <= (str)[0] && (str)[0] <= 'z') || \ - ('A' <= (str)[0] && (str)[0] <= 'Z')) && \ - ((str)[1] == ':' || (str)[1] == '|') && \ - ((str)[2] == '/' || (str)[2] == '\\' || (str)[2] == 0)) - - /* Don't mistake a drive letter for a scheme if the default protocol is file. - curld --proto-default file c:/foo/bar.txt */ - if(STARTS_WITH_DRIVE_PREFIX(data->change.url) && - data->set.str[STRING_DEFAULT_PROTOCOL] && - strcasecompare(data->set.str[STRING_DEFAULT_PROTOCOL], "file")) { - ; /* do nothing */ - } - else { /* check for a scheme */ - for(i = 0; i < 16 && data->change.url[i]; ++i) { - if(data->change.url[i] == '/') - break; - if(data->change.url[i] == ':') { - url_has_scheme = TRUE; - break; - } - } + if(data->set.str[STRING_DEFAULT_PROTOCOL] && + !Curl_is_absolute_url(data->change.url, NULL, MAX_SCHEME_LEN)) { + char *url; + if(data->change.url_alloc) + free(data->change.url); + url = aprintf("%s://%s", data->set.str[STRING_DEFAULT_PROTOCOL], + data->change.url); + if(!url) + return CURLE_OUT_OF_MEMORY; + data->change.url = url; + data->change.url_alloc = TRUE; } - /* handle the file: scheme */ - if((url_has_scheme && strncasecompare(data->change.url, "file:", 5)) || - (!url_has_scheme && data->set.str[STRING_DEFAULT_PROTOCOL] && - strcasecompare(data->set.str[STRING_DEFAULT_PROTOCOL], "file"))) { - if(url_has_scheme) - rc = sscanf(data->change.url, "%*15[^\n/:]:%[^\n]", path); - else - rc = sscanf(data->change.url, "%[^\n]", path); - - if(rc != 1) { - failf(data, "Bad URL"); - return CURLE_URL_MALFORMAT; - } - - /* Extra handling URLs with an authority component (i.e. that start with - * "file://") - * - * We allow omitted hostname (e.g. file:/<path>) -- valid according to - * RFC 8089, but not the (current) WHAT-WG URL spec. - */ - if(url_has_scheme && path[0] == '/' && path[1] == '/') { - /* swallow the two slashes */ - char *ptr = &path[2]; - - /* - * According to RFC 8089, a file: URL can be reliably dereferenced if: - * - * o it has no/blank hostname, or - * - * o the hostname matches "localhost" (case-insensitively), or - * - * o the hostname is a FQDN that resolves to this machine. - * - * For brevity, we only consider URLs with empty, "localhost", or - * "127.0.0.1" hostnames as local. - * - * Additionally, there is an exception for URLs with a Windows drive - * letter in the authority (which was accidentally omitted from RFC 8089 - * Appendix E, but believe me, it was meant to be there. --MK) - */ - if(ptr[0] != '/' && !STARTS_WITH_URL_DRIVE_PREFIX(ptr)) { - /* the URL includes a host name, it must match "localhost" or - "127.0.0.1" to be valid */ - if(!checkprefix("localhost/", ptr) && - !checkprefix("127.0.0.1/", ptr)) { - failf(data, "Invalid file://hostname/, " - "expected localhost or 127.0.0.1 or none"); - return CURLE_URL_MALFORMAT; - } - ptr += 9; /* now points to the slash after the host */ - } + uc = curl_url_set(uh, CURLUPART_URL, data->change.url, + CURLU_GUESS_SCHEME | + CURLU_NON_SUPPORT_SCHEME | + (data->set.disallow_username_in_url ? + CURLU_DISALLOW_USER : 0) | + (data->set.path_as_is ? CURLU_PATH_AS_IS : 0)); + if(uc) + return Curl_uc_to_curlcode(uc); - /* This cannot be done with strcpy, as the memory chunks overlap! */ - memmove(path, ptr, strlen(ptr) + 1); - } + uc = curl_url_get(uh, CURLUPART_SCHEME, &data->state.up.scheme, 0); + if(uc) + return Curl_uc_to_curlcode(uc); -#if !defined(MSDOS) && !defined(WIN32) && !defined(__CYGWIN__) - /* Don't allow Windows drive letters when not in Windows. - * This catches both "file:/c:" and "file:c:" */ - if(('/' == path[0] && STARTS_WITH_URL_DRIVE_PREFIX(&path[1])) || - STARTS_WITH_URL_DRIVE_PREFIX(path)) { - failf(data, "File drive letters are only accepted in MSDOS/Windows."); - return CURLE_URL_MALFORMAT; - } -#else - /* If the path starts with a slash and a drive letter, ditch the slash */ - if('/' == path[0] && STARTS_WITH_URL_DRIVE_PREFIX(&path[1])) { - /* This cannot be done with strcpy, as the memory chunks overlap! */ - memmove(path, &path[1], strlen(&path[1]) + 1); - } -#endif + result = findprotocol(data, conn, data->state.up.scheme); + if(result) + return result; - protop = "file"; /* protocol string */ - *prot_missing = !url_has_scheme; + uc = curl_url_get(uh, CURLUPART_USER, &data->state.up.user, + CURLU_URLDECODE); + if(!uc) { + conn->user = strdup(data->state.up.user); + if(!conn->user) + return CURLE_OUT_OF_MEMORY; + conn->bits.user_passwd = TRUE; } - else { - /* clear path */ - char slashbuf[4]; - path[0] = 0; - - rc = sscanf(data->change.url, - "%15[^\n/:]:%3[/]%[^\n/?#]%[^\n]", - protobuf, slashbuf, conn->host.name, path); - if(2 == rc) { - failf(data, "Bad URL"); - return CURLE_URL_MALFORMAT; - } - if(3 > rc) { - - /* - * The URL was badly formatted, let's try the browser-style _without_ - * protocol specified like 'http://'. - */ - rc = sscanf(data->change.url, "%[^\n/?#]%[^\n]", conn->host.name, path); - if(1 > rc) { - /* - * We couldn't even get this format. - * djgpp 2.04 has a sscanf() bug where 'conn->host.name' is - * assigned, but the return value is EOF! - */ -#if defined(__DJGPP__) && (DJGPP_MINOR == 4) - if(!(rc == -1 && *conn->host.name)) -#endif - { - failf(data, "<url> malformed"); - return CURLE_URL_MALFORMAT; - } - } - - /* - * Since there was no protocol part specified in the URL use the - * user-specified default protocol. If we weren't given a default make a - * guess by matching some protocols against the host's outermost - * sub-domain name. Finally if there was no match use HTTP. - */ - - protop = data->set.str[STRING_DEFAULT_PROTOCOL]; - if(!protop) { - /* Note: if you add a new protocol, please update the list in - * lib/version.c too! */ - if(checkprefix("FTP.", conn->host.name)) - protop = "ftp"; - else if(checkprefix("DICT.", conn->host.name)) - protop = "DICT"; - else if(checkprefix("LDAP.", conn->host.name)) - protop = "LDAP"; - else if(checkprefix("IMAP.", conn->host.name)) - protop = "IMAP"; - else if(checkprefix("SMTP.", conn->host.name)) - protop = "smtp"; - else if(checkprefix("POP3.", conn->host.name)) - protop = "pop3"; - else - protop = "http"; - } + else if(uc != CURLUE_NO_USER) + return Curl_uc_to_curlcode(uc); - *prot_missing = TRUE; /* not given in URL */ - } - else { - size_t s = strlen(slashbuf); - protop = protobuf; - if(s != 2) { - infof(data, "Unwillingly accepted illegal URL using %d slash%s!\n", - s, s>1?"es":""); - - if(data->change.url_alloc) - free(data->change.url); - /* repair the URL to use two slashes */ - data->change.url = aprintf("%s://%s%s", - protobuf, conn->host.name, path); - if(!data->change.url) - return CURLE_OUT_OF_MEMORY; - data->change.url_alloc = TRUE; - } - } + uc = curl_url_get(uh, CURLUPART_PASSWORD, &data->state.up.password, + CURLU_URLDECODE); + if(!uc) { + conn->passwd = strdup(data->state.up.password); + if(!conn->passwd) + return CURLE_OUT_OF_MEMORY; + conn->bits.user_passwd = TRUE; } + else if(uc != CURLUE_NO_PASSWORD) + return Curl_uc_to_curlcode(uc); - /* We search for '?' in the host name (but only on the right side of a - * @-letter to allow ?-letters in username and password) to handle things - * like http://example.com?param= (notice the missing '/'). - */ - at = strchr(conn->host.name, '@'); - if(at) - query = strchr(at + 1, '?'); - else - query = strchr(conn->host.name, '?'); - - if(query) { - /* We must insert a slash before the '?'-letter in the URL. If the URL had - a slash after the '?', that is where the path currently begins and the - '?string' is still part of the host name. - - We must move the trailing part from the host name and put it first in - the path. And have it all prefixed with a slash. - */ - - size_t hostlen = strlen(query); - size_t pathlen = strlen(path); - - /* move the existing path plus the zero byte forward, to make room for - the host-name part */ - memmove(path + hostlen + 1, path, pathlen + 1); - - /* now copy the trailing host part in front of the existing path */ - memcpy(path + 1, query, hostlen); - - path[0]='/'; /* prepend the missing slash */ - rebuild_url = TRUE; - - *query = 0; /* now cut off the hostname at the ? */ - } - else if(!path[0]) { - /* if there's no path set, use a single slash */ - strcpy(path, "/"); - rebuild_url = TRUE; + uc = curl_url_get(uh, CURLUPART_OPTIONS, &data->state.up.options, + CURLU_URLDECODE); + if(!uc) { + conn->options = strdup(data->state.up.options); + if(!conn->options) + return CURLE_OUT_OF_MEMORY; } + else if(uc != CURLUE_NO_OPTIONS) + return Curl_uc_to_curlcode(uc); - /* If the URL is malformatted (missing a '/' after hostname before path) we - * insert a slash here. The only letters except '/' that can start a path is - * '?' and '#' - as controlled by the two sscanf() patterns above. - */ - if(path[0] != '/') { - /* We need this function to deal with overlapping memory areas. We know - that the memory area 'path' points to is 'urllen' bytes big and that - is bigger than the path. Use +1 to move the zero byte too. */ - memmove(&path[1], path, strlen(path) + 1); - path[0] = '/'; - rebuild_url = TRUE; - } - else if(!data->set.path_as_is) { - /* sanitise paths and remove ../ and ./ sequences according to RFC3986 */ - char *newp = Curl_dedotdotify(path); - if(!newp) + uc = curl_url_get(uh, CURLUPART_HOST, &data->state.up.hostname, 0); + if(uc) { + if(!strcasecompare("file", data->state.up.scheme)) return CURLE_OUT_OF_MEMORY; - - if(strcmp(newp, path)) { - rebuild_url = TRUE; - free(data->state.pathbuffer); - data->state.pathbuffer = newp; - data->state.path = newp; - path = newp; - } - else - free(newp); } - /* - * "rebuild_url" means that one or more URL components have been modified so - * we need to generate an updated full version. We need the corrected URL - * when communicating over HTTP proxy and we don't know at this point if - * we're using a proxy or not. - */ - if(rebuild_url) { - char *reurl; - - size_t plen = strlen(path); /* new path, should be 1 byte longer than - the original */ - size_t prefixlen = strlen(conn->host.name); - - if(!*prot_missing) { - size_t protolen = strlen(protop); - - if(curl_strnequal(protop, data->change.url, protolen)) - prefixlen += protolen; - else { - failf(data, "<url> malformed"); - return CURLE_URL_MALFORMAT; - } - - if(curl_strnequal("://", &data->change.url[protolen], 3)) - prefixlen += 3; - /* only file: is allowed to omit one or both slashes */ - else if(curl_strnequal("file:", data->change.url, 5)) - prefixlen += 1 + (data->change.url[5] == '/'); - else { - failf(data, "<url> malformed"); - return CURLE_URL_MALFORMAT; - } - } + uc = curl_url_get(uh, CURLUPART_PATH, &data->state.up.path, 0); + if(uc) + return Curl_uc_to_curlcode(uc); - reurl = malloc(prefixlen + plen + 1); - if(!reurl) + uc = curl_url_get(uh, CURLUPART_PORT, &data->state.up.port, + CURLU_DEFAULT_PORT); + if(uc) { + if(!strcasecompare("file", data->state.up.scheme)) return CURLE_OUT_OF_MEMORY; - - /* copy the prefix */ - memcpy(reurl, data->change.url, prefixlen); - - /* append the trailing piece + zerobyte */ - memcpy(&reurl[prefixlen], path, plen + 1); - - /* possible free the old one */ - if(data->change.url_alloc) { - Curl_safefree(data->change.url); - data->change.url_alloc = FALSE; - } - - infof(data, "Rebuilt URL to: %s\n", reurl); - - data->change.url = reurl; - data->change.url_alloc = TRUE; /* free this later */ + } + else { + unsigned long port = strtoul(data->state.up.port, NULL, 10); + conn->remote_port = curlx_ultous(port); } - result = findprotocol(data, conn, protop); - if(result) - return result; + (void)curl_url_get(uh, CURLUPART_QUERY, &data->state.up.query, 0); - /* - * Parse the login details from the URL and strip them out of - * the host name - */ - result = parse_url_login(data, conn, userp, passwdp, optionsp); - if(result) - return result; + hostname = data->state.up.hostname; + if(!hostname) + /* this is for file:// transfers, get a dummy made */ + hostname = (char *)""; - if(conn->host.name[0] == '[') { + if(hostname[0] == '[') { /* This looks like an IPv6 address literal. See if there is an address - scope if there is no location header */ - char *percent = strchr(conn->host.name, '%'); + scope. */ + char *percent = strchr(++hostname, '%'); + conn->bits.ipv6_ip = TRUE; if(percent) { unsigned int identifier_offset = 3; char *endp; @@ -2406,33 +2172,22 @@ static CURLcode parseurlandfillconn(struct Curl_easy *data, infof(data, "Invalid IPv6 address format\n"); } } + percent = strchr(hostname, ']'); + if(percent) + /* terminate IPv6 numerical at end bracket */ + *percent = 0; } + /* make sure the connect struct gets its own copy of the host name */ + conn->host.rawalloc = strdup(hostname); + if(!conn->host.rawalloc) + return CURLE_OUT_OF_MEMORY; + conn->host.name = conn->host.rawalloc; + if(data->set.scope_id) /* Override any scope that was set above. */ conn->scope_id = data->set.scope_id; - /* Remove the fragment part of the path. Per RFC 2396, this is always the - last part of the URI. We are looking for the first '#' so that we deal - gracefully with non conformant URI such as http://example.com#foo#bar. */ - fragment = strchr(path, '#'); - if(fragment) { - *fragment = 0; - - /* we know the path part ended with a fragment, so we know the full URL - string does too and we need to cut it off from there so it isn't used - over proxy */ - fragment = strchr(data->change.url, '#'); - if(fragment) - *fragment = 0; - } - - /* - * So if the URL was A://B/C#D, - * protop is A - * conn->host.name is B - * data->state.path is /C - */ return CURLE_OK; } @@ -2449,7 +2204,7 @@ static CURLcode setup_range(struct Curl_easy *data) free(s->range); if(s->resume_from) - s->range = aprintf("%" CURL_FORMAT_CURL_OFF_TU "-", s->resume_from); + s->range = aprintf("%" CURL_FORMAT_CURL_OFF_T "-", s->resume_from); else s->range = strdup(data->set.str[STRING_SET_RANGE]); @@ -2481,18 +2236,6 @@ static CURLcode setup_connection_internals(struct connectdata *conn) { const struct Curl_handler * p; CURLcode result; - struct Curl_easy *data = conn->data; - - /* in some case in the multi state-machine, we go back to the CONNECT state - and then a second (or third or...) call to this function will be made - without doing a DISCONNECT or DONE in between (since the connection is - yet in place) and therefore this function needs to first make sure - there's no lingering previous data allocated. */ - Curl_free_request_state(data); - - memset(&data->req, 0, sizeof(struct SingleRequest)); - data->req.maxdownload = -1; - conn->socktype = SOCK_STREAM; /* most of them are TCP streams */ /* Perform setup complement if some. */ @@ -2539,14 +2282,13 @@ static bool check_noproxy(const char *name, const char *no_proxy) * not be proxied, or an asterisk to override * all proxy variables) */ - size_t tok_start; - size_t tok_end; - const char *separator = ", "; - size_t no_proxy_len; - size_t namelen; - char *endptr; - if(no_proxy && no_proxy[0]) { + size_t tok_start; + size_t tok_end; + const char *separator = ", "; + size_t no_proxy_len; + size_t namelen; + char *endptr; if(strcasecompare("*", no_proxy)) { return TRUE; } @@ -2560,11 +2302,8 @@ static bool check_noproxy(const char *name, const char *no_proxy) if(!endptr) return FALSE; name++; - } - else - endptr = strchr(name, ':'); - if(endptr) namelen = endptr - name; + } else namelen = strlen(name); @@ -3097,124 +2836,6 @@ out: #endif /* CURL_DISABLE_PROXY */ /* - * parse_url_login() - * - * Parse the login details (user name, password and options) from the URL and - * strip them out of the host name - * - * Inputs: data->set.use_netrc (CURLOPT_NETRC) - * conn->host.name - * - * Outputs: (almost :- all currently undefined) - * conn->bits.user_passwd - non-zero if non-default passwords exist - * user - non-zero length if defined - * passwd - non-zero length if defined - * options - non-zero length if defined - * conn->host.name - remove user name and password - */ -static CURLcode parse_url_login(struct Curl_easy *data, - struct connectdata *conn, - char **user, char **passwd, char **options) -{ - CURLcode result = CURLE_OK; - char *userp = NULL; - char *passwdp = NULL; - char *optionsp = NULL; - - /* At this point, we're hoping all the other special cases have - * been taken care of, so conn->host.name is at most - * [user[:password][;options]]@]hostname - * - * We need somewhere to put the embedded details, so do that first. - */ - - char *ptr = strchr(conn->host.name, '@'); - char *login = conn->host.name; - - DEBUGASSERT(!**user); - DEBUGASSERT(!**passwd); - DEBUGASSERT(!**options); - DEBUGASSERT(conn->handler); - - if(!ptr) - goto out; - - /* We will now try to extract the - * possible login information in a string like: - * ftp://user:password@ftp.my.site:8021/README */ - conn->host.name = ++ptr; - - /* So the hostname is sane. Only bother interpreting the - * results if we could care. It could still be wasted - * work because it might be overtaken by the programmatically - * set user/passwd, but doing that first adds more cases here :-( - */ - - if(data->set.use_netrc == CURL_NETRC_REQUIRED) - goto out; - - /* We could use the login information in the URL so extract it. Only parse - options if the handler says we should. */ - result = - Curl_parse_login_details(login, ptr - login - 1, - &userp, &passwdp, - (conn->handler->flags & PROTOPT_URLOPTIONS)? - &optionsp:NULL); - if(result) - goto out; - - if(userp) { - char *newname; - - /* We have a user in the URL */ - conn->bits.userpwd_in_url = TRUE; - conn->bits.user_passwd = TRUE; /* enable user+password */ - - /* Decode the user */ - result = Curl_urldecode(data, userp, 0, &newname, NULL, FALSE); - if(result) { - goto out; - } - - free(*user); - *user = newname; - } - - if(passwdp) { - /* We have a password in the URL so decode it */ - char *newpasswd; - result = Curl_urldecode(data, passwdp, 0, &newpasswd, NULL, FALSE); - if(result) { - goto out; - } - - free(*passwd); - *passwd = newpasswd; - } - - if(optionsp) { - /* We have an options list in the URL so decode it */ - char *newoptions; - result = Curl_urldecode(data, optionsp, 0, &newoptions, NULL, FALSE); - if(result) { - goto out; - } - - free(*options); - *options = newoptions; - } - - - out: - - free(userp); - free(passwdp); - free(optionsp); - - return result; -} - -/* * Curl_parse_login_details() * * This is used to parse a login string for user name, password and options in @@ -3236,7 +2857,7 @@ static CURLcode parse_url_login(struct Curl_easy *data, * len [in] - The length of the login string. * userp [in/out] - The address where a pointer to newly allocated memory * holding the user will be stored upon completion. - * passdwp [in/out] - The address where a pointer to newly allocated memory + * passwdp [in/out] - The address where a pointer to newly allocated memory * holding the password will be stored upon completion. * optionsp [in/out] - The address where a pointer to newly allocated memory * holding the options will be stored upon completion. @@ -3347,131 +2968,23 @@ CURLcode Curl_parse_login_details(const char *login, const size_t len, * No matter if we use a proxy or not, we have to figure out the remote * port number of various reasons. * - * To be able to detect port number flawlessly, we must not confuse them - * IPv6-specified addresses in the [0::1] style. (RFC2732) - * - * The conn->host.name is currently [user:passwd@]host[:port] where host - * could be a hostname, IPv4 address or IPv6 address. - * * The port number embedded in the URL is replaced, if necessary. *************************************************************/ static CURLcode parse_remote_port(struct Curl_easy *data, struct connectdata *conn) { - char *portptr; - char endbracket; - - /* Note that at this point, the IPv6 address cannot contain any scope - suffix as that has already been removed in the parseurlandfillconn() - function */ - if((1 == sscanf(conn->host.name, "[%*45[0123456789abcdefABCDEF:.]%c", - &endbracket)) && - (']' == endbracket)) { - /* this is a RFC2732-style specified IP-address */ - conn->bits.ipv6_ip = TRUE; - - conn->host.name++; /* skip over the starting bracket */ - portptr = strchr(conn->host.name, ']'); - if(portptr) { - *portptr++ = '\0'; /* zero terminate, killing the bracket */ - if(*portptr) { - if (*portptr != ':') { - failf(data, "IPv6 closing bracket followed by '%c'", *portptr); - return CURLE_URL_MALFORMAT; - } - } - else - portptr = NULL; /* no port number available */ - } - } - else { -#ifdef ENABLE_IPV6 - struct in6_addr in6; - if(Curl_inet_pton(AF_INET6, conn->host.name, &in6) > 0) { - /* This is a numerical IPv6 address, meaning this is a wrongly formatted - URL */ - failf(data, "IPv6 numerical address used in URL without brackets"); - return CURLE_URL_MALFORMAT; - } -#endif - - portptr = strchr(conn->host.name, ':'); - } if(data->set.use_port && data->state.allow_port) { - /* if set, we use this and ignore the port possibly given in the URL */ + /* if set, we use this instead of the port possibly given in the URL */ + char portbuf[16]; + CURLUcode uc; conn->remote_port = (unsigned short)data->set.use_port; - if(portptr) - *portptr = '\0'; /* cut off the name there anyway - if there was a port - number - since the port number is to be ignored! */ - if(conn->bits.httpproxy) { - /* we need to create new URL with the new port number */ - char *url; - char type[12]=""; - - if(conn->bits.type_set) - snprintf(type, sizeof(type), ";type=%c", - data->set.prefer_ascii?'A': - (data->set.ftp_list_only?'D':'I')); - - /* - * This synthesized URL isn't always right--suffixes like ;type=A are - * stripped off. It would be better to work directly from the original - * URL and simply replace the port part of it. - */ - url = aprintf("%s://%s%s%s:%d%s%s%s", conn->given->scheme, - conn->bits.ipv6_ip?"[":"", conn->host.name, - conn->bits.ipv6_ip?"]":"", conn->remote_port, - data->state.slash_removed?"/":"", data->state.path, - type); - if(!url) - return CURLE_OUT_OF_MEMORY; - - if(data->change.url_alloc) { - Curl_safefree(data->change.url); - data->change.url_alloc = FALSE; - } - - data->change.url = url; - data->change.url_alloc = TRUE; - } - } - else if(portptr) { - /* no CURLOPT_PORT given, extract the one from the URL */ - - char *rest; - long port; - - port = strtol(portptr + 1, &rest, 10); /* Port number must be decimal */ - - if((port < 0) || (port > 0xffff)) { - /* Single unix standard says port numbers are 16 bits long */ - failf(data, "Port number out of range"); - return CURLE_URL_MALFORMAT; - } - - if(rest[0]) { - failf(data, "Port number ended with '%c'", rest[0]); - return CURLE_URL_MALFORMAT; - } - - if(rest != &portptr[1]) { - *portptr = '\0'; /* cut off the name there */ - conn->remote_port = curlx_ultous(port); - } - else { - /* Browser behavior adaptation. If there's a colon with no digits after, - just cut off the name there which makes us ignore the colon and just - use the default port. Firefox and Chrome both do that. */ - *portptr = '\0'; - } + snprintf(portbuf, sizeof(portbuf), "%u", conn->remote_port); + uc = curl_url_set(data->state.uh, CURLUPART_PORT, portbuf, 0); + if(uc) + return CURLE_OUT_OF_MEMORY; } - /* only if remote_port was not already parsed off the URL we use the - default port number */ - if(conn->remote_port < 0) - conn->remote_port = (unsigned short)conn->given->defport; - return CURLE_OK; } @@ -3483,11 +2996,16 @@ static CURLcode override_login(struct Curl_easy *data, struct connectdata *conn, char **userp, char **passwdp, char **optionsp) { + bool user_changed = FALSE; + bool passwd_changed = FALSE; + CURLUcode uc; if(data->set.str[STRING_USERNAME]) { free(*userp); *userp = strdup(data->set.str[STRING_USERNAME]); if(!*userp) return CURLE_OUT_OF_MEMORY; + conn->bits.user_passwd = TRUE; /* enable user+password */ + user_changed = TRUE; } if(data->set.str[STRING_PASSWORD]) { @@ -3495,6 +3013,8 @@ static CURLcode override_login(struct Curl_easy *data, *passwdp = strdup(data->set.str[STRING_PASSWORD]); if(!*passwdp) return CURLE_OUT_OF_MEMORY; + conn->bits.user_passwd = TRUE; /* enable user+password */ + passwd_changed = TRUE; } if(data->set.str[STRING_OPTIONS]) { @@ -3506,9 +3026,16 @@ static CURLcode override_login(struct Curl_easy *data, conn->bits.netrc = FALSE; if(data->set.use_netrc != CURL_NETRC_IGNORED) { - int ret = Curl_parsenetrc(conn->host.name, - userp, passwdp, - data->set.str[STRING_NETRC_FILE]); + char *nuser = NULL; + char *npasswd = NULL; + int ret; + + if(data->set.use_netrc == CURL_NETRC_OPTIONAL) + nuser = *userp; /* to separate otherwise identical machines */ + + ret = Curl_parsenetrc(conn->host.name, + &nuser, &npasswd, + data->set.str[STRING_NETRC_FILE]); if(ret > 0) { infof(data, "Couldn't find host %s in the " DOT_CHAR "netrc file; using defaults\n", @@ -3522,55 +3049,85 @@ static CURLcode override_login(struct Curl_easy *data, file, so that it is safe to use even if we followed a Location: to a different host or similar. */ conn->bits.netrc = TRUE; - conn->bits.user_passwd = TRUE; /* enable user+password */ + + if(data->set.use_netrc == CURL_NETRC_OPTIONAL) { + /* prefer credentials outside netrc */ + if(nuser && !*userp) { + free(*userp); + *userp = nuser; + user_changed = TRUE; + } + if(npasswd && !*passwdp) { + free(*passwdp); + *passwdp = npasswd; + passwd_changed = TRUE; + } + } + else { + /* prefer netrc credentials */ + if(nuser) { + free(*userp); + *userp = nuser; + user_changed = TRUE; + } + if(npasswd) { + free(*passwdp); + *passwdp = npasswd; + passwd_changed = TRUE; + } + } } } + /* for updated strings, we update them in the URL */ + if(user_changed) { + uc = curl_url_set(data->state.uh, CURLUPART_USER, *userp, 0); + if(uc) + return Curl_uc_to_curlcode(uc); + } + if(passwd_changed) { + uc = curl_url_set(data->state.uh, CURLUPART_PASSWORD, *passwdp, 0); + if(uc) + return Curl_uc_to_curlcode(uc); + } return CURLE_OK; } /* * Set the login details so they're available in the connection */ -static CURLcode set_login(struct connectdata *conn, - const char *user, const char *passwd, - const char *options) +static CURLcode set_login(struct connectdata *conn) { CURLcode result = CURLE_OK; + const char *setuser = CURL_DEFAULT_USER; + const char *setpasswd = CURL_DEFAULT_PASSWORD; /* If our protocol needs a password and we have none, use the defaults */ - if((conn->handler->flags & PROTOPT_NEEDSPWD) && !conn->bits.user_passwd) { - /* Store the default user */ - conn->user = strdup(CURL_DEFAULT_USER); - - /* Store the default password */ - if(conn->user) - conn->passwd = strdup(CURL_DEFAULT_PASSWORD); - else - conn->passwd = NULL; - - /* This is the default password, so DON'T set conn->bits.user_passwd */ - } + if((conn->handler->flags & PROTOPT_NEEDSPWD) && !conn->bits.user_passwd) + ; else { - /* Store the user, zero-length if not set */ - conn->user = strdup(user); - - /* Store the password (only if user is present), zero-length if not set */ - if(conn->user) - conn->passwd = strdup(passwd); - else - conn->passwd = NULL; + setuser = ""; + setpasswd = ""; + } + /* Store the default user */ + if(!conn->user) { + conn->user = strdup(setuser); + if(!conn->user) + return CURLE_OUT_OF_MEMORY; } - if(!conn->user || !conn->passwd) - result = CURLE_OUT_OF_MEMORY; - - /* Store the options, null if not set */ - if(!result && options[0]) { - conn->options = strdup(options); + /* Store the default password */ + if(!conn->passwd) { + conn->passwd = strdup(setpasswd); + if(!conn->passwd) + result = CURLE_OUT_OF_MEMORY; + } - if(!conn->options) + /* if there's a user without password, consider password blank */ + if(conn->user && !conn->passwd) { + conn->passwd = strdup(""); + if(!conn->passwd) result = CURLE_OUT_OF_MEMORY; } @@ -3982,6 +3539,7 @@ static void reuse_conn(struct connectdata *old_conn, Curl_safefree(old_conn->user); Curl_safefree(old_conn->passwd); + Curl_safefree(old_conn->options); Curl_safefree(old_conn->http_proxy.user); Curl_safefree(old_conn->socks_proxy.user); Curl_safefree(old_conn->http_proxy.passwd); @@ -4021,12 +3579,7 @@ static CURLcode create_conn(struct Curl_easy *data, CURLcode result = CURLE_OK; struct connectdata *conn; struct connectdata *conn_temp = NULL; - size_t urllen; - char *user = NULL; - char *passwd = NULL; - char *options = NULL; bool reuse; - bool prot_missing = FALSE; bool connections_available = TRUE; bool force_reuse = FALSE; bool waitpipe = FALSE; @@ -4038,7 +3591,6 @@ static CURLcode create_conn(struct Curl_easy *data, /************************************************************* * Check input data *************************************************************/ - if(!data->change.url) { result = CURLE_URL_MALFORMAT; goto out; @@ -4060,107 +3612,10 @@ static CURLcode create_conn(struct Curl_easy *data, any failure */ *in_connect = conn; - /* This initing continues below, see the comment "Continue connectdata - * initialization here" */ - - /*********************************************************** - * We need to allocate memory to store the path in. We get the size of the - * full URL to be sure, and we need to make it at least 256 bytes since - * other parts of the code will rely on this fact - ***********************************************************/ -#define LEAST_PATH_ALLOC 256 - urllen = strlen(data->change.url); - if(urllen < LEAST_PATH_ALLOC) - urllen = LEAST_PATH_ALLOC; - - /* - * We malloc() the buffers below urllen+2 to make room for 2 possibilities: - * 1 - an extra terminating zero - * 2 - an extra slash (in case a syntax like "www.host.com?moo" is used) - */ - - Curl_safefree(data->state.pathbuffer); - data->state.path = NULL; - - data->state.pathbuffer = malloc(urllen + 2); - if(NULL == data->state.pathbuffer) { - result = CURLE_OUT_OF_MEMORY; /* really bad error */ - goto out; - } - data->state.path = data->state.pathbuffer; - - conn->host.rawalloc = malloc(urllen + 2); - if(NULL == conn->host.rawalloc) { - Curl_safefree(data->state.pathbuffer); - data->state.path = NULL; - result = CURLE_OUT_OF_MEMORY; - goto out; - } - - conn->host.name = conn->host.rawalloc; - conn->host.name[0] = 0; - - user = strdup(""); - passwd = strdup(""); - options = strdup(""); - if(!user || !passwd || !options) { - result = CURLE_OUT_OF_MEMORY; - goto out; - } - - result = parseurlandfillconn(data, conn, &prot_missing, &user, &passwd, - &options); + result = parseurlandfillconn(data, conn); if(result) goto out; - /************************************************************* - * No protocol part in URL was used, add it! - *************************************************************/ - if(prot_missing) { - /* We're guessing prefixes here and if we're told to use a proxy or if - we're going to follow a Location: later or... then we need the protocol - part added so that we have a valid URL. */ - char *reurl; - char *ch_lower; - - reurl = aprintf("%s://%s", conn->handler->scheme, data->change.url); - - if(!reurl) { - result = CURLE_OUT_OF_MEMORY; - goto out; - } - - /* Change protocol prefix to lower-case */ - for(ch_lower = reurl; *ch_lower != ':'; ch_lower++) - *ch_lower = (char)TOLOWER(*ch_lower); - - if(data->change.url_alloc) { - Curl_safefree(data->change.url); - data->change.url_alloc = FALSE; - } - - data->change.url = reurl; - data->change.url_alloc = TRUE; /* free this later */ - } - - /************************************************************* - * If the protocol can't handle url query strings, then cut - * off the unhandable part - *************************************************************/ - if((conn->given->flags&PROTOPT_NOURLQUERY)) { - char *path_q_sep = strchr(conn->data->state.path, '?'); - if(path_q_sep) { - /* according to rfc3986, allow the query (?foo=bar) - also on protocols that can't handle it. - - cut the string-part after '?' - */ - - /* terminate the string */ - path_q_sep[0] = 0; - } - } - if(data->set.str[STRING_BEARER]) { conn->oauth_bearer = strdup(data->set.str[STRING_BEARER]); if(!conn->oauth_bearer) { @@ -4204,10 +3659,12 @@ static CURLcode create_conn(struct Curl_easy *data, /* Check for overridden login details and set them accordingly so they they are known when protocol->setup_connection is called! */ - result = override_login(data, conn, &user, &passwd, &options); + result = override_login(data, conn, &conn->user, &conn->passwd, + &conn->options); if(result) goto out; - result = set_login(conn, user, passwd, options); + + result = set_login(conn); /* default credentials */ if(result) goto out; @@ -4290,6 +3747,7 @@ static CURLcode create_conn(struct Curl_easy *data, /* this is supposed to be the connect function so we better at least check that the file is present here! */ DEBUGASSERT(conn->handler->connect_it); + Curl_persistconninfo(conn); result = conn->handler->connect_it(conn, &done); /* Setup a "faked" transfer that'll do nothing */ @@ -4297,7 +3755,9 @@ static CURLcode create_conn(struct Curl_easy *data, conn->data = data; conn->bits.tcpconnect[FIRSTSOCKET] = TRUE; /* we are "connected */ - Curl_conncache_add_conn(data->state.conn_cache, conn); + result = Curl_conncache_add_conn(data->state.conn_cache, conn); + if(result) + goto out; /* * Setup whatever necessary for a resumed transfer @@ -4343,6 +3803,10 @@ static CURLcode create_conn(struct Curl_easy *data, data->set.str[STRING_SSL_CIPHER_LIST_ORIG]; data->set.proxy_ssl.primary.cipher_list = data->set.str[STRING_SSL_CIPHER_LIST_PROXY]; + data->set.ssl.primary.cipher_list13 = + data->set.str[STRING_SSL_CIPHER13_LIST_ORIG]; + data->set.proxy_ssl.primary.cipher_list13 = + data->set.str[STRING_SSL_CIPHER13_LIST_PROXY]; data->set.ssl.CRLfile = data->set.str[STRING_SSL_CRLFILE_ORIG]; data->set.proxy_ssl.CRLfile = data->set.str[STRING_SSL_CRLFILE_PROXY]; @@ -4387,6 +3851,9 @@ static CURLcode create_conn(struct Curl_easy *data, * new one. *************************************************************/ + DEBUGASSERT(conn->user); + DEBUGASSERT(conn->passwd); + /* reuse_fresh is TRUE if we are told to use a new connection by force, but we only acknowledge this option if this is not a re-used connection already (which happens due to follow-location or during a HTTP @@ -4468,13 +3935,11 @@ static CURLcode create_conn(struct Curl_easy *data, conn_candidate = Curl_conncache_extract_bundle(data, bundle); Curl_conncache_unlock(conn); - if(conn_candidate) { - /* Set the connection's owner correctly, then kill it */ - conn_candidate->data = data; - (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); - } + if(conn_candidate) + (void)Curl_disconnect(data, conn_candidate, + /* dead_connection */ FALSE); else { - infof(data, "No more connections allowed to host: %d\n", + infof(data, "No more connections allowed to host: %zu\n", max_host_connections); connections_available = FALSE; } @@ -4491,12 +3956,9 @@ static CURLcode create_conn(struct Curl_easy *data, /* The cache is full. Let's see if we can kill a connection. */ conn_candidate = Curl_conncache_extract_oldest(data); - - if(conn_candidate) { - /* Set the connection's owner correctly, then kill it */ - conn_candidate->data = data; - (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); - } + if(conn_candidate) + (void)Curl_disconnect(data, conn_candidate, + /* dead_connection */ FALSE); else { infof(data, "No connections available in cache\n"); connections_available = FALSE; @@ -4513,14 +3975,13 @@ static CURLcode create_conn(struct Curl_easy *data, goto out; } else { - /* Mark the connection as used, before we add it */ - conn->inuse = TRUE; - /* * This is a brand new connection, so let's store it in the connection * cache of ours! */ - Curl_conncache_add_conn(data->state.conn_cache, conn); + result = Curl_conncache_add_conn(data->state.conn_cache, conn); + if(result) + goto out; } #if defined(USE_NTLM) @@ -4568,10 +4029,6 @@ static CURLcode create_conn(struct Curl_easy *data, result = resolve_server(data, conn, async); out: - - free(options); - free(passwd); - free(user); return result; } @@ -4656,12 +4113,16 @@ CURLcode Curl_connect(struct Curl_easy *data, *asyncp = FALSE; /* assume synchronous resolves by default */ + /* init the single-transfer specific data */ + Curl_free_request_state(data); + memset(&data->req, 0, sizeof(struct SingleRequest)); + data->req.maxdownload = -1; + /* call the stuff that needs to be called */ result = create_conn(data, in_connect, asyncp); if(!result) { - /* no error */ - if((*in_connect)->send_pipe.size || (*in_connect)->recv_pipe.size) + if(CONN_INUSE(*in_connect)) /* pipelining */ *protocol_done = TRUE; else if(!*asyncp) { @@ -4676,12 +4137,11 @@ CURLcode Curl_connect(struct Curl_easy *data, *in_connect = NULL; return result; } - - if(result && *in_connect) { - /* We're not allowed to return failure with memory left allocated - in the connectdata struct, free those here */ - Curl_disconnect(*in_connect, FALSE); /* close the connection */ - *in_connect = NULL; /* return a NULL */ + else if(result && *in_connect) { + /* We're not allowed to return failure with memory left allocated in the + connectdata struct, free those here */ + Curl_disconnect(data, *in_connect, TRUE); + *in_connect = NULL; /* return a NULL */ } return result; @@ -4701,16 +4161,18 @@ CURLcode Curl_init_do(struct Curl_easy *data, struct connectdata *conn) { struct SingleRequest *k = &data->req; - conn->bits.do_more = FALSE; /* by default there's no curl_do_more() to - use */ + if(conn) { + conn->bits.do_more = FALSE; /* by default there's no curl_do_more() to + use */ + /* if the protocol used doesn't support wildcards, switch it off */ + if(data->state.wildcardmatch && + !(conn->handler->flags & PROTOPT_WILDCARD)) + data->state.wildcardmatch = FALSE; + } data->state.done = FALSE; /* *_done() is not called yet */ data->state.expect100header = FALSE; - /* if the protocol used doesn't support wildcards, switch it off */ - if(data->state.wildcardmatch && - !(conn->handler->flags & PROTOPT_WILDCARD)) - data->state.wildcardmatch = FALSE; if(data->set.opt_no_body) /* in HTTP lingo, no body means using the HEAD request... */ @@ -4850,3 +4312,34 @@ static unsigned int get_protocol_family(unsigned int protocol) return family; } + + +/* + * Wrapper to call functions in Curl_conncache_foreach() + * + * Returns always 0. + */ +static int conn_upkeep(struct connectdata *conn, + void *param) +{ + /* Param is unused. */ + (void)param; + + if(conn->handler->connection_check) { + /* Do a protocol-specific keepalive check on the connection. */ + conn->handler->connection_check(conn, CONNCHECK_KEEPALIVE); + } + + return 0; /* continue iteration */ +} + +CURLcode Curl_upkeep(struct conncache *conn_cache, + void *data) +{ + /* Loop over every connection and make connection alive. */ + Curl_conncache_foreach(data, + conn_cache, + data, + conn_upkeep); + return CURLE_OK; +} diff --git a/Utilities/cmcurl/lib/url.h b/Utilities/cmcurl/lib/url.h index a70bd5466..095d63833 100644 --- a/Utilities/cmcurl/lib/url.h +++ b/Utilities/cmcurl/lib/url.h @@ -27,6 +27,18 @@ #define READBUFFER_MAX CURL_MAX_READ_SIZE #define READBUFFER_MIN 1024 +/* The default upload buffer size, should not be smaller than + CURL_MAX_WRITE_SIZE, as it needs to hold a full buffer as could be sent in + a write callback. + + The size was 16KB for many years but was bumped to 64KB because it makes + libcurl able to do significantly faster uploads in some circumstances. Even + larger buffers can help further, but this is deemed a fair memory/speed + compromise. */ +#define UPLOADBUFFER_DEFAULT 65536 +#define UPLOADBUFFER_MAX (2*1024*1024) +#define UPLOADBUFFER_MIN CURL_MAX_WRITE_SIZE + /* * Prototypes for library-wide functions provided by url.c */ @@ -34,12 +46,16 @@ CURLcode Curl_init_do(struct Curl_easy *data, struct connectdata *conn); CURLcode Curl_open(struct Curl_easy **curl); CURLcode Curl_init_userdefined(struct Curl_easy *data); -CURLcode Curl_dupset(struct Curl_easy * dst, struct Curl_easy * src); + void Curl_freeset(struct Curl_easy * data); +/* free the URL pieces */ +void Curl_up_free(struct Curl_easy *data); +CURLcode Curl_uc_to_curlcode(CURLUcode uc); CURLcode Curl_close(struct Curl_easy *data); /* opposite of curl_open() */ CURLcode Curl_connect(struct Curl_easy *, struct connectdata **, bool *async, bool *protocol_connect); -CURLcode Curl_disconnect(struct connectdata *, bool dead_connection); +CURLcode Curl_disconnect(struct Curl_easy *data, + struct connectdata *, bool dead_connection); CURLcode Curl_protocol_connect(struct connectdata *conn, bool *done); CURLcode Curl_protocol_connecting(struct connectdata *conn, bool *done); CURLcode Curl_protocol_doing(struct connectdata *conn, bool *done); @@ -56,9 +72,7 @@ int Curl_doing_getsock(struct connectdata *conn, CURLcode Curl_parse_login_details(const char *login, const size_t len, char **userptr, char **passwdptr, char **optionsptr); -bool Curl_isPipeliningEnabled(const struct Curl_easy *handle); -CURLcode Curl_addHandleToPipeline(struct Curl_easy *handle, - struct curl_llist *pipeline); + int Curl_removeHandleFromPipeline(struct Curl_easy *handle, struct curl_llist *pipeline); /* remove the specified connection from all (possible) pipelines and related @@ -66,7 +80,9 @@ int Curl_removeHandleFromPipeline(struct Curl_easy *handle, void Curl_getoff_all_pipelines(struct Curl_easy *data, struct connectdata *conn); -void Curl_close_connections(struct Curl_easy *data); +CURLcode Curl_upkeep(struct conncache *conn_cache, void *data); + +const struct Curl_handler *Curl_builtin_scheme(const char *scheme); #define CURL_DEFAULT_PROXY_PORT 1080 /* default proxy port unless specified */ #define CURL_DEFAULT_HTTPS_PROXY_PORT 443 /* default https proxy port unless diff --git a/Utilities/cmcurl/lib/urlapi-int.h b/Utilities/cmcurl/lib/urlapi-int.h new file mode 100644 index 000000000..a57d2e22b --- /dev/null +++ b/Utilities/cmcurl/lib/urlapi-int.h @@ -0,0 +1,33 @@ +#ifndef HEADER_CURL_URLAPI_INT_H +#define HEADER_CURL_URLAPI_INT_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#include "curl_setup.h" +/* scheme is not URL encoded, the longest libcurl supported ones are 6 + letters */ +#define MAX_SCHEME_LEN 8 + +bool Curl_is_absolute_url(const char *url, char *scheme, size_t buflen); +char *Curl_concat_url(const char *base, const char *relurl); +size_t Curl_strlen_url(const char *url, bool relative); +void Curl_strcpy_url(char *output, const char *url, bool relative); +#endif /* HEADER_CURL_URLAPI_INT_H */ diff --git a/Utilities/cmcurl/lib/urlapi.c b/Utilities/cmcurl/lib/urlapi.c new file mode 100644 index 000000000..c53e52343 --- /dev/null +++ b/Utilities/cmcurl/lib/urlapi.c @@ -0,0 +1,1340 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "curl_setup.h" + +#include "urldata.h" +#include "urlapi-int.h" +#include "strcase.h" +#include "dotdot.h" +#include "url.h" +#include "escape.h" +#include "curl_ctype.h" + +/* The last 3 #include files should be in this order */ +#include "curl_printf.h" +#include "curl_memory.h" +#include "memdebug.h" + + /* MSDOS/Windows style drive prefix, eg c: in c:foo */ +#define STARTS_WITH_DRIVE_PREFIX(str) \ + ((('a' <= str[0] && str[0] <= 'z') || \ + ('A' <= str[0] && str[0] <= 'Z')) && \ + (str[1] == ':')) + + /* MSDOS/Windows style drive prefix, optionally with + * a '|' instead of ':', followed by a slash or NUL */ +#define STARTS_WITH_URL_DRIVE_PREFIX(str) \ + ((('a' <= (str)[0] && (str)[0] <= 'z') || \ + ('A' <= (str)[0] && (str)[0] <= 'Z')) && \ + ((str)[1] == ':' || (str)[1] == '|') && \ + ((str)[2] == '/' || (str)[2] == '\\' || (str)[2] == 0)) + +/* Internal representation of CURLU. Point to URL-encoded strings. */ +struct Curl_URL { + char *scheme; + char *user; + char *password; + char *options; /* IMAP only? */ + char *host; + char *port; + char *path; + char *query; + char *fragment; + + char *scratch; /* temporary scratch area */ + long portnum; /* the numerical version */ +}; + +#define DEFAULT_SCHEME "https" + +static void free_urlhandle(struct Curl_URL *u) +{ + free(u->scheme); + free(u->user); + free(u->password); + free(u->options); + free(u->host); + free(u->port); + free(u->path); + free(u->query); + free(u->fragment); + free(u->scratch); +} + +/* move the full contents of one handle onto another and + free the original */ +static void mv_urlhandle(struct Curl_URL *from, + struct Curl_URL *to) +{ + free_urlhandle(to); + *to = *from; + free(from); +} + +/* + * Find the separator at the end of the host name, or the '?' in cases like + * http://www.url.com?id=2380 + */ +static const char *find_host_sep(const char *url) +{ + const char *sep; + const char *query; + + /* Find the start of the hostname */ + sep = strstr(url, "//"); + if(!sep) + sep = url; + else + sep += 2; + + query = strchr(sep, '?'); + sep = strchr(sep, '/'); + + if(!sep) + sep = url + strlen(url); + + if(!query) + query = url + strlen(url); + + return sep < query ? sep : query; +} + +/* + * Decide in an encoding-independent manner whether a character in an + * URL must be escaped. The same criterion must be used in strlen_url() + * and strcpy_url(). + */ +static bool urlchar_needs_escaping(int c) +{ + return !(ISCNTRL(c) || ISSPACE(c) || ISGRAPH(c)); +} + +/* + * strlen_url() returns the length of the given URL if the spaces within the + * URL were properly URL encoded. + * URL encoding should be skipped for host names, otherwise IDN resolution + * will fail. + */ +size_t Curl_strlen_url(const char *url, bool relative) +{ + const unsigned char *ptr; + size_t newlen = 0; + bool left = TRUE; /* left side of the ? */ + const unsigned char *host_sep = (const unsigned char *) url; + + if(!relative) + host_sep = (const unsigned char *) find_host_sep(url); + + for(ptr = (unsigned char *)url; *ptr; ptr++) { + + if(ptr < host_sep) { + ++newlen; + continue; + } + + switch(*ptr) { + case '?': + left = FALSE; + /* FALLTHROUGH */ + default: + if(urlchar_needs_escaping(*ptr)) + newlen += 2; + newlen++; + break; + case ' ': + if(left) + newlen += 3; + else + newlen++; + break; + } + } + return newlen; +} + +/* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in + * the source URL accordingly. + * URL encoding should be skipped for host names, otherwise IDN resolution + * will fail. + */ +void Curl_strcpy_url(char *output, const char *url, bool relative) +{ + /* we must add this with whitespace-replacing */ + bool left = TRUE; + const unsigned char *iptr; + char *optr = output; + const unsigned char *host_sep = (const unsigned char *) url; + + if(!relative) + host_sep = (const unsigned char *) find_host_sep(url); + + for(iptr = (unsigned char *)url; /* read from here */ + *iptr; /* until zero byte */ + iptr++) { + + if(iptr < host_sep) { + *optr++ = *iptr; + continue; + } + + switch(*iptr) { + case '?': + left = FALSE; + /* FALLTHROUGH */ + default: + if(urlchar_needs_escaping(*iptr)) { + snprintf(optr, 4, "%%%02x", *iptr); + optr += 3; + } + else + *optr++=*iptr; + break; + case ' ': + if(left) { + *optr++='%'; /* add a '%' */ + *optr++='2'; /* add a '2' */ + *optr++='0'; /* add a '0' */ + } + else + *optr++='+'; /* add a '+' here */ + break; + } + } + *optr = 0; /* zero terminate output buffer */ + +} + +/* + * Returns true if the given URL is absolute (as opposed to relative) within + * the buffer size. Returns the scheme in the buffer if TRUE and 'buf' is + * non-NULL. + */ +bool Curl_is_absolute_url(const char *url, char *buf, size_t buflen) +{ + size_t i; +#ifdef WIN32 + if(STARTS_WITH_DRIVE_PREFIX(url)) + return FALSE; +#endif + for(i = 0; i < buflen && url[i]; ++i) { + char s = url[i]; + if(s == ':') { + if(buf) + buf[i] = 0; + return TRUE; + } + /* RFC 3986 3.1 explains: + scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + */ + else if(ISALNUM(s) || (s == '+') || (s == '-') || (s == '.') ) { + if(buf) + buf[i] = (char)TOLOWER(s); + } + else + break; + } + return FALSE; +} + +/* + * Concatenate a relative URL to a base URL making it absolute. + * URL-encodes any spaces. + * The returned pointer must be freed by the caller unless NULL + * (returns NULL on out of memory). + */ +char *Curl_concat_url(const char *base, const char *relurl) +{ + /*** + TRY to append this new path to the old URL + to the right of the host part. Oh crap, this is doomed to cause + problems in the future... + */ + char *newest; + char *protsep; + char *pathsep; + size_t newlen; + bool host_changed = FALSE; + + const char *useurl = relurl; + size_t urllen; + + /* we must make our own copy of the URL to play with, as it may + point to read-only data */ + char *url_clone = strdup(base); + + if(!url_clone) + return NULL; /* skip out of this NOW */ + + /* protsep points to the start of the host name */ + protsep = strstr(url_clone, "//"); + if(!protsep) + protsep = url_clone; + else + protsep += 2; /* pass the slashes */ + + if('/' != relurl[0]) { + int level = 0; + + /* First we need to find out if there's a ?-letter in the URL, + and cut it and the right-side of that off */ + pathsep = strchr(protsep, '?'); + if(pathsep) + *pathsep = 0; + + /* we have a relative path to append to the last slash if there's one + available, or if the new URL is just a query string (starts with a + '?') we append the new one at the end of the entire currently worked + out URL */ + if(useurl[0] != '?') { + pathsep = strrchr(protsep, '/'); + if(pathsep) + *pathsep = 0; + } + + /* Check if there's any slash after the host name, and if so, remember + that position instead */ + pathsep = strchr(protsep, '/'); + if(pathsep) + protsep = pathsep + 1; + else + protsep = NULL; + + /* now deal with one "./" or any amount of "../" in the newurl + and act accordingly */ + + if((useurl[0] == '.') && (useurl[1] == '/')) + useurl += 2; /* just skip the "./" */ + + while((useurl[0] == '.') && + (useurl[1] == '.') && + (useurl[2] == '/')) { + level++; + useurl += 3; /* pass the "../" */ + } + + if(protsep) { + while(level--) { + /* cut off one more level from the right of the original URL */ + pathsep = strrchr(protsep, '/'); + if(pathsep) + *pathsep = 0; + else { + *protsep = 0; + break; + } + } + } + } + else { + /* We got a new absolute path for this server */ + + if((relurl[0] == '/') && (relurl[1] == '/')) { + /* the new URL starts with //, just keep the protocol part from the + original one */ + *protsep = 0; + useurl = &relurl[2]; /* we keep the slashes from the original, so we + skip the new ones */ + host_changed = TRUE; + } + else { + /* cut off the original URL from the first slash, or deal with URLs + without slash */ + pathsep = strchr(protsep, '/'); + if(pathsep) { + /* When people use badly formatted URLs, such as + "http://www.url.com?dir=/home/daniel" we must not use the first + slash, if there's a ?-letter before it! */ + char *sep = strchr(protsep, '?'); + if(sep && (sep < pathsep)) + pathsep = sep; + *pathsep = 0; + } + else { + /* There was no slash. Now, since we might be operating on a badly + formatted URL, such as "http://www.url.com?id=2380" which doesn't + use a slash separator as it is supposed to, we need to check for a + ?-letter as well! */ + pathsep = strchr(protsep, '?'); + if(pathsep) + *pathsep = 0; + } + } + } + + /* If the new part contains a space, this is a mighty stupid redirect + but we still make an effort to do "right". To the left of a '?' + letter we replace each space with %20 while it is replaced with '+' + on the right side of the '?' letter. + */ + newlen = Curl_strlen_url(useurl, !host_changed); + + urllen = strlen(url_clone); + + newest = malloc(urllen + 1 + /* possible slash */ + newlen + 1 /* zero byte */); + + if(!newest) { + free(url_clone); /* don't leak this */ + return NULL; + } + + /* copy over the root url part */ + memcpy(newest, url_clone, urllen); + + /* check if we need to append a slash */ + if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0])) + ; + else + newest[urllen++]='/'; + + /* then append the new piece on the right side */ + Curl_strcpy_url(&newest[urllen], useurl, !host_changed); + + free(url_clone); + + return newest; +} + +/* + * parse_hostname_login() + * + * Parse the login details (user name, password and options) from the URL and + * strip them out of the host name + * + */ +static CURLUcode parse_hostname_login(struct Curl_URL *u, + const struct Curl_handler *h, + char **hostname, + unsigned int flags) +{ + CURLUcode result = CURLUE_OK; + CURLcode ccode; + char *userp = NULL; + char *passwdp = NULL; + char *optionsp = NULL; + + /* At this point, we're hoping all the other special cases have + * been taken care of, so conn->host.name is at most + * [user[:password][;options]]@]hostname + * + * We need somewhere to put the embedded details, so do that first. + */ + + char *ptr = strchr(*hostname, '@'); + char *login = *hostname; + + if(!ptr) + goto out; + + /* We will now try to extract the + * possible login information in a string like: + * ftp://user:password@ftp.my.site:8021/README */ + *hostname = ++ptr; + + /* We could use the login information in the URL so extract it. Only parse + options if the handler says we should. Note that 'h' might be NULL! */ + ccode = Curl_parse_login_details(login, ptr - login - 1, + &userp, &passwdp, + (h && (h->flags & PROTOPT_URLOPTIONS)) ? + &optionsp:NULL); + if(ccode) { + result = CURLUE_MALFORMED_INPUT; + goto out; + } + + if(userp) { + if(flags & CURLU_DISALLOW_USER) { + /* Option DISALLOW_USER is set and url contains username. */ + result = CURLUE_USER_NOT_ALLOWED; + goto out; + } + + u->user = userp; + } + + if(passwdp) + u->password = passwdp; + + if(optionsp) + u->options = optionsp; + + return CURLUE_OK; + out: + + free(userp); + free(passwdp); + free(optionsp); + + return result; +} + +static CURLUcode parse_port(struct Curl_URL *u, char *hostname) +{ + char *portptr; + char endbracket; + int len; + + if((1 == sscanf(hostname, "[%*45[0123456789abcdefABCDEF:.%%]%c%n", + &endbracket, &len)) && + (']' == endbracket)) { + /* this is a RFC2732-style specified IP-address */ + portptr = &hostname[len]; + if (*portptr != ':') + return CURLUE_MALFORMED_INPUT; + } + else + portptr = strchr(hostname, ':'); + + if(portptr) { + char *rest; + long port; + char portbuf[7]; + + if(!ISDIGIT(portptr[1])) + return CURLUE_BAD_PORT_NUMBER; + + port = strtol(portptr + 1, &rest, 10); /* Port number must be decimal */ + + if((port <= 0) || (port > 0xffff)) + /* Single unix standard says port numbers are 16 bits long, but we don't + treat port zero as OK. */ + return CURLUE_BAD_PORT_NUMBER; + + if(rest[0]) + return CURLUE_BAD_PORT_NUMBER; + + if(rest != &portptr[1]) { + *portptr++ = '\0'; /* cut off the name there */ + *rest = 0; + /* generate a new to get rid of leading zeroes etc */ + snprintf(portbuf, sizeof(portbuf), "%ld", port); + u->portnum = port; + u->port = strdup(portbuf); + if(!u->port) + return CURLUE_OUT_OF_MEMORY; + } + else { + /* Browser behavior adaptation. If there's a colon with no digits after, + just cut off the name there which makes us ignore the colon and just + use the default port. Firefox and Chrome both do that. */ + *portptr = '\0'; + } + } + + return CURLUE_OK; +} + +/* scan for byte values < 31 or 127 */ +static CURLUcode junkscan(char *part) +{ + char badbytes[]={ + /* */ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x7f, + 0x00 /* zero terminate */ + }; + if(part) { + size_t n = strlen(part); + size_t nfine = strcspn(part, badbytes); + if(nfine != n) + /* since we don't know which part is scanned, return a generic error + code */ + return CURLUE_MALFORMED_INPUT; + } + return CURLUE_OK; +} + +static CURLUcode hostname_check(char *hostname, unsigned int flags) +{ + const char *l = NULL; /* accepted characters */ + size_t len; + size_t hlen = strlen(hostname); + (void)flags; + + if(hostname[0] == '[') { + hostname++; + l = "0123456789abcdefABCDEF::.%"; + hlen -= 2; + } + + if(l) { + /* only valid letters are ok */ + len = strspn(hostname, l); + if(hlen != len) + /* hostname with bad content */ + return CURLUE_MALFORMED_INPUT; + } + else { + /* letters from the second string is not ok */ + len = strcspn(hostname, " "); + if(hlen != len) + /* hostname with bad content */ + return CURLUE_MALFORMED_INPUT; + } + return CURLUE_OK; +} + +#define HOSTNAME_END(x) (((x) == '/') || ((x) == '?') || ((x) == '#')) + +static CURLUcode seturl(const char *url, CURLU *u, unsigned int flags) +{ + char *path; + bool path_alloced = FALSE; + char *hostname; + char *query = NULL; + char *fragment = NULL; + CURLUcode result; + bool url_has_scheme = FALSE; + char schemebuf[MAX_SCHEME_LEN]; + char *schemep = NULL; + size_t schemelen = 0; + size_t urllen; + const struct Curl_handler *h = NULL; + + if(!url) + return CURLUE_MALFORMED_INPUT; + + /************************************************************* + * Parse the URL. + ************************************************************/ + /* allocate scratch area */ + urllen = strlen(url); + path = u->scratch = malloc(urllen * 2 + 2); + if(!path) + return CURLUE_OUT_OF_MEMORY; + + hostname = &path[urllen + 1]; + hostname[0] = 0; + + if(Curl_is_absolute_url(url, schemebuf, sizeof(schemebuf))) { + url_has_scheme = TRUE; + schemelen = strlen(schemebuf); + } + + /* handle the file: scheme */ + if(url_has_scheme && strcasecompare(schemebuf, "file")) { + /* path has been allocated large enough to hold this */ + strcpy(path, &url[5]); + + hostname = NULL; /* no host for file: URLs */ + u->scheme = strdup("file"); + if(!u->scheme) + return CURLUE_OUT_OF_MEMORY; + + /* Extra handling URLs with an authority component (i.e. that start with + * "file://") + * + * We allow omitted hostname (e.g. file:/<path>) -- valid according to + * RFC 8089, but not the (current) WHAT-WG URL spec. + */ + if(path[0] == '/' && path[1] == '/') { + /* swallow the two slashes */ + char *ptr = &path[2]; + + /* + * According to RFC 8089, a file: URL can be reliably dereferenced if: + * + * o it has no/blank hostname, or + * + * o the hostname matches "localhost" (case-insensitively), or + * + * o the hostname is a FQDN that resolves to this machine. + * + * For brevity, we only consider URLs with empty, "localhost", or + * "127.0.0.1" hostnames as local. + * + * Additionally, there is an exception for URLs with a Windows drive + * letter in the authority (which was accidentally omitted from RFC 8089 + * Appendix E, but believe me, it was meant to be there. --MK) + */ + if(ptr[0] != '/' && !STARTS_WITH_URL_DRIVE_PREFIX(ptr)) { + /* the URL includes a host name, it must match "localhost" or + "127.0.0.1" to be valid */ + if(!checkprefix("localhost/", ptr) && + !checkprefix("127.0.0.1/", ptr)) { + /* Invalid file://hostname/, expected localhost or 127.0.0.1 or + none */ + return CURLUE_MALFORMED_INPUT; + } + ptr += 9; /* now points to the slash after the host */ + } + + path = ptr; + } + +#if !defined(MSDOS) && !defined(WIN32) && !defined(__CYGWIN__) + /* Don't allow Windows drive letters when not in Windows. + * This catches both "file:/c:" and "file:c:" */ + if(('/' == path[0] && STARTS_WITH_URL_DRIVE_PREFIX(&path[1])) || + STARTS_WITH_URL_DRIVE_PREFIX(path)) { + /* File drive letters are only accepted in MSDOS/Windows */ + return CURLUE_MALFORMED_INPUT; + } +#else + /* If the path starts with a slash and a drive letter, ditch the slash */ + if('/' == path[0] && STARTS_WITH_URL_DRIVE_PREFIX(&path[1])) { + /* This cannot be done with strcpy, as the memory chunks overlap! */ + memmove(path, &path[1], strlen(&path[1]) + 1); + } +#endif + + } + else { + /* clear path */ + const char *p; + const char *hostp; + size_t len; + path[0] = 0; + + if(url_has_scheme) { + int i = 0; + p = &url[schemelen + 1]; + while(p && (*p == '/') && (i < 4)) { + p++; + i++; + } + if((i < 1) || (i>3)) + /* less than one or more than three slashes */ + return CURLUE_MALFORMED_INPUT; + + schemep = schemebuf; + if(!Curl_builtin_scheme(schemep) && + !(flags & CURLU_NON_SUPPORT_SCHEME)) + return CURLUE_UNSUPPORTED_SCHEME; + + if(junkscan(schemep)) + return CURLUE_MALFORMED_INPUT; + } + else { + /* no scheme! */ + + if(!(flags & (CURLU_DEFAULT_SCHEME|CURLU_GUESS_SCHEME))) + return CURLUE_MALFORMED_INPUT; + if(flags & CURLU_DEFAULT_SCHEME) + schemep = (char *) DEFAULT_SCHEME; + + /* + * The URL was badly formatted, let's try without scheme specified. + */ + p = url; + } + hostp = p; /* host name starts here */ + + while(*p && !HOSTNAME_END(*p)) /* find end of host name */ + p++; + + len = p - hostp; + if(!len) + return CURLUE_MALFORMED_INPUT; + + memcpy(hostname, hostp, len); + hostname[len] = 0; + + if((flags & CURLU_GUESS_SCHEME) && !schemep) { + /* legacy curl-style guess based on host name */ + if(checkprefix("ftp.", hostname)) + schemep = (char *)"ftp"; + else if(checkprefix("dict.", hostname)) + schemep = (char *)"dict"; + else if(checkprefix("ldap.", hostname)) + schemep = (char *)"ldap"; + else if(checkprefix("imap.", hostname)) + schemep = (char *)"imap"; + else if(checkprefix("smtp.", hostname)) + schemep = (char *)"smtp"; + else if(checkprefix("pop3.", hostname)) + schemep = (char *)"pop3"; + else + schemep = (char *)"http"; + } + + len = strlen(p); + memcpy(path, p, len); + path[len] = 0; + + u->scheme = strdup(schemep); + if(!u->scheme) + return CURLUE_OUT_OF_MEMORY; + } + + /* if this is a known scheme, get some details */ + h = Curl_builtin_scheme(u->scheme); + + if(junkscan(path)) + return CURLUE_MALFORMED_INPUT; + + query = strchr(path, '?'); + if(query) + *query++ = 0; + + fragment = strchr(query?query:path, '#'); + if(fragment) + *fragment++ = 0; + + if(!path[0]) + /* if there's no path set, unset */ + path = NULL; + else if(!(flags & CURLU_PATH_AS_IS)) { + /* sanitise paths and remove ../ and ./ sequences according to RFC3986 */ + char *newp = Curl_dedotdotify(path); + if(!newp) + return CURLUE_OUT_OF_MEMORY; + + if(strcmp(newp, path)) { + /* if we got a new version */ + path = newp; + path_alloced = TRUE; + } + else + free(newp); + } + if(path) { + u->path = path_alloced?path:strdup(path); + if(!u->path) + return CURLUE_OUT_OF_MEMORY; + } + + if(hostname) { + /* + * Parse the login details and strip them out of the host name. + */ + if(junkscan(hostname)) + return CURLUE_MALFORMED_INPUT; + + result = parse_hostname_login(u, h, &hostname, flags); + if(result) + return result; + + result = parse_port(u, hostname); + if(result) + return result; + + result = hostname_check(hostname, flags); + if(result) + return result; + + u->host = strdup(hostname); + if(!u->host) + return CURLUE_OUT_OF_MEMORY; + } + + if(query && query[0]) { + u->query = strdup(query); + if(!u->query) + return CURLUE_OUT_OF_MEMORY; + } + if(fragment && fragment[0]) { + u->fragment = strdup(fragment); + if(!u->fragment) + return CURLUE_OUT_OF_MEMORY; + } + + free(u->scratch); + u->scratch = NULL; + + return CURLUE_OK; +} + +/* + * Parse the URL and set the relevant members of the Curl_URL struct. + */ +static CURLUcode parseurl(const char *url, CURLU *u, unsigned int flags) +{ + CURLUcode result = seturl(url, u, flags); + if(result) { + free_urlhandle(u); + memset(u, 0, sizeof(struct Curl_URL)); + } + return result; +} + +/* + */ +CURLU *curl_url(void) +{ + return calloc(sizeof(struct Curl_URL), 1); +} + +void curl_url_cleanup(CURLU *u) +{ + if(u) { + free_urlhandle(u); + free(u); + } +} + +#define DUP(dest, src, name) \ + if(src->name) { \ + dest->name = strdup(src->name); \ + if(!dest->name) \ + goto fail; \ + } + +CURLU *curl_url_dup(CURLU *in) +{ + struct Curl_URL *u = calloc(sizeof(struct Curl_URL), 1); + if(u) { + DUP(u, in, scheme); + DUP(u, in, user); + DUP(u, in, password); + DUP(u, in, options); + DUP(u, in, host); + DUP(u, in, port); + DUP(u, in, path); + DUP(u, in, query); + DUP(u, in, fragment); + u->portnum = in->portnum; + } + return u; + fail: + curl_url_cleanup(u); + return NULL; +} + +CURLUcode curl_url_get(CURLU *u, CURLUPart what, + char **part, unsigned int flags) +{ + char *ptr; + CURLUcode ifmissing = CURLUE_UNKNOWN_PART; + char portbuf[7]; + bool urldecode = (flags & CURLU_URLDECODE)?1:0; + bool plusdecode = FALSE; + (void)flags; + if(!u) + return CURLUE_BAD_HANDLE; + if(!part) + return CURLUE_BAD_PARTPOINTER; + *part = NULL; + + switch(what) { + case CURLUPART_SCHEME: + ptr = u->scheme; + ifmissing = CURLUE_NO_SCHEME; + urldecode = FALSE; /* never for schemes */ + break; + case CURLUPART_USER: + ptr = u->user; + ifmissing = CURLUE_NO_USER; + break; + case CURLUPART_PASSWORD: + ptr = u->password; + ifmissing = CURLUE_NO_PASSWORD; + break; + case CURLUPART_OPTIONS: + ptr = u->options; + ifmissing = CURLUE_NO_OPTIONS; + break; + case CURLUPART_HOST: + ptr = u->host; + ifmissing = CURLUE_NO_HOST; + break; + case CURLUPART_PORT: + ptr = u->port; + ifmissing = CURLUE_NO_PORT; + urldecode = FALSE; /* never for port */ + if(!ptr && (flags & CURLU_DEFAULT_PORT) && u->scheme) { + /* there's no stored port number, but asked to deliver + a default one for the scheme */ + const struct Curl_handler *h = + Curl_builtin_scheme(u->scheme); + if(h) { + snprintf(portbuf, sizeof(portbuf), "%ld", h->defport); + ptr = portbuf; + } + } + else if(ptr && u->scheme) { + /* there is a stored port number, but ask to inhibit if + it matches the default one for the scheme */ + const struct Curl_handler *h = + Curl_builtin_scheme(u->scheme); + if(h && (h->defport == u->portnum) && + (flags & CURLU_NO_DEFAULT_PORT)) + ptr = NULL; + } + break; + case CURLUPART_PATH: + ptr = u->path; + if(!ptr) { + ptr = u->path = strdup("/"); + if(!u->path) + return CURLUE_OUT_OF_MEMORY; + } + break; + case CURLUPART_QUERY: + ptr = u->query; + ifmissing = CURLUE_NO_QUERY; + plusdecode = urldecode; + break; + case CURLUPART_FRAGMENT: + ptr = u->fragment; + ifmissing = CURLUE_NO_FRAGMENT; + break; + case CURLUPART_URL: { + char *url; + char *scheme; + char *options = u->options; + char *port = u->port; + if(u->scheme && strcasecompare("file", u->scheme)) { + url = aprintf("file://%s%s%s", + u->path, + u->fragment? "#": "", + u->fragment? u->fragment : ""); + } + else if(!u->host) + return CURLUE_NO_HOST; + else { + const struct Curl_handler *h = NULL; + if(u->scheme) + scheme = u->scheme; + else if(flags & CURLU_DEFAULT_SCHEME) + scheme = (char *) DEFAULT_SCHEME; + else + return CURLUE_NO_SCHEME; + + if(scheme) { + h = Curl_builtin_scheme(scheme); + if(!port && (flags & CURLU_DEFAULT_PORT)) { + /* there's no stored port number, but asked to deliver + a default one for the scheme */ + if(h) { + snprintf(portbuf, sizeof(portbuf), "%ld", h->defport); + port = portbuf; + } + } + else if(port) { + /* there is a stored port number, but asked to inhibit if it matches + the default one for the scheme */ + if(h && (h->defport == u->portnum) && + (flags & CURLU_NO_DEFAULT_PORT)) + port = NULL; + } + } + if(h && !(h->flags & PROTOPT_URLOPTIONS)) + options = NULL; + + url = aprintf("%s://%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", + scheme, + u->user ? u->user : "", + u->password ? ":": "", + u->password ? u->password : "", + options ? ";" : "", + options ? options : "", + (u->user || u->password || options) ? "@": "", + u->host, + port ? ":": "", + port ? port : "", + (u->path && (u->path[0] != '/')) ? "/": "", + u->path ? u->path : "/", + u->query? "?": "", + u->query? u->query : "", + u->fragment? "#": "", + u->fragment? u->fragment : ""); + } + if(!url) + return CURLUE_OUT_OF_MEMORY; + *part = url; + return CURLUE_OK; + break; + } + default: + ptr = NULL; + } + if(ptr) { + *part = strdup(ptr); + if(!*part) + return CURLUE_OUT_OF_MEMORY; + if(plusdecode) { + /* convert + to space */ + char *plus; + for(plus = *part; *plus; ++plus) { + if(*plus == '+') + *plus = ' '; + } + } + if(urldecode) { + char *decoded; + size_t dlen; + CURLcode res = Curl_urldecode(NULL, *part, 0, &decoded, &dlen, TRUE); + free(*part); + if(res) { + *part = NULL; + return CURLUE_URLDECODE; + } + *part = decoded; + } + return CURLUE_OK; + } + else + return ifmissing; +} + +CURLUcode curl_url_set(CURLU *u, CURLUPart what, + const char *part, unsigned int flags) +{ + char **storep = NULL; + long port = 0; + bool urlencode = (flags & CURLU_URLENCODE)? 1 : 0; + bool plusencode = FALSE; + bool urlskipslash = FALSE; + bool appendquery = FALSE; + + if(!u) + return CURLUE_BAD_HANDLE; + if(!part) { + /* setting a part to NULL clears it */ + switch(what) { + case CURLUPART_URL: + break; + case CURLUPART_SCHEME: + storep = &u->scheme; + break; + case CURLUPART_USER: + storep = &u->user; + break; + case CURLUPART_PASSWORD: + storep = &u->password; + break; + case CURLUPART_OPTIONS: + storep = &u->options; + break; + case CURLUPART_HOST: + storep = &u->host; + break; + case CURLUPART_PORT: + storep = &u->port; + break; + case CURLUPART_PATH: + storep = &u->path; + break; + case CURLUPART_QUERY: + storep = &u->query; + break; + case CURLUPART_FRAGMENT: + storep = &u->fragment; + break; + default: + return CURLUE_UNKNOWN_PART; + } + if(storep && *storep) { + free(*storep); + *storep = NULL; + } + return CURLUE_OK; + } + + switch(what) { + case CURLUPART_SCHEME: + if(!(flags & CURLU_NON_SUPPORT_SCHEME) && + /* verify that it is a fine scheme */ + !Curl_builtin_scheme(part)) + return CURLUE_UNSUPPORTED_SCHEME; + storep = &u->scheme; + urlencode = FALSE; /* never */ + break; + case CURLUPART_USER: + storep = &u->user; + break; + case CURLUPART_PASSWORD: + storep = &u->password; + break; + case CURLUPART_OPTIONS: + storep = &u->options; + break; + case CURLUPART_HOST: + storep = &u->host; + break; + case CURLUPART_PORT: + urlencode = FALSE; /* never */ + port = strtol(part, NULL, 10); /* Port number must be decimal */ + if((port <= 0) || (port > 0xffff)) + return CURLUE_BAD_PORT_NUMBER; + storep = &u->port; + break; + case CURLUPART_PATH: + urlskipslash = TRUE; + storep = &u->path; + break; + case CURLUPART_QUERY: + plusencode = urlencode; + appendquery = (flags & CURLU_APPENDQUERY)?1:0; + storep = &u->query; + break; + case CURLUPART_FRAGMENT: + storep = &u->fragment; + break; + case CURLUPART_URL: { + /* + * Allow a new URL to replace the existing (if any) contents. + * + * If the existing contents is enough for a URL, allow a relative URL to + * replace it. + */ + CURLUcode result; + char *oldurl; + char *redired_url; + CURLU *handle2; + + if(Curl_is_absolute_url(part, NULL, MAX_SCHEME_LEN)) { + handle2 = curl_url(); + if(!handle2) + return CURLUE_OUT_OF_MEMORY; + result = parseurl(part, handle2, flags); + if(!result) + mv_urlhandle(handle2, u); + else + curl_url_cleanup(handle2); + return result; + } + /* extract the full "old" URL to do the redirect on */ + result = curl_url_get(u, CURLUPART_URL, &oldurl, flags); + if(result) { + /* couldn't get the old URL, just use the new! */ + handle2 = curl_url(); + if(!handle2) + return CURLUE_OUT_OF_MEMORY; + result = parseurl(part, handle2, flags); + if(!result) + mv_urlhandle(handle2, u); + else + curl_url_cleanup(handle2); + return result; + } + + /* apply the relative part to create a new URL */ + redired_url = Curl_concat_url(oldurl, part); + free(oldurl); + if(!redired_url) + return CURLUE_OUT_OF_MEMORY; + + /* now parse the new URL */ + handle2 = curl_url(); + if(!handle2) { + free(redired_url); + return CURLUE_OUT_OF_MEMORY; + } + result = parseurl(redired_url, handle2, flags); + free(redired_url); + if(!result) + mv_urlhandle(handle2, u); + else + curl_url_cleanup(handle2); + return result; + } + default: + return CURLUE_UNKNOWN_PART; + } + if(storep) { + const char *newp = part; + size_t nalloc = strlen(part); + + if(urlencode) { + const char *i; + char *o; + bool free_part = FALSE; + char *enc = malloc(nalloc * 3 + 1); /* for worst case! */ + if(!enc) + return CURLUE_OUT_OF_MEMORY; + if(plusencode) { + /* space to plus */ + i = part; + for(o = enc; *i; ++o, ++i) + *o = (*i == ' ') ? '+' : *i; + *o = 0; /* zero terminate */ + part = strdup(enc); + if(!part) { + free(enc); + return CURLUE_OUT_OF_MEMORY; + } + free_part = TRUE; + } + for(i = part, o = enc; *i; i++) { + if(Curl_isunreserved(*i) || + ((*i == '/') && urlskipslash) || + ((*i == '=') && appendquery) || + ((*i == '+') && plusencode)) { + *o = *i; + o++; + } + else { + snprintf(o, 4, "%%%02x", *i); + o += 3; + } + } + *o = 0; /* zero terminate */ + newp = enc; + if(free_part) + free((char *)part); + } + else { + char *p; + newp = strdup(part); + if(!newp) + return CURLUE_OUT_OF_MEMORY; + p = (char *)newp; + while(*p) { + /* make sure percent encoded are lower case */ + if((*p == '%') && ISXDIGIT(p[1]) && ISXDIGIT(p[2]) && + (ISUPPER(p[1]) || ISUPPER(p[2]))) { + p[1] = (char)TOLOWER(p[1]); + p[2] = (char)TOLOWER(p[2]); + p += 3; + } + else + p++; + } + } + + if(appendquery) { + /* Append the string onto the old query. Add a '&' separator if none is + present at the end of the exsting query already */ + size_t querylen = u->query ? strlen(u->query) : 0; + bool addamperand = querylen && (u->query[querylen -1] != '&'); + if(querylen) { + size_t newplen = strlen(newp); + char *p = malloc(querylen + addamperand + newplen + 1); + if(!p) { + free((char *)newp); + return CURLUE_OUT_OF_MEMORY; + } + strcpy(p, u->query); /* original query */ + if(addamperand) + p[querylen] = '&'; /* ampersand */ + strcpy(&p[querylen + addamperand], newp); /* new suffix */ + free((char *)newp); + free(*storep); + *storep = p; + return CURLUE_OK; + } + } + + free(*storep); + *storep = (char *)newp; + } + /* set after the string, to make it not assigned if the allocation above + fails */ + if(port) + u->portnum = port; + return CURLUE_OK; +} diff --git a/Utilities/cmcurl/lib/urldata.h b/Utilities/cmcurl/lib/urldata.h index 7fae00fd9..11a6a22c6 100644 --- a/Utilities/cmcurl/lib/urldata.h +++ b/Utilities/cmcurl/lib/urldata.h @@ -80,6 +80,7 @@ #define RESP_TIMEOUT (1800*1000) #include "cookie.h" +#include "psl.h" #include "formdata.h" #ifdef HAVE_NETINET_IN_H @@ -141,9 +142,6 @@ typedef ssize_t (Curl_recv)(struct connectdata *conn, /* connection data */ #include <libssh2_sftp.h> #endif /* HAVE_LIBSSH2_H */ -/* The upload buffer size, should not be smaller than CURL_MAX_WRITE_SIZE, as - it needs to hold a full buffer as could be sent in a write callback */ -#define UPLOAD_BUFSIZE CURL_MAX_WRITE_SIZE /* The "master buffer" is for HTTP pipelining */ #define MASTERBUF_SIZE 16384 @@ -156,11 +154,6 @@ typedef ssize_t (Curl_recv)(struct connectdata *conn, /* connection data */ #define GOOD_EASY_HANDLE(x) \ ((x) && ((x)->magic == CURLEASY_MAGIC_NUMBER)) -/* Some convenience macros to get the larger/smaller value out of two given. - We prefix with CURL to prevent name collisions. */ -#define CURLMAX(x,y) ((x)>(y)?(x):(y)) -#define CURLMIN(x,y) ((x)<(y)?(x):(y)) - #ifdef HAVE_GSSAPI /* Types needed for krb5-ftp connections */ struct krb5buffer { @@ -226,6 +219,7 @@ struct ssl_primary_config { char *random_file; /* path to file containing "random" data */ char *egdsocket; /* path to file containing the EGD daemon socket */ char *cipher_list; /* list of ciphers to use */ + char *cipher_list13; /* list of TLS 1.3 cipher suites to use */ }; struct ssl_config_data { @@ -474,7 +468,6 @@ struct hostname { #define KEEP_SENDBITS (KEEP_SEND | KEEP_SEND_HOLD | KEEP_SEND_PAUSE) -#ifdef CURLRES_ASYNCH struct Curl_async { char *hostname; int port; @@ -483,7 +476,6 @@ struct Curl_async { int status; /* if done is TRUE, this is the status from the callback */ void *os_specific; /* 'struct thread_data' for Windows */ }; -#endif #define FIRSTSOCKET 0 #define SECONDARYSOCKET 1 @@ -509,6 +501,28 @@ enum upgrade101 { UPGR101_WORKING /* talking upgraded protocol */ }; +struct dohresponse { + unsigned char *memory; + size_t size; +}; + +/* one of these for each DoH request */ +struct dnsprobe { + CURL *easy; + int dnstype; + unsigned char dohbuffer[512]; + size_t dohlen; + struct dohresponse serverdoh; +}; + +struct dohdata { + struct curl_slist *headers; + struct dnsprobe probe[2]; + unsigned int pending; /* still outstanding requests */ + const char *host; + int port; +}; + /* * Request specific data in the easy handle (Curl_easy). Previously, * these members were on the connectdata struct but since a conn struct may @@ -604,6 +618,7 @@ struct SingleRequest { void *protop; /* Allocated protocol-specific data. Each protocol handler makes sure this points to data it needs. */ + struct dohdata doh; /* DoH specific data for this request */ }; /* @@ -634,7 +649,7 @@ struct Curl_handler { */ CURLcode (*connect_it)(struct connectdata *, bool *done); - /* See above. Currently only used for FTP. */ + /* See above. */ CURLcode (*connecting)(struct connectdata *, bool *done); CURLcode (*doing)(struct connectdata *, bool *done); @@ -714,6 +729,7 @@ struct Curl_handler { #define CONNCHECK_NONE 0 /* No checks */ #define CONNCHECK_ISDEAD (1<<0) /* Check if the connection is dead. */ +#define CONNCHECK_KEEPALIVE (1<<1) /* Perform any keepalive function. */ #define CONNRESULT_NONE 0 /* No extra information. */ #define CONNRESULT_DEAD (1<<0) /* The connection is dead. */ @@ -779,11 +795,12 @@ struct connectdata { curl_closesocket_callback fclosesocket; /* function closing the socket(s) */ void *closesocket_client; - bool inuse; /* This is a marker for the connection cache logic. If this is - TRUE this handle is being used by one or more easy handles - and can only used by any other easy handle without careful - consideration (== only for pipelining/multiplexing) and it - cannot be used by another multi handle! */ + /* This is used by the connection cache logic. If this returns TRUE, this + handle is being used by one or more easy handles and can only used by any + other easy handle without careful consideration (== only for + pipelining/multiplexing) and it cannot be used by another multi + handle! */ +#define CONN_INUSE(c) ((c)->send_pipe.size + (c)->recv_pipe.size) /**** Fields set when inited and not modified again */ long connection_id; /* Contains a unique number to make it easier to @@ -889,6 +906,13 @@ struct connectdata { long ip_version; /* copied from the Curl_easy at creation time */ + /* Protocols can use a custom keepalive mechanism to keep connections alive. + This allows those protocols to track the last time the keepalive mechanism + was used on this connection. */ + struct curltime keepalive; + + long upkeep_interval_ms; /* Time between calls for connection upkeep. */ + /**** curl_get() phase fields */ curl_socket_t sockfd; /* socket to read from or CURL_SOCKET_BAD */ @@ -966,11 +990,8 @@ struct connectdata { #endif char syserr_buf [256]; /* buffer for Curl_strerror() */ - -#ifdef CURLRES_ASYNCH /* data used for the asynch name resolve callback */ struct Curl_async async; -#endif /* These three are used for chunked-encoding trailer support */ char *trailer; /* allocated buffer to store trailer in */ @@ -1203,6 +1224,18 @@ struct time_node { expire_id eid; }; +/* individual pieces of the URL */ +struct urlpieces { + char *scheme; + char *hostname; + char *port; + char *user; + char *password; + char *options; + char *path; + char *query; +}; + struct UrlState { /* Points to the connection cache */ @@ -1222,7 +1255,7 @@ struct UrlState { size_t headersize; /* size of the allocation */ char *buffer; /* download buffer */ - char uploadbuffer[UPLOAD_BUFSIZE + 1]; /* upload buffer */ + char *ulbuf; /* allocated upload buffer or NULL */ curl_off_t current_speed; /* the ProgressShow() function sets this, bytes / second */ bool this_is_a_follow; /* this is a followed Location: request */ @@ -1265,7 +1298,7 @@ struct UrlState { void *resolver; /* resolver state, if it is used in the URL state - ares_channel f.e. */ -#if defined(USE_OPENSSL) && defined(HAVE_OPENSSL_ENGINE_H) +#if defined(USE_OPENSSL) /* void instead of ENGINE to avoid bleeding OpenSSL into this header */ void *engine; #endif /* USE_OPENSSL */ @@ -1284,9 +1317,6 @@ struct UrlState { involved in this request */ bool expect100header; /* TRUE if we added Expect: 100-continue */ - bool pipe_broke; /* TRUE if the connection we were pipelined on broke - and we need to restart from the beginning */ - #if !defined(WIN32) && !defined(MSDOS) && !defined(__EMX__) && \ !defined(__SYMBIAN32__) /* do FTP line-end conversions on most platforms */ @@ -1296,9 +1326,6 @@ struct UrlState { /* for FTP downloads: how many CRLFs did we converted to LFs? */ curl_off_t crlf_conversions; #endif - char *pathbuffer;/* allocated buffer to store the URL's path part in */ - char *path; /* path to use, points to somewhere within the pathbuffer - area */ bool slash_removed; /* set TRUE if the 'path' points to a path where the initial URL slash separator has been taken off */ bool use_range; @@ -1332,6 +1359,8 @@ struct UrlState { #ifdef CURLDEBUG bool conncache_lock; #endif + CURLU *uh; /* URL handle for the current parsed URL */ + struct urlpieces up; }; @@ -1346,7 +1375,7 @@ struct DynamicStatic { char *url; /* work URL, copied from UserDefined */ bool url_alloc; /* URL string is malloc()'ed */ char *referer; /* referer string */ - bool referer_alloc; /* referer sting is malloc()ed */ + bool referer_alloc; /* referer string is malloc()ed */ struct curl_slist *cookielist; /* list of cookie files set by curl_easy_setopt(COOKIEFILE) calls */ struct curl_slist *resolve; /* set to point to the set.resolve list when @@ -1400,6 +1429,8 @@ enum dupstring { STRING_SSL_PINNEDPUBLICKEY_PROXY, /* public key file to verify proxy */ STRING_SSL_CIPHER_LIST_ORIG, /* list of ciphers to use */ STRING_SSL_CIPHER_LIST_PROXY, /* list of ciphers to use */ + STRING_SSL_CIPHER13_LIST_ORIG, /* list of TLS 1.3 ciphers to use */ + STRING_SSL_CIPHER13_LIST_PROXY, /* list of TLS 1.3 ciphers to use */ STRING_SSL_EGDSOCKET, /* path to file containing the EGD daemon socket */ STRING_SSL_RANDOM_FILE, /* path to file containing "random" data */ STRING_USERAGENT, /* User-Agent string */ @@ -1407,6 +1438,7 @@ enum dupstring { STRING_SSL_CRLFILE_PROXY, /* crl file to check certificate */ STRING_SSL_ISSUERCERT_ORIG, /* issuer cert file to check certificate */ STRING_SSL_ISSUERCERT_PROXY, /* issuer cert file to check certificate */ + STRING_SSL_ENGINE, /* name of ssl engine */ STRING_USERNAME, /* <username>, if used */ STRING_PASSWORD, /* <password>, if used */ STRING_OPTIONS, /* <options>, if used */ @@ -1439,18 +1471,23 @@ enum dupstring { STRING_UNIX_SOCKET_PATH, /* path to Unix socket, if used */ #endif STRING_TARGET, /* CURLOPT_REQUEST_TARGET */ + STRING_DOH, /* CURLOPT_DOH_URL */ /* -- end of zero-terminated strings -- */ STRING_LASTZEROTERMINATED, - /* -- below this are pointers to binary data that cannot be strdup'ed. - Each such pointer must be added manually to Curl_dupset() --- */ + /* -- below this are pointers to binary data that cannot be strdup'ed. --- */ STRING_COPYPOSTFIELDS, /* if POST, set the fields' values here */ STRING_LAST /* not used, just an end-of-list marker */ }; +/* callback that gets called when this easy handle is completed within a multi + handle. Only used for internally created transfers, like for example + DoH. */ +typedef int (*multidone_func)(struct Curl_easy *easy, CURLcode result); + struct UserDefined { FILE *err; /* the stderr user data goes here */ void *debugdata; /* the data that will be passed to fdebug */ @@ -1559,6 +1596,8 @@ struct UserDefined { curl_proxytype proxytype; /* what kind of proxy that is in use */ long dns_cache_timeout; /* DNS cache timeout */ long buffer_size; /* size of receive buffer to use */ + size_t upload_buffer_size; /* size of upload buffer to use, + keep it >= CURL_MAX_WRITE_SIZE */ void *private_data; /* application-private data */ struct curl_slist *http200aliases; /* linked list of aliases for http200 */ @@ -1581,8 +1620,6 @@ struct UserDefined { /* Here follows boolean settings that define how to behave during this session. They are STATIC, set by libcurl users or at least initially and they don't change during operations. */ - - bool printhost; /* printing host name in debug info */ bool get_filetime; /* get the time and get of the remote file */ bool tunnel_thru_httpproxy; /* use CONNECT through a HTTP proxy */ bool prefer_ascii; /* ASCII rather than binary */ @@ -1676,7 +1713,7 @@ struct UserDefined { bool stream_depends_e; /* set or don't set the Exclusive bit */ int stream_weight; - bool haproxyprotocol; /* whether to send HAProxy PROXY protocol header */ + bool haproxyprotocol; /* whether to send HAProxy PROXY protocol v1 header */ struct Curl_http2_dep *stream_dependents; @@ -1685,6 +1722,12 @@ struct UserDefined { curl_resolver_start_callback resolver_start; /* optional callback called before resolver start */ void *resolver_start_client; /* pointer to pass to resolver start callback */ + bool disallow_username_in_url; /* disallow username in url */ + long upkeep_interval_ms; /* Time between calls for connection upkeep. */ + bool doh; /* DNS-over-HTTPS enabled */ + bool doh_get; /* use GET for DoH requests, instead of POST */ + multidone_func fmultidone; + struct Curl_easy *dohfor; /* this is a DoH request for that transfer */ }; struct Names { @@ -1736,6 +1779,9 @@ struct Curl_easy { struct to which this "belongs" when used by the easy interface */ struct Curl_share *share; /* Share, handles global variable mutexing */ +#ifdef USE_LIBPSL + struct PslCache *psl; /* The associated PSL cache. */ +#endif struct SingleRequest req; /* Request-specific data */ struct UserDefined set; /* values set by the libcurl user */ struct DynamicStatic change; /* possibly modified userdefined data */ diff --git a/Utilities/cmcurl/lib/vauth/cleartext.c b/Utilities/cmcurl/lib/vauth/cleartext.c index 5d61ce6dc..be6d6111e 100644 --- a/Utilities/cmcurl/lib/vauth/cleartext.c +++ b/Utilities/cmcurl/lib/vauth/cleartext.c @@ -50,7 +50,7 @@ * * data [in] - The session handle. * userp [in] - The user name. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * outptr [in/out] - The address where a pointer to newly allocated memory * holding the result will be stored upon completion. * outlen [out] - The length of the output message. @@ -74,7 +74,7 @@ CURLcode Curl_auth_create_plain_message(struct Curl_easy *data, plen = strlen(passwdp); /* Compute binary message length. Check for overflows. */ - if((ulen > SIZE_T_MAX/2) || (plen > (SIZE_T_MAX/2 - 2))) + if((ulen > SIZE_T_MAX/4) || (plen > (SIZE_T_MAX/2 - 2))) return CURLE_OUT_OF_MEMORY; plainlen = 2 * ulen + plen + 2; diff --git a/Utilities/cmcurl/lib/vauth/cram.c b/Utilities/cmcurl/lib/vauth/cram.c index 3074a163a..d148618b0 100644 --- a/Utilities/cmcurl/lib/vauth/cram.c +++ b/Utilities/cmcurl/lib/vauth/cram.c @@ -81,7 +81,7 @@ CURLcode Curl_auth_decode_cram_md5_message(const char *chlg64, char **outptr, * data [in] - The session handle. * chlg [in] - The challenge. * userp [in] - The user name. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * outptr [in/out] - The address where a pointer to newly allocated memory * holding the result will be stored upon completion. * outlen [out] - The length of the output message. diff --git a/Utilities/cmcurl/lib/vauth/digest.c b/Utilities/cmcurl/lib/vauth/digest.c index 131d9da8c..ab5156eb7 100644 --- a/Utilities/cmcurl/lib/vauth/digest.c +++ b/Utilities/cmcurl/lib/vauth/digest.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -158,7 +158,7 @@ static void auth_digest_sha256_to_ascii(unsigned char *source, /* 32 bytes */ /* Perform quoted-string escaping as described in RFC2616 and its errata */ static char *auth_digest_string_quoted(const char *source) { - char *dest, *d; + char *dest; const char *s = source; size_t n = 1; /* null terminator */ @@ -173,8 +173,8 @@ static char *auth_digest_string_quoted(const char *source) dest = malloc(n); if(dest) { + char *d = dest; s = source; - d = dest; while(*s) { if(*s == '"' || *s == '\\') { *d++ = '\\'; @@ -342,7 +342,7 @@ bool Curl_auth_is_digest_supported(void) * data [in] - The session handle. * chlg64 [in] - The base64 encoded challenge message. * userp [in] - The user name. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * service [in] - The service type such as http, smtp, pop or imap. * outptr [in/out] - The address where a pointer to newly allocated memory * holding the result will be stored upon completion. @@ -668,7 +668,7 @@ CURLcode Curl_auth_decode_digest_http_message(const char *chlg, * * data [in] - The session handle. * userp [in] - The user name. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * request [in] - The HTTP request. * uripath [in] - The path of the HTTP uri. * digest [in/out] - The digest data struct being used and modified. @@ -696,7 +696,6 @@ static CURLcode _Curl_auth_create_digest_http_message( unsigned char ha1[65]; /* 64 digits and 1 zero byte */ unsigned char ha2[65]; /* 64 digits and 1 zero byte */ char userh[65]; - char cnoncebuf[33]; char *cnonce = NULL; size_t cnonce_sz = 0; char *userp_quoted; @@ -707,6 +706,7 @@ static CURLcode _Curl_auth_create_digest_http_message( digest->nc = 1; if(!digest->cnonce) { + char cnoncebuf[33]; result = Curl_rand_hex(data, (unsigned char *)cnoncebuf, sizeof(cnoncebuf)); if(result) @@ -781,6 +781,8 @@ static CURLcode _Curl_auth_create_digest_http_message( */ hashthis = (unsigned char *) aprintf("%s:%s", request, uripath); + if(!hashthis) + return CURLE_OUT_OF_MEMORY; if(digest->qop && strcasecompare(digest->qop, "auth-int")) { /* We don't support auth-int for PUT or POST at the moment. @@ -932,7 +934,7 @@ static CURLcode _Curl_auth_create_digest_http_message( * * data [in] - The session handle. * userp [in] - The user name. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * request [in] - The HTTP request. * uripath [in] - The path of the HTTP uri. * digest [in/out] - The digest data struct being used and modified. diff --git a/Utilities/cmcurl/lib/vauth/digest_sspi.c b/Utilities/cmcurl/lib/vauth/digest_sspi.c index a3f96ed24..928755735 100644 --- a/Utilities/cmcurl/lib/vauth/digest_sspi.c +++ b/Utilities/cmcurl/lib/vauth/digest_sspi.c @@ -75,7 +75,7 @@ bool Curl_auth_is_digest_supported(void) * data [in] - The session handle. * chlg64 [in] - The base64 encoded challenge message. * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * service [in] - The service type such as http, smtp, pop or imap. * outptr [in/out] - The address where a pointer to newly allocated memory * holding the result will be stored upon completion. @@ -391,7 +391,7 @@ CURLcode Curl_auth_decode_digest_http_message(const char *chlg, * * data [in] - The session handle. * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * request [in] - The HTTP request. * uripath [in] - The path of the HTTP uri. * digest [in/out] - The digest data struct being used and modified. diff --git a/Utilities/cmcurl/lib/vauth/krb5_gssapi.c b/Utilities/cmcurl/lib/vauth/krb5_gssapi.c index 560ecc5bc..55daec1ff 100644 --- a/Utilities/cmcurl/lib/vauth/krb5_gssapi.c +++ b/Utilities/cmcurl/lib/vauth/krb5_gssapi.c @@ -65,7 +65,7 @@ bool Curl_auth_is_gssapi_supported(void) * * data [in] - The session handle. * userp [in] - The user name. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * service [in] - The service type such as http, smtp, pop or imap. * host [in[ - The host name. * mutual_auth [in] - Flag specifying whether or not mutual authentication diff --git a/Utilities/cmcurl/lib/vauth/krb5_sspi.c b/Utilities/cmcurl/lib/vauth/krb5_sspi.c index 9afb971fc..cb11ed901 100644 --- a/Utilities/cmcurl/lib/vauth/krb5_sspi.c +++ b/Utilities/cmcurl/lib/vauth/krb5_sspi.c @@ -71,7 +71,7 @@ bool Curl_auth_is_gssapi_supported(void) * * data [in] - The session handle. * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * service [in] - The service type such as http, smtp, pop or imap. * host [in] - The host name. * mutual_auth [in] - Flag specifying whether or not mutual authentication diff --git a/Utilities/cmcurl/lib/vauth/ntlm.c b/Utilities/cmcurl/lib/vauth/ntlm.c index cdb8d8f0d..11f42f504 100644 --- a/Utilities/cmcurl/lib/vauth/ntlm.c +++ b/Utilities/cmcurl/lib/vauth/ntlm.c @@ -354,7 +354,7 @@ static void unicodecpy(unsigned char *dest, const char *src, size_t length) * * data [in] - The session handle. * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * service [in] - The service type such as http, smtp, pop or imap. * host [in] - The host name. * ntlm [in/out] - The NTLM data struct being used and modified. @@ -481,7 +481,7 @@ CURLcode Curl_auth_create_ntlm_type1_message(struct Curl_easy *data, * * data [in] - The session handle. * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * ntlm [in/out] - The NTLM data struct being used and modified. * outptr [in/out] - The address where a pointer to newly allocated memory * holding the result will be stored upon completion. diff --git a/Utilities/cmcurl/lib/vauth/ntlm_sspi.c b/Utilities/cmcurl/lib/vauth/ntlm_sspi.c index 089c1a6d4..b66cfe737 100644 --- a/Utilities/cmcurl/lib/vauth/ntlm_sspi.c +++ b/Utilities/cmcurl/lib/vauth/ntlm_sspi.c @@ -69,7 +69,7 @@ bool Curl_auth_is_ntlm_supported(void) * * data [in] - The session handle. * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * service [in] - The service type such as http, smtp, pop or imap. * host [in] - The host name. * ntlm [in/out] - The NTLM data struct being used and modified. @@ -234,7 +234,7 @@ CURLcode Curl_auth_decode_ntlm_type2_message(struct Curl_easy *data, * * data [in] - The session handle. * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * ntlm [in/out] - The NTLM data struct being used and modified. * outptr [in/out] - The address where a pointer to newly allocated memory * holding the result will be stored upon completion. diff --git a/Utilities/cmcurl/lib/vauth/spnego_gssapi.c b/Utilities/cmcurl/lib/vauth/spnego_gssapi.c index 5196c2704..4a48bdd20 100644 --- a/Utilities/cmcurl/lib/vauth/spnego_gssapi.c +++ b/Utilities/cmcurl/lib/vauth/spnego_gssapi.c @@ -64,7 +64,7 @@ bool Curl_auth_is_spnego_supported(void) * * data [in] - The session handle. * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * passwdp [in] - The user's password. * service [in] - The service type such as http, smtp, pop or imap. * host [in] - The host name. * chlg64 [in] - The optional base64 encoded challenge message. diff --git a/Utilities/cmcurl/lib/vauth/spnego_sspi.c b/Utilities/cmcurl/lib/vauth/spnego_sspi.c index 1fe19e3f9..77d1895a5 100644 --- a/Utilities/cmcurl/lib/vauth/spnego_sspi.c +++ b/Utilities/cmcurl/lib/vauth/spnego_sspi.c @@ -71,8 +71,8 @@ bool Curl_auth_is_spnego_supported(void) * Parameters: * * data [in] - The session handle. - * userp [in] - The user name in the format User or Domain\User. - * passdwp [in] - The user's password. + * user [in] - The user name in the format User or Domain\User. + * password [in] - The user's password. * service [in] - The service type such as http, smtp, pop or imap. * host [in] - The host name. * chlg64 [in] - The optional base64 encoded challenge message. diff --git a/Utilities/cmcurl/lib/version.c b/Utilities/cmcurl/lib/version.c index 5b0d05a15..05c2cd8b0 100644 --- a/Utilities/cmcurl/lib/version.c +++ b/Utilities/cmcurl/lib/version.c @@ -445,7 +445,7 @@ curl_version_info_data *curl_version_info(CURLversion stamp) #ifdef HAVE_BROTLI version_info.brotli_ver_num = BrotliDecoderVersion(); - brotli_version(brotli_buffer, sizeof brotli_buffer); + brotli_version(brotli_buffer, sizeof(brotli_buffer)); version_info.brotli_version = brotli_buffer; #endif diff --git a/Utilities/cmcurl/lib/vtls/axtls.c b/Utilities/cmcurl/lib/vtls/axtls.c index 5ed898b4f..b262392a0 100644 --- a/Utilities/cmcurl/lib/vtls/axtls.c +++ b/Utilities/cmcurl/lib/vtls/axtls.c @@ -29,6 +29,11 @@ #include "curl_setup.h" #ifdef USE_AXTLS + +#error axTLS support has been disabled in curl due to doubts about quality, +#error user dedication and a lack of use/testing. We urge users to consider +#error using a more established TLS backend instead. + #include <axTLS/config.h> #include <axTLS/ssl.h> #include "axtls.h" diff --git a/Utilities/cmcurl/lib/vtls/axtls.h b/Utilities/cmcurl/lib/vtls/axtls.h index 3f1e129c2..cb8187272 100644 --- a/Utilities/cmcurl/lib/vtls/axtls.h +++ b/Utilities/cmcurl/lib/vtls/axtls.h @@ -31,4 +31,3 @@ extern const struct Curl_ssl Curl_ssl_axtls; #endif /* USE_AXTLS */ #endif /* HEADER_CURL_AXTLS_H */ - diff --git a/Utilities/cmcurl/lib/vtls/cyassl.c b/Utilities/cmcurl/lib/vtls/cyassl.c index 20ce460e8..e10398ac3 100644 --- a/Utilities/cmcurl/lib/vtls/cyassl.c +++ b/Utilities/cmcurl/lib/vtls/cyassl.c @@ -569,7 +569,7 @@ cyassl_connect_step2(struct connectdata *conn, return CURLE_SSL_PINNEDPUBKEYNOTMATCH; } - memset(&x509_parsed, 0, sizeof x509_parsed); + memset(&x509_parsed, 0, sizeof(x509_parsed)); if(Curl_parseX509(&x509_parsed, x509_der, x509_der + x509_der_len)) return CURLE_SSL_PINNEDPUBKEYNOTMATCH; @@ -968,6 +968,8 @@ static CURLcode Curl_cyassl_random(struct Curl_easy *data, return CURLE_FAILED_INIT; if(RNG_GenerateBlock(&rng, entropy, (unsigned)length)) return CURLE_FAILED_INIT; + if(FreeRng(&rng)) + return CURLE_FAILED_INIT; return CURLE_OK; } diff --git a/Utilities/cmcurl/lib/vtls/darwinssl.c b/Utilities/cmcurl/lib/vtls/darwinssl.c index 45fe49d82..e8116b8a1 100644 --- a/Utilities/cmcurl/lib/vtls/darwinssl.c +++ b/Utilities/cmcurl/lib/vtls/darwinssl.c @@ -64,6 +64,7 @@ #define CURL_BUILD_IOS 0 #define CURL_BUILD_IOS_7 0 +#define CURL_BUILD_IOS_9 0 #define CURL_BUILD_IOS_11 0 #define CURL_BUILD_MAC 1 /* This is the maximum API level we are allowed to use when building: */ @@ -72,6 +73,7 @@ #define CURL_BUILD_MAC_10_7 MAC_OS_X_VERSION_MAX_ALLOWED >= 1070 #define CURL_BUILD_MAC_10_8 MAC_OS_X_VERSION_MAX_ALLOWED >= 1080 #define CURL_BUILD_MAC_10_9 MAC_OS_X_VERSION_MAX_ALLOWED >= 1090 +#define CURL_BUILD_MAC_10_11 MAC_OS_X_VERSION_MAX_ALLOWED >= 101100 #define CURL_BUILD_MAC_10_13 MAC_OS_X_VERSION_MAX_ALLOWED >= 101300 /* These macros mean "the following code is present to allow runtime backward compatibility with at least this cat or earlier": @@ -86,6 +88,7 @@ #elif TARGET_OS_EMBEDDED || TARGET_OS_IPHONE #define CURL_BUILD_IOS 1 #define CURL_BUILD_IOS_7 __IPHONE_OS_VERSION_MAX_ALLOWED >= 70000 +#define CURL_BUILD_IOS_9 __IPHONE_OS_VERSION_MAX_ALLOWED >= 90000 #define CURL_BUILD_IOS_11 __IPHONE_OS_VERSION_MAX_ALLOWED >= 110000 #define CURL_BUILD_MAC 0 #define CURL_BUILD_MAC_10_5 0 @@ -93,6 +96,7 @@ #define CURL_BUILD_MAC_10_7 0 #define CURL_BUILD_MAC_10_8 0 #define CURL_BUILD_MAC_10_9 0 +#define CURL_BUILD_MAC_10_11 0 #define CURL_BUILD_MAC_10_13 0 #define CURL_SUPPORT_MAC_10_5 0 #define CURL_SUPPORT_MAC_10_6 0 @@ -116,6 +120,7 @@ #include "vtls.h" #include "darwinssl.h" #include "curl_printf.h" +#include "strdup.h" #include "curl_memory.h" /* The last #include file should be: */ @@ -945,7 +950,7 @@ static CURLcode CopyCertSubject(struct Curl_easy *data, if(!c) { failf(data, "SSL: invalid CA certificate subject"); - return CURLE_OUT_OF_MEMORY; + return CURLE_SSL_CACERT; } /* If the subject is already available as UTF-8 encoded (ie 'direct') then @@ -1252,14 +1257,13 @@ static CURLcode darwinssl_version_from_curl(SSLProtocol *darwinver, return CURLE_OK; case CURL_SSLVERSION_TLSv1_3: /* TLS 1.3 support first appeared in iOS 11 and macOS 10.13 */ -#if CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 - /* We can assume __builtin_available() will always work in the - 10.13/11.0 SDK: */ +#if (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 if(__builtin_available(macOS 10.13, iOS 11.0, *)) { *darwinver = kTLSProtocol13; return CURLE_OK; } -#endif /* CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 */ +#endif /* (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && + HAVE_BUILTIN_AVAILABLE == 1 */ break; } return CURLE_SSL_CONNECT_ERROR; @@ -1278,7 +1282,7 @@ set_ssl_version_min_max(struct connectdata *conn, int sockindex) /* macOS 10.5-10.7 supported TLS 1.0 only. macOS 10.8 and later, and iOS 5 and later, added TLS 1.1 and 1.2. macOS 10.13 and later, and iOS 11 and later, added TLS 1.3. */ -#if CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 +#if (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 if(__builtin_available(macOS 10.13, iOS 11.0, *)) { max_supported_version_by_os = CURL_SSLVERSION_MAX_TLSv1_3; } @@ -1287,7 +1291,8 @@ set_ssl_version_min_max(struct connectdata *conn, int sockindex) } #else max_supported_version_by_os = CURL_SSLVERSION_MAX_TLSv1_2; -#endif /* CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 */ +#endif /* (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && + HAVE_BUILTIN_AVAILABLE == 1 */ switch(ssl_version) { case CURL_SSLVERSION_DEFAULT: @@ -1299,8 +1304,6 @@ set_ssl_version_min_max(struct connectdata *conn, int sockindex) switch(ssl_version_max) { case CURL_SSLVERSION_MAX_NONE: - ssl_version_max = ssl_version << 16; - break; case CURL_SSLVERSION_MAX_DEFAULT: ssl_version_max = max_supported_version_by_os; break; @@ -1430,7 +1433,7 @@ static CURLcode darwinssl_connect_step1(struct connectdata *conn, case CURL_SSLVERSION_DEFAULT: case CURL_SSLVERSION_TLSv1: (void)SSLSetProtocolVersionMin(BACKEND->ssl_ctx, kTLSProtocol1); -#if CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 +#if (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 if(__builtin_available(macOS 10.13, iOS 11.0, *)) { (void)SSLSetProtocolVersionMax(BACKEND->ssl_ctx, kTLSProtocol13); } @@ -1439,7 +1442,8 @@ static CURLcode darwinssl_connect_step1(struct connectdata *conn, } #else (void)SSLSetProtocolVersionMax(BACKEND->ssl_ctx, kTLSProtocol12); -#endif /* CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11 */ +#endif /* (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && + HAVE_BUILTIN_AVAILABLE == 1 */ break; case CURL_SSLVERSION_TLSv1_0: case CURL_SSLVERSION_TLSv1_1: @@ -1572,6 +1576,35 @@ static CURLcode darwinssl_connect_step1(struct connectdata *conn, } #endif /* CURL_BUILD_MAC_10_8 || CURL_BUILD_IOS */ +#if (CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 + if(conn->bits.tls_enable_alpn) { + if(__builtin_available(macOS 10.13.4, iOS 11, *)) { + CFMutableArrayRef alpnArr = CFArrayCreateMutable(NULL, 0, + &kCFTypeArrayCallBacks); + +#ifdef USE_NGHTTP2 + if(data->set.httpversion >= CURL_HTTP_VERSION_2 && + (!SSL_IS_PROXY() || !conn->bits.tunnel_proxy)) { + CFArrayAppendValue(alpnArr, CFSTR(NGHTTP2_PROTO_VERSION_ID)); + infof(data, "ALPN, offering %s\n", NGHTTP2_PROTO_VERSION_ID); + } +#endif + + CFArrayAppendValue(alpnArr, CFSTR(ALPN_HTTP_1_1)); + infof(data, "ALPN, offering %s\n", ALPN_HTTP_1_1); + + /* expects length prefixed preference ordered list of protocols in wire + * format + */ + err = SSLSetALPNProtocols(BACKEND->ssl_ctx, alpnArr); + if(err != noErr) + infof(data, "WARNING: failed to set ALPN protocols; OSStatus %d\n", + err); + CFRelease(alpnArr); + } + } +#endif + if(SSL_SET_OPTION(key)) { infof(data, "WARNING: SSL: CURLOPT_SSLKEY is ignored by Secure " "Transport. The private key must be in the Keychain.\n"); @@ -1616,6 +1649,8 @@ static CURLcode darwinssl_connect_step1(struct connectdata *conn, } CFRelease(cert); + if(result == CURLE_SSL_CACERT) + return CURLE_SSL_CERTPROBLEM; if(result) return result; } @@ -1751,107 +1786,118 @@ static CURLcode darwinssl_connect_step1(struct connectdata *conn, higher priority, but it's probably better that we not connect at all than to give the user a false sense of security if the server only supports insecure ciphers. (Note: We don't care about SSLv2-only ciphers.) */ - (void)SSLGetNumberSupportedCiphers(BACKEND->ssl_ctx, &all_ciphers_count); + err = SSLGetNumberSupportedCiphers(BACKEND->ssl_ctx, &all_ciphers_count); + if(err != noErr) { + failf(data, "SSL: SSLGetNumberSupportedCiphers() failed: OSStatus %d", + err); + return CURLE_SSL_CIPHER; + } all_ciphers = malloc(all_ciphers_count*sizeof(SSLCipherSuite)); - allowed_ciphers = malloc(all_ciphers_count*sizeof(SSLCipherSuite)); - if(all_ciphers && allowed_ciphers && - SSLGetSupportedCiphers(BACKEND->ssl_ctx, all_ciphers, - &all_ciphers_count) == noErr) { - for(i = 0UL ; i < all_ciphers_count ; i++) { -#if CURL_BUILD_MAC - /* There's a known bug in early versions of Mountain Lion where ST's ECC - ciphers (cipher suite 0xC001 through 0xC032) simply do not work. - Work around the problem here by disabling those ciphers if we are - running in an affected version of OS X. */ - if(darwinver_maj == 12 && darwinver_min <= 3 && - all_ciphers[i] >= 0xC001 && all_ciphers[i] <= 0xC032) { - continue; - } -#endif /* CURL_BUILD_MAC */ - switch(all_ciphers[i]) { - /* Disable NULL ciphersuites: */ - case SSL_NULL_WITH_NULL_NULL: - case SSL_RSA_WITH_NULL_MD5: - case SSL_RSA_WITH_NULL_SHA: - case 0x003B: /* TLS_RSA_WITH_NULL_SHA256 */ - case SSL_FORTEZZA_DMS_WITH_NULL_SHA: - case 0xC001: /* TLS_ECDH_ECDSA_WITH_NULL_SHA */ - case 0xC006: /* TLS_ECDHE_ECDSA_WITH_NULL_SHA */ - case 0xC00B: /* TLS_ECDH_RSA_WITH_NULL_SHA */ - case 0xC010: /* TLS_ECDHE_RSA_WITH_NULL_SHA */ - case 0x002C: /* TLS_PSK_WITH_NULL_SHA */ - case 0x002D: /* TLS_DHE_PSK_WITH_NULL_SHA */ - case 0x002E: /* TLS_RSA_PSK_WITH_NULL_SHA */ - case 0x00B0: /* TLS_PSK_WITH_NULL_SHA256 */ - case 0x00B1: /* TLS_PSK_WITH_NULL_SHA384 */ - case 0x00B4: /* TLS_DHE_PSK_WITH_NULL_SHA256 */ - case 0x00B5: /* TLS_DHE_PSK_WITH_NULL_SHA384 */ - case 0x00B8: /* TLS_RSA_PSK_WITH_NULL_SHA256 */ - case 0x00B9: /* TLS_RSA_PSK_WITH_NULL_SHA384 */ - /* Disable anonymous ciphersuites: */ - case SSL_DH_anon_EXPORT_WITH_RC4_40_MD5: - case SSL_DH_anon_WITH_RC4_128_MD5: - case SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA: - case SSL_DH_anon_WITH_DES_CBC_SHA: - case SSL_DH_anon_WITH_3DES_EDE_CBC_SHA: - case TLS_DH_anon_WITH_AES_128_CBC_SHA: - case TLS_DH_anon_WITH_AES_256_CBC_SHA: - case 0xC015: /* TLS_ECDH_anon_WITH_NULL_SHA */ - case 0xC016: /* TLS_ECDH_anon_WITH_RC4_128_SHA */ - case 0xC017: /* TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA */ - case 0xC018: /* TLS_ECDH_anon_WITH_AES_128_CBC_SHA */ - case 0xC019: /* TLS_ECDH_anon_WITH_AES_256_CBC_SHA */ - case 0x006C: /* TLS_DH_anon_WITH_AES_128_CBC_SHA256 */ - case 0x006D: /* TLS_DH_anon_WITH_AES_256_CBC_SHA256 */ - case 0x00A6: /* TLS_DH_anon_WITH_AES_128_GCM_SHA256 */ - case 0x00A7: /* TLS_DH_anon_WITH_AES_256_GCM_SHA384 */ - /* Disable weak key ciphersuites: */ - case SSL_RSA_EXPORT_WITH_RC4_40_MD5: - case SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5: - case SSL_RSA_EXPORT_WITH_DES40_CBC_SHA: - case SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA: - case SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA: - case SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA: - case SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA: - case SSL_RSA_WITH_DES_CBC_SHA: - case SSL_DH_DSS_WITH_DES_CBC_SHA: - case SSL_DH_RSA_WITH_DES_CBC_SHA: - case SSL_DHE_DSS_WITH_DES_CBC_SHA: - case SSL_DHE_RSA_WITH_DES_CBC_SHA: - /* Disable IDEA: */ - case SSL_RSA_WITH_IDEA_CBC_SHA: - case SSL_RSA_WITH_IDEA_CBC_MD5: - /* Disable RC4: */ - case SSL_RSA_WITH_RC4_128_MD5: - case SSL_RSA_WITH_RC4_128_SHA: - case 0xC002: /* TLS_ECDH_ECDSA_WITH_RC4_128_SHA */ - case 0xC007: /* TLS_ECDHE_ECDSA_WITH_RC4_128_SHA*/ - case 0xC00C: /* TLS_ECDH_RSA_WITH_RC4_128_SHA */ - case 0xC011: /* TLS_ECDHE_RSA_WITH_RC4_128_SHA */ - case 0x008A: /* TLS_PSK_WITH_RC4_128_SHA */ - case 0x008E: /* TLS_DHE_PSK_WITH_RC4_128_SHA */ - case 0x0092: /* TLS_RSA_PSK_WITH_RC4_128_SHA */ - break; - default: /* enable everything else */ - allowed_ciphers[allowed_ciphers_count++] = all_ciphers[i]; - break; - } - } - err = SSLSetEnabledCiphers(BACKEND->ssl_ctx, allowed_ciphers, - allowed_ciphers_count); - if(err != noErr) { - failf(data, "SSL: SSLSetEnabledCiphers() failed: OSStatus %d", err); - return CURLE_SSL_CONNECT_ERROR; - } + if(!all_ciphers) { + failf(data, "SSL: Failed to allocate memory for all ciphers"); + return CURLE_OUT_OF_MEMORY; } - else { + allowed_ciphers = malloc(all_ciphers_count*sizeof(SSLCipherSuite)); + if(!allowed_ciphers) { Curl_safefree(all_ciphers); - Curl_safefree(allowed_ciphers); failf(data, "SSL: Failed to allocate memory for allowed ciphers"); return CURLE_OUT_OF_MEMORY; } + err = SSLGetSupportedCiphers(BACKEND->ssl_ctx, all_ciphers, + &all_ciphers_count); + if(err != noErr) { + Curl_safefree(all_ciphers); + Curl_safefree(allowed_ciphers); + return CURLE_SSL_CIPHER; + } + for(i = 0UL ; i < all_ciphers_count ; i++) { +#if CURL_BUILD_MAC + /* There's a known bug in early versions of Mountain Lion where ST's ECC + ciphers (cipher suite 0xC001 through 0xC032) simply do not work. + Work around the problem here by disabling those ciphers if we are + running in an affected version of OS X. */ + if(darwinver_maj == 12 && darwinver_min <= 3 && + all_ciphers[i] >= 0xC001 && all_ciphers[i] <= 0xC032) { + continue; + } +#endif /* CURL_BUILD_MAC */ + switch(all_ciphers[i]) { + /* Disable NULL ciphersuites: */ + case SSL_NULL_WITH_NULL_NULL: + case SSL_RSA_WITH_NULL_MD5: + case SSL_RSA_WITH_NULL_SHA: + case 0x003B: /* TLS_RSA_WITH_NULL_SHA256 */ + case SSL_FORTEZZA_DMS_WITH_NULL_SHA: + case 0xC001: /* TLS_ECDH_ECDSA_WITH_NULL_SHA */ + case 0xC006: /* TLS_ECDHE_ECDSA_WITH_NULL_SHA */ + case 0xC00B: /* TLS_ECDH_RSA_WITH_NULL_SHA */ + case 0xC010: /* TLS_ECDHE_RSA_WITH_NULL_SHA */ + case 0x002C: /* TLS_PSK_WITH_NULL_SHA */ + case 0x002D: /* TLS_DHE_PSK_WITH_NULL_SHA */ + case 0x002E: /* TLS_RSA_PSK_WITH_NULL_SHA */ + case 0x00B0: /* TLS_PSK_WITH_NULL_SHA256 */ + case 0x00B1: /* TLS_PSK_WITH_NULL_SHA384 */ + case 0x00B4: /* TLS_DHE_PSK_WITH_NULL_SHA256 */ + case 0x00B5: /* TLS_DHE_PSK_WITH_NULL_SHA384 */ + case 0x00B8: /* TLS_RSA_PSK_WITH_NULL_SHA256 */ + case 0x00B9: /* TLS_RSA_PSK_WITH_NULL_SHA384 */ + /* Disable anonymous ciphersuites: */ + case SSL_DH_anon_EXPORT_WITH_RC4_40_MD5: + case SSL_DH_anon_WITH_RC4_128_MD5: + case SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA: + case SSL_DH_anon_WITH_DES_CBC_SHA: + case SSL_DH_anon_WITH_3DES_EDE_CBC_SHA: + case TLS_DH_anon_WITH_AES_128_CBC_SHA: + case TLS_DH_anon_WITH_AES_256_CBC_SHA: + case 0xC015: /* TLS_ECDH_anon_WITH_NULL_SHA */ + case 0xC016: /* TLS_ECDH_anon_WITH_RC4_128_SHA */ + case 0xC017: /* TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA */ + case 0xC018: /* TLS_ECDH_anon_WITH_AES_128_CBC_SHA */ + case 0xC019: /* TLS_ECDH_anon_WITH_AES_256_CBC_SHA */ + case 0x006C: /* TLS_DH_anon_WITH_AES_128_CBC_SHA256 */ + case 0x006D: /* TLS_DH_anon_WITH_AES_256_CBC_SHA256 */ + case 0x00A6: /* TLS_DH_anon_WITH_AES_128_GCM_SHA256 */ + case 0x00A7: /* TLS_DH_anon_WITH_AES_256_GCM_SHA384 */ + /* Disable weak key ciphersuites: */ + case SSL_RSA_EXPORT_WITH_RC4_40_MD5: + case SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5: + case SSL_RSA_EXPORT_WITH_DES40_CBC_SHA: + case SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA: + case SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA: + case SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA: + case SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA: + case SSL_RSA_WITH_DES_CBC_SHA: + case SSL_DH_DSS_WITH_DES_CBC_SHA: + case SSL_DH_RSA_WITH_DES_CBC_SHA: + case SSL_DHE_DSS_WITH_DES_CBC_SHA: + case SSL_DHE_RSA_WITH_DES_CBC_SHA: + /* Disable IDEA: */ + case SSL_RSA_WITH_IDEA_CBC_SHA: + case SSL_RSA_WITH_IDEA_CBC_MD5: + /* Disable RC4: */ + case SSL_RSA_WITH_RC4_128_MD5: + case SSL_RSA_WITH_RC4_128_SHA: + case 0xC002: /* TLS_ECDH_ECDSA_WITH_RC4_128_SHA */ + case 0xC007: /* TLS_ECDHE_ECDSA_WITH_RC4_128_SHA*/ + case 0xC00C: /* TLS_ECDH_RSA_WITH_RC4_128_SHA */ + case 0xC011: /* TLS_ECDHE_RSA_WITH_RC4_128_SHA */ + case 0x008A: /* TLS_PSK_WITH_RC4_128_SHA */ + case 0x008E: /* TLS_DHE_PSK_WITH_RC4_128_SHA */ + case 0x0092: /* TLS_RSA_PSK_WITH_RC4_128_SHA */ + break; + default: /* enable everything else */ + allowed_ciphers[allowed_ciphers_count++] = all_ciphers[i]; + break; + } + } + err = SSLSetEnabledCiphers(BACKEND->ssl_ctx, allowed_ciphers, + allowed_ciphers_count); Curl_safefree(all_ciphers); Curl_safefree(allowed_ciphers); + if(err != noErr) { + failf(data, "SSL: SSLSetEnabledCiphers() failed: OSStatus %d", err); + return CURLE_SSL_CIPHER; + } #if CURL_BUILD_MAC_10_9 || CURL_BUILD_IOS_7 /* We want to enable 1/n-1 when using a CBC cipher unless the user @@ -2009,7 +2055,7 @@ static int read_cert(const char *file, unsigned char **out, size_t *outlen) if(len + n >= cap) { cap *= 2; - data = realloc(data, cap); + data = Curl_saferealloc(data, cap); if(!data) { close(fd); return -1; @@ -2027,35 +2073,6 @@ static int read_cert(const char *file, unsigned char **out, size_t *outlen) return 0; } -static int sslerr_to_curlerr(struct Curl_easy *data, int err) -{ - switch(err) { - case errSSLXCertChainInvalid: - failf(data, "SSL certificate problem: Invalid certificate chain"); - return CURLE_SSL_CACERT; - case errSSLUnknownRootCert: - failf(data, "SSL certificate problem: Untrusted root certificate"); - return CURLE_SSL_CACERT; - case errSSLNoRootCert: - failf(data, "SSL certificate problem: No root certificate"); - return CURLE_SSL_CACERT; - case errSSLCertExpired: - failf(data, "SSL certificate problem: Certificate chain had an " - "expired certificate"); - return CURLE_SSL_CACERT; - case errSSLBadCert: - failf(data, "SSL certificate problem: Couldn't understand the server " - "certificate format"); - return CURLE_SSL_CONNECT_ERROR; - case errSSLHostNameMismatch: - failf(data, "SSL certificate peer hostname mismatch"); - return CURLE_PEER_FAILED_VERIFICATION; - default: - failf(data, "SSL unexpected certificate error %d", err); - return CURLE_SSL_CACERT; - } -} - static int append_cert_to_array(struct Curl_easy *data, unsigned char *buf, size_t buflen, CFMutableArrayRef array) @@ -2073,13 +2090,20 @@ static int append_cert_to_array(struct Curl_easy *data, CFRelease(certdata); if(!cacert) { failf(data, "SSL: failed to create SecCertificate from CA certificate"); - return CURLE_SSL_CACERT; + return CURLE_SSL_CACERT_BADFILE; } /* Check if cacert is valid. */ result = CopyCertSubject(data, cacert, &certp); - if(result) - return result; + switch(result) { + case CURLE_OK: + break; + case CURLE_PEER_FAILED_VERIFICATION: + return CURLE_SSL_CACERT_BADFILE; + case CURLE_OUT_OF_MEMORY: + default: + return result; + } free(certp); CFArrayAppendValue(array, cacert); @@ -2098,7 +2122,7 @@ static int verify_cert(const char *cafile, struct Curl_easy *data, if(read_cert(cafile, &certbuf, &buflen) < 0) { failf(data, "SSL: failed to read or invalid CA certificate"); - return CURLE_SSL_CACERT; + return CURLE_SSL_CACERT_BADFILE; } /* @@ -2131,7 +2155,7 @@ static int verify_cert(const char *cafile, struct Curl_easy *data, CFRelease(array); failf(data, "SSL: invalid CA certificate #%d (offset %d) in bundle", n, offset); - return CURLE_SSL_CACERT; + return CURLE_SSL_CACERT_BADFILE; } offset += res; @@ -2165,22 +2189,27 @@ static int verify_cert(const char *cafile, struct Curl_easy *data, if(trust == NULL) { failf(data, "SSL: error getting certificate chain"); CFRelease(array); - return CURLE_OUT_OF_MEMORY; + return CURLE_PEER_FAILED_VERIFICATION; } else if(ret != noErr) { CFRelease(array); - return sslerr_to_curlerr(data, ret); + failf(data, "SSLCopyPeerTrust() returned error %d", ret); + return CURLE_PEER_FAILED_VERIFICATION; } ret = SecTrustSetAnchorCertificates(trust, array); if(ret != noErr) { + CFRelease(array); CFRelease(trust); - return sslerr_to_curlerr(data, ret); + failf(data, "SecTrustSetAnchorCertificates() returned error %d", ret); + return CURLE_PEER_FAILED_VERIFICATION; } ret = SecTrustSetAnchorCertificatesOnly(trust, true); if(ret != noErr) { + CFRelease(array); CFRelease(trust); - return sslerr_to_curlerr(data, ret); + failf(data, "SecTrustSetAnchorCertificatesOnly() returned error %d", ret); + return CURLE_PEER_FAILED_VERIFICATION; } SecTrustResultType trust_eval = 0; @@ -2188,7 +2217,8 @@ static int verify_cert(const char *cafile, struct Curl_easy *data, CFRelease(array); CFRelease(trust); if(ret != noErr) { - return sslerr_to_curlerr(data, ret); + failf(data, "SecTrustEvaluate() returned error %d", ret); + return CURLE_PEER_FAILED_VERIFICATION; } switch(trust_eval) { @@ -2349,6 +2379,53 @@ darwinssl_connect_step2(struct connectdata *conn, int sockindex) /* the documentation says we need to call SSLHandshake() again */ return darwinssl_connect_step2(conn, sockindex); + /* Problem with encrypt / decrypt */ + case errSSLPeerDecodeError: + failf(data, "Decode failed"); + break; + case errSSLDecryptionFail: + case errSSLPeerDecryptionFail: + failf(data, "Decryption failed"); + break; + case errSSLPeerDecryptError: + failf(data, "A decryption error occurred"); + break; + case errSSLBadCipherSuite: + failf(data, "A bad SSL cipher suite was encountered"); + break; + case errSSLCrypto: + failf(data, "An underlying cryptographic error was encountered"); + break; +#if CURL_BUILD_MAC_10_11 || CURL_BUILD_IOS_9 + case errSSLWeakPeerEphemeralDHKey: + failf(data, "Indicates a weak ephemeral Diffie-Hellman key"); + break; +#endif + + /* Problem with the message record validation */ + case errSSLBadRecordMac: + case errSSLPeerBadRecordMac: + failf(data, "A record with a bad message authentication code (MAC) " + "was encountered"); + break; + case errSSLRecordOverflow: + case errSSLPeerRecordOverflow: + failf(data, "A record overflow occurred"); + break; + + /* Problem with zlib decompression */ + case errSSLPeerDecompressFail: + failf(data, "Decompression failed"); + break; + + /* Problem with access */ + case errSSLPeerAccessDenied: + failf(data, "Access was denied"); + break; + case errSSLPeerInsufficientSecurity: + failf(data, "There is insufficient security for this operation"); + break; + /* These are all certificate problems with the server: */ case errSSLXCertChainInvalid: failf(data, "SSL certificate problem: Invalid certificate chain"); @@ -2359,28 +2436,44 @@ darwinssl_connect_step2(struct connectdata *conn, int sockindex) case errSSLNoRootCert: failf(data, "SSL certificate problem: No root certificate"); return CURLE_SSL_CACERT; + case errSSLCertNotYetValid: + failf(data, "SSL certificate problem: The certificate chain had a " + "certificate that is not yet valid"); + return CURLE_SSL_CACERT; case errSSLCertExpired: + case errSSLPeerCertExpired: failf(data, "SSL certificate problem: Certificate chain had an " "expired certificate"); return CURLE_SSL_CACERT; case errSSLBadCert: + case errSSLPeerBadCert: failf(data, "SSL certificate problem: Couldn't understand the server " "certificate format"); - return CURLE_SSL_CONNECT_ERROR; + return CURLE_SSL_CACERT; + case errSSLPeerUnsupportedCert: + failf(data, "SSL certificate problem: An unsupported certificate " + "format was encountered"); + return CURLE_SSL_CACERT; + case errSSLPeerCertRevoked: + failf(data, "SSL certificate problem: The certificate was revoked"); + return CURLE_SSL_CACERT; + case errSSLPeerCertUnknown: + failf(data, "SSL certificate problem: The certificate is unknown"); + return CURLE_SSL_CACERT; /* These are all certificate problems with the client: */ case errSecAuthFailed: failf(data, "SSL authentication failed"); - return CURLE_SSL_CONNECT_ERROR; + break; case errSSLPeerHandshakeFail: failf(data, "SSL peer handshake failed, the server most likely " "requires a client certificate to connect"); - return CURLE_SSL_CONNECT_ERROR; + break; case errSSLPeerUnknownCA: failf(data, "SSL server rejected the client certificate due to " "the certificate being signed by an unknown certificate " "authority"); - return CURLE_SSL_CONNECT_ERROR; + break; /* This error is raised if the server's cert didn't match the server's host name: */ @@ -2389,30 +2482,98 @@ darwinssl_connect_step2(struct connectdata *conn, int sockindex) "certificate did not match \"%s\"\n", conn->host.dispname); return CURLE_PEER_FAILED_VERIFICATION; + /* Problem with SSL / TLS negotiation */ + case errSSLNegotiation: + failf(data, "Could not negotiate an SSL cipher suite with the server"); + break; + case errSSLBadConfiguration: + failf(data, "A configuration error occurred"); + break; + case errSSLProtocol: + failf(data, "SSL protocol error"); + break; + case errSSLPeerProtocolVersion: + failf(data, "A bad protocol version was encountered"); + break; + case errSSLPeerNoRenegotiation: + failf(data, "No renegotiation is allowed"); + break; + /* Generic handshake errors: */ case errSSLConnectionRefused: failf(data, "Server dropped the connection during the SSL handshake"); - return CURLE_SSL_CONNECT_ERROR; + break; case errSSLClosedAbort: failf(data, "Server aborted the SSL handshake"); - return CURLE_SSL_CONNECT_ERROR; - case errSSLNegotiation: - failf(data, "Could not negotiate an SSL cipher suite with the server"); - return CURLE_SSL_CONNECT_ERROR; + break; + case errSSLClosedGraceful: + failf(data, "The connection closed gracefully"); + break; + case errSSLClosedNoNotify: + failf(data, "The server closed the session with no notification"); + break; /* Sometimes paramErr happens with buggy ciphers: */ - case paramErr: case errSSLInternal: + case paramErr: + case errSSLInternal: + case errSSLPeerInternalError: failf(data, "Internal SSL engine error encountered during the " "SSL handshake"); - return CURLE_SSL_CONNECT_ERROR; + break; case errSSLFatalAlert: failf(data, "Fatal SSL engine error encountered during the SSL " "handshake"); - return CURLE_SSL_CONNECT_ERROR; + break; + /* Unclassified error */ + case errSSLBufferOverflow: + failf(data, "An insufficient buffer was provided"); + break; + case errSSLIllegalParam: + failf(data, "An illegal parameter was encountered"); + break; + case errSSLModuleAttach: + failf(data, "Module attach failure"); + break; + case errSSLSessionNotFound: + failf(data, "An attempt to restore an unknown session failed"); + break; + case errSSLPeerExportRestriction: + failf(data, "An export restriction occurred"); + break; + case errSSLPeerUserCancelled: + failf(data, "The user canceled the operation"); + break; + case errSSLPeerUnexpectedMsg: + failf(data, "Peer rejected unexpected message"); + break; +#if CURL_BUILD_MAC_10_11 || CURL_BUILD_IOS_9 + /* Treaing non-fatal error as fatal like before */ + case errSSLClientHelloReceived: + failf(data, "A non-fatal result for providing a server name " + "indication"); + break; +#endif + + /* Error codes defined in the enum but should never be returned. + We list them here just in case. */ +#if CURL_BUILD_MAC_10_6 + /* Only returned when kSSLSessionOptionBreakOnCertRequested is set */ + case errSSLClientCertRequested: + failf(data, "The server has requested a client certificate"); + break; +#endif +#if CURL_BUILD_MAC_10_9 + /* Alias for errSSLLast, end of error range */ + case errSSLUnexpectedRecord: + failf(data, "Unexpected (skipped) record in DTLS"); + break; +#endif default: + /* May also return codes listed in Security Framework Result Codes */ failf(data, "Unknown SSL protocol error in connection to %s:%d", hostname, err); - return CURLE_SSL_CONNECT_ERROR; + break; } + return CURLE_SSL_CONNECT_ERROR; } else { /* we have been connected fine, we're not waiting for anything else. */ @@ -2466,6 +2627,39 @@ darwinssl_connect_step2(struct connectdata *conn, int sockindex) break; } +#if(CURL_BUILD_MAC_10_13 || CURL_BUILD_IOS_11) && HAVE_BUILTIN_AVAILABLE == 1 + if(conn->bits.tls_enable_alpn) { + if(__builtin_available(macOS 10.13.4, iOS 11, *)) { + CFArrayRef alpnArr = NULL; + CFStringRef chosenProtocol = NULL; + err = SSLCopyALPNProtocols(BACKEND->ssl_ctx, &alpnArr); + + if(err == noErr && alpnArr && CFArrayGetCount(alpnArr) >= 1) + chosenProtocol = CFArrayGetValueAtIndex(alpnArr, 0); + +#ifdef USE_NGHTTP2 + if(chosenProtocol && + !CFStringCompare(chosenProtocol, CFSTR(NGHTTP2_PROTO_VERSION_ID), + 0)) { + conn->negnpn = CURL_HTTP_VERSION_2; + } + else +#endif + if(chosenProtocol && + !CFStringCompare(chosenProtocol, CFSTR(ALPN_HTTP_1_1), 0)) { + conn->negnpn = CURL_HTTP_VERSION_1_1; + } + else + infof(data, "ALPN, server did not agree to a protocol\n"); + + /* chosenProtocol is a reference to the string within alpnArr + and doesn't need to be freed separately */ + if(alpnArr) + CFRelease(alpnArr); + } + } +#endif + return CURLE_OK; } } diff --git a/Utilities/cmcurl/lib/vtls/gskit.c b/Utilities/cmcurl/lib/vtls/gskit.c index b0856cdf4..8d1b3d6af 100644 --- a/Utilities/cmcurl/lib/vtls/gskit.c +++ b/Utilities/cmcurl/lib/vtls/gskit.c @@ -328,7 +328,7 @@ static CURLcode set_ciphers(struct connectdata *conn, GSKit tokens are always shorter than their cipher names, allocated buffers will always be large enough to accommodate the result. */ l = strlen(cipherlist) + 1; - memset((char *) ciphers, 0, sizeof ciphers); + memset((char *) ciphers, 0, sizeof(ciphers)); for(i = 0; i < CURL_GSKPROTO_LAST; i++) { ciphers[i].buf = malloc(l); if(!ciphers[i].buf) { @@ -536,18 +536,18 @@ inetsocketpair(int sv[2]) lfd = socket(AF_INET, SOCK_STREAM, 0); if(lfd < 0) return -1; - memset((char *) &addr1, 0, sizeof addr1); + memset((char *) &addr1, 0, sizeof(addr1)); addr1.sin_family = AF_INET; addr1.sin_addr.s_addr = htonl(INADDR_LOOPBACK); addr1.sin_port = 0; - if(bind(lfd, (struct sockaddr *) &addr1, sizeof addr1) || + if(bind(lfd, (struct sockaddr *) &addr1, sizeof(addr1)) || listen(lfd, 2) < 0) { close(lfd); return -1; } /* Get the allocated port. */ - len = sizeof addr1; + len = sizeof(addr1); if(getsockname(lfd, (struct sockaddr *) &addr1, &len) < 0) { close(lfd); return -1; @@ -562,7 +562,7 @@ inetsocketpair(int sv[2]) /* Request unblocking connection to the listening socket. */ curlx_nonblock(cfd, TRUE); - if(connect(cfd, (struct sockaddr *) &addr1, sizeof addr1) < 0 && + if(connect(cfd, (struct sockaddr *) &addr1, sizeof(addr1)) < 0 && errno != EINPROGRESS) { close(lfd); close(cfd); @@ -570,7 +570,7 @@ inetsocketpair(int sv[2]) } /* Get the client dynamic port for intrusion check below. */ - len = sizeof addr2; + len = sizeof(addr2); if(getsockname(cfd, (struct sockaddr *) &addr2, &len) < 0) { close(lfd); close(cfd); @@ -580,7 +580,7 @@ inetsocketpair(int sv[2]) /* Accept the incoming connection and get the server socket. */ curlx_nonblock(lfd, TRUE); for(;;) { - len = sizeof addr1; + len = sizeof(addr1); sfd = accept(lfd, (struct sockaddr *) &addr1, &len); if(sfd < 0) { close(lfd); @@ -644,7 +644,7 @@ static int pipe_ssloverssl(struct connectdata *conn, int sockindex, /* Try getting data from HTTPS proxy and pipe it upstream. */ n = 0; i = gsk_secure_soc_read(connproxyssl->backend->handle, - buf, sizeof buf, &n); + buf, sizeof(buf), &n); switch(i) { case GSK_OK: if(n) { @@ -665,7 +665,7 @@ static int pipe_ssloverssl(struct connectdata *conn, int sockindex, if(FD_ISSET(BACKEND->remotefd, &fds_read) && FD_ISSET(conn->sock[sockindex], &fds_write)) { /* Pipe data to HTTPS proxy. */ - n = read(BACKEND->remotefd, buf, sizeof buf); + n = read(BACKEND->remotefd, buf, sizeof(buf)); if(n < 0) return -1; if(n) { @@ -766,8 +766,6 @@ set_ssl_version_min_max(unsigned int *protoflags, struct connectdata *conn) long i = ssl_version; switch(ssl_version_max) { case CURL_SSLVERSION_MAX_NONE: - ssl_version_max = ssl_version; - break; case CURL_SSLVERSION_MAX_DEFAULT: ssl_version_max = CURL_SSLVERSION_TLSv1_2; break; @@ -864,13 +862,13 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex) BACKEND->localfd = sockpair[0]; BACKEND->remotefd = sockpair[1]; setsockopt(BACKEND->localfd, SOL_SOCKET, SO_RCVBUF, - (void *) sobufsize, sizeof sobufsize); + (void *) sobufsize, sizeof(sobufsize)); setsockopt(BACKEND->remotefd, SOL_SOCKET, SO_RCVBUF, - (void *) sobufsize, sizeof sobufsize); + (void *) sobufsize, sizeof(sobufsize)); setsockopt(BACKEND->localfd, SOL_SOCKET, SO_SNDBUF, - (void *) sobufsize, sizeof sobufsize); + (void *) sobufsize, sizeof(sobufsize)); setsockopt(BACKEND->remotefd, SOL_SOCKET, SO_SNDBUF, - (void *) sobufsize, sizeof sobufsize); + (void *) sobufsize, sizeof(sobufsize)); curlx_nonblock(BACKEND->localfd, TRUE); curlx_nonblock(BACKEND->remotefd, TRUE); } @@ -977,7 +975,7 @@ static CURLcode gskit_connect_step1(struct connectdata *conn, int sockindex) if(!result) { /* Start handshake. Try asynchronous first. */ - memset(&commarea, 0, sizeof commarea); + memset(&commarea, 0, sizeof(commarea)); BACKEND->iocport = QsoCreateIOCompletionPort(); if(BACKEND->iocport != -1) { result = gskit_status(data, @@ -1316,8 +1314,7 @@ static int Curl_gskit_shutdown(struct connectdata *conn, int sockindex) static size_t Curl_gskit_version(char *buffer, size_t size) { - strncpy(buffer, "GSKit", size); - return strlen(buffer); + return snprintf(buffer, size, "GSKit"); } @@ -1333,11 +1330,11 @@ static int Curl_gskit_check_cxn(struct connectdata *cxn) return 0; /* connection has been closed */ err = 0; - errlen = sizeof err; + errlen = sizeof(err); if(getsockopt(cxn->sock[FIRSTSOCKET], SOL_SOCKET, SO_ERROR, (unsigned char *) &err, &errlen) || - errlen != sizeof err || err) + errlen != sizeof(err) || err) return 0; /* connection has been closed */ return -1; /* connection status unknown */ diff --git a/Utilities/cmcurl/lib/vtls/gtls.c b/Utilities/cmcurl/lib/vtls/gtls.c index 207b0fd1b..37662a748 100644 --- a/Utilities/cmcurl/lib/vtls/gtls.c +++ b/Utilities/cmcurl/lib/vtls/gtls.c @@ -94,6 +94,10 @@ static bool gtls_inited = FALSE; # endif #endif +#if (GNUTLS_VERSION_NUMBER >= 0x030603) +#define HAS_TLS13 +#endif + #ifdef HAS_OCSP # include <gnutls/ocsp.h> #endif @@ -390,9 +394,10 @@ set_ssl_version_min_max(int *list, size_t list_size, struct connectdata *conn) switch(ssl_version_max) { case CURL_SSLVERSION_MAX_NONE: - ssl_version_max = ssl_version << 16; - break; case CURL_SSLVERSION_MAX_DEFAULT: +#ifdef HAS_TLS13 + ssl_version_max = CURL_SSLVERSION_MAX_TLSv1_3; +#endif ssl_version_max = CURL_SSLVERSION_MAX_TLSv1_2; break; } @@ -410,8 +415,13 @@ set_ssl_version_min_max(int *list, size_t list_size, struct connectdata *conn) protocol_priority[protocol_priority_idx++] = GNUTLS_TLS1_2; break; case CURL_SSLVERSION_TLSv1_3: +#ifdef HAS_TLS13 + protocol_priority[protocol_priority_idx++] = GNUTLS_TLS1_3; + break; +#else failf(data, "GnuTLS: TLS 1.3 is not yet supported"); return CURLE_SSL_CONNECT_ERROR; +#endif } } return CURLE_OK; @@ -429,13 +439,9 @@ set_ssl_version_min_max(const char **prioritylist, struct connectdata *conn) struct Curl_easy *data = conn->data; long ssl_version = SSL_CONN_CONFIG(version); long ssl_version_max = SSL_CONN_CONFIG(version_max); - if(ssl_version == CURL_SSLVERSION_TLSv1_3 || - ssl_version_max == CURL_SSLVERSION_MAX_TLSv1_3) { - failf(data, "GnuTLS: TLS 1.3 is not yet supported"); - return CURLE_SSL_CONNECT_ERROR; - } + if(ssl_version_max == CURL_SSLVERSION_MAX_NONE) { - ssl_version_max = ssl_version << 16; + ssl_version_max = CURL_SSLVERSION_MAX_DEFAULT; } switch(ssl_version | ssl_version_max) { case CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_TLSv1_0: @@ -447,7 +453,6 @@ set_ssl_version_min_max(const char **prioritylist, struct connectdata *conn) "+VERS-TLS1.0:+VERS-TLS1.1:" GNUTLS_SRP; return CURLE_OK; case CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_TLSv1_2: - case CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_DEFAULT: *prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:-VERS-TLS-ALL:" "+VERS-TLS1.0:+VERS-TLS1.1:+VERS-TLS1.2:" GNUTLS_SRP; return CURLE_OK; @@ -456,15 +461,54 @@ set_ssl_version_min_max(const char **prioritylist, struct connectdata *conn) "+VERS-TLS1.1:" GNUTLS_SRP; return CURLE_OK; case CURL_SSLVERSION_TLSv1_1 | CURL_SSLVERSION_MAX_TLSv1_2: - case CURL_SSLVERSION_TLSv1_1 | CURL_SSLVERSION_MAX_DEFAULT: *prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:-VERS-TLS-ALL:" "+VERS-TLS1.1:+VERS-TLS1.2:" GNUTLS_SRP; return CURLE_OK; case CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_TLSv1_2: - case CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT: *prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:-VERS-TLS-ALL:" "+VERS-TLS1.2:" GNUTLS_SRP; return CURLE_OK; + case CURL_SSLVERSION_TLSv1_3 | CURL_SSLVERSION_MAX_TLSv1_3: +#ifdef HAS_TLS13 + *prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:-VERS-TLS-ALL:" + "+VERS-TLS1.3:" GNUTLS_SRP; + return CURLE_OK; +#else + failf(data, "GnuTLS: TLS 1.3 is not yet supported"); + return CURLE_SSL_CONNECT_ERROR; +#endif + case CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_DEFAULT: + *prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:-VERS-TLS-ALL:" + "+VERS-TLS1.0:+VERS-TLS1.1:+VERS-TLS1.2:" +#ifdef HAS_TLS13 + "+VERS-TLS1.3:" +#endif + GNUTLS_SRP; + return CURLE_OK; + case CURL_SSLVERSION_TLSv1_1 | CURL_SSLVERSION_MAX_DEFAULT: + *prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:-VERS-TLS-ALL:" + "+VERS-TLS1.1:+VERS-TLS1.2:" +#ifdef HAS_TLS13 + "+VERS-TLS1.3:" +#endif + GNUTLS_SRP; + return CURLE_OK; + case CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT: + *prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:-VERS-TLS-ALL:" + "+VERS-TLS1.2:" +#ifdef HAS_TLS13 + "+VERS-TLS1.3:" +#endif + GNUTLS_SRP; + return CURLE_OK; + case CURL_SSLVERSION_TLSv1_3 | CURL_SSLVERSION_MAX_DEFAULT: + *prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:-VERS-TLS-ALL:" + "+VERS-TLS1.2:" +#ifdef HAS_TLS13 + "+VERS-TLS1.3:" +#endif + GNUTLS_SRP; + return CURLE_OK; } failf(data, "GnuTLS: cannot set ssl protocol"); @@ -677,6 +721,9 @@ gtls_connect_step1(struct connectdata *conn, protocol_priority[0] = GNUTLS_TLS1_0; protocol_priority[1] = GNUTLS_TLS1_1; protocol_priority[2] = GNUTLS_TLS1_2; +#ifdef HAS_TLS13 + protocol_priority[3] = GNUTLS_TLS1_3; +#endif break; case CURL_SSLVERSION_TLSv1_0: case CURL_SSLVERSION_TLSv1_1: @@ -709,11 +756,14 @@ gtls_connect_step1(struct connectdata *conn, switch(SSL_CONN_CONFIG(version)) { case CURL_SSLVERSION_SSLv3: prioritylist = GNUTLS_CIPHERS ":-VERS-TLS-ALL:+VERS-SSL3.0"; - sni = false; break; case CURL_SSLVERSION_DEFAULT: case CURL_SSLVERSION_TLSv1: - prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:" GNUTLS_SRP; + prioritylist = GNUTLS_CIPHERS ":-VERS-SSL3.0:" +#ifdef HAS_TLS13 + "+VERS-TLS1.3:" +#endif + GNUTLS_SRP; break; case CURL_SSLVERSION_TLSv1_0: case CURL_SSLVERSION_TLSv1_1: @@ -1102,8 +1152,8 @@ gtls_connect_step3(struct connectdata *conn, return CURLE_SSL_INVALIDCERTSTATUS; } - rc = gnutls_ocsp_resp_get_single(ocsp_resp, 0, NULL, NULL, NULL, NULL, - &status, NULL, NULL, NULL, &reason); + (void)gnutls_ocsp_resp_get_single(ocsp_resp, 0, NULL, NULL, NULL, NULL, + &status, NULL, NULL, NULL, &reason); switch(status) { case GNUTLS_OCSP_CERT_GOOD: @@ -1589,7 +1639,7 @@ static int Curl_gtls_shutdown(struct connectdata *conn, int sockindex) ssize_t result; int retval = 0; struct Curl_easy *data = conn->data; - int done = 0; + bool done = FALSE; char buf[120]; /* This has only been tested on the proftpd server, and the mod_tls code @@ -1613,7 +1663,7 @@ static int Curl_gtls_shutdown(struct connectdata *conn, int sockindex) case 0: /* This is the expected response. There was no data but only the close notify alert */ - done = 1; + done = TRUE; break; case GNUTLS_E_AGAIN: case GNUTLS_E_INTERRUPTED: @@ -1621,21 +1671,20 @@ static int Curl_gtls_shutdown(struct connectdata *conn, int sockindex) break; default: retval = -1; - done = 1; + done = TRUE; break; } } else if(0 == what) { /* timeout */ failf(data, "SSL shutdown timeout"); - done = 1; - break; + done = TRUE; } else { /* anything that gets here is fatally bad */ failf(data, "select/poll on SSL socket, errno: %d", SOCKERRNO); retval = -1; - done = 1; + done = TRUE; } } gnutls_deinit(BACKEND->session); diff --git a/Utilities/cmcurl/lib/vtls/mbedtls.c b/Utilities/cmcurl/lib/vtls/mbedtls.c index d7759dc84..c5ed8872e 100644 --- a/Utilities/cmcurl/lib/vtls/mbedtls.c +++ b/Utilities/cmcurl/lib/vtls/mbedtls.c @@ -205,14 +205,11 @@ set_ssl_version_min_max(struct connectdata *conn, int sockindex) case CURL_SSLVERSION_DEFAULT: case CURL_SSLVERSION_TLSv1: ssl_version = CURL_SSLVERSION_TLSv1_0; - ssl_version_max = CURL_SSLVERSION_MAX_TLSv1_2; break; } switch(ssl_version_max) { case CURL_SSLVERSION_MAX_NONE: - ssl_version_max = ssl_version << 16; - break; case CURL_SSLVERSION_MAX_DEFAULT: ssl_version_max = CURL_SSLVERSION_MAX_TLSv1_2; break; diff --git a/Utilities/cmcurl/lib/vtls/mesalink.c b/Utilities/cmcurl/lib/vtls/mesalink.c new file mode 100644 index 000000000..6a2b67e63 --- /dev/null +++ b/Utilities/cmcurl/lib/vtls/mesalink.c @@ -0,0 +1,627 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2017-2018, Yiming Jing, <jingyiming@baidu.com> + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* + * Source file for all MesaLink-specific code for the TLS/SSL layer. No code + * but vtls.c should ever call or use these functions. + * + */ + +/* + * Based upon the CyaSSL implementation in cyassl.c and cyassl.h: + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * Thanks for code and inspiration! + */ + +#include "curl_setup.h" + +#ifdef USE_MESALINK + +#include <mesalink/options.h> +#include <mesalink/version.h> + +#include "urldata.h" +#include "sendf.h" +#include "inet_pton.h" +#include "vtls.h" +#include "parsedate.h" +#include "connect.h" /* for the connect timeout */ +#include "select.h" +#include "strcase.h" +#include "x509asn1.h" +#include "curl_printf.h" + +#include "mesalink.h" +#include <mesalink/openssl/ssl.h> +#include <mesalink/openssl/err.h> + +/* The last #include files should be: */ +#include "curl_memory.h" +#include "memdebug.h" + +#define MESALINK_MAX_ERROR_SZ 80 + +struct ssl_backend_data +{ + SSL_CTX *ctx; + SSL *handle; +}; + +#define BACKEND connssl->backend + +static Curl_recv mesalink_recv; +static Curl_send mesalink_send; + +/* + * This function loads all the client/CA certificates and CRLs. Setup the TLS + * layer and do all necessary magic. + */ +static CURLcode +mesalink_connect_step1(struct connectdata *conn, int sockindex) +{ + char *ciphers; + struct Curl_easy *data = conn->data; + struct ssl_connect_data *connssl = &conn->ssl[sockindex]; + const bool verifypeer = SSL_CONN_CONFIG(verifypeer); + const char *const ssl_cafile = SSL_CONN_CONFIG(CAfile); + const char *const ssl_capath = SSL_CONN_CONFIG(CApath); + struct in_addr addr4; +#ifdef ENABLE_IPV6 + struct in6_addr addr6; +#endif + const char *const hostname = + SSL_IS_PROXY() ? conn->http_proxy.host.name : conn->host.name; + size_t hostname_len = strlen(hostname); + + SSL_METHOD *req_method = NULL; + curl_socket_t sockfd = conn->sock[sockindex]; + + if(connssl->state == ssl_connection_complete) + return CURLE_OK; + + if(SSL_CONN_CONFIG(version_max) != CURL_SSLVERSION_MAX_NONE) { + failf(data, "MesaLink does not support to set maximum SSL/TLS version"); + return CURLE_SSL_CONNECT_ERROR; + } + + switch(SSL_CONN_CONFIG(version)) { + case CURL_SSLVERSION_SSLv3: + case CURL_SSLVERSION_TLSv1: + case CURL_SSLVERSION_TLSv1_0: + case CURL_SSLVERSION_TLSv1_1: + failf(data, "MesaLink does not support SSL 3.0, TLS 1.0, or TLS 1.1"); + return CURLE_NOT_BUILT_IN; + case CURL_SSLVERSION_DEFAULT: + case CURL_SSLVERSION_TLSv1_2: + req_method = TLSv1_2_client_method(); + break; + case CURL_SSLVERSION_TLSv1_3: + req_method = TLSv1_3_client_method(); + break; + case CURL_SSLVERSION_SSLv2: + failf(data, "MesaLink does not support SSLv2"); + return CURLE_SSL_CONNECT_ERROR; + default: + failf(data, "Unrecognized parameter passed via CURLOPT_SSLVERSION"); + return CURLE_SSL_CONNECT_ERROR; + } + + if(!req_method) { + failf(data, "SSL: couldn't create a method!"); + return CURLE_OUT_OF_MEMORY; + } + + if(BACKEND->ctx) + SSL_CTX_free(BACKEND->ctx); + BACKEND->ctx = SSL_CTX_new(req_method); + + if(!BACKEND->ctx) { + failf(data, "SSL: couldn't create a context!"); + return CURLE_OUT_OF_MEMORY; + } + + SSL_CTX_set_verify( + BACKEND->ctx, verifypeer ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, NULL); + + if(ssl_cafile || ssl_capath) { + if(!SSL_CTX_load_verify_locations(BACKEND->ctx, ssl_cafile, ssl_capath)) { + if(verifypeer) { + failf(data, + "error setting certificate verify locations:\n" + " CAfile: %s\n CApath: %s", + ssl_cafile ? ssl_cafile : "none", + ssl_capath ? ssl_capath : "none"); + return CURLE_SSL_CACERT_BADFILE; + } + infof(data, + "error setting certificate verify locations," + " continuing anyway:\n"); + } + else { + infof(data, "successfully set certificate verify locations:\n"); + } + infof(data, + " CAfile: %s\n" + " CApath: %s\n", + ssl_cafile ? ssl_cafile : "none", + ssl_capath ? ssl_capath : "none"); + } + + ciphers = SSL_CONN_CONFIG(cipher_list); + if(ciphers) { +#ifdef MESALINK_HAVE_CIPHER + if(!SSL_CTX_set_cipher_list(BACKEND->ctx, ciphers)) { + failf(data, "failed setting cipher list: %s", ciphers); + return CURLE_SSL_CIPHER; + } +#endif + infof(data, "Cipher selection: %s\n", ciphers); + } + + if(BACKEND->handle) + SSL_free(BACKEND->handle); + BACKEND->handle = SSL_new(BACKEND->ctx); + if(!BACKEND->handle) { + failf(data, "SSL: couldn't create a context (handle)!"); + return CURLE_OUT_OF_MEMORY; + } + + if((hostname_len < USHRT_MAX) && + (0 == Curl_inet_pton(AF_INET, hostname, &addr4)) +#ifdef ENABLE_IPV6 + && (0 == Curl_inet_pton(AF_INET6, hostname, &addr6)) +#endif + ) { + /* hostname is not a valid IP address */ + if(SSL_set_tlsext_host_name(BACKEND->handle, hostname) != SSL_SUCCESS) { + failf(data, + "WARNING: failed to configure server name indication (SNI) " + "TLS extension\n"); + return CURLE_SSL_CONNECT_ERROR; + } + } + else { +#ifdef CURLDEBUG + /* Check if the hostname is 127.0.0.1 or [::1]; + * otherwise reject because MesaLink always wants a valid DNS Name + * specified in RFC 5280 Section 7.2 */ + if(strncmp(hostname, "127.0.0.1", 9) == 0 +#ifdef ENABLE_IPV6 + || strncmp(hostname, "[::1]", 5) == 0 +#endif + ) { + SSL_set_tlsext_host_name(BACKEND->handle, "localhost"); + } + else +#endif + { + failf(data, + "ERROR: MesaLink does not accept an IP address as a hostname\n"); + return CURLE_SSL_CONNECT_ERROR; + } + } + +#ifdef MESALINK_HAVE_SESSION + if(SSL_SET_OPTION(primary.sessionid)) { + void *ssl_sessionid = NULL; + + Curl_ssl_sessionid_lock(conn); + if(!Curl_ssl_getsessionid(conn, &ssl_sessionid, NULL, sockindex)) { + /* we got a session id, use it! */ + if(!SSL_set_session(BACKEND->handle, ssl_sessionid)) { + Curl_ssl_sessionid_unlock(conn); + failf( + data, + "SSL: SSL_set_session failed: %s", + ERR_error_string(SSL_get_error(BACKEND->handle, 0), error_buffer)); + return CURLE_SSL_CONNECT_ERROR; + } + /* Informational message */ + infof(data, "SSL re-using session ID\n"); + } + Curl_ssl_sessionid_unlock(conn); + } +#endif /* MESALINK_HAVE_SESSION */ + + if(SSL_set_fd(BACKEND->handle, (int)sockfd) != SSL_SUCCESS) { + failf(data, "SSL: SSL_set_fd failed"); + return CURLE_SSL_CONNECT_ERROR; + } + + connssl->connecting_state = ssl_connect_2; + return CURLE_OK; +} + +static CURLcode +mesalink_connect_step2(struct connectdata *conn, int sockindex) +{ + int ret = -1; + struct Curl_easy *data = conn->data; + struct ssl_connect_data *connssl = &conn->ssl[sockindex]; + + conn->recv[sockindex] = mesalink_recv; + conn->send[sockindex] = mesalink_send; + + ret = SSL_connect(BACKEND->handle); + if(ret != SSL_SUCCESS) { + char error_buffer[MESALINK_MAX_ERROR_SZ]; + int detail = SSL_get_error(BACKEND->handle, ret); + + if(SSL_ERROR_WANT_CONNECT == detail) { + connssl->connecting_state = ssl_connect_2_reading; + return CURLE_OK; + } + else { + failf(data, + "SSL_connect failed with error %d: %s", + detail, + ERR_error_string_n(detail, error_buffer, sizeof(error_buffer))); + ERR_print_errors_fp(stderr); + if(detail && SSL_CONN_CONFIG(verifypeer)) { + detail &= ~0xFF; + if(detail == TLS_ERROR_WEBPKI_ERRORS) { + failf(data, "Cert verify failed"); + return CURLE_PEER_FAILED_VERIFICATION; + } + } + return CURLE_SSL_CONNECT_ERROR; + } + } + + connssl->connecting_state = ssl_connect_3; + infof(data, + "SSL connection using %s / %s\n", + SSL_get_version(BACKEND->handle), + SSL_get_cipher_name(BACKEND->handle)); + + return CURLE_OK; +} + +static CURLcode +mesalink_connect_step3(struct connectdata *conn, int sockindex) +{ + CURLcode result = CURLE_OK; + struct ssl_connect_data *connssl = &conn->ssl[sockindex]; + + DEBUGASSERT(ssl_connect_3 == connssl->connecting_state); + +#ifdef MESALINK_HAVE_SESSION + if(SSL_SET_OPTION(primary.sessionid)) { + bool incache; + SSL_SESSION *our_ssl_sessionid; + void *old_ssl_sessionid = NULL; + + our_ssl_sessionid = SSL_get_session(BACKEND->handle); + + Curl_ssl_sessionid_lock(conn); + incache = + !(Curl_ssl_getsessionid(conn, &old_ssl_sessionid, NULL, sockindex)); + if(incache) { + if(old_ssl_sessionid != our_ssl_sessionid) { + infof(data, "old SSL session ID is stale, removing\n"); + Curl_ssl_delsessionid(conn, old_ssl_sessionid); + incache = FALSE; + } + } + + if(!incache) { + result = Curl_ssl_addsessionid( + conn, our_ssl_sessionid, 0 /* unknown size */, sockindex); + if(result) { + Curl_ssl_sessionid_unlock(conn); + failf(data, "failed to store ssl session"); + return result; + } + } + Curl_ssl_sessionid_unlock(conn); + } +#endif /* MESALINK_HAVE_SESSION */ + + connssl->connecting_state = ssl_connect_done; + + return result; +} + +static ssize_t +mesalink_send(struct connectdata *conn, int sockindex, const void *mem, + size_t len, CURLcode *curlcode) +{ + struct ssl_connect_data *connssl = &conn->ssl[sockindex]; + char error_buffer[MESALINK_MAX_ERROR_SZ]; + int memlen = (len > (size_t)INT_MAX) ? INT_MAX : (int)len; + int rc = SSL_write(BACKEND->handle, mem, memlen); + + if(rc < 0) { + int err = SSL_get_error(BACKEND->handle, rc); + switch(err) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + /* there's data pending, re-invoke SSL_write() */ + *curlcode = CURLE_AGAIN; + return -1; + default: + failf(conn->data, + "SSL write: %s, errno %d", + ERR_error_string_n(err, error_buffer, sizeof(error_buffer)), + SOCKERRNO); + *curlcode = CURLE_SEND_ERROR; + return -1; + } + } + return rc; +} + +static void +Curl_mesalink_close(struct connectdata *conn, int sockindex) +{ + struct ssl_connect_data *connssl = &conn->ssl[sockindex]; + + if(BACKEND->handle) { + (void)SSL_shutdown(BACKEND->handle); + SSL_free(BACKEND->handle); + BACKEND->handle = NULL; + } + if(BACKEND->ctx) { + SSL_CTX_free(BACKEND->ctx); + BACKEND->ctx = NULL; + } +} + +static ssize_t +mesalink_recv(struct connectdata *conn, int num, char *buf, size_t buffersize, + CURLcode *curlcode) +{ + struct ssl_connect_data *connssl = &conn->ssl[num]; + char error_buffer[MESALINK_MAX_ERROR_SZ]; + int buffsize = (buffersize > (size_t)INT_MAX) ? INT_MAX : (int)buffersize; + int nread = SSL_read(BACKEND->handle, buf, buffsize); + + if(nread <= 0) { + int err = SSL_get_error(BACKEND->handle, nread); + + switch(err) { + case SSL_ERROR_ZERO_RETURN: /* no more data */ + case IO_ERROR_CONNECTION_ABORTED: + break; + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + /* there's data pending, re-invoke SSL_read() */ + *curlcode = CURLE_AGAIN; + return -1; + default: + failf(conn->data, + "SSL read: %s, errno %d", + ERR_error_string_n(err, error_buffer, sizeof(error_buffer)), + SOCKERRNO); + *curlcode = CURLE_RECV_ERROR; + return -1; + } + } + return nread; +} + +static size_t +Curl_mesalink_version(char *buffer, size_t size) +{ + return snprintf(buffer, size, "MesaLink/%s", MESALINK_VERSION_STRING); +} + +static int +Curl_mesalink_init(void) +{ + return (SSL_library_init() == SSL_SUCCESS); +} + +/* + * This function is called to shut down the SSL layer but keep the + * socket open (CCC - Clear Command Channel) + */ +static int +Curl_mesalink_shutdown(struct connectdata *conn, int sockindex) +{ + int retval = 0; + struct ssl_connect_data *connssl = &conn->ssl[sockindex]; + + if(BACKEND->handle) { + SSL_free(BACKEND->handle); + BACKEND->handle = NULL; + } + return retval; +} + +static CURLcode +mesalink_connect_common(struct connectdata *conn, int sockindex, + bool nonblocking, bool *done) +{ + CURLcode result; + struct Curl_easy *data = conn->data; + struct ssl_connect_data *connssl = &conn->ssl[sockindex]; + curl_socket_t sockfd = conn->sock[sockindex]; + time_t timeout_ms; + int what; + + /* check if the connection has already been established */ + if(ssl_connection_complete == connssl->state) { + *done = TRUE; + return CURLE_OK; + } + + if(ssl_connect_1 == connssl->connecting_state) { + /* Find out how much more time we're allowed */ + timeout_ms = Curl_timeleft(data, NULL, TRUE); + + if(timeout_ms < 0) { + /* no need to continue if time already is up */ + failf(data, "SSL connection timeout"); + return CURLE_OPERATION_TIMEDOUT; + } + + result = mesalink_connect_step1(conn, sockindex); + if(result) + return result; + } + + while(ssl_connect_2 == connssl->connecting_state || + ssl_connect_2_reading == connssl->connecting_state || + ssl_connect_2_writing == connssl->connecting_state) { + + /* check allowed time left */ + timeout_ms = Curl_timeleft(data, NULL, TRUE); + + if(timeout_ms < 0) { + /* no need to continue if time already is up */ + failf(data, "SSL connection timeout"); + return CURLE_OPERATION_TIMEDOUT; + } + + /* if ssl is expecting something, check if it's available. */ + if(connssl->connecting_state == ssl_connect_2_reading || + connssl->connecting_state == ssl_connect_2_writing) { + + curl_socket_t writefd = + ssl_connect_2_writing == connssl->connecting_state ? sockfd + : CURL_SOCKET_BAD; + curl_socket_t readfd = ssl_connect_2_reading == connssl->connecting_state + ? sockfd + : CURL_SOCKET_BAD; + + what = Curl_socket_check( + readfd, CURL_SOCKET_BAD, writefd, nonblocking ? 0 : timeout_ms); + if(what < 0) { + /* fatal error */ + failf(data, "select/poll on SSL socket, errno: %d", SOCKERRNO); + return CURLE_SSL_CONNECT_ERROR; + } + else if(0 == what) { + if(nonblocking) { + *done = FALSE; + return CURLE_OK; + } + else { + /* timeout */ + failf(data, "SSL connection timeout"); + return CURLE_OPERATION_TIMEDOUT; + } + } + /* socket is readable or writable */ + } + + /* Run transaction, and return to the caller if it failed or if + * this connection is part of a multi handle and this loop would + * execute again. This permits the owner of a multi handle to + * abort a connection attempt before step2 has completed while + * ensuring that a client using select() or epoll() will always + * have a valid fdset to wait on. + */ + result = mesalink_connect_step2(conn, sockindex); + + if(result || + (nonblocking && (ssl_connect_2 == connssl->connecting_state || + ssl_connect_2_reading == connssl->connecting_state || + ssl_connect_2_writing == connssl->connecting_state))) { + return result; + } + } /* repeat step2 until all transactions are done. */ + + if(ssl_connect_3 == connssl->connecting_state) { + result = mesalink_connect_step3(conn, sockindex); + if(result) + return result; + } + + if(ssl_connect_done == connssl->connecting_state) { + connssl->state = ssl_connection_complete; + conn->recv[sockindex] = mesalink_recv; + conn->send[sockindex] = mesalink_send; + *done = TRUE; + } + else + *done = FALSE; + + /* Reset our connect state machine */ + connssl->connecting_state = ssl_connect_1; + + return CURLE_OK; +} + +static CURLcode +Curl_mesalink_connect_nonblocking(struct connectdata *conn, int sockindex, + bool *done) +{ + return mesalink_connect_common(conn, sockindex, TRUE, done); +} + +static CURLcode +Curl_mesalink_connect(struct connectdata *conn, int sockindex) +{ + CURLcode result; + bool done = FALSE; + + result = mesalink_connect_common(conn, sockindex, FALSE, &done); + if(result) + return result; + + DEBUGASSERT(done); + + return CURLE_OK; +} + +static void * +Curl_mesalink_get_internals(struct ssl_connect_data *connssl, + CURLINFO info UNUSED_PARAM) +{ + (void)info; + return BACKEND->handle; +} + +const struct Curl_ssl Curl_ssl_mesalink = { + { CURLSSLBACKEND_MESALINK, "MesaLink" }, /* info */ + + SSLSUPP_SSL_CTX, + + sizeof(struct ssl_backend_data), + + Curl_mesalink_init, /* init */ + Curl_none_cleanup, /* cleanup */ + Curl_mesalink_version, /* version */ + Curl_none_check_cxn, /* check_cxn */ + Curl_mesalink_shutdown, /* shutdown */ + Curl_none_data_pending, /* data_pending */ + Curl_none_random, /* random */ + Curl_none_cert_status_request, /* cert_status_request */ + Curl_mesalink_connect, /* connect */ + Curl_mesalink_connect_nonblocking, /* connect_nonblocking */ + Curl_mesalink_get_internals, /* get_internals */ + Curl_mesalink_close, /* close_one */ + Curl_none_close_all, /* close_all */ + Curl_none_session_free, /* session_free */ + Curl_none_set_engine, /* set_engine */ + Curl_none_set_engine_default, /* set_engine_default */ + Curl_none_engines_list, /* engines_list */ + Curl_none_false_start, /* false_start */ + Curl_none_md5sum, /* md5sum */ + NULL /* sha256sum */ +}; + +#endif diff --git a/Utilities/cmcurl/lib/vtls/mesalink.h b/Utilities/cmcurl/lib/vtls/mesalink.h new file mode 100644 index 000000000..54cb94ad3 --- /dev/null +++ b/Utilities/cmcurl/lib/vtls/mesalink.h @@ -0,0 +1,32 @@ +#ifndef HEADER_CURL_MESALINK_H +#define HEADER_CURL_MESALINK_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2017-2018, Yiming Jing, <jingyiming@baidu.com> + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#include "curl_setup.h" + +#ifdef USE_MESALINK + +extern const struct Curl_ssl Curl_ssl_mesalink; + +#endif /* USE_MESALINK */ +#endif /* HEADER_CURL_MESALINK_H */ diff --git a/Utilities/cmcurl/lib/vtls/nss.c b/Utilities/cmcurl/lib/vtls/nss.c index 7cd450cda..a3d3e58bb 100644 --- a/Utilities/cmcurl/lib/vtls/nss.c +++ b/Utilities/cmcurl/lib/vtls/nss.c @@ -217,10 +217,15 @@ static const cipher_s cipherlist[] = { #endif }; +#ifdef WIN32 +static const char *pem_library = "nsspem.dll"; +static const char *trust_library = "nssckbi.dll"; +#else static const char *pem_library = "libnsspem.so"; -static SECMODModule *pem_module = NULL; - static const char *trust_library = "libnssckbi.so"; +#endif + +static SECMODModule *pem_module = NULL; static SECMODModule *trust_module = NULL; /* NSPR I/O layer we use to detect blocking direction during SSL handshake */ @@ -394,7 +399,7 @@ static PK11SlotInfo* nss_find_slot_by_name(const char *slot_name) /* wrap 'ptr' as list node and tail-insert into 'list' */ static CURLcode insert_wrapped_ptr(struct curl_llist *list, void *ptr) { - struct ptr_list_wrap *wrap = malloc(sizeof *wrap); + struct ptr_list_wrap *wrap = malloc(sizeof(*wrap)); if(!wrap) return CURLE_OUT_OF_MEMORY; @@ -914,11 +919,11 @@ static CURLcode display_conn_info(struct connectdata *conn, PRFileDesc *sock) PRTime now; int i; - if(SSL_GetChannelInfo(sock, &channel, sizeof channel) == - SECSuccess && channel.length == sizeof channel && + if(SSL_GetChannelInfo(sock, &channel, sizeof(channel)) == + SECSuccess && channel.length == sizeof(channel) && channel.cipherSuite) { if(SSL_GetCipherSuiteInfo(channel.cipherSuite, - &suite, sizeof suite) == SECSuccess) { + &suite, sizeof(suite)) == SECSuccess) { infof(conn->data, "SSL connection using %s\n", suite.cipherSuiteName); } } @@ -1345,7 +1350,8 @@ static CURLcode nss_init(struct Curl_easy *data) return CURLE_OUT_OF_MEMORY; /* the default methods just call down to the lower I/O layer */ - memcpy(&nspr_io_methods, PR_GetDefaultIOMethods(), sizeof nspr_io_methods); + memcpy(&nspr_io_methods, PR_GetDefaultIOMethods(), + sizeof(nspr_io_methods)); /* override certain methods in the table by our wrappers */ nspr_io_methods.recv = nspr_io_recv; @@ -1521,7 +1527,6 @@ static bool is_nss_error(CURLcode err) { switch(err) { case CURLE_PEER_FAILED_VERIFICATION: - case CURLE_SSL_CACERT: case CURLE_SSL_CERTPROBLEM: case CURLE_SSL_CONNECT_ERROR: case CURLE_SSL_ISSUER_ERROR: @@ -1578,8 +1583,9 @@ static CURLcode nss_load_ca_certificates(struct connectdata *conn, infof(data, "%s %s\n", (result) ? "failed to load" : "loaded", trust_library); if(result == CURLE_FAILED_INIT) - /* make the error non-fatal if we are not going to verify peer */ - result = CURLE_SSL_CACERT_BADFILE; + /* If libnssckbi.so is not available (or fails to load), one can still + use CA certificates stored in NSS database. Ignore the failure. */ + result = CURLE_OK; } else if(!use_trust_module && trust_module) { /* libnssckbi.so not needed but already loaded --> unload it! */ @@ -1714,8 +1720,6 @@ static CURLcode nss_init_sslver(SSLVersionRange *sslver, failf(data, "unsupported min version passed via CURLOPT_SSLVERSION"); return result; } - if(max == CURL_SSLVERSION_MAX_NONE) - sslver->max = sslver->min; } switch(max) { diff --git a/Utilities/cmcurl/lib/vtls/openssl.c b/Utilities/cmcurl/lib/vtls/openssl.c index f6a4bd3fb..4c5e8c19c 100644 --- a/Utilities/cmcurl/lib/vtls/openssl.c +++ b/Utilities/cmcurl/lib/vtls/openssl.c @@ -50,9 +50,6 @@ #include "hostcheck.h" #include "curl_printf.h" #include <openssl/ssl.h> -#ifdef HAVE_OPENSSL_ENGINE_H -#include <openssl/engine.h> -#endif #include <openssl/rand.h> #include <openssl/x509v3.h> #ifndef OPENSSL_NO_DSA @@ -72,6 +69,12 @@ #include <openssl/ocsp.h> #endif +#if (OPENSSL_VERSION_NUMBER >= 0x0090800fL) && /* 0.9.8 or later */ \ + !defined(OPENSSL_NO_ENGINE) +#define USE_OPENSSL_ENGINE +#include <openssl/engine.h> +#endif + #include "warnless.h" #include "non-ascii.h" /* for Curl_convert_from_utf8 prototype */ @@ -83,7 +86,7 @@ #error "OPENSSL_VERSION_NUMBER not defined" #endif -#if defined(HAVE_OPENSSL_ENGINE_H) +#ifdef USE_OPENSSL_ENGINE #include <openssl/ui.h> #endif @@ -126,16 +129,15 @@ #define X509_get0_notBefore(x) X509_get_notBefore(x) #define X509_get0_notAfter(x) X509_get_notAfter(x) #define CONST_EXTS /* nope */ -#ifdef LIBRESSL_VERSION_NUMBER -static unsigned long OpenSSL_version_num(void) -{ - return LIBRESSL_VERSION_NUMBER; -} -#else +#ifndef LIBRESSL_VERSION_NUMBER #define OpenSSL_version_num() SSLeay() #endif #endif +#ifdef LIBRESSL_VERSION_NUMBER +#define OpenSSL_version_num() LIBRESSL_VERSION_NUMBER +#endif + #if (OPENSSL_VERSION_NUMBER >= 0x1000200fL) && /* 1.0.2 or later */ \ !(defined(LIBRESSL_VERSION_NUMBER) && \ LIBRESSL_VERSION_NUMBER < 0x20700000L) @@ -166,6 +168,18 @@ static unsigned long OpenSSL_version_num(void) #define HAVE_KEYLOG_CALLBACK #endif +/* Whether SSL_CTX_set_ciphersuites is available. + * OpenSSL: supported since 1.1.1 (commit a53b5be6a05) + * BoringSSL: no + * LibreSSL: no + */ +#if ((OPENSSL_VERSION_NUMBER >= 0x10101000L) && \ + !defined(LIBRESSL_VERSION_NUMBER) && \ + !defined(OPENSSL_IS_BORINGSSL)) +#define HAVE_SSL_CTX_SET_CIPHERSUITES +#define HAVE_SSL_CTX_SET_POST_HANDSHAKE_AUTH +#endif + #if defined(LIBRESSL_VERSION_NUMBER) #define OSSL_PACKAGE "LibreSSL" #elif defined(OPENSSL_IS_BORINGSSL) @@ -239,7 +253,7 @@ static void ossl_keylog_callback(const SSL *ssl, const char *line) if(!buf) return; } - strncpy(buf, line, linelen); + memcpy(buf, line, linelen); buf[linelen] = '\n'; buf[linelen + 1] = '\0'; @@ -504,7 +518,7 @@ static int do_file_type(const char *type) return -1; } -#if defined(HAVE_OPENSSL_ENGINE_H) +#ifdef USE_OPENSSL_ENGINE /* * Supply default password to the engine user interface conversation. * The password is passed by OpenSSL engine from ENGINE_load_private_key() @@ -544,8 +558,20 @@ static int ssl_ui_writer(UI *ui, UI_STRING *uis) } return (UI_method_get_writer(UI_OpenSSL()))(ui, uis); } + +/* + * Check if a given string is a PKCS#11 URI + */ +static bool is_pkcs11_uri(const char *string) +{ + return (string && strncasecompare(string, "pkcs11:", 7)); +} + #endif +static CURLcode Curl_ossl_set_engine(struct Curl_easy *data, + const char *engine); + static int cert_stuff(struct connectdata *conn, SSL_CTX* ctx, @@ -606,8 +632,18 @@ int cert_stuff(struct connectdata *conn, } break; case SSL_FILETYPE_ENGINE: -#if defined(HAVE_OPENSSL_ENGINE_H) && defined(ENGINE_CTRL_GET_CMD_FROM_NAME) +#if defined(USE_OPENSSL_ENGINE) && defined(ENGINE_CTRL_GET_CMD_FROM_NAME) { + /* Implicitly use pkcs11 engine if none was provided and the + * cert_file is a PKCS#11 URI */ + if(!data->state.engine) { + if(is_pkcs11_uri(cert_file)) { + if(Curl_ossl_set_engine(data, "pkcs11") != CURLE_OK) { + return 0; + } + } + } + if(data->state.engine) { const char *cmd_name = "LOAD_CERT_CTRL"; struct { @@ -781,9 +817,20 @@ int cert_stuff(struct connectdata *conn, } break; case SSL_FILETYPE_ENGINE: -#ifdef HAVE_OPENSSL_ENGINE_H +#ifdef USE_OPENSSL_ENGINE { /* XXXX still needs some work */ EVP_PKEY *priv_key = NULL; + + /* Implicitly use pkcs11 engine if none was provided and the + * key_file is a PKCS#11 URI */ + if(!data->state.engine) { + if(is_pkcs11_uri(key_file)) { + if(Curl_ossl_set_engine(data, "pkcs11") != CURLE_OK) { + return 0; + } + } + } + if(data->state.engine) { UI_METHOD *ui_method = UI_create_method((char *)"curl user interface"); @@ -931,7 +978,7 @@ static int Curl_ossl_init(void) OPENSSL_load_builtin_modules(); -#ifdef HAVE_ENGINE_LOAD_BUILTIN_ENGINES +#ifdef USE_OPENSSL_ENGINE ENGINE_load_builtin_engines(); #endif @@ -947,9 +994,11 @@ static int Curl_ossl_init(void) #define CONF_MFLAGS_DEFAULT_SECTION 0x0 #endif +#ifndef CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG CONF_modules_load_file(NULL, NULL, CONF_MFLAGS_DEFAULT_SECTION| CONF_MFLAGS_IGNORE_MISSING_FILE); +#endif #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && \ !defined(LIBRESSL_VERSION_NUMBER) @@ -1088,7 +1137,7 @@ static int Curl_ossl_check_cxn(struct connectdata *conn) static CURLcode Curl_ossl_set_engine(struct Curl_easy *data, const char *engine) { -#if defined(USE_OPENSSL) && defined(HAVE_OPENSSL_ENGINE_H) +#ifdef USE_OPENSSL_ENGINE ENGINE *e; #if OPENSSL_VERSION_NUMBER >= 0x00909000L @@ -1133,7 +1182,7 @@ static CURLcode Curl_ossl_set_engine(struct Curl_easy *data, */ static CURLcode Curl_ossl_set_engine_default(struct Curl_easy *data) { -#ifdef HAVE_OPENSSL_ENGINE_H +#ifdef USE_OPENSSL_ENGINE if(data->state.engine) { if(ENGINE_set_default(data->state.engine, ENGINE_METHOD_ALL) > 0) { infof(data, "set default crypto engine '%s'\n", @@ -1156,7 +1205,7 @@ static CURLcode Curl_ossl_set_engine_default(struct Curl_easy *data) static struct curl_slist *Curl_ossl_engines_list(struct Curl_easy *data) { struct curl_slist *list = NULL; -#if defined(USE_OPENSSL) && defined(HAVE_OPENSSL_ENGINE_H) +#ifdef USE_OPENSSL_ENGINE struct curl_slist *beg; ENGINE *e; @@ -1213,7 +1262,7 @@ static int Curl_ossl_shutdown(struct connectdata *conn, int sockindex) ssize_t nread; int buffsize; int err; - int done = 0; + bool done = FALSE; /* This has only been tested on the proftpd server, and the mod_tls code sends a close notify alert without waiting for a close notify alert in @@ -1241,7 +1290,7 @@ static int Curl_ossl_shutdown(struct connectdata *conn, int sockindex) case SSL_ERROR_ZERO_RETURN: /* no more data */ /* This is the expected response. There was no data but only the close notify alert */ - done = 1; + done = TRUE; break; case SSL_ERROR_WANT_READ: /* there's data pending, re-invoke SSL_read() */ @@ -1250,7 +1299,7 @@ static int Curl_ossl_shutdown(struct connectdata *conn, int sockindex) case SSL_ERROR_WANT_WRITE: /* SSL wants a write. Really odd. Let's bail out. */ infof(data, "SSL_ERROR_WANT_WRITE\n"); - done = 1; + done = TRUE; break; default: /* openssl/ssl.h says "look at error stack/return value/errno" */ @@ -1260,20 +1309,20 @@ static int Curl_ossl_shutdown(struct connectdata *conn, int sockindex) ossl_strerror(sslerror, buf, sizeof(buf)) : SSL_ERROR_to_str(err)), SOCKERRNO); - done = 1; + done = TRUE; break; } } else if(0 == what) { /* timeout */ failf(data, "SSL shutdown timeout"); - done = 1; + done = TRUE; } else { /* anything that gets here is fatally bad */ failf(data, "select/poll on SSL socket, errno: %d", SOCKERRNO); retval = -1; - done = 1; + done = TRUE; } } /* while()-loop for the select() */ @@ -1312,7 +1361,7 @@ static void Curl_ossl_session_free(void *ptr) */ static void Curl_ossl_close_all(struct Curl_easy *data) { -#ifdef HAVE_OPENSSL_ENGINE_H +#ifdef USE_OPENSSL_ENGINE if(data->state.engine) { ENGINE_finish(data->state.engine); ENGINE_free(data->state.engine); @@ -1369,6 +1418,10 @@ static bool subj_alt_hostcheck(struct Curl_easy *data, } #else { +#ifdef CURL_DISABLE_VERBOSE_STRINGS + (void)dispname; + (void)data; +#endif if(Curl_cert_hostcheck(match_pattern, hostname)) { infof(data, " subjectAltName: host \"%s\" matched cert's \"%s\"\n", dispname, match_pattern); @@ -1922,7 +1975,15 @@ static void ssl_tls_trace(int direction, int ssl_ver, int content_type, } else #endif - { + if(content_type == SSL3_RT_CHANGE_CIPHER_SPEC) { + msg_type = *(char *)buf; + msg_name = "Change cipher spec"; + } + else if(content_type == SSL3_RT_ALERT) { + msg_type = (((char *)buf)[0] << 8) + ((char *)buf)[1]; + msg_name = SSL_alert_desc_string_long(msg_type); + } + else { msg_type = *(char *)buf; msg_name = ssl_msg_type(ssl_ver, msg_type); } @@ -1931,12 +1992,12 @@ static void ssl_tls_trace(int direction, int ssl_ver, int content_type, verstr, direction?"OUT":"IN", tls_rt_name, msg_name, msg_type); if(0 <= txt_len && (unsigned)txt_len < sizeof(ssl_buf)) { - Curl_debug(data, CURLINFO_TEXT, ssl_buf, (size_t)txt_len, NULL); + Curl_debug(data, CURLINFO_TEXT, ssl_buf, (size_t)txt_len); } } Curl_debug(data, (direction == 1) ? CURLINFO_SSL_DATA_OUT : - CURLINFO_SSL_DATA_IN, (char *)buf, len, NULL); + CURLINFO_SSL_DATA_IN, (char *)buf, len); (void) ssl; } #endif @@ -2025,6 +2086,7 @@ select_next_proto_cb(SSL *ssl, } #endif /* HAS_NPN */ +#ifndef CURL_DISABLE_VERBOSE_STRINGS static const char * get_ssl_version_txt(SSL *ssl) { @@ -2051,6 +2113,7 @@ get_ssl_version_txt(SSL *ssl) } return "unknown"; } +#endif static CURLcode set_ssl_version_min_max(long *ctx_options, struct connectdata *conn, @@ -2064,10 +2127,6 @@ set_ssl_version_min_max(long *ctx_options, struct connectdata *conn, long ssl_version = SSL_CONN_CONFIG(version); long ssl_version_max = SSL_CONN_CONFIG(version_max); - if(ssl_version_max == CURL_SSLVERSION_MAX_NONE) { - ssl_version_max = ssl_version << 16; - } - switch(ssl_version) { case CURL_SSLVERSION_TLSv1_3: #ifdef TLS1_3_VERSION @@ -2099,8 +2158,7 @@ set_ssl_version_min_max(long *ctx_options, struct connectdata *conn, #endif /* FALLTHROUGH */ case CURL_SSLVERSION_TLSv1_0: - *ctx_options |= SSL_OP_NO_SSLv2; - *ctx_options |= SSL_OP_NO_SSLv3; + case CURL_SSLVERSION_TLSv1: break; } @@ -2116,7 +2174,6 @@ set_ssl_version_min_max(long *ctx_options, struct connectdata *conn, #endif /* FALLTHROUGH */ case CURL_SSLVERSION_MAX_TLSv1_2: - case CURL_SSLVERSION_MAX_DEFAULT: #ifdef TLS1_3_VERSION *ctx_options |= SSL_OP_NO_TLSv1_3; #endif @@ -2305,11 +2362,6 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) switch(ssl_version) { case CURL_SSLVERSION_SSLv3: -#ifdef USE_TLS_SRP - if(ssl_authtype == CURL_TLSAUTH_SRP) { - infof(data, "Set version TLSv1.x for SRP authorisation\n"); - } -#endif ctx_options |= SSL_OP_NO_SSLv2; ctx_options |= SSL_OP_NO_TLSv1; #if OPENSSL_VERSION_NUMBER >= 0x1000100FL @@ -2323,21 +2375,20 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) case CURL_SSLVERSION_DEFAULT: case CURL_SSLVERSION_TLSv1: - ctx_options |= SSL_OP_NO_SSLv2; - ctx_options |= SSL_OP_NO_SSLv3; - break; - case CURL_SSLVERSION_TLSv1_0: case CURL_SSLVERSION_TLSv1_1: case CURL_SSLVERSION_TLSv1_2: case CURL_SSLVERSION_TLSv1_3: + /* asking for any TLS version as the minimum, means no SSL versions + allowed */ + ctx_options |= SSL_OP_NO_SSLv2; + ctx_options |= SSL_OP_NO_SSLv3; result = set_ssl_version_min_max(&ctx_options, conn, sockindex); if(result != CURLE_OK) return result; break; case CURL_SSLVERSION_SSLv2: -#ifndef OPENSSL_NO_SSL2 ctx_options |= SSL_OP_NO_SSLv3; ctx_options |= SSL_OP_NO_TLSv1; #if OPENSSL_VERSION_NUMBER >= 0x1000100FL @@ -2348,10 +2399,6 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) #endif #endif break; -#else - failf(data, OSSL_PACKAGE " was built without SSLv2 support"); - return CURLE_NOT_BUILT_IN; -#endif default: failf(data, "Unrecognized parameter passed via CURLOPT_SSLVERSION"); @@ -2414,6 +2461,24 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) infof(data, "Cipher selection: %s\n", ciphers); } +#ifdef HAVE_SSL_CTX_SET_CIPHERSUITES + { + char *ciphers13 = SSL_CONN_CONFIG(cipher_list13); + if(ciphers13) { + if(!SSL_CTX_set_ciphersuites(BACKEND->ctx, ciphers13)) { + failf(data, "failed setting TLS 1.3 cipher suite: %s", ciphers13); + return CURLE_SSL_CIPHER; + } + infof(data, "TLS 1.3 cipher selection: %s\n", ciphers13); + } + } +#endif + +#ifdef HAVE_SSL_CTX_SET_POST_HANDSHAKE_AUTH + /* OpenSSL 1.1.1 requires clients to opt-in for PHA */ + SSL_CTX_set_post_handshake_auth(BACKEND->ctx, 1); +#endif + #ifdef USE_TLS_SRP if(ssl_authtype == CURL_TLSAUTH_SRP) { char * const ssl_username = SSL_SET_OPTION(username); @@ -2468,7 +2533,7 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex) } #ifdef CURL_CA_FALLBACK else if(verifypeer) { - /* verfying the peer without any CA certificates won't + /* verifying the peer without any CA certificates won't work so use openssl's built in default as fallback */ SSL_CTX_set_default_verify_paths(BACKEND->ctx); } @@ -3134,7 +3199,7 @@ static CURLcode servercert(struct connectdata *conn, { CURLcode result = CURLE_OK; int rc; - long lerr, len; + long lerr; struct Curl_easy *data = conn->data; X509 *issuer; BIO *fp = NULL; @@ -3157,7 +3222,7 @@ static CURLcode servercert(struct connectdata *conn, ossl_strerror(ERR_get_error(), error_buffer, sizeof(error_buffer)) ); BIO_free(mem); - return 0; + return CURLE_OUT_OF_MEMORY; } BACKEND->server_cert = SSL_get_peer_certificate(BACKEND->handle); @@ -3177,15 +3242,20 @@ static CURLcode servercert(struct connectdata *conn, buffer, sizeof(buffer)); infof(data, " subject: %s\n", rc?"[NONE]":buffer); - ASN1_TIME_print(mem, X509_get0_notBefore(BACKEND->server_cert)); - len = BIO_get_mem_data(mem, (char **) &ptr); - infof(data, " start date: %.*s\n", len, ptr); - (void)BIO_reset(mem); +#ifndef CURL_DISABLE_VERBOSE_STRINGS + { + long len; + ASN1_TIME_print(mem, X509_get0_notBefore(BACKEND->server_cert)); + len = BIO_get_mem_data(mem, (char **) &ptr); + infof(data, " start date: %.*s\n", len, ptr); + (void)BIO_reset(mem); - ASN1_TIME_print(mem, X509_get0_notAfter(BACKEND->server_cert)); - len = BIO_get_mem_data(mem, (char **) &ptr); - infof(data, " expire date: %.*s\n", len, ptr); - (void)BIO_reset(mem); + ASN1_TIME_print(mem, X509_get0_notAfter(BACKEND->server_cert)); + len = BIO_get_mem_data(mem, (char **) &ptr); + infof(data, " expire date: %.*s\n", len, ptr); + (void)BIO_reset(mem); + } +#endif BIO_free(mem); @@ -3204,7 +3274,7 @@ static CURLcode servercert(struct connectdata *conn, if(rc) { if(strict) failf(data, "SSL: couldn't get X509-issuer name!"); - result = CURLE_SSL_CONNECT_ERROR; + result = CURLE_PEER_FAILED_VERIFICATION; } else { infof(data, " issuer: %s\n", buffer); @@ -3754,6 +3824,9 @@ const struct Curl_ssl Curl_ssl_openssl = { SSLSUPP_CERTINFO | SSLSUPP_PINNEDPUBKEY | SSLSUPP_SSL_CTX | +#ifdef HAVE_SSL_CTX_SET_CIPHERSUITES + SSLSUPP_TLS13_CIPHERSUITES | +#endif SSLSUPP_HTTPS_PROXY, sizeof(struct ssl_backend_data), diff --git a/Utilities/cmcurl/lib/vtls/polarssl.c b/Utilities/cmcurl/lib/vtls/polarssl.c index 604cb4c86..27af0ccf3 100644 --- a/Utilities/cmcurl/lib/vtls/polarssl.c +++ b/Utilities/cmcurl/lib/vtls/polarssl.c @@ -185,14 +185,11 @@ set_ssl_version_min_max(struct connectdata *conn, int sockindex) case CURL_SSLVERSION_DEFAULT: case CURL_SSLVERSION_TLSv1: ssl_version = CURL_SSLVERSION_TLSv1_0; - ssl_version_max = CURL_SSLVERSION_MAX_TLSv1_2; break; } switch(ssl_version_max) { case CURL_SSLVERSION_MAX_NONE: - ssl_version_max = ssl_version << 16; - break; case CURL_SSLVERSION_MAX_DEFAULT: ssl_version_max = CURL_SSLVERSION_MAX_TLSv1_2; break; diff --git a/Utilities/cmcurl/lib/vtls/schannel.c b/Utilities/cmcurl/lib/vtls/schannel.c index 2cfd5c19f..e4426924b 100644 --- a/Utilities/cmcurl/lib/vtls/schannel.c +++ b/Utilities/cmcurl/lib/vtls/schannel.c @@ -90,11 +90,17 @@ #endif #endif +#if defined(CryptStringToBinary) && defined(CRYPT_STRING_HEX) +#define HAS_CLIENT_CERT_PATH +#endif + +#ifdef HAS_CLIENT_CERT_PATH #ifdef UNICODE #define CURL_CERT_STORE_PROV_SYSTEM CERT_STORE_PROV_SYSTEM_W #else #define CURL_CERT_STORE_PROV_SYSTEM CERT_STORE_PROV_SYSTEM_A #endif +#endif #ifndef SP_PROT_SSL2_CLIENT #define SP_PROT_SSL2_CLIENT 0x00000008 @@ -174,8 +180,6 @@ set_ssl_version_min_max(SCHANNEL_CRED *schannel_cred, struct connectdata *conn) switch(ssl_version_max) { case CURL_SSLVERSION_MAX_NONE: - ssl_version_max = ssl_version << 16; - break; case CURL_SSLVERSION_MAX_DEFAULT: ssl_version_max = CURL_SSLVERSION_MAX_TLSv1_2; break; @@ -199,6 +203,155 @@ set_ssl_version_min_max(SCHANNEL_CRED *schannel_cred, struct connectdata *conn) return CURLE_OK; } +/*longest is 26, buffer is slightly bigger*/ +#define LONGEST_ALG_ID 32 +#define CIPHEROPTION(X) \ +if(strcmp(#X, tmp) == 0) \ + return X + +static int +get_alg_id_by_name(char *name) +{ + char tmp[LONGEST_ALG_ID] = { 0 }; + char *nameEnd = strchr(name, ':'); + size_t n = nameEnd ? min((size_t)(nameEnd - name), LONGEST_ALG_ID - 1) : \ + min(strlen(name), LONGEST_ALG_ID - 1); + strncpy(tmp, name, n); + tmp[n] = 0; + CIPHEROPTION(CALG_MD2); + CIPHEROPTION(CALG_MD4); + CIPHEROPTION(CALG_MD5); + CIPHEROPTION(CALG_SHA); + CIPHEROPTION(CALG_SHA1); + CIPHEROPTION(CALG_MAC); + CIPHEROPTION(CALG_RSA_SIGN); + CIPHEROPTION(CALG_DSS_SIGN); +/*ifdefs for the options that are defined conditionally in wincrypt.h*/ +#ifdef CALG_NO_SIGN + CIPHEROPTION(CALG_NO_SIGN); +#endif + CIPHEROPTION(CALG_RSA_KEYX); + CIPHEROPTION(CALG_DES); +#ifdef CALG_3DES_112 + CIPHEROPTION(CALG_3DES_112); +#endif + CIPHEROPTION(CALG_3DES); + CIPHEROPTION(CALG_DESX); + CIPHEROPTION(CALG_RC2); + CIPHEROPTION(CALG_RC4); + CIPHEROPTION(CALG_SEAL); +#ifdef CALG_DH_SF + CIPHEROPTION(CALG_DH_SF); +#endif + CIPHEROPTION(CALG_DH_EPHEM); +#ifdef CALG_AGREEDKEY_ANY + CIPHEROPTION(CALG_AGREEDKEY_ANY); +#endif +#ifdef CALG_HUGHES_MD5 + CIPHEROPTION(CALG_HUGHES_MD5); +#endif + CIPHEROPTION(CALG_SKIPJACK); +#ifdef CALG_TEK + CIPHEROPTION(CALG_TEK); +#endif + CIPHEROPTION(CALG_CYLINK_MEK); + CIPHEROPTION(CALG_SSL3_SHAMD5); +#ifdef CALG_SSL3_MASTER + CIPHEROPTION(CALG_SSL3_MASTER); +#endif +#ifdef CALG_SCHANNEL_MASTER_HASH + CIPHEROPTION(CALG_SCHANNEL_MASTER_HASH); +#endif +#ifdef CALG_SCHANNEL_MAC_KEY + CIPHEROPTION(CALG_SCHANNEL_MAC_KEY); +#endif +#ifdef CALG_SCHANNEL_ENC_KEY + CIPHEROPTION(CALG_SCHANNEL_ENC_KEY); +#endif +#ifdef CALG_PCT1_MASTER + CIPHEROPTION(CALG_PCT1_MASTER); +#endif +#ifdef CALG_SSL2_MASTER + CIPHEROPTION(CALG_SSL2_MASTER); +#endif +#ifdef CALG_TLS1_MASTER + CIPHEROPTION(CALG_TLS1_MASTER); +#endif +#ifdef CALG_RC5 + CIPHEROPTION(CALG_RC5); +#endif +#ifdef CALG_HMAC + CIPHEROPTION(CALG_HMAC); +#endif +#if !defined(__W32API_MAJOR_VERSION) || \ + !defined(__W32API_MINOR_VERSION) || \ + defined(__MINGW64_VERSION_MAJOR) || \ + (__W32API_MAJOR_VERSION > 5) || \ + ((__W32API_MAJOR_VERSION == 5) && (__W32API_MINOR_VERSION > 0)) + /* CALG_TLS1PRF has a syntax error in MinGW's w32api up to version 5.0, + see https://osdn.net/projects/mingw/ticket/38391 */ + CIPHEROPTION(CALG_TLS1PRF); +#endif +#ifdef CALG_HASH_REPLACE_OWF + CIPHEROPTION(CALG_HASH_REPLACE_OWF); +#endif +#ifdef CALG_AES_128 + CIPHEROPTION(CALG_AES_128); +#endif +#ifdef CALG_AES_192 + CIPHEROPTION(CALG_AES_192); +#endif +#ifdef CALG_AES_256 + CIPHEROPTION(CALG_AES_256); +#endif +#ifdef CALG_AES + CIPHEROPTION(CALG_AES); +#endif +#ifdef CALG_SHA_256 + CIPHEROPTION(CALG_SHA_256); +#endif +#ifdef CALG_SHA_384 + CIPHEROPTION(CALG_SHA_384); +#endif +#ifdef CALG_SHA_512 + CIPHEROPTION(CALG_SHA_512); +#endif +#ifdef CALG_ECDH + CIPHEROPTION(CALG_ECDH); +#endif +#ifdef CALG_ECMQV + CIPHEROPTION(CALG_ECMQV); +#endif +#ifdef CALG_ECDSA + CIPHEROPTION(CALG_ECDSA); +#endif + return 0; +} + +static CURLcode +set_ssl_ciphers(SCHANNEL_CRED *schannel_cred, char *ciphers) +{ + char *startCur = ciphers; + int algCount = 0; + static ALG_ID algIds[45]; /*There are 45 listed in the MS headers*/ + while(startCur && (0 != *startCur) && (algCount < 45)) { + long alg = strtol(startCur, 0, 0); + if(!alg) + alg = get_alg_id_by_name(startCur); + if(alg) + algIds[algCount++] = alg; + else + return CURLE_SSL_CIPHER; + startCur = strchr(startCur, ':'); + if(startCur) + startCur++; + } + schannel_cred->palgSupportedAlgs = algIds; + schannel_cred->cSupportedAlgs = algCount; + return CURLE_OK; +} + +#ifdef HAS_CLIENT_CERT_PATH static CURLcode get_cert_location(TCHAR *path, DWORD *store_name, TCHAR **store_path, TCHAR **thumbprint) @@ -208,7 +361,7 @@ get_cert_location(TCHAR *path, DWORD *store_name, TCHAR **store_path, sep = _tcschr(path, TEXT('\\')); if(sep == NULL) - return CURLE_SSL_CONNECT_ERROR; + return CURLE_SSL_CERTPROBLEM; store_name_len = sep - path; @@ -232,22 +385,23 @@ get_cert_location(TCHAR *path, DWORD *store_name, TCHAR **store_path, store_name_len) == 0) *store_name = CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE; else - return CURLE_SSL_CONNECT_ERROR; + return CURLE_SSL_CERTPROBLEM; *store_path = sep + 1; sep = _tcschr(*store_path, TEXT('\\')); if(sep == NULL) - return CURLE_SSL_CONNECT_ERROR; + return CURLE_SSL_CERTPROBLEM; *sep = 0; *thumbprint = sep + 1; if(_tcslen(*thumbprint) != CERT_THUMBPRINT_STR_LEN) - return CURLE_SSL_CONNECT_ERROR; + return CURLE_SSL_CERTPROBLEM; return CURLE_OK; } +#endif static CURLcode schannel_connect_step1(struct connectdata *conn, int sockindex) @@ -299,10 +453,15 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) #endif #ifdef _WIN32_WCE +#ifdef HAS_MANUAL_VERIFY_API /* certificate validation on CE doesn't seem to work right; we'll * do it following a more manual process. */ BACKEND->use_manual_cred_validation = true; #else +#error "compiler too old to support requisite manual cert verify for Win CE" +#endif +#else +#ifdef HAS_MANUAL_VERIFY_API if(SSL_CONN_CONFIG(CAfile)) { if(Curl_verify_windows_version(6, 1, PLATFORM_WINNT, VERSION_GREATER_THAN_EQUAL)) { @@ -316,6 +475,12 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) } else BACKEND->use_manual_cred_validation = false; +#else + if(SSL_CONN_CONFIG(CAfile)) { + failf(data, "schannel: CA cert support not built in"); + return CURLE_NOT_BUILT_IN; + } +#endif #endif BACKEND->cred = NULL; @@ -341,9 +506,11 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) schannel_cred.dwVersion = SCHANNEL_CRED_VERSION; if(conn->ssl_config.verifypeer) { +#ifdef HAS_MANUAL_VERIFY_API if(BACKEND->use_manual_cred_validation) schannel_cred.dwFlags = SCH_CRED_MANUAL_CRED_VALIDATION; else +#endif schannel_cred.dwFlags = SCH_CRED_AUTO_CRED_VALIDATION; /* TODO s/data->set.ssl.no_revoke/SSL_SET_OPTION(no_revoke)/g */ @@ -401,6 +568,16 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) return CURLE_SSL_CONNECT_ERROR; } + if(SSL_CONN_CONFIG(cipher_list)) { + result = set_ssl_ciphers(&schannel_cred, SSL_CONN_CONFIG(cipher_list)); + if(CURLE_OK != result) { + failf(data, "Unable to set ciphers to passed via SSL_CONN_CONFIG"); + return result; + } + } + + +#ifdef HAS_CLIENT_CERT_PATH /* client certificate */ if(data->set.ssl.cert) { DWORD cert_store_name; @@ -417,16 +594,23 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) result = get_cert_location(cert_path, &cert_store_name, &cert_store_path, &cert_thumbprint_str); if(result != CURLE_OK) { + failf(data, "schannel: Failed to get certificate location for %s", + cert_path); Curl_unicodefree(cert_path); return result; } - cert_store = CertOpenStore(CURL_CERT_STORE_PROV_SYSTEM, 0, - (HCRYPTPROV)NULL, - cert_store_name, cert_store_path); + cert_store = + CertOpenStore(CURL_CERT_STORE_PROV_SYSTEM, 0, + (HCRYPTPROV)NULL, + CERT_STORE_OPEN_EXISTING_FLAG | cert_store_name, + cert_store_path); if(!cert_store) { + failf(data, "schannel: Failed to open cert store %x %s, " + "last error is %x", + cert_store_name, cert_store_path, GetLastError()); Curl_unicodefree(cert_path); - return CURLE_SSL_CONNECT_ERROR; + return CURLE_SSL_CERTPROBLEM; } cert_thumbprint.pbData = cert_thumbprint_data; @@ -437,7 +621,7 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) cert_thumbprint_data, &cert_thumbprint.cbData, NULL, NULL)) { Curl_unicodefree(cert_path); - return CURLE_SSL_CONNECT_ERROR; + return CURLE_SSL_CERTPROBLEM; } client_certs[0] = CertFindCertificateInStore( @@ -450,9 +634,19 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) schannel_cred.cCreds = 1; schannel_cred.paCred = client_certs; } + else { + /* CRYPT_E_NOT_FOUND / E_INVALIDARG */ + return CURLE_SSL_CERTPROBLEM; + } CertCloseStore(cert_store, 0); } +#else + if(data->set.ssl.cert) { + failf(data, "schannel: client cert support not built in"); + return CURLE_NOT_BUILT_IN; + } +#endif /* allocate memory for the re-usable credential handle */ BACKEND->cred = (struct curl_schannel_cred *) @@ -480,14 +674,20 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) CertFreeCertificateContext(client_certs[0]); if(sspi_status != SEC_E_OK) { - if(sspi_status == SEC_E_WRONG_PRINCIPAL) - failf(data, "schannel: SNI or certificate check failed: %s", - Curl_sspi_strerror(conn, sspi_status)); - else - failf(data, "schannel: AcquireCredentialsHandle failed: %s", - Curl_sspi_strerror(conn, sspi_status)); + failf(data, "schannel: AcquireCredentialsHandle failed: %s", + Curl_sspi_strerror(conn, sspi_status)); Curl_safefree(BACKEND->cred); - return CURLE_SSL_CONNECT_ERROR; + switch(sspi_status) { + case SEC_E_INSUFFICIENT_MEMORY: + return CURLE_OUT_OF_MEMORY; + case SEC_E_NO_CREDENTIALS: + case SEC_E_SECPKG_NOT_FOUND: + case SEC_E_NOT_OWNER: + case SEC_E_UNKNOWN_CREDENTIALS: + case SEC_E_INTERNAL_ERROR: + default: + return CURLE_SSL_CONNECT_ERROR; + } } } @@ -590,14 +790,32 @@ schannel_connect_step1(struct connectdata *conn, int sockindex) Curl_unicodefree(host_name); if(sspi_status != SEC_I_CONTINUE_NEEDED) { - if(sspi_status == SEC_E_WRONG_PRINCIPAL) - failf(data, "schannel: SNI or certificate check failed: %s", - Curl_sspi_strerror(conn, sspi_status)); - else - failf(data, "schannel: initial InitializeSecurityContext failed: %s", - Curl_sspi_strerror(conn, sspi_status)); Curl_safefree(BACKEND->ctxt); - return CURLE_SSL_CONNECT_ERROR; + switch(sspi_status) { + case SEC_E_INSUFFICIENT_MEMORY: + failf(data, "schannel: initial InitializeSecurityContext failed: %s", + Curl_sspi_strerror(conn, sspi_status)); + return CURLE_OUT_OF_MEMORY; + case SEC_E_WRONG_PRINCIPAL: + failf(data, "schannel: SNI or certificate check failed: %s", + Curl_sspi_strerror(conn, sspi_status)); + return CURLE_PEER_FAILED_VERIFICATION; + /* + case SEC_E_INVALID_HANDLE: + case SEC_E_INVALID_TOKEN: + case SEC_E_LOGON_DENIED: + case SEC_E_TARGET_UNKNOWN: + case SEC_E_NO_AUTHENTICATING_AUTHORITY: + case SEC_E_INTERNAL_ERROR: + case SEC_E_NO_CREDENTIALS: + case SEC_E_UNSUPPORTED_FUNCTION: + case SEC_E_APPLICATION_PROTOCOL_MISMATCH: + */ + default: + failf(data, "schannel: initial InitializeSecurityContext failed: %s", + Curl_sspi_strerror(conn, sspi_status)); + return CURLE_SSL_CONNECT_ERROR; + } } infof(data, "schannel: sending initial handshake data: " @@ -812,14 +1030,31 @@ schannel_connect_step2(struct connectdata *conn, int sockindex) } } else { - if(sspi_status == SEC_E_WRONG_PRINCIPAL) - failf(data, "schannel: SNI or certificate check failed: %s", - Curl_sspi_strerror(conn, sspi_status)); - else - failf(data, "schannel: next InitializeSecurityContext failed: %s", - Curl_sspi_strerror(conn, sspi_status)); - return sspi_status == SEC_E_UNTRUSTED_ROOT ? - CURLE_SSL_CACERT : CURLE_SSL_CONNECT_ERROR; + switch(sspi_status) { + case SEC_E_INSUFFICIENT_MEMORY: + failf(data, "schannel: next InitializeSecurityContext failed: %s", + Curl_sspi_strerror(conn, sspi_status)); + return CURLE_OUT_OF_MEMORY; + case SEC_E_WRONG_PRINCIPAL: + failf(data, "schannel: SNI or certificate check failed: %s", + Curl_sspi_strerror(conn, sspi_status)); + return CURLE_PEER_FAILED_VERIFICATION; + /* + case SEC_E_INVALID_HANDLE: + case SEC_E_INVALID_TOKEN: + case SEC_E_LOGON_DENIED: + case SEC_E_TARGET_UNKNOWN: + case SEC_E_NO_AUTHENTICATING_AUTHORITY: + case SEC_E_INTERNAL_ERROR: + case SEC_E_NO_CREDENTIALS: + case SEC_E_UNSUPPORTED_FUNCTION: + case SEC_E_APPLICATION_PROTOCOL_MISMATCH: + */ + default: + failf(data, "schannel: next InitializeSecurityContext failed: %s", + Curl_sspi_strerror(conn, sspi_status)); + return CURLE_SSL_CONNECT_ERROR; + } } /* check if there was additional remaining encrypted data */ @@ -877,9 +1112,11 @@ schannel_connect_step2(struct connectdata *conn, int sockindex) } } +#ifdef HAS_MANUAL_VERIFY_API if(conn->ssl_config.verifypeer && BACKEND->use_manual_cred_validation) { return verify_certificate(conn, sockindex); } +#endif return CURLE_OK; } @@ -998,7 +1235,7 @@ schannel_connect_step3(struct connectdata *conn, int sockindex) if((sspi_status != SEC_E_OK) || (ccert_context == NULL)) { failf(data, "schannel: failed to retrieve remote cert context"); - return CURLE_SSL_CONNECT_ERROR; + return CURLE_PEER_FAILED_VERIFICATION; } result = Curl_ssl_init_certinfo(data, 1); @@ -1819,7 +2056,7 @@ static CURLcode pkp_pin_peer_pubkey(struct connectdata *conn, int sockindex, x509_der = (const char *)pCertContextServer->pbCertEncoded; x509_der_len = pCertContextServer->cbCertEncoded; - memset(&x509_parsed, 0, sizeof x509_parsed); + memset(&x509_parsed, 0, sizeof(x509_parsed)); if(Curl_parseX509(&x509_parsed, x509_der, x509_der + x509_der_len)) break; @@ -1870,7 +2107,8 @@ static void Curl_schannel_checksum(const unsigned char *input, if(!CryptCreateHash(hProv, algId, 0, 0, &hHash)) break; /* failed */ - if(!CryptHashData(hHash, (const BYTE*)input, (DWORD)inputlen, 0)) + /* workaround for original MinGW, should be (const BYTE*) */ + if(!CryptHashData(hHash, (BYTE*)input, (DWORD)inputlen, 0)) break; /* failed */ /* get hash size */ diff --git a/Utilities/cmcurl/lib/vtls/schannel.h b/Utilities/cmcurl/lib/vtls/schannel.h index 447690027..e491bd431 100644 --- a/Utilities/cmcurl/lib/vtls/schannel.h +++ b/Utilities/cmcurl/lib/vtls/schannel.h @@ -32,12 +32,44 @@ #include "urldata.h" +/* <wincrypt.h> has been included via the above <schnlsp.h>. + * Or in case of ldap.c, it was included via <winldap.h>. + * And since <wincrypt.h> has this: + * #define X509_NAME ((LPCSTR) 7) + * + * And in BoringSSL's <openssl/base.h> there is: + * typedef struct X509_name_st X509_NAME; + * etc. + * + * this will cause all kinds of C-preprocessing paste errors in + * BoringSSL's <openssl/x509.h>: So just undefine those defines here + * (and only here). + */ +#if defined(HAVE_BORINGSSL) || defined(OPENSSL_IS_BORINGSSL) +# undef X509_NAME +# undef X509_CERT_PAIR +# undef X509_EXTENSIONS +#endif + extern const struct Curl_ssl Curl_ssl_schannel; CURLcode verify_certificate(struct connectdata *conn, int sockindex); /* structs to expose only in schannel.c and schannel_verify.c */ #ifdef EXPOSE_SCHANNEL_INTERNAL_STRUCTS + +#ifdef __MINGW32__ +#include <_mingw.h> +#ifdef __MINGW64_VERSION_MAJOR +#define HAS_MANUAL_VERIFY_API +#endif +#else +#include <wincrypt.h> +#ifdef CERT_CHAIN_REVOCATION_CHECK_CHAIN +#define HAS_MANUAL_VERIFY_API +#endif +#endif + struct curl_schannel_cred { CredHandle cred_handle; TimeStamp time_stamp; @@ -66,7 +98,9 @@ struct ssl_backend_data { bool recv_sspi_close_notify; /* true if connection closed by close_notify */ bool recv_connection_closed; /* true if connection closed, regardless how */ bool use_alpn; /* true if ALPN is used for this connection */ +#ifdef HAS_MANUAL_VERIFY_API bool use_manual_cred_validation; /* true if manual cred validation is used */ +#endif }; #endif /* EXPOSE_SCHANNEL_INTERNAL_STRUCTS */ diff --git a/Utilities/cmcurl/lib/vtls/schannel_verify.c b/Utilities/cmcurl/lib/vtls/schannel_verify.c index db187dd6b..2516f5665 100644 --- a/Utilities/cmcurl/lib/vtls/schannel_verify.c +++ b/Utilities/cmcurl/lib/vtls/schannel_verify.c @@ -30,14 +30,15 @@ #include "curl_setup.h" #ifdef USE_SCHANNEL - -#define EXPOSE_SCHANNEL_INTERNAL_STRUCTS - #ifndef USE_WINDOWS_SSPI # error "Can't compile SCHANNEL support without SSPI." #endif +#define EXPOSE_SCHANNEL_INTERNAL_STRUCTS #include "schannel.h" + +#ifdef HAS_MANUAL_VERIFY_API + #include "vtls.h" #include "sendf.h" #include "strerror.h" @@ -53,7 +54,7 @@ #define BACKEND connssl->backend #define MAX_CAFILE_SIZE 1048576 /* 1 MiB */ -#define BEGIN_CERT "-----BEGIN CERTIFICATE-----\n" +#define BEGIN_CERT "-----BEGIN CERTIFICATE-----" #define END_CERT "\n-----END CERTIFICATE-----" typedef struct { @@ -71,6 +72,10 @@ typedef struct { HCERTSTORE hExclusiveTrustedPeople; } CERT_CHAIN_ENGINE_CONFIG_WIN7, *PCERT_CHAIN_ENGINE_CONFIG_WIN7; +static int is_cr_or_lf(char c) +{ + return c == '\r' || c == '\n'; +} static CURLcode add_certs_to_store(HCERTSTORE trust_store, const char *ca_file, @@ -130,7 +135,7 @@ static CURLcode add_certs_to_store(HCERTSTORE trust_store, failf(data, "schannel: CA file exceeds max size of %u bytes", MAX_CAFILE_SIZE); - result = CURLE_OUT_OF_MEMORY; + result = CURLE_SSL_CACERT_BADFILE; goto cleanup; } @@ -177,7 +182,7 @@ static CURLcode add_certs_to_store(HCERTSTORE trust_store, current_ca_file_ptr = ca_file_buffer; while(more_certs && *current_ca_file_ptr != '\0') { char *begin_cert_ptr = strstr(current_ca_file_ptr, BEGIN_CERT); - if(!begin_cert_ptr) { + if(!begin_cert_ptr || !is_cr_or_lf(begin_cert_ptr[strlen(BEGIN_CERT)])) { more_certs = 0; } else { @@ -209,7 +214,7 @@ static CURLcode add_certs_to_store(HCERTSTORE trust_store, NULL, NULL, NULL, - &cert_context)) { + (const void **)&cert_context)) { failf(data, "schannel: failed to extract certificate from CA file " @@ -239,7 +244,7 @@ static CURLcode add_certs_to_store(HCERTSTORE trust_store, CertFreeCertificateContext(cert_context); if(!add_cert_result) { failf(data, - "schannel: failed to add certificate from CA file '%s'" + "schannel: failed to add certificate from CA file '%s' " "to certificate store: %s", ca_file, Curl_strerror(conn, GetLastError())); result = CURLE_SSL_CACERT_BADFILE; @@ -314,6 +319,10 @@ static CURLcode verify_host(struct Curl_easy *data, * embedded null bytes. This appears to be undocumented behavior. */ cert_hostname_buff = (LPTSTR)malloc(len * sizeof(TCHAR)); + if(!cert_hostname_buff) { + result = CURLE_OUT_OF_MEMORY; + goto cleanup; + } actual_len = CertGetNameString(pCertContextServer, CERT_NAME_DNS_TYPE, name_flags, @@ -548,4 +557,5 @@ CURLcode verify_certificate(struct connectdata *conn, int sockindex) return result; } +#endif /* HAS_MANUAL_VERIFY_API */ #endif /* USE_SCHANNEL */ diff --git a/Utilities/cmcurl/lib/vtls/vtls.c b/Utilities/cmcurl/lib/vtls/vtls.c index ee5bc7a0a..6af39feab 100644 --- a/Utilities/cmcurl/lib/vtls/vtls.c +++ b/Utilities/cmcurl/lib/vtls/vtls.c @@ -96,7 +96,8 @@ Curl_ssl_config_matches(struct ssl_primary_config* data, Curl_safe_strcasecompare(data->clientcert, needle->clientcert) && Curl_safe_strcasecompare(data->random_file, needle->random_file) && Curl_safe_strcasecompare(data->egdsocket, needle->egdsocket) && - Curl_safe_strcasecompare(data->cipher_list, needle->cipher_list)) + Curl_safe_strcasecompare(data->cipher_list, needle->cipher_list) && + Curl_safe_strcasecompare(data->cipher_list13, needle->cipher_list13)) return TRUE; return FALSE; @@ -119,6 +120,7 @@ Curl_clone_primary_ssl_config(struct ssl_primary_config *source, CLONE_STRING(random_file); CLONE_STRING(egdsocket); CLONE_STRING(cipher_list); + CLONE_STRING(cipher_list13); return TRUE; } @@ -131,6 +133,7 @@ void Curl_free_primary_ssl_config(struct ssl_primary_config* sslc) Curl_safefree(sslc->random_file); Curl_safefree(sslc->egdsocket); Curl_safefree(sslc->cipher_list); + Curl_safefree(sslc->cipher_list13); } #ifdef USE_SSL @@ -980,6 +983,14 @@ bool Curl_ssl_false_start(void) } /* + * Check whether the SSL backend supports setting TLS 1.3 cipher suites + */ +bool Curl_ssl_tls13_ciphersuites(void) +{ + return Curl_ssl->supports & SSLSUPP_TLS13_CIPHERSUITES; +} + +/* * Default implementations for unsupported functions. */ @@ -1179,6 +1190,8 @@ const struct Curl_ssl *Curl_ssl = &Curl_ssl_polarssl; #elif defined(USE_SCHANNEL) &Curl_ssl_schannel; +#elif defined(USE_MESALINK) + &Curl_ssl_mesalink; #else #error "Missing struct Curl_ssl for selected SSL backend" #endif @@ -1214,6 +1227,9 @@ static const struct Curl_ssl *available_backends[] = { #if defined(USE_SCHANNEL) &Curl_ssl_schannel, #endif +#if defined(USE_MESALINK) + &Curl_ssl_mesalink, +#endif NULL }; diff --git a/Utilities/cmcurl/lib/vtls/vtls.h b/Utilities/cmcurl/lib/vtls/vtls.h index e7b87c4d3..5cd11602e 100644 --- a/Utilities/cmcurl/lib/vtls/vtls.h +++ b/Utilities/cmcurl/lib/vtls/vtls.h @@ -31,6 +31,7 @@ struct ssl_connect_data; #define SSLSUPP_PINNEDPUBKEY (1<<2) /* supports CURLOPT_PINNEDPUBLICKEY */ #define SSLSUPP_SSL_CTX (1<<3) /* supports CURLOPT_SSL_CTX */ #define SSLSUPP_HTTPS_PROXY (1<<4) /* supports access via HTTPS proxies */ +#define SSLSUPP_TLS13_CIPHERSUITES (1<<5) /* supports TLS 1.3 ciphersuites */ struct Curl_ssl { /* @@ -93,6 +94,7 @@ CURLcode Curl_none_set_engine(struct Curl_easy *data, const char *engine); CURLcode Curl_none_set_engine_default(struct Curl_easy *data); struct curl_slist *Curl_none_engines_list(struct Curl_easy *data); bool Curl_none_false_start(void); +bool Curl_ssl_tls13_ciphersuites(void); CURLcode Curl_none_md5sum(unsigned char *input, size_t inputlen, unsigned char *md5sum, size_t md5len); @@ -106,6 +108,7 @@ CURLcode Curl_none_md5sum(unsigned char *input, size_t inputlen, #include "schannel.h" /* Schannel SSPI version */ #include "darwinssl.h" /* SecureTransport (Darwin) version */ #include "mbedtls.h" /* mbedTLS versions */ +#include "mesalink.h" /* MesaLink versions */ #ifndef MAX_PINNED_PUBKEY_SIZE #define MAX_PINNED_PUBKEY_SIZE 1048576 /* 1MB */ @@ -246,7 +249,7 @@ bool Curl_ssl_false_start(void); #define SSL_SHUTDOWN_TIMEOUT 10000 /* ms */ -#else +#else /* if not USE_SSL */ /* When SSL support is not present, just define away these function calls */ #define Curl_ssl_init() 1 @@ -270,6 +273,7 @@ bool Curl_ssl_false_start(void); #define Curl_ssl_random(x,y,z) ((void)x, CURLE_NOT_BUILT_IN) #define Curl_ssl_cert_status_request() FALSE #define Curl_ssl_false_start() FALSE +#define Curl_ssl_tls13_ciphersuites() FALSE #endif #endif /* HEADER_CURL_VTLS_H */ diff --git a/Utilities/cmcurl/lib/warnless.h b/Utilities/cmcurl/lib/warnless.h index f6a2d744b..284ea1e75 100644 --- a/Utilities/cmcurl/lib/warnless.h +++ b/Utilities/cmcurl/lib/warnless.h @@ -26,6 +26,9 @@ #include <curl/curl.h> /* for curl_socket_t */ #endif +#define CURLX_FUNCTION_CAST(target_type, func) \ + (target_type)(void (*) (void))(func) + unsigned short curlx_ultous(unsigned long ulnum); unsigned char curlx_ultouc(unsigned long ulnum); diff --git a/Utilities/cmcurl/lib/x509asn1.c b/Utilities/cmcurl/lib/x509asn1.c index bba20233f..a576fc703 100644 --- a/Utilities/cmcurl/lib/x509asn1.c +++ b/Utilities/cmcurl/lib/x509asn1.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -40,10 +40,6 @@ #include "curl_memory.h" #include "memdebug.h" -/* For overflow checks. */ -#define CURL_SIZE_T_MAX ((size_t)-1) - - /* ASN.1 OIDs. */ static const char cnOID[] = "2.5.4.3"; /* Common name. */ static const char sanOID[] = "2.5.29.17"; /* Subject alternative name. */ @@ -107,9 +103,12 @@ static const curl_OID OIDtable[] = { * Please note there is no pretention here to rewrite a full SSL library. */ +static const char *getASN1Element(curl_asn1Element *elem, + const char *beg, const char *end) + WARN_UNUSED_RESULT; -const char *Curl_getASN1Element(curl_asn1Element *elem, - const char *beg, const char *end) +static const char *getASN1Element(curl_asn1Element *elem, + const char *beg, const char *end) { unsigned char b; unsigned long len; @@ -146,7 +145,7 @@ const char *Curl_getASN1Element(curl_asn1Element *elem, return (const char *) NULL; elem->beg = beg; while(beg < end && *beg) { - beg = Curl_getASN1Element(&lelem, beg, end); + beg = getASN1Element(&lelem, beg, end); if(!beg) return (const char *) NULL; } @@ -206,7 +205,7 @@ static const char *octet2str(const char *beg, const char *end) /* Convert an ASN.1 octet string to a printable string. Return the dynamically allocated string, or NULL if an error occurs. */ - if(n <= (CURL_SIZE_T_MAX - 1) / 3) { + if(n <= (SIZE_T_MAX - 1) / 3) { buf = malloc(3 * n + 1); if(buf) for(n = 0; beg < end; n += 3) @@ -227,7 +226,7 @@ static const char *bit2str(const char *beg, const char *end) static const char *int2str(const char *beg, const char *end) { - long val = 0; + unsigned long val = 0; size_t n = end - beg; /* Convert an ASN.1 integer value into its string representation. @@ -247,7 +246,7 @@ static const char *int2str(const char *beg, const char *end) do val = (val << 8) | *(const unsigned char *) beg++; while(beg < end); - return curl_maprintf("%s%lx", (val < 0 || val >= 10)? "0x": "", val); + return curl_maprintf("%s%lx", val >= 10? "0x": "", val); } static ssize_t @@ -287,7 +286,7 @@ utf8asn1str(char **to, int type, const char *from, const char *end) if(inlength % size) return -1; /* Length inconsistent with character size. */ - if(inlength / size > (CURL_SIZE_T_MAX - 1) / 4) + if(inlength / size > (SIZE_T_MAX - 1) / 4) return -1; /* Too big. */ buf = malloc(4 * (inlength / size) + 1); if(!buf) @@ -306,10 +305,10 @@ utf8asn1str(char **to, int type, const char *from, const char *end) case 4: wc = (wc << 8) | *(const unsigned char *) from++; wc = (wc << 8) | *(const unsigned char *) from++; - /* fallthrough */ + /* FALLTHROUGH */ case 2: wc = (wc << 8) | *(const unsigned char *) from++; - /* fallthrough */ + /* FALLTHROUGH */ default: /* case 1: */ wc = (wc << 8) | *(const unsigned char *) from++; } @@ -546,7 +545,7 @@ static const char *UTime2str(const char *beg, const char *end) tzl, tzp); } -const char *Curl_ASN1tostr(curl_asn1Element *elem, int type) +static const char *ASN1tostr(curl_asn1Element *elem, int type) { /* Convert an ASN.1 element to a printable string. Return the dynamically allocated string, or NULL if an error occurs. */ @@ -605,12 +604,19 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn) Return the total string length, even if larger than `n'. */ for(p1 = dn->beg; p1 < dn->end;) { - p1 = Curl_getASN1Element(&rdn, p1, dn->end); + p1 = getASN1Element(&rdn, p1, dn->end); + if(!p1) + return -1; for(p2 = rdn.beg; p2 < rdn.end;) { - p2 = Curl_getASN1Element(&atv, p2, rdn.end); - p3 = Curl_getASN1Element(&oid, atv.beg, atv.end); - Curl_getASN1Element(&value, p3, atv.end); - str = Curl_ASN1tostr(&oid, 0); + p2 = getASN1Element(&atv, p2, rdn.end); + if(!p2) + return -1; + p3 = getASN1Element(&oid, atv.beg, atv.end); + if(!p3) + return -1; + if(!getASN1Element(&value, p3, atv.end)) + return -1; + str = ASN1tostr(&oid, 0); if(!str) return -1; @@ -640,7 +646,7 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn) l++; /* Generate value. */ - str = Curl_ASN1tostr(&value, 0); + str = ASN1tostr(&value, 0); if(!str) return -1; for(p3 = str; *p3; p3++) { @@ -655,7 +661,7 @@ static ssize_t encodeDN(char *buf, size_t n, curl_asn1Element *dn) return l; } -const char *Curl_DNtostr(curl_asn1Element *dn) +static const char *DNtostr(curl_asn1Element *dn) { char *buf = (char *) NULL; ssize_t n = encodeDN(buf, 0, dn); @@ -694,17 +700,22 @@ int Curl_parseX509(curl_X509certificate *cert, cert->certificate.end = end; /* Get the sequence content. */ - if(!Curl_getASN1Element(&elem, beg, end)) + if(!getASN1Element(&elem, beg, end)) return -1; /* Invalid bounds/size. */ beg = elem.beg; end = elem.end; /* Get tbsCertificate. */ - beg = Curl_getASN1Element(&tbsCertificate, beg, end); + beg = getASN1Element(&tbsCertificate, beg, end); + if(!beg) + return -1; /* Skip the signatureAlgorithm. */ - beg = Curl_getASN1Element(&cert->signatureAlgorithm, beg, end); + beg = getASN1Element(&cert->signatureAlgorithm, beg, end); + if(!beg) + return -1; /* Get the signatureValue. */ - Curl_getASN1Element(&cert->signature, beg, end); + if(!getASN1Element(&cert->signature, beg, end)) + return -1; /* Parse TBSCertificate. */ beg = tbsCertificate.beg; @@ -712,30 +723,49 @@ int Curl_parseX509(curl_X509certificate *cert, /* Get optional version, get serialNumber. */ cert->version.header = NULL; cert->version.beg = &defaultVersion; - cert->version.end = &defaultVersion + sizeof defaultVersion;; - beg = Curl_getASN1Element(&elem, beg, end); + cert->version.end = &defaultVersion + sizeof(defaultVersion); + beg = getASN1Element(&elem, beg, end); + if(!beg) + return -1; if(elem.tag == 0) { - Curl_getASN1Element(&cert->version, elem.beg, elem.end); - beg = Curl_getASN1Element(&elem, beg, end); + if(!getASN1Element(&cert->version, elem.beg, elem.end)) + return -1; + beg = getASN1Element(&elem, beg, end); + if(!beg) + return -1; } cert->serialNumber = elem; /* Get signature algorithm. */ - beg = Curl_getASN1Element(&cert->signatureAlgorithm, beg, end); + beg = getASN1Element(&cert->signatureAlgorithm, beg, end); /* Get issuer. */ - beg = Curl_getASN1Element(&cert->issuer, beg, end); + beg = getASN1Element(&cert->issuer, beg, end); + if(!beg) + return -1; /* Get notBefore and notAfter. */ - beg = Curl_getASN1Element(&elem, beg, end); - ccp = Curl_getASN1Element(&cert->notBefore, elem.beg, elem.end); - Curl_getASN1Element(&cert->notAfter, ccp, elem.end); + beg = getASN1Element(&elem, beg, end); + if(!beg) + return -1; + ccp = getASN1Element(&cert->notBefore, elem.beg, elem.end); + if(!ccp) + return -1; + if(!getASN1Element(&cert->notAfter, ccp, elem.end)) + return -1; /* Get subject. */ - beg = Curl_getASN1Element(&cert->subject, beg, end); + beg = getASN1Element(&cert->subject, beg, end); + if(!beg) + return -1; /* Get subjectPublicKeyAlgorithm and subjectPublicKey. */ - beg = Curl_getASN1Element(&cert->subjectPublicKeyInfo, beg, end); - ccp = Curl_getASN1Element(&cert->subjectPublicKeyAlgorithm, - cert->subjectPublicKeyInfo.beg, - cert->subjectPublicKeyInfo.end); - Curl_getASN1Element(&cert->subjectPublicKey, ccp, - cert->subjectPublicKeyInfo.end); + beg = getASN1Element(&cert->subjectPublicKeyInfo, beg, end); + if(!beg) + return -1; + ccp = getASN1Element(&cert->subjectPublicKeyAlgorithm, + cert->subjectPublicKeyInfo.beg, + cert->subjectPublicKeyInfo.end); + if(!ccp) + return -1; + if(!getASN1Element(&cert->subjectPublicKey, ccp, + cert->subjectPublicKeyInfo.end)) + return -1; /* Get optional issuerUiqueID, subjectUniqueID and extensions. */ cert->issuerUniqueID.tag = cert->subjectUniqueID.tag = 0; cert->extensions.tag = elem.tag = 0; @@ -744,20 +774,30 @@ int Curl_parseX509(curl_X509certificate *cert, cert->subjectUniqueID.beg = cert->subjectUniqueID.end = ""; cert->extensions.header = NULL; cert->extensions.beg = cert->extensions.end = ""; - if(beg < end) - beg = Curl_getASN1Element(&elem, beg, end); + if(beg < end) { + beg = getASN1Element(&elem, beg, end); + if(!beg) + return -1; + } if(elem.tag == 1) { cert->issuerUniqueID = elem; - if(beg < end) - beg = Curl_getASN1Element(&elem, beg, end); + if(beg < end) { + beg = getASN1Element(&elem, beg, end); + if(!beg) + return -1; + } } if(elem.tag == 2) { cert->subjectUniqueID = elem; - if(beg < end) - beg = Curl_getASN1Element(&elem, beg, end); + if(beg < end) { + beg = getASN1Element(&elem, beg, end); + if(!beg) + return -1; + } } if(elem.tag == 3) - Curl_getASN1Element(&cert->extensions, elem.beg, elem.end); + if(!getASN1Element(&cert->extensions, elem.beg, elem.end)) + return -1; return 0; } @@ -785,12 +825,15 @@ static const char *dumpAlgo(curl_asn1Element *param, /* Get algorithm parameters and return algorithm name. */ - beg = Curl_getASN1Element(&oid, beg, end); + beg = getASN1Element(&oid, beg, end); + if(!beg) + return NULL; param->header = NULL; param->tag = 0; param->beg = param->end = end; if(beg < end) - Curl_getASN1Element(param, beg, end); + if(!getASN1Element(param, beg, end)) + return NULL; return OID2str(oid.beg, oid.end, TRUE); } @@ -801,7 +844,7 @@ static void do_pubkey_field(struct Curl_easy *data, int certnum, /* Generate a certificate information record for the public key. */ - output = Curl_ASN1tostr(elem, 0); + output = ASN1tostr(elem, 0); if(output) { if(data->set.ssl.certinfo) Curl_ssl_push_certinfo(data, certnum, label, output); @@ -825,10 +868,14 @@ static void do_pubkey(struct Curl_easy *data, int certnum, /* Generate all information records for the public key. */ /* Get the public key (single element). */ - Curl_getASN1Element(&pk, pubkey->beg + 1, pubkey->end); + if(!getASN1Element(&pk, pubkey->beg + 1, pubkey->end)) + return; if(strcasecompare(algo, "rsaEncryption")) { - p = Curl_getASN1Element(&elem, pk.beg, pk.end); + p = getASN1Element(&elem, pk.beg, pk.end); + if(!p) + return; + /* Compute key length. */ for(q = elem.beg; !*q && q < elem.end; q++) ; @@ -849,30 +896,34 @@ static void do_pubkey(struct Curl_easy *data, int certnum, } /* Generate coefficients. */ do_pubkey_field(data, certnum, "rsa(n)", &elem); - Curl_getASN1Element(&elem, p, pk.end); + if(!getASN1Element(&elem, p, pk.end)) + return; do_pubkey_field(data, certnum, "rsa(e)", &elem); } else if(strcasecompare(algo, "dsa")) { - p = Curl_getASN1Element(&elem, param->beg, param->end); - do_pubkey_field(data, certnum, "dsa(p)", &elem); - p = Curl_getASN1Element(&elem, p, param->end); - do_pubkey_field(data, certnum, "dsa(q)", &elem); - Curl_getASN1Element(&elem, p, param->end); - do_pubkey_field(data, certnum, "dsa(g)", &elem); - do_pubkey_field(data, certnum, "dsa(pub_key)", &pk); + p = getASN1Element(&elem, param->beg, param->end); + if(p) { + do_pubkey_field(data, certnum, "dsa(p)", &elem); + p = getASN1Element(&elem, p, param->end); + if(p) { + do_pubkey_field(data, certnum, "dsa(q)", &elem); + if(getASN1Element(&elem, p, param->end)) { + do_pubkey_field(data, certnum, "dsa(g)", &elem); + do_pubkey_field(data, certnum, "dsa(pub_key)", &pk); + } + } + } } else if(strcasecompare(algo, "dhpublicnumber")) { - p = Curl_getASN1Element(&elem, param->beg, param->end); - do_pubkey_field(data, certnum, "dh(p)", &elem); - Curl_getASN1Element(&elem, param->beg, param->end); - do_pubkey_field(data, certnum, "dh(g)", &elem); - do_pubkey_field(data, certnum, "dh(pub_key)", &pk); - } -#if 0 /* Patent-encumbered. */ - else if(strcasecompare(algo, "ecPublicKey")) { - /* Left TODO. */ + p = getASN1Element(&elem, param->beg, param->end); + if(p) { + do_pubkey_field(data, certnum, "dh(p)", &elem); + if(getASN1Element(&elem, param->beg, param->end)) { + do_pubkey_field(data, certnum, "dh(g)", &elem); + do_pubkey_field(data, certnum, "dh(pub_key)", &pk); + } + } } -#endif } CURLcode Curl_extract_certinfo(struct connectdata *conn, @@ -900,10 +951,10 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, /* Extract the certificate ASN.1 elements. */ if(Curl_parseX509(&cert, beg, end)) - return CURLE_OUT_OF_MEMORY; + return CURLE_PEER_FAILED_VERIFICATION; /* Subject. */ - ccp = Curl_DNtostr(&cert.subject); + ccp = DNtostr(&cert.subject); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -913,7 +964,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, free((char *) ccp); /* Issuer. */ - ccp = Curl_DNtostr(&cert.issuer); + ccp = DNtostr(&cert.issuer); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -937,7 +988,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, infof(data, " Version: %lu (0x%lx)\n", version + 1, version); /* Serial number. */ - ccp = Curl_ASN1tostr(&cert.serialNumber, 0); + ccp = ASN1tostr(&cert.serialNumber, 0); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -958,7 +1009,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, free((char *) ccp); /* Start Date. */ - ccp = Curl_ASN1tostr(&cert.notBefore, 0); + ccp = ASN1tostr(&cert.notBefore, 0); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -968,7 +1019,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, free((char *) ccp); /* Expire Date. */ - ccp = Curl_ASN1tostr(&cert.notAfter, 0); + ccp = ASN1tostr(&cert.notAfter, 0); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -992,7 +1043,7 @@ CURLcode Curl_extract_certinfo(struct connectdata *conn, /* TODO: extensions. */ /* Signature. */ - ccp = Curl_ASN1tostr(&cert.signature, 0); + ccp = ASN1tostr(&cert.signature, 0); if(!ccp) return CURLE_OUT_OF_MEMORY; if(data->set.ssl.certinfo) @@ -1051,7 +1102,7 @@ static const char *checkOID(const char *beg, const char *end, /* Check if first ASN.1 element at `beg' is the given OID. Return a pointer in the source after the OID if found, else NULL. */ - ccp = Curl_getASN1Element(&e, beg, end); + ccp = getASN1Element(&e, beg, end); if(!ccp || e.tag != CURL_ASN1_OBJECT_IDENTIFIER) return (const char *) NULL; @@ -1110,19 +1161,30 @@ CURLcode Curl_verifyhost(struct connectdata *conn, /* Process extensions. */ for(p = cert.extensions.beg; p < cert.extensions.end && matched != 1;) { - p = Curl_getASN1Element(&ext, p, cert.extensions.end); + p = getASN1Element(&ext, p, cert.extensions.end); + if(!p) + return CURLE_PEER_FAILED_VERIFICATION; + /* Check if extension is a subjectAlternativeName. */ ext.beg = checkOID(ext.beg, ext.end, sanOID); if(ext.beg) { - ext.beg = Curl_getASN1Element(&elem, ext.beg, ext.end); + ext.beg = getASN1Element(&elem, ext.beg, ext.end); + if(!ext.beg) + return CURLE_PEER_FAILED_VERIFICATION; /* Skip critical if present. */ - if(elem.tag == CURL_ASN1_BOOLEAN) - ext.beg = Curl_getASN1Element(&elem, ext.beg, ext.end); + if(elem.tag == CURL_ASN1_BOOLEAN) { + ext.beg = getASN1Element(&elem, ext.beg, ext.end); + if(!ext.beg) + return CURLE_PEER_FAILED_VERIFICATION; + } /* Parse the octet string contents: is a single sequence. */ - Curl_getASN1Element(&elem, elem.beg, elem.end); + if(!getASN1Element(&elem, elem.beg, elem.end)) + return CURLE_PEER_FAILED_VERIFICATION; /* Check all GeneralNames. */ for(q = elem.beg; matched != 1 && q < elem.end;) { - q = Curl_getASN1Element(&name, q, elem.end); + q = getASN1Element(&name, q, elem.end); + if(!q) + break; switch(name.tag) { case 2: /* DNS name. */ len = utf8asn1str(&dnsname, CURL_ASN1_IA5_STRING, @@ -1135,8 +1197,8 @@ CURLcode Curl_verifyhost(struct connectdata *conn, break; case 7: /* IP address. */ - matched = (size_t) (name.end - q) == addrlen && - !memcmp(&addr, q, addrlen); + matched = (size_t) (name.end - name.beg) == addrlen && + !memcmp(&addr, name.beg, addrlen); break; } } @@ -1162,9 +1224,13 @@ CURLcode Curl_verifyhost(struct connectdata *conn, /* we have to look to the last occurrence of a commonName in the distinguished one to get the most significant one. */ while(q < cert.subject.end) { - q = Curl_getASN1Element(&dn, q, cert.subject.end); + q = getASN1Element(&dn, q, cert.subject.end); + if(!q) + break; for(p = dn.beg; p < dn.end;) { - p = Curl_getASN1Element(&elem, p, dn.end); + p = getASN1Element(&elem, p, dn.end); + if(!p) + return CURLE_PEER_FAILED_VERIFICATION; /* We have a DN's AttributeTypeAndValue: check it in case it's a CN. */ elem.beg = checkOID(elem.beg, elem.end, cnOID); if(elem.beg) @@ -1173,7 +1239,7 @@ CURLcode Curl_verifyhost(struct connectdata *conn, } /* Check the CN if found. */ - if(!Curl_getASN1Element(&elem, name.beg, name.end)) + if(!getASN1Element(&elem, name.beg, name.end)) failf(data, "SSL: unable to obtain common name from peer certificate"); else { len = utf8asn1str(&dnsname, elem.tag, elem.beg, elem.end); diff --git a/Utilities/cmlibarchive/CMakeLists.txt b/Utilities/cmlibarchive/CMakeLists.txt index 206f3c6a5..d7af6e2b6 100644 --- a/Utilities/cmlibarchive/CMakeLists.txt +++ b/Utilities/cmlibarchive/CMakeLists.txt @@ -71,6 +71,7 @@ include(CTest) OPTION(ENABLE_NETTLE "Enable use of Nettle" ON) OPTION(ENABLE_OPENSSL "Enable use of OpenSSL" ON) +OPTION(ENABLE_LZ4 "Enable the use of the system LZ4 library if found" ON) OPTION(ENABLE_LZO "Enable the use of the system LZO library if found" OFF) OPTION(ENABLE_LZMA "Enable the use of the system LZMA library if found" ON) @@ -328,7 +329,7 @@ IF(ENABLE_LZO) INCLUDE(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZO2 DEFAULT_MSG LZO2_LIBRARY LZO2_INCLUDE_DIR) ELSE(ENABLE_LZO) - SET(LIBZMA_FOUND FALSE) # Override cached value + SET(LZO2_FOUND FALSE) # Override cached value ENDIF(ENABLE_LZO) IF(LZO2_FOUND) SET(HAVE_LIBLZO2 1) @@ -347,15 +348,19 @@ IF(0) # CMake does not need LZ4 support in libarchive # # Find LZ4 # -IF (LZ4_INCLUDE_DIR) - # Already in cache, be silent - SET(LZ4_FIND_QUIETLY TRUE) -ENDIF (LZ4_INCLUDE_DIR) +IF(ENABLE_LZ4) + IF (LZ4_INCLUDE_DIR) + # Already in cache, be silent + SET(LZ4_FIND_QUIETLY TRUE) + ENDIF (LZ4_INCLUDE_DIR) -FIND_PATH(LZ4_INCLUDE_DIR lz4.h) -FIND_LIBRARY(LZ4_LIBRARY NAMES lz4 liblz4) -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZ4 DEFAULT_MSG LZ4_LIBRARY LZ4_INCLUDE_DIR) + FIND_PATH(LZ4_INCLUDE_DIR lz4.h) + FIND_LIBRARY(LZ4_LIBRARY NAMES lz4 liblz4) + INCLUDE(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZ4 DEFAULT_MSG LZ4_LIBRARY LZ4_INCLUDE_DIR) +ELSE(ENABLE_LZ4) + SET(LZ4_FOUND FALSE) # Override cached value +ENDIF(ENABLE_LZ4) IF(LZ4_FOUND) SET(HAVE_LIBLZ4 1) SET(HAVE_LZ4_H 1) @@ -371,6 +376,31 @@ IF(LZ4_FOUND) ENDIF(LZ4_FOUND) MARK_AS_ADVANCED(CLEAR LZ4_INCLUDE_DIR) MARK_AS_ADVANCED(CLEAR LZ4_LIBRARY) +# +# Find Zstd +# +IF (ZSTD_INCLUDE_DIR) + # Already in cache, be silent + SET(ZSTD_FIND_QUIETLY TRUE) +ENDIF (ZSTD_INCLUDE_DIR) + +FIND_PATH(ZSTD_INCLUDE_DIR zstd.h) +FIND_LIBRARY(ZSTD_LIBRARY NAMES zstd libzstd) +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) +IF(ZSTD_FOUND) + SET(HAVE_ZSTD_H 1) + INCLUDE_DIRECTORIES(${ZSTD_INCLUDE_DIR}) + LIST(APPEND ADDITIONAL_LIBS ${ZSTD_LIBRARY}) + SET(CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY}) + SET(CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR}) + CHECK_FUNCTION_EXISTS(ZSTD_compressStream HAVE_LIBZSTD) + # + # TODO: test for static library. + # +ENDIF(ZSTD_FOUND) +MARK_AS_ADVANCED(CLEAR ZSTD_INCLUDE_DIR) +MARK_AS_ADVANCED(CLEAR ZSTD_LIBRARY) ENDIF() # @@ -451,6 +481,7 @@ LA_CHECK_INCLUDE_FILE("sys/select.h" HAVE_SYS_SELECT_H) LA_CHECK_INCLUDE_FILE("sys/stat.h" HAVE_SYS_STAT_H) LA_CHECK_INCLUDE_FILE("sys/statfs.h" HAVE_SYS_STATFS_H) LA_CHECK_INCLUDE_FILE("sys/statvfs.h" HAVE_SYS_STATVFS_H) +LA_CHECK_INCLUDE_FILE("sys/sysmacros.h" HAVE_SYS_SYSMACROS_H) LA_CHECK_INCLUDE_FILE("sys/time.h" HAVE_SYS_TIME_H) LA_CHECK_INCLUDE_FILE("sys/utime.h" HAVE_SYS_UTIME_H) LA_CHECK_INCLUDE_FILE("sys/utsname.h" HAVE_SYS_UTSNAME_H) diff --git a/Utilities/cmlibarchive/build/cmake/config.h.in b/Utilities/cmlibarchive/build/cmake/config.h.in index 368a451e2..1851d81d0 100644 --- a/Utilities/cmlibarchive/build/cmake/config.h.in +++ b/Utilities/cmlibarchive/build/cmake/config.h.in @@ -728,6 +728,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `z' library (-lz). */ #cmakedefine HAVE_LIBZ 1 +/* Define to 1 if you have the `zstd' library (-lzstd). */ +#cmakedefine HAVE_LIBZSTD 1 + /* Define to 1 if you have the <limits.h> header file. */ #cmakedefine HAVE_LIMITS_H 1 @@ -1071,6 +1074,10 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the <sys/stat.h> header file. */ #cmakedefine HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the <sys/sysmacros.h> header file. */ +#cmakedefine HAVE_SYS_SYSMACROS_H 1 + /* Define to 1 if you have the <sys/time.h> header file. */ #cmakedefine HAVE_SYS_TIME_H 1 @@ -1185,6 +1192,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the <zlib.h> header file. */ #cmakedefine HAVE_ZLIB_H 1 +/* Define to 1 if you have the <zstd.h> header file. */ +#cmakedefine HAVE_ZSTD_H 1 + /* Define to 1 if you have the `_ctime64_s' function. */ #cmakedefine HAVE__CTIME64_S 1 diff --git a/Utilities/cmlibarchive/build/pkgconfig/libarchive.pc.in b/Utilities/cmlibarchive/build/pkgconfig/libarchive.pc.in index 95d715951..4b631e635 100644 --- a/Utilities/cmlibarchive/build/pkgconfig/libarchive.pc.in +++ b/Utilities/cmlibarchive/build/pkgconfig/libarchive.pc.in @@ -7,5 +7,6 @@ Name: libarchive Description: library that can create and read several streaming archive formats Version: @VERSION@ Cflags: -I${includedir} +Cflags.private: -DLIBARCHIVE_STATIC Libs: -L${libdir} -larchive Libs.private: @LIBS@ diff --git a/Utilities/cmlibarchive/build/version b/Utilities/cmlibarchive/build/version index 2dd083974..2427eab0f 100644 --- a/Utilities/cmlibarchive/build/version +++ b/Utilities/cmlibarchive/build/version @@ -1 +1 @@ -3003002 +3003003 diff --git a/Utilities/cmlibarchive/libarchive/CMakeLists.txt b/Utilities/cmlibarchive/libarchive/CMakeLists.txt index b02ae828e..e38d66453 100644 --- a/Utilities/cmlibarchive/libarchive/CMakeLists.txt +++ b/Utilities/cmlibarchive/libarchive/CMakeLists.txt @@ -88,6 +88,7 @@ SET(libarchive_SOURCES archive_read_support_filter_rpm.c archive_read_support_filter_uu.c archive_read_support_filter_xz.c + archive_read_support_filter_zstd.c archive_read_support_format_7zip.c archive_read_support_format_all.c archive_read_support_format_ar.c @@ -134,6 +135,7 @@ SET(libarchive_SOURCES archive_write_add_filter_program.c archive_write_add_filter_uuencode.c archive_write_add_filter_xz.c + archive_write_add_filter_zstd.c archive_write_set_format.c archive_write_set_format_7zip.c archive_write_set_format_ar.c diff --git a/Utilities/cmlibarchive/libarchive/archive.h b/Utilities/cmlibarchive/libarchive/archive.h index 0f94d2e6c..f3ebbfe42 100644 --- a/Utilities/cmlibarchive/libarchive/archive.h +++ b/Utilities/cmlibarchive/libarchive/archive.h @@ -36,7 +36,7 @@ * assert that ARCHIVE_VERSION_NUMBER >= 2012108. */ /* Note: Compiler will complain if this does not match archive_entry.h! */ -#define ARCHIVE_VERSION_NUMBER 3003002 +#define ARCHIVE_VERSION_NUMBER 3003003 #include <sys/stat.h> #include <stddef.h> /* for wchar_t */ @@ -152,7 +152,7 @@ __LA_DECL int archive_version_number(void); /* * Textual name/version of the library, useful for version displays. */ -#define ARCHIVE_VERSION_ONLY_STRING "3.3.2" +#define ARCHIVE_VERSION_ONLY_STRING "3.3.3" #define ARCHIVE_VERSION_STRING "libarchive " ARCHIVE_VERSION_ONLY_STRING __LA_DECL const char * archive_version_string(void); @@ -174,6 +174,7 @@ __LA_DECL const char * archive_zlib_version(void); __LA_DECL const char * archive_liblzma_version(void); __LA_DECL const char * archive_bzlib_version(void); __LA_DECL const char * archive_liblz4_version(void); +__LA_DECL const char * archive_libzstd_version(void); /* Declare our basic types. */ struct archive; @@ -273,6 +274,7 @@ typedef const char *archive_passphrase_callback(struct archive *, #define ARCHIVE_FILTER_LZOP 11 #define ARCHIVE_FILTER_GRZIP 12 #define ARCHIVE_FILTER_LZ4 13 +#define ARCHIVE_FILTER_ZSTD 14 #if ARCHIVE_VERSION_NUMBER < 4000000 #define ARCHIVE_COMPRESSION_NONE ARCHIVE_FILTER_NONE @@ -430,6 +432,7 @@ __LA_DECL int archive_read_support_filter_program_signature __LA_DECL int archive_read_support_filter_rpm(struct archive *); __LA_DECL int archive_read_support_filter_uu(struct archive *); __LA_DECL int archive_read_support_filter_xz(struct archive *); +__LA_DECL int archive_read_support_filter_zstd(struct archive *); __LA_DECL int archive_read_support_format_7zip(struct archive *); __LA_DECL int archive_read_support_format_all(struct archive *); @@ -775,6 +778,7 @@ __LA_DECL int archive_write_add_filter_program(struct archive *, const char *cmd); __LA_DECL int archive_write_add_filter_uuencode(struct archive *); __LA_DECL int archive_write_add_filter_xz(struct archive *); +__LA_DECL int archive_write_add_filter_zstd(struct archive *); /* A convenience function to set the format based on the code or name. */ diff --git a/Utilities/cmlibarchive/libarchive/archive_acl.c b/Utilities/cmlibarchive/libarchive/archive_acl.c index b8b6b6364..4736531af 100644 --- a/Utilities/cmlibarchive/libarchive/archive_acl.c +++ b/Utilities/cmlibarchive/libarchive/archive_acl.c @@ -1159,6 +1159,7 @@ archive_acl_from_text_w(struct archive_acl *acl, const wchar_t *text, switch (want_type) { case ARCHIVE_ENTRY_ACL_TYPE_POSIX1E: want_type = ARCHIVE_ENTRY_ACL_TYPE_ACCESS; + __LA_FALLTHROUGH; case ARCHIVE_ENTRY_ACL_TYPE_ACCESS: case ARCHIVE_ENTRY_ACL_TYPE_DEFAULT: numfields = 5; @@ -1626,6 +1627,7 @@ archive_acl_from_text_l(struct archive_acl *acl, const char *text, switch (want_type) { case ARCHIVE_ENTRY_ACL_TYPE_POSIX1E: want_type = ARCHIVE_ENTRY_ACL_TYPE_ACCESS; + __LA_FALLTHROUGH; case ARCHIVE_ENTRY_ACL_TYPE_ACCESS: case ARCHIVE_ENTRY_ACL_TYPE_DEFAULT: numfields = 5; diff --git a/Utilities/cmlibarchive/libarchive/archive_cmdline.c b/Utilities/cmlibarchive/libarchive/archive_cmdline.c index 7d3bac53b..5c519cd17 100644 --- a/Utilities/cmlibarchive/libarchive/archive_cmdline.c +++ b/Utilities/cmlibarchive/libarchive/archive_cmdline.c @@ -100,10 +100,10 @@ get_argument(struct archive_string *as, const char *p) /* * Set up command line arguments. - * Returns ARChIVE_OK if everything okey. - * Returns ARChIVE_FAILED if there is a lack of the `"' terminator or an + * Returns ARCHIVE_OK if everything okey. + * Returns ARCHIVE_FAILED if there is a lack of the `"' terminator or an * empty command line. - * Returns ARChIVE_FATAL if no memory. + * Returns ARCHIVE_FATAL if no memory. */ int __archive_cmdline_parse(struct archive_cmdline *data, const char *cmd) diff --git a/Utilities/cmlibarchive/libarchive/archive_cryptor.c b/Utilities/cmlibarchive/libarchive/archive_cryptor.c index ced52fd70..71967c9d4 100644 --- a/Utilities/cmlibarchive/libarchive/archive_cryptor.c +++ b/Utilities/cmlibarchive/libarchive/archive_cryptor.c @@ -153,7 +153,7 @@ aes_ctr_encrypt_counter(archive_crypto_ctx *ctx) CCCryptorStatus r; r = CCCryptorReset(ref, NULL); - if (r != kCCSuccess) + if (r != kCCSuccess && r != kCCUnimplemented) return -1; r = CCCryptorUpdate(ref, ctx->nonce, AES_BLOCK_SIZE, ctx->encr_buf, AES_BLOCK_SIZE, NULL); diff --git a/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h b/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h index 0ca544b57..b9759220d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h @@ -64,7 +64,7 @@ typedef struct { } archive_crypto_ctx; #elif defined(_WIN32) && !defined(__CYGWIN__) && defined(HAVE_BCRYPT_H) -#include <Bcrypt.h> +#include <bcrypt.h> /* Common in other bcrypt implementations, but missing from VS2008. */ #ifndef BCRYPT_SUCCESS diff --git a/Utilities/cmlibarchive/libarchive/archive_disk_acl_freebsd.c b/Utilities/cmlibarchive/libarchive/archive_disk_acl_freebsd.c index 07d08ff96..aba41e5da 100644 --- a/Utilities/cmlibarchive/libarchive/archive_disk_acl_freebsd.c +++ b/Utilities/cmlibarchive/libarchive/archive_disk_acl_freebsd.c @@ -93,7 +93,9 @@ static const acl_perm_map_t acl_nfs4_flag_map[] = { {ARCHIVE_ENTRY_ACL_ENTRY_INHERIT_ONLY, ACL_ENTRY_INHERIT_ONLY}, {ARCHIVE_ENTRY_ACL_ENTRY_SUCCESSFUL_ACCESS, ACL_ENTRY_SUCCESSFUL_ACCESS}, {ARCHIVE_ENTRY_ACL_ENTRY_FAILED_ACCESS, ACL_ENTRY_FAILED_ACCESS}, +#ifdef ACL_ENTRY_INHERITED {ARCHIVE_ENTRY_ACL_ENTRY_INHERITED, ACL_ENTRY_INHERITED} +#endif }; static const int acl_nfs4_flag_map_size = diff --git a/Utilities/cmlibarchive/libarchive/archive_entry.c b/Utilities/cmlibarchive/libarchive/archive_entry.c index 30fb4566e..f722bbe85 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry.c +++ b/Utilities/cmlibarchive/libarchive/archive_entry.c @@ -1491,7 +1491,7 @@ archive_entry_acl_next(struct archive_entry *entry, int want_type, int *type, * the style of the generated ACL. */ wchar_t * -archive_entry_acl_to_text_w(struct archive_entry *entry, ssize_t *len, +archive_entry_acl_to_text_w(struct archive_entry *entry, la_ssize_t *len, int flags) { return (archive_acl_to_text_w(&entry->acl, len, flags, @@ -1499,7 +1499,7 @@ archive_entry_acl_to_text_w(struct archive_entry *entry, ssize_t *len, } char * -archive_entry_acl_to_text(struct archive_entry *entry, ssize_t *len, +archive_entry_acl_to_text(struct archive_entry *entry, la_ssize_t *len, int flags) { return (archive_acl_to_text_l(&entry->acl, len, flags, NULL)); diff --git a/Utilities/cmlibarchive/libarchive/archive_entry.h b/Utilities/cmlibarchive/libarchive/archive_entry.h index 8c8e75a1b..a9134f6b6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry.h +++ b/Utilities/cmlibarchive/libarchive/archive_entry.h @@ -30,7 +30,7 @@ #define ARCHIVE_ENTRY_H_INCLUDED /* Note: Compiler will complain if this does not match archive.h! */ -#define ARCHIVE_VERSION_NUMBER 3003002 +#define ARCHIVE_VERSION_NUMBER 3003003 /* * Note: archive_entry.h is for use outside of libarchive; the @@ -42,6 +42,7 @@ #include <sys/types.h> #include <stddef.h> /* for wchar_t */ +#include <stdint.h> #include <time.h> #if defined(_WIN32) && !defined(__CYGWIN__) diff --git a/Utilities/cmlibarchive/libarchive/archive_match.c b/Utilities/cmlibarchive/libarchive/archive_match.c index be72066ea..f150e8224 100644 --- a/Utilities/cmlibarchive/libarchive/archive_match.c +++ b/Utilities/cmlibarchive/libarchive/archive_match.c @@ -1582,7 +1582,7 @@ time_excluded(struct archive_match *a, struct archive_entry *entry) */ int -archive_match_include_uid(struct archive *_a, int64_t uid) +archive_match_include_uid(struct archive *_a, la_int64_t uid) { struct archive_match *a; @@ -1593,7 +1593,7 @@ archive_match_include_uid(struct archive *_a, int64_t uid) } int -archive_match_include_gid(struct archive *_a, int64_t gid) +archive_match_include_gid(struct archive *_a, la_int64_t gid) { struct archive_match *a; diff --git a/Utilities/cmlibarchive/libarchive/archive_pack_dev.c b/Utilities/cmlibarchive/libarchive/archive_pack_dev.c index 098881b67..53bddd790 100644 --- a/Utilities/cmlibarchive/libarchive/archive_pack_dev.c +++ b/Utilities/cmlibarchive/libarchive/archive_pack_dev.c @@ -57,6 +57,9 @@ __RCSID("$NetBSD$"); #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif +#ifdef HAVE_SYS_SYSMACROS_H +#include <sys/sysmacros.h> +#endif #ifdef HAVE_UNISTD_H #include <unistd.h> #endif diff --git a/Utilities/cmlibarchive/libarchive/archive_platform.h b/Utilities/cmlibarchive/libarchive/archive_platform.h index f33208c9a..e161e6406 100644 --- a/Utilities/cmlibarchive/libarchive/archive_platform.h +++ b/Utilities/cmlibarchive/libarchive/archive_platform.h @@ -196,4 +196,10 @@ #define ARCHIVE_ERRNO_MISC (-1) #endif +#if defined(__GNUC__) && (__GNUC__ >= 7) +#define __LA_FALLTHROUGH __attribute__((fallthrough)) +#else +#define __LA_FALLTHROUGH +#endif + #endif /* !ARCHIVE_PLATFORM_H_INCLUDED */ diff --git a/Utilities/cmlibarchive/libarchive/archive_ppmd7.c b/Utilities/cmlibarchive/libarchive/archive_ppmd7.c index 1aed922db..d0bacc68c 100644 --- a/Utilities/cmlibarchive/libarchive/archive_ppmd7.c +++ b/Utilities/cmlibarchive/libarchive/archive_ppmd7.c @@ -115,14 +115,14 @@ static void Ppmd7_Construct(CPpmd7 *p) memset(p->HB2Flag + 0x40, 8, 0x100 - 0x40); } -static void Ppmd7_Free(CPpmd7 *p, ISzAlloc *alloc) +static void Ppmd7_Free(CPpmd7 *p) { - alloc->Free(alloc, p->Base); + free(p->Base); p->Size = 0; p->Base = 0; } -static Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc) +static Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size) { if (p->Base == 0 || p->Size != size) { @@ -131,14 +131,14 @@ static Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc) if (size < UNIT_SIZE) { return False; } - Ppmd7_Free(p, alloc); + Ppmd7_Free(p); p->AlignOffset = #ifdef PPMD_32BIT (4 - size) & 3; #else 4 - (size & 3); #endif - if ((p->Base = (Byte *)alloc->Alloc(alloc, p->AlignOffset + size + if ((p->Base = (Byte *)malloc(p->AlignOffset + size #ifndef PPMD_32BIT + UNIT_SIZE #endif diff --git a/Utilities/cmlibarchive/libarchive/archive_ppmd7_private.h b/Utilities/cmlibarchive/libarchive/archive_ppmd7_private.h index 06c99e828..577d6fb43 100644 --- a/Utilities/cmlibarchive/libarchive/archive_ppmd7_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_ppmd7_private.h @@ -95,8 +95,8 @@ typedef struct { /* Base Functions */ void (*Ppmd7_Construct)(CPpmd7 *p); - Bool (*Ppmd7_Alloc)(CPpmd7 *p, UInt32 size, ISzAlloc *alloc); - void (*Ppmd7_Free)(CPpmd7 *p, ISzAlloc *alloc); + Bool (*Ppmd7_Alloc)(CPpmd7 *p, UInt32 size); + void (*Ppmd7_Free)(CPpmd7 *p); void (*Ppmd7_Init)(CPpmd7 *p, unsigned maxOrder); #define Ppmd7_WasAllocated(p) ((p)->Base != NULL) diff --git a/Utilities/cmlibarchive/libarchive/archive_ppmd_private.h b/Utilities/cmlibarchive/libarchive/archive_ppmd_private.h index e78bde594..a83b8514d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_ppmd_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_ppmd_private.h @@ -69,13 +69,6 @@ typedef struct void (*Write)(void *p, Byte b); } IByteOut; - -typedef struct -{ - void *(*Alloc)(void *p, size_t size); - void (*Free)(void *p, void *address); /* address can be 0 */ -} ISzAlloc; - /*** End defined in Types.h ***/ /*** Begin defined in CpuArch.h ***/ diff --git a/Utilities/cmlibarchive/libarchive/archive_read.c b/Utilities/cmlibarchive/libarchive/archive_read.c index a642a336d..0e56e76e7 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read.c +++ b/Utilities/cmlibarchive/libarchive/archive_read.c @@ -120,7 +120,8 @@ archive_read_new(void) * Record the do-not-extract-to file. This belongs in archive_read_extract.c. */ void -archive_read_extract_set_skip_file(struct archive *_a, int64_t d, int64_t i) +archive_read_extract_set_skip_file(struct archive *_a, la_int64_t d, + la_int64_t i) { struct archive_read *a = (struct archive_read *)_a; @@ -747,7 +748,7 @@ choose_format(struct archive_read *a) * Return the file offset (within the uncompressed data stream) where * the last header started. */ -int64_t +la_int64_t archive_read_header_position(struct archive *_a) { struct archive_read *a = (struct archive_read *)_a; @@ -820,7 +821,7 @@ archive_read_format_capabilities(struct archive *_a) * DO NOT intermingle calls to this function and archive_read_data_block * to read a single entry body. */ -ssize_t +la_ssize_t archive_read_data(struct archive *_a, void *buff, size_t s) { struct archive *a = (struct archive *)_a; @@ -943,7 +944,7 @@ archive_read_data_skip(struct archive *_a) return (r); } -int64_t +la_int64_t archive_seek_data(struct archive *_a, int64_t offset, int whence) { struct archive_read *a = (struct archive_read *)_a; @@ -1626,7 +1627,8 @@ __archive_read_filter_seek(struct archive_read_filter *filter, int64_t offset, switch (whence) { case SEEK_CUR: /* Adjust the offset and use SEEK_SET instead */ - offset += filter->position; + offset += filter->position; + __LA_FALLTHROUGH; case SEEK_SET: cursor = 0; while (1) diff --git a/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c b/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c index 5e4d16307..da7c55b9b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c @@ -89,6 +89,10 @@ archive_read_append_filter(struct archive *_a, int code) strcpy(str, "lz4"); r1 = archive_read_support_filter_lz4(_a); break; + case ARCHIVE_FILTER_ZSTD: + strcpy(str, "zstd"); + r1 = archive_read_support_filter_zstd(_a); + break; case ARCHIVE_FILTER_LZIP: strcpy(str, "lzip"); r1 = archive_read_support_filter_lzip(_a); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c index 548ba89ef..1786cff38 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c @@ -127,7 +127,7 @@ archive_read_disk_entry_setup_acls(struct archive_read_disk *a, /* * Enter working directory and return working pathname of archive_entry. * If a pointer to an integer is provided and its value is below zero - * open a file descriptor on this pahtname. + * open a file descriptor on this pathname. */ const char * archive_read_disk_entry_setup_path(struct archive_read_disk *a, diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c index 6961ae6a4..cdf754123 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c @@ -387,7 +387,7 @@ archive_read_disk_vtable(void) } const char * -archive_read_disk_gname(struct archive *_a, int64_t gid) +archive_read_disk_gname(struct archive *_a, la_int64_t gid) { struct archive_read_disk *a = (struct archive_read_disk *)_a; if (ARCHIVE_OK != __archive_check_magic(_a, ARCHIVE_READ_DISK_MAGIC, @@ -399,7 +399,7 @@ archive_read_disk_gname(struct archive *_a, int64_t gid) } const char * -archive_read_disk_uname(struct archive *_a, int64_t uid) +archive_read_disk_uname(struct archive *_a, la_int64_t uid) { struct archive_read_disk *a = (struct archive_read_disk *)_a; if (ARCHIVE_OK != __archive_check_magic(_a, ARCHIVE_READ_DISK_MAGIC, @@ -413,7 +413,7 @@ archive_read_disk_uname(struct archive *_a, int64_t uid) int archive_read_disk_set_gname_lookup(struct archive *_a, void *private_data, - const char * (*lookup_gname)(void *private, int64_t gid), + const char * (*lookup_gname)(void *private, la_int64_t gid), void (*cleanup_gname)(void *private)) { struct archive_read_disk *a = (struct archive_read_disk *)_a; @@ -432,7 +432,7 @@ archive_read_disk_set_gname_lookup(struct archive *_a, int archive_read_disk_set_uname_lookup(struct archive *_a, void *private_data, - const char * (*lookup_uname)(void *private, int64_t uid), + const char * (*lookup_uname)(void *private, la_int64_t uid), void (*cleanup_uname)(void *private)) { struct archive_read_disk *a = (struct archive_read_disk *)_a; diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c index 3b903304f..d82048de2 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c @@ -117,7 +117,7 @@ struct filesystem { */ #define MAX_OVERLAPPED 8 -#define BUFFER_SIZE (1024 * 8) +#define READ_BUFFER_SIZE (1024 * 64) /* Default to 64KB per https://technet.microsoft.com/en-us/library/cc938632.aspx */ #define DIRECT_IO 0/* Disabled */ #define ASYNC_IO 1/* Enabled */ @@ -320,7 +320,7 @@ archive_read_disk_vtable(void) } const char * -archive_read_disk_gname(struct archive *_a, int64_t gid) +archive_read_disk_gname(struct archive *_a, la_int64_t gid) { struct archive_read_disk *a = (struct archive_read_disk *)_a; if (ARCHIVE_OK != __archive_check_magic(_a, ARCHIVE_READ_DISK_MAGIC, @@ -332,7 +332,7 @@ archive_read_disk_gname(struct archive *_a, int64_t gid) } const char * -archive_read_disk_uname(struct archive *_a, int64_t uid) +archive_read_disk_uname(struct archive *_a, la_int64_t uid) { struct archive_read_disk *a = (struct archive_read_disk *)_a; if (ARCHIVE_OK != __archive_check_magic(_a, ARCHIVE_READ_DISK_MAGIC, @@ -346,7 +346,7 @@ archive_read_disk_uname(struct archive *_a, int64_t uid) int archive_read_disk_set_gname_lookup(struct archive *_a, void *private_data, - const char * (*lookup_gname)(void *private, int64_t gid), + const char * (*lookup_gname)(void *private, la_int64_t gid), void (*cleanup_gname)(void *private)) { struct archive_read_disk *a = (struct archive_read_disk *)_a; @@ -567,7 +567,7 @@ start_next_async_read(struct archive_read_disk *a, struct tree *t) /* Allocate read buffer. */ if (olp->buff == NULL) { void *p; - size_t s = (size_t)align_num_per_sector(t, BUFFER_SIZE); + size_t s = (size_t)align_num_per_sector(t, READ_BUFFER_SIZE); p = VirtualAlloc(NULL, s, MEM_COMMIT, PAGE_READWRITE); if (p == NULL) { archive_set_error(&a->archive, ENOMEM, @@ -683,7 +683,7 @@ _archive_read_data_block(struct archive *_a, const void **buff, break; } while (r == ARCHIVE_OK && t->ol_num_doing < MAX_OVERLAPPED); } else { - if (start_next_async_read(a, t) == ARCHIVE_FATAL) + if ((r = start_next_async_read(a, t)) == ARCHIVE_FATAL) goto abort_read_data; } @@ -923,6 +923,7 @@ next_entry(struct archive_read_disk *a, struct tree *t, t->entry_fh = CreateFileW(tree_current_access_path(t), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, flags, NULL); if (t->entry_fh == INVALID_HANDLE_VALUE) { + la_dosmaperr(GetLastError()); archive_set_error(&a->archive, errno, "Couldn't open %ls", tree_current_path(a->tree)); return (ARCHIVE_FAILED); @@ -2275,10 +2276,10 @@ setup_sparse_from_disk(struct archive_read_disk *a, if (range.Length.QuadPart > 0) continue; } else { - /* The remaining data is hole. */ + /* The entire file is a hole. Add one data block of size 0 at the end. */ archive_entry_sparse_add_entry(entry, - range.FileOffset.QuadPart, - range.Length.QuadPart); + entry_size, + 0); } break; } else { diff --git a/Utilities/cmlibarchive/libarchive/archive_read_filter.3 b/Utilities/cmlibarchive/libarchive/archive_read_filter.3 index 7f020e373..ef0a70175 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_filter.3 +++ b/Utilities/cmlibarchive/libarchive/archive_read_filter.3 @@ -38,6 +38,7 @@ .Nm archive_read_support_filter_rpm , .Nm archive_read_support_filter_uu , .Nm archive_read_support_filter_xz , +.Nm archive_read_support_filter_zstd , .Nm archive_read_support_filter_program , .Nm archive_read_support_filter_program_signature .Nd functions for reading streaming archives @@ -73,6 +74,8 @@ Streaming Archive Library (libarchive, -larchive) .Ft int .Fn archive_read_support_filter_xz "struct archive *" .Ft int +.Fn archive_read_support_filter_zstd "struct archive *" +.Ft int .Fo archive_read_support_filter_program .Fa "struct archive *" .Fa "const char *cmd" @@ -99,7 +102,8 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_read_support_filter_none , .Fn archive_read_support_filter_rpm , .Fn archive_read_support_filter_uu , -.Fn archive_read_support_filter_xz +.Fn archive_read_support_filter_xz , +.Fn archive_read_support_filter_zstd , .Xc Enables auto-detection code and decompression support for the specified compression. diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c index 68c53de41..edb508c1d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c @@ -71,6 +71,8 @@ archive_read_support_filter_all(struct archive *a) archive_read_support_filter_grzip(a); /* Lz4 falls back to "lz4 -d" command-line program. */ archive_read_support_filter_lz4(a); + /* Zstd falls back to "zstd -d" command-line program. */ + archive_read_support_filter_zstd(a); /* Note: We always return ARCHIVE_OK here, even if some of the * above return ARCHIVE_WARN. The intent here is to enable diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_zstd.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_zstd.c new file mode 100644 index 000000000..c8bb36be7 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_zstd.c @@ -0,0 +1,292 @@ +/*- + * Copyright (c) 2009-2011 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#include <stdio.h> +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#if HAVE_ZSTD_H +#include <zstd.h> +#endif + +#include "archive.h" +#include "archive_endian.h" +#include "archive_private.h" +#include "archive_read_private.h" + +#if HAVE_ZSTD_H && HAVE_LIBZSTD + +struct private_data { + ZSTD_DStream *dstream; + unsigned char *out_block; + size_t out_block_size; + int64_t total_out; + char in_frame; /* True = in the middle of a zstd frame. */ + char eof; /* True = found end of compressed data. */ +}; + +/* Zstd Filter. */ +static ssize_t zstd_filter_read(struct archive_read_filter *, const void**); +static int zstd_filter_close(struct archive_read_filter *); +#endif + +/* + * Note that we can detect zstd compressed files even if we can't decompress + * them. (In fact, we like detecting them because we can give better error + * messages.) So the bid framework here gets compiled even if no zstd library + * is available. + */ +static int zstd_bidder_bid(struct archive_read_filter_bidder *, + struct archive_read_filter *); +static int zstd_bidder_init(struct archive_read_filter *); + +int +archive_read_support_filter_zstd(struct archive *_a) +{ + struct archive_read *a = (struct archive_read *)_a; + struct archive_read_filter_bidder *bidder; + + archive_check_magic(_a, ARCHIVE_READ_MAGIC, + ARCHIVE_STATE_NEW, "archive_read_support_filter_zstd"); + + if (__archive_read_get_bidder(a, &bidder) != ARCHIVE_OK) + return (ARCHIVE_FATAL); + + bidder->data = NULL; + bidder->name = "zstd"; + bidder->bid = zstd_bidder_bid; + bidder->init = zstd_bidder_init; + bidder->options = NULL; + bidder->free = NULL; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + return (ARCHIVE_OK); +#else + archive_set_error(_a, ARCHIVE_ERRNO_MISC, + "Using external zstd program for zstd decompression"); + return (ARCHIVE_WARN); +#endif +} + +/* + * Test whether we can handle this data. + */ +static int +zstd_bidder_bid(struct archive_read_filter_bidder *self, + struct archive_read_filter *filter) +{ + const unsigned char *buffer; + ssize_t avail; + unsigned prefix; + + /* Zstd frame magic values */ + const unsigned zstd_magic = 0xFD2FB528U; + + (void) self; /* UNUSED */ + + buffer = __archive_read_filter_ahead(filter, 4, &avail); + if (buffer == NULL) + return (0); + + prefix = archive_le32dec(buffer); + if (prefix == zstd_magic) + return (32); + + return (0); +} + +#if !(HAVE_ZSTD_H && HAVE_LIBZSTD) + +/* + * If we don't have the library on this system, we can't do the + * decompression directly. We can, however, try to run "zstd -d" + * in case that's available. + */ +static int +zstd_bidder_init(struct archive_read_filter *self) +{ + int r; + + r = __archive_read_program(self, "zstd -d -qq"); + /* Note: We set the format here even if __archive_read_program() + * above fails. We do, after all, know what the format is + * even if we weren't able to read it. */ + self->code = ARCHIVE_FILTER_ZSTD; + self->name = "zstd"; + return (r); +} + +#else + +/* + * Initialize the filter object + */ +static int +zstd_bidder_init(struct archive_read_filter *self) +{ + struct private_data *state; + const size_t out_block_size = ZSTD_DStreamOutSize(); + void *out_block; + ZSTD_DStream *dstream; + + self->code = ARCHIVE_FILTER_ZSTD; + self->name = "zstd"; + + state = (struct private_data *)calloc(sizeof(*state), 1); + out_block = (unsigned char *)malloc(out_block_size); + dstream = ZSTD_createDStream(); + + if (state == NULL || out_block == NULL || dstream == NULL) { + free(out_block); + free(state); + ZSTD_freeDStream(dstream); /* supports free on NULL */ + archive_set_error(&self->archive->archive, ENOMEM, + "Can't allocate data for zstd decompression"); + return (ARCHIVE_FATAL); + } + + self->data = state; + + state->out_block_size = out_block_size; + state->out_block = out_block; + state->dstream = dstream; + self->read = zstd_filter_read; + self->skip = NULL; /* not supported */ + self->close = zstd_filter_close; + + state->eof = 0; + state->in_frame = 0; + + return (ARCHIVE_OK); +} + +static ssize_t +zstd_filter_read(struct archive_read_filter *self, const void **p) +{ + struct private_data *state; + size_t decompressed; + ssize_t avail_in; + ZSTD_outBuffer out; + ZSTD_inBuffer in; + + state = (struct private_data *)self->data; + + out = (ZSTD_outBuffer) { state->out_block, state->out_block_size, 0 }; + + /* Try to fill the output buffer. */ + while (out.pos < out.size && !state->eof) { + if (!state->in_frame) { + const size_t ret = ZSTD_initDStream(state->dstream); + if (ZSTD_isError(ret)) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Error initializing zstd decompressor: %s", + ZSTD_getErrorName(ret)); + return (ARCHIVE_FATAL); + } + } + in.src = __archive_read_filter_ahead(self->upstream, 1, + &avail_in); + if (avail_in < 0) { + return avail_in; + } + if (in.src == NULL && avail_in == 0) { + if (!state->in_frame) { + /* end of stream */ + state->eof = 1; + break; + } else { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Truncated zstd input"); + return (ARCHIVE_FATAL); + } + } + in.size = avail_in; + in.pos = 0; + + { + const size_t ret = + ZSTD_decompressStream(state->dstream, &out, &in); + + if (ZSTD_isError(ret)) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Zstd decompression failed: %s", + ZSTD_getErrorName(ret)); + return (ARCHIVE_FATAL); + } + + /* Decompressor made some progress */ + __archive_read_filter_consume(self->upstream, in.pos); + + /* ret guaranteed to be > 0 if frame isn't done yet */ + state->in_frame = (ret != 0); + } + } + + decompressed = out.pos; + state->total_out += decompressed; + if (decompressed == 0) + *p = NULL; + else + *p = state->out_block; + return (decompressed); +} + +/* + * Clean up the decompressor. + */ +static int +zstd_filter_close(struct archive_read_filter *self) +{ + struct private_data *state; + + state = (struct private_data *)self->data; + + ZSTD_freeDStream(state->dstream); + free(state->out_block); + free(state); + + return (ARCHIVE_OK); +} + +#endif /* HAVE_ZLIB_H && HAVE_LIBZSTD */ diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c index 96774f4e3..a885a4c45 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c @@ -975,18 +975,6 @@ decode_codec_id(const unsigned char *codecId, size_t id_size) return (id); } -static void * -ppmd_alloc(void *p, size_t size) -{ - (void)p; - return malloc(size); -} -static void -ppmd_free(void *p, void *address) -{ - (void)p; - free(address); -} static Byte ppmd_read(void *p) { @@ -1006,8 +994,6 @@ ppmd_read(void *p) return (b); } -static ISzAlloc g_szalloc = { ppmd_alloc, ppmd_free }; - static int init_decompression(struct archive_read *a, struct _7zip *zip, const struct _7z_coder *coder1, const struct _7z_coder *coder2) @@ -1237,7 +1223,7 @@ init_decompression(struct archive_read *a, struct _7zip *zip, if (zip->ppmd7_valid) { __archive_ppmd7_functions.Ppmd7_Free( - &zip->ppmd7_context, &g_szalloc); + &zip->ppmd7_context); zip->ppmd7_valid = 0; } @@ -1256,7 +1242,7 @@ init_decompression(struct archive_read *a, struct _7zip *zip, } __archive_ppmd7_functions.Ppmd7_Construct(&zip->ppmd7_context); r = __archive_ppmd7_functions.Ppmd7_Alloc( - &zip->ppmd7_context, msize, &g_szalloc); + &zip->ppmd7_context, msize); if (r == 0) { archive_set_error(&a->archive, ENOMEM, "Coludn't allocate memory for PPMd"); @@ -1636,7 +1622,7 @@ free_decompression(struct archive_read *a, struct _7zip *zip) #endif if (zip->ppmd7_valid) { __archive_ppmd7_functions.Ppmd7_Free( - &zip->ppmd7_context, &g_szalloc); + &zip->ppmd7_context); zip->ppmd7_valid = 0; } return (r); @@ -2569,6 +2555,7 @@ read_Header(struct archive_read *a, struct _7z_header_info *h, case kDummy: if (ll == 0) break; + __LA_FALLTHROUGH; default: if (header_bytes(a, ll) == NULL) return (-1); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c index ad9f782de..67d5b21ee 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c @@ -633,6 +633,13 @@ header_newc(struct archive_read *a, struct cpio *cpio, /* Pad name to 2 more than a multiple of 4. */ *name_pad = (2 - *namelength) & 3; + /* Make sure that the padded name length fits into size_t. */ + if (*name_pad > SIZE_MAX - *namelength) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "cpio archive has invalid namelength"); + return (ARCHIVE_FATAL); + } + /* * Note: entry_bytes_remaining is at least 64 bits and * therefore guaranteed to be big enough for a 33-bit file diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c index 4b436d1c4..93649f862 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c @@ -409,7 +409,8 @@ static int next_entry_seek(struct archive_read *, struct iso9660 *, struct file_info **); static struct file_info * parse_file_info(struct archive_read *a, - struct file_info *parent, const unsigned char *isodirrec); + struct file_info *parent, const unsigned char *isodirrec, + size_t reclen); static int parse_rockridge(struct archive_read *a, struct file_info *file, const unsigned char *start, const unsigned char *end); @@ -1022,7 +1023,7 @@ read_children(struct archive_read *a, struct file_info *parent) if (*(p + DR_name_len_offset) == 1 && *(p + DR_name_offset) == '\001') continue; - child = parse_file_info(a, parent, p); + child = parse_file_info(a, parent, p, b - p); if (child == NULL) { __archive_read_consume(a, skip_size); return (ARCHIVE_FATAL); @@ -1112,7 +1113,7 @@ choose_volume(struct archive_read *a, struct iso9660 *iso9660) */ seenJoliet = iso9660->seenJoliet;/* Save flag. */ iso9660->seenJoliet = 0; - file = parse_file_info(a, NULL, block); + file = parse_file_info(a, NULL, block, vd->size); if (file == NULL) return (ARCHIVE_FATAL); iso9660->seenJoliet = seenJoliet; @@ -1144,7 +1145,7 @@ choose_volume(struct archive_read *a, struct iso9660 *iso9660) return (ARCHIVE_FATAL); } iso9660->seenJoliet = 0; - file = parse_file_info(a, NULL, block); + file = parse_file_info(a, NULL, block, vd->size); if (file == NULL) return (ARCHIVE_FATAL); iso9660->seenJoliet = seenJoliet; @@ -1749,7 +1750,7 @@ archive_read_format_iso9660_cleanup(struct archive_read *a) */ static struct file_info * parse_file_info(struct archive_read *a, struct file_info *parent, - const unsigned char *isodirrec) + const unsigned char *isodirrec, size_t reclen) { struct iso9660 *iso9660; struct file_info *file, *filep; @@ -1763,16 +1764,20 @@ parse_file_info(struct archive_read *a, struct file_info *parent, iso9660 = (struct iso9660 *)(a->format->data); - dr_len = (size_t)isodirrec[DR_length_offset]; - name_len = (size_t)isodirrec[DR_name_len_offset]; - location = archive_le32dec(isodirrec + DR_extent_offset); - fsize = toi(isodirrec + DR_size_offset, DR_size_size); - /* Sanity check that dr_len needs at least 34. */ - if (dr_len < 34) { + if (reclen != 0) + dr_len = (size_t)isodirrec[DR_length_offset]; + /* + * Sanity check that reclen is not zero and dr_len is greater than + * reclen but at least 34 + */ + if (reclen == 0 || reclen < dr_len || dr_len < 34) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Invalid length of directory record"); + "Invalid length of directory record"); return (NULL); } + name_len = (size_t)isodirrec[DR_name_len_offset]; + location = archive_le32dec(isodirrec + DR_extent_offset); + fsize = toi(isodirrec + DR_size_offset, DR_size_size); /* Sanity check that name_len doesn't exceed dr_len. */ if (dr_len - 33 < name_len || name_len == 0) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c index 1995e9c66..98d02c62d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c @@ -701,6 +701,12 @@ archive_read_format_lha_read_header(struct archive_read *a, * Prepare variables used to read a file content. */ lha->entry_bytes_remaining = lha->compsize; + if (lha->entry_bytes_remaining < 0) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Invalid LHa entry size"); + return (ARCHIVE_FATAL); + } lha->entry_offset = 0; lha->entry_crc_calculated = 0; diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c index 44b6083cb..5b0eadc08 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c @@ -49,6 +49,7 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_mtree.c 2011 #include "archive.h" #include "archive_entry.h" #include "archive_private.h" +#include "archive_rb.h" #include "archive_read_private.h" #include "archive_string.h" #include "archive_pack_dev.h" @@ -75,7 +76,7 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_mtree.c 2011 #define MTREE_HAS_OPTIONAL 0x0800 #define MTREE_HAS_NOCHANGE 0x1000 /* FreeBSD specific */ -#define MTREE_HASHTABLE_SIZE 1024 +#define MAX_LINE_LEN (1024 * 1024) struct mtree_option { struct mtree_option *next; @@ -83,13 +84,13 @@ struct mtree_option { }; struct mtree_entry { + struct archive_rb_node rbnode; + struct mtree_entry *next_dup; struct mtree_entry *next; struct mtree_option *options; char *name; char full; char used; - unsigned int name_hash; - struct mtree_entry *hashtable_next; }; struct mtree { @@ -102,11 +103,12 @@ struct mtree { const char *archive_format_name; struct mtree_entry *entries; struct mtree_entry *this_entry; - struct mtree_entry *entry_hashtable[MTREE_HASHTABLE_SIZE]; + struct archive_rb_tree entry_rbtree; struct archive_string current_dir; struct archive_string contents_name; struct archive_entry_linkresolver *resolver; + struct archive_rb_tree rbtree; int64_t cur_size; char checkfs; @@ -115,7 +117,6 @@ struct mtree { static int bid_keycmp(const char *, const char *, ssize_t); static int cleanup(struct archive_read *); static int detect_form(struct archive_read *, int *); -static unsigned int hash(const char *); static int mtree_bid(struct archive_read *, int); static int parse_file(struct archive_read *, struct archive_entry *, struct mtree *, struct mtree_entry *, int *); @@ -217,9 +218,30 @@ free_options(struct mtree_option *head) } } +static int +mtree_cmp_node(const struct archive_rb_node *n1, + const struct archive_rb_node *n2) +{ + const struct mtree_entry *e1 = (const struct mtree_entry *)n1; + const struct mtree_entry *e2 = (const struct mtree_entry *)n2; + + return (strcmp(e1->name, e2->name)); +} + +static int +mtree_cmp_key(const struct archive_rb_node *n, const void *key) +{ + const struct mtree_entry *e = (const struct mtree_entry *)n; + + return (strcmp(e->name, key)); +} + int archive_read_support_format_mtree(struct archive *_a) { + static const struct archive_rb_tree_ops rb_ops = { + mtree_cmp_node, mtree_cmp_key, + }; struct archive_read *a = (struct archive_read *)_a; struct mtree *mtree; int r; @@ -235,6 +257,8 @@ archive_read_support_format_mtree(struct archive *_a) } mtree->fd = -1; + __archive_rb_tree_init(&mtree->rbtree, &rb_ops); + r = __archive_read_register_format(a, mtree, "mtree", mtree_bid, archive_read_format_mtree_options, read_header, read_data, skip, NULL, cleanup, NULL, NULL); @@ -334,6 +358,14 @@ next_line(struct archive_read *a, size_t nbytes_req = (*ravail+1023) & ~1023U; ssize_t tested; + /* + * Place an arbitrary limit on the line length. + * mtree is almost free-form input and without line length limits, + * it can consume a lot of memory. + */ + if (len >= MAX_LINE_LEN) + return (-1); + /* Increase reading bytes if it is not enough to at least * new two lines. */ if (nbytes_req < (size_t)*ravail + 160) @@ -865,12 +897,11 @@ process_add_entry(struct archive_read *a, struct mtree *mtree, struct mtree_option **global, const char *line, ssize_t line_len, struct mtree_entry **last_entry, int is_form_d) { - struct mtree_entry *entry, *ht_iter; + struct mtree_entry *entry; struct mtree_option *iter; const char *next, *eq, *name, *end; size_t name_len, len; int r, i; - unsigned int ht_idx; if ((entry = malloc(sizeof(*entry))) == NULL) { archive_set_error(&a->archive, errno, "Can't allocate memory"); @@ -881,8 +912,6 @@ process_add_entry(struct archive_read *a, struct mtree *mtree, entry->name = NULL; entry->used = 0; entry->full = 0; - entry->name_hash = 0; - entry->hashtable_next = NULL; /* Add this entry to list. */ if (*last_entry == NULL) @@ -935,15 +964,17 @@ process_add_entry(struct archive_read *a, struct mtree *mtree, memcpy(entry->name, name, name_len); entry->name[name_len] = '\0'; parse_escapes(entry->name, entry); - entry->name_hash = hash(entry->name); - ht_idx = entry->name_hash % MTREE_HASHTABLE_SIZE; - if ((ht_iter = mtree->entry_hashtable[ht_idx]) != NULL) { - while (ht_iter->hashtable_next) - ht_iter = ht_iter->hashtable_next; - ht_iter->hashtable_next = entry; - } else { - mtree->entry_hashtable[ht_idx] = entry; + entry->next_dup = NULL; + if (entry->full) { + if (!__archive_rb_tree_insert_node(&mtree->rbtree, &entry->rbnode)) { + struct mtree_entry *alt; + alt = (struct mtree_entry *)__archive_rb_tree_find_node( + &mtree->rbtree, entry->name); + while (alt->next_dup) + alt = alt->next_dup; + alt->next_dup = entry; + } } for (iter = *global; iter != NULL; iter = iter->next) { @@ -1138,14 +1169,13 @@ parse_file(struct archive_read *a, struct archive_entry *entry, * with pathname canonicalization, which is a very * tricky subject.) */ - for (mp = mentry->hashtable_next; mp != NULL; mp = mp->hashtable_next) { - if (mp->full && !mp->used - && mentry->name_hash == mp->name_hash - && strcmp(mentry->name, mp->name) == 0) { + mp = (struct mtree_entry *)__archive_rb_tree_find_node( + &mtree->rbtree, mentry->name); + for (; mp; mp = mp->next_dup) { + if (mp->full && !mp->used) { /* Later lines override earlier ones. */ mp->used = 1; - r1 = parse_line(a, entry, mtree, mp, - &parsed_kws); + r1 = parse_line(a, entry, mtree, mp, &parsed_kws); if (r1 < r) r = r1; } @@ -1489,6 +1519,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, } if (strcmp(key, "cksum") == 0) break; + __LA_FALLTHROUGH; case 'd': if (strcmp(key, "device") == 0) { /* stat(2) st_rdev field, e.g. the major/minor IDs @@ -1502,12 +1533,14 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_set_rdev(entry, dev); return r; } + __LA_FALLTHROUGH; case 'f': if (strcmp(key, "flags") == 0) { *parsed_kws |= MTREE_HAS_FFLAGS; archive_entry_copy_fflags_text(entry, val); break; } + __LA_FALLTHROUGH; case 'g': if (strcmp(key, "gid") == 0) { *parsed_kws |= MTREE_HAS_GID; @@ -1519,16 +1552,19 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_copy_gname(entry, val); break; } + __LA_FALLTHROUGH; case 'i': if (strcmp(key, "inode") == 0) { archive_entry_set_ino(entry, mtree_atol(&val, 10)); break; } + __LA_FALLTHROUGH; case 'l': if (strcmp(key, "link") == 0) { archive_entry_copy_symlink(entry, val); break; } + __LA_FALLTHROUGH; case 'm': if (strcmp(key, "md5") == 0 || strcmp(key, "md5digest") == 0) break; @@ -1545,6 +1581,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, } break; } + __LA_FALLTHROUGH; case 'n': if (strcmp(key, "nlink") == 0) { *parsed_kws |= MTREE_HAS_NLINK; @@ -1552,6 +1589,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, (unsigned int)mtree_atol(&val, 10)); break; } + __LA_FALLTHROUGH; case 'r': if (strcmp(key, "resdevice") == 0) { /* stat(2) st_dev field, e.g. the device ID where the @@ -1567,6 +1605,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, if (strcmp(key, "rmd160") == 0 || strcmp(key, "rmd160digest") == 0) break; + __LA_FALLTHROUGH; case 's': if (strcmp(key, "sha1") == 0 || strcmp(key, "sha1digest") == 0) break; @@ -1583,6 +1622,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_set_size(entry, mtree_atol(&val, 10)); break; } + __LA_FALLTHROUGH; case 't': if (strcmp(key, "tags") == 0) { /* @@ -1625,18 +1665,21 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_set_filetype(entry, AE_IFBLK); break; } + __LA_FALLTHROUGH; case 'c': if (strcmp(val, "char") == 0) { archive_entry_set_filetype(entry, AE_IFCHR); break; } + __LA_FALLTHROUGH; case 'd': if (strcmp(val, "dir") == 0) { archive_entry_set_filetype(entry, AE_IFDIR); break; } + __LA_FALLTHROUGH; case 'f': if (strcmp(val, "fifo") == 0) { archive_entry_set_filetype(entry, @@ -1648,12 +1691,14 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, AE_IFREG); break; } + __LA_FALLTHROUGH; case 'l': if (strcmp(val, "link") == 0) { archive_entry_set_filetype(entry, AE_IFLNK); break; } + __LA_FALLTHROUGH; default: archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, @@ -1665,6 +1710,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, *parsed_kws |= MTREE_HAS_TYPE; break; } + __LA_FALLTHROUGH; case 'u': if (strcmp(key, "uid") == 0) { *parsed_kws |= MTREE_HAS_UID; @@ -1676,6 +1722,7 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, archive_entry_copy_uname(entry, val); break; } + __LA_FALLTHROUGH; default: archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, "Unrecognized key %s=%s", key, val); @@ -1962,19 +2009,3 @@ readline(struct archive_read *a, struct mtree *mtree, char **start, find_off = u - mtree->line.s; } } - -static unsigned int -hash(const char *p) -{ - /* A 32-bit version of Peter Weinberger's (PJW) hash algorithm, - as used by ELF for hashing function names. */ - unsigned g, h = 0; - while (*p != '\0') { - h = (h << 4) + *p++; - if ((g = h & 0xF0000000) != 0) { - h ^= g >> 24; - h &= 0x0FFFFFFF; - } - } - return h; -} diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c index a0e641e35..c68d77b49 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c @@ -604,20 +604,6 @@ lzss_emit_match(struct rar *rar, int offset, int length) rar->lzss.position += length; } -static void * -ppmd_alloc(void *p, size_t size) -{ - (void)p; - return malloc(size); -} -static void -ppmd_free(void *p, void *address) -{ - (void)p; - free(address); -} -static ISzAlloc g_szalloc = { ppmd_alloc, ppmd_free }; - static Byte ppmd_read(void *p) { @@ -1038,7 +1024,7 @@ archive_read_format_rar_read_data(struct archive_read *a, const void **buff, case COMPRESS_METHOD_BEST: ret = read_data_compressed(a, buff, size, offset); if (ret != ARCHIVE_OK && ret != ARCHIVE_WARN) - __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context); break; default: @@ -1253,7 +1239,7 @@ archive_read_format_rar_cleanup(struct archive_read *a) free(rar->dbo); free(rar->unp_buffer); free(rar->lzss.window); - __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context); free(rar); (a->format->data) = NULL; return (ARCHIVE_OK); @@ -1496,7 +1482,11 @@ read_header(struct archive_read *a, struct archive_entry *entry, return (ARCHIVE_FATAL); } filename[filename_size++] = '\0'; - filename[filename_size++] = '\0'; + /* + * Do not increment filename_size here as the computations below + * add the space for the terminating NUL explicitly. + */ + filename[filename_size] = '\0'; /* Decoded unicode form is UTF-16BE, so we have to update a string * conversion object for it. */ @@ -1654,7 +1644,7 @@ read_header(struct archive_read *a, struct archive_entry *entry, rar->unp_offset = 0; rar->unp_buffer_size = UNP_BUFFER_SIZE; memset(rar->lengthtable, 0, sizeof(rar->lengthtable)); - __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context); rar->ppmd_valid = rar->ppmd_eod = 0; /* Don't set any archive entries for non-file header types */ @@ -2118,7 +2108,7 @@ parse_codes(struct archive_read *a) /* Make sure ppmd7_contest is freed before Ppmd7_Construct * because reading a broken file cause this abnormal sequence. */ - __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&rar->ppmd7_context); rar->bytein.a = a; rar->bytein.Read = &ppmd_read; @@ -2133,7 +2123,7 @@ parse_codes(struct archive_read *a) } if (!__archive_ppmd7_functions.Ppmd7_Alloc(&rar->ppmd7_context, - rar->dictionary_size, &g_szalloc)) + rar->dictionary_size)) { archive_set_error(&a->archive, ENOMEM, "Out of memory"); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c index 30d5bc83c..60800bb81 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c @@ -251,15 +251,15 @@ archive_read_support_format_tar(struct archive *_a) ARCHIVE_STATE_NEW, "archive_read_support_format_tar"); tar = (struct tar *)calloc(1, sizeof(*tar)); -#ifdef HAVE_COPYFILE_H - /* Set this by default on Mac OS. */ - tar->process_mac_extensions = 1; -#endif if (tar == NULL) { archive_set_error(&a->archive, ENOMEM, "Can't allocate tar data"); return (ARCHIVE_FATAL); } +#ifdef HAVE_COPYFILE_H + /* Set this by default on Mac OS. */ + tar->process_mac_extensions = 1; +#endif r = __archive_read_register_format(a, tar, "tar", archive_read_format_tar_bid, @@ -2241,7 +2241,7 @@ gnu_add_sparse_entry(struct archive_read *a, struct tar *tar, else tar->sparse_list = p; tar->sparse_last = p; - if (remaining < 0 || offset < 0) { + if (remaining < 0 || offset < 0 || offset > INT64_MAX - remaining) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Malformed sparse map data"); return (ARCHIVE_FATAL); } diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c index eeac1c888..9292ed784 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c @@ -1040,6 +1040,9 @@ atol10(const char *p, size_t char_cnt) uint64_t l; int digit; + if (char_cnt == 0) + return (0); + l = 0; digit = *p - '0'; while (digit >= 0 && digit < 10 && char_cnt-- > 0) { @@ -1054,7 +1057,10 @@ atol8(const char *p, size_t char_cnt) { int64_t l; int digit; - + + if (char_cnt == 0) + return (0); + l = 0; while (char_cnt-- > 0) { if (*p >= '0' && *p <= '7') @@ -2623,6 +2629,14 @@ strappend_base64(struct xar *xar, archive_strncat(as, (const char *)buff, len); } +static int +is_string(const char *known, const char *data, size_t len) +{ + if (strlen(known) != len) + return -1; + return memcmp(data, known, len); +} + static void xml_data(void *userData, const char *s, int len) { @@ -2674,26 +2688,26 @@ xml_data(void *userData, const char *s, int len) archive_strncpy(&(xar->file->symlink), s, len); break; case FILE_TYPE: - if (strncmp("file", s, len) == 0 || - strncmp("hardlink", s, len) == 0) + if (is_string("file", s, len) == 0 || + is_string("hardlink", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFREG; - if (strncmp("directory", s, len) == 0) + if (is_string("directory", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFDIR; - if (strncmp("symlink", s, len) == 0) + if (is_string("symlink", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFLNK; - if (strncmp("character special", s, len) == 0) + if (is_string("character special", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFCHR; - if (strncmp("block special", s, len) == 0) + if (is_string("block special", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFBLK; - if (strncmp("socket", s, len) == 0) + if (is_string("socket", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFSOCK; - if (strncmp("fifo", s, len) == 0) + if (is_string("fifo", s, len) == 0) xar->file->mode = (xar->file->mode & ~AE_IFMT) | AE_IFIFO; xar->file->has |= HAS_TYPE; diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c index ddd445833..7e99b1275 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c @@ -511,7 +511,13 @@ process_extra(struct archive_read *a, const char *p, size_t extra_length, struct case 0x5455: { /* Extended time field "UT". */ - int flags = p[offset]; + int flags; + if (datasize == 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Incomplete extended time field"); + return ARCHIVE_FAILED; + } + flags = p[offset]; offset++; datasize--; /* Flag bits indicate which dates are present. */ @@ -723,6 +729,11 @@ process_extra(struct archive_read *a, const char *p, size_t extra_length, struct } case 0x9901: /* WinZip AES extra data field. */ + if (datasize < 6) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Incomplete AES field"); + return ARCHIVE_FAILED; + } if (p[offset + 2] == 'A' && p[offset + 3] == 'E') { /* Vendor version. */ zip_entry->aes_extra.vendor = @@ -881,6 +892,24 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, zip_entry->mode |= 0664; } + /* Windows archivers sometimes use backslash as the directory separator. + Normalize to slash. */ + if (zip_entry->system == 0 && + (wp = archive_entry_pathname_w(entry)) != NULL) { + if (wcschr(wp, L'/') == NULL && wcschr(wp, L'\\') != NULL) { + size_t i; + struct archive_wstring s; + archive_string_init(&s); + archive_wstrcpy(&s, wp); + for (i = 0; i < archive_strlen(&s); i++) { + if (s.s[i] == '\\') + s.s[i] = '/'; + } + archive_entry_copy_pathname_w(entry, s.s); + archive_wstring_free(&s); + } + } + /* Make sure that entries with a trailing '/' are marked as directories * even if the External File Attributes contains bogus values. If this * is not a directory and there is no type, assume regularfile. */ @@ -1056,6 +1085,7 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, zip->end_of_entry = 1; /* Set up a more descriptive format name. */ + archive_string_empty(&zip->format_name); archive_string_sprintf(&zip->format_name, "ZIP %d.%d (%s)", version / 10, version % 10, compression_name(zip->entry->compression)); diff --git a/Utilities/cmlibarchive/libarchive/archive_string.c b/Utilities/cmlibarchive/libarchive/archive_string.c index 5ae09b626..554533ecb 100644 --- a/Utilities/cmlibarchive/libarchive/archive_string.c +++ b/Utilities/cmlibarchive/libarchive/archive_string.c @@ -214,7 +214,8 @@ archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) { if (archive_wstring_ensure(as, as->length + s + 1) == NULL) return (NULL); - wmemmove(as->s + as->length, p, s); + if (s) + wmemmove(as->s + as->length, p, s); as->length += s; as->s[as->length] = 0; return (as); diff --git a/Utilities/cmlibarchive/libarchive/archive_util.c b/Utilities/cmlibarchive/libarchive/archive_util.c index 1e36ad7ee..e5c6e3bff 100644 --- a/Utilities/cmlibarchive/libarchive/archive_util.c +++ b/Utilities/cmlibarchive/libarchive/archive_util.c @@ -140,7 +140,7 @@ archive_compression_name(struct archive *a) /* * Return a count of the number of compressed bytes processed. */ -int64_t +la_int64_t archive_position_compressed(struct archive *a) { return archive_filter_bytes(a, -1); @@ -149,7 +149,7 @@ archive_position_compressed(struct archive *a) /* * Return a count of the number of uncompressed bytes processed. */ -int64_t +la_int64_t archive_position_uncompressed(struct archive *a) { return archive_filter_bytes(a, 0); diff --git a/Utilities/cmlibarchive/libarchive/archive_version_details.c b/Utilities/cmlibarchive/libarchive/archive_version_details.c index 9289bf1ac..e773e5e45 100644 --- a/Utilities/cmlibarchive/libarchive/archive_version_details.c +++ b/Utilities/cmlibarchive/libarchive/archive_version_details.c @@ -45,6 +45,9 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_util.c 201098 2009-12-28 02:58:1 #ifdef HAVE_LZ4_H #include <lz4.h> #endif +#ifdef HAVE_ZSTD_H +#include <zstd.h> +#endif #include "archive.h" #include "archive_private.h" @@ -59,6 +62,7 @@ archive_version_details(void) const char *liblzma = archive_liblzma_version(); const char *bzlib = archive_bzlib_version(); const char *liblz4 = archive_liblz4_version(); + const char *libzstd = archive_libzstd_version(); if (!init) { archive_string_init(&str); @@ -84,6 +88,10 @@ archive_version_details(void) archive_strcat(&str, " liblz4/"); archive_strcat(&str, liblz4); } + if (libzstd) { + archive_strcat(&str, " libzstd/"); + archive_strcat(&str, libzstd); + } } return str.s; } @@ -131,3 +139,13 @@ archive_liblz4_version(void) return NULL; #endif } + +const char * +archive_libzstd_version(void) +{ +#if HAVE_ZSTD_H && HAVE_LIBZSTD + return ZSTD_VERSION_STRING; +#else + return NULL; +#endif +} diff --git a/Utilities/cmlibarchive/libarchive/archive_virtual.c b/Utilities/cmlibarchive/libarchive/archive_virtual.c index de2595a9e..f509ee5c6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_virtual.c +++ b/Utilities/cmlibarchive/libarchive/archive_virtual.c @@ -48,7 +48,7 @@ archive_filter_name(struct archive *a, int n) return ((a->vtable->archive_filter_name)(a, n)); } -int64_t +la_int64_t archive_filter_bytes(struct archive *a, int n) { return ((a->vtable->archive_filter_bytes)(a, n)); @@ -124,14 +124,15 @@ archive_write_finish_entry(struct archive *a) return ((a->vtable->archive_write_finish_entry)(a)); } -ssize_t +la_ssize_t archive_write_data(struct archive *a, const void *buff, size_t s) { return ((a->vtable->archive_write_data)(a, buff, s)); } -ssize_t -archive_write_data_block(struct archive *a, const void *buff, size_t s, int64_t o) +la_ssize_t +archive_write_data_block(struct archive *a, const void *buff, size_t s, + la_int64_t o) { if (a->vtable->archive_write_data_block == NULL) { archive_set_error(a, ARCHIVE_ERRNO_MISC, @@ -156,7 +157,7 @@ archive_read_next_header2(struct archive *a, struct archive_entry *entry) int archive_read_data_block(struct archive *a, - const void **buff, size_t *s, int64_t *o) + const void **buff, size_t *s, la_int64_t *o) { return ((a->vtable->archive_read_data_block)(a, buff, s, o)); } diff --git a/Utilities/cmlibarchive/libarchive/archive_write.3 b/Utilities/cmlibarchive/libarchive/archive_write.3 index 376d71dee..c1164f5b5 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write.3 @@ -71,7 +71,7 @@ support. .\" .Ss Set options See -.Xr archive_read_set_options 3 . +.Xr archive_write_set_options 3 . .\" .Ss Open archive See diff --git a/Utilities/cmlibarchive/libarchive/archive_write.c b/Utilities/cmlibarchive/libarchive/archive_write.c index 0634a2296..e8daf530d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write.c +++ b/Utilities/cmlibarchive/libarchive/archive_write.c @@ -190,7 +190,7 @@ archive_write_get_bytes_in_last_block(struct archive *_a) * an archive to itself recursively. */ int -archive_write_set_skip_file(struct archive *_a, int64_t d, int64_t i) +archive_write_set_skip_file(struct archive *_a, la_int64_t d, la_int64_t i) { struct archive_write *a = (struct archive_write *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c index 08f518ade..203f4142b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c @@ -53,6 +53,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] = { ARCHIVE_FILTER_LZOP, archive_write_add_filter_lzip }, { ARCHIVE_FILTER_UU, archive_write_add_filter_uuencode }, { ARCHIVE_FILTER_XZ, archive_write_add_filter_xz }, + { ARCHIVE_FILTER_ZSTD, archive_write_add_filter_zstd }, { -1, NULL } }; diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c index 85a8d4753..ffa633c96 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c @@ -57,6 +57,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] = { "lzop", archive_write_add_filter_lzop }, { "uuencode", archive_write_add_filter_uuencode }, { "xz", archive_write_add_filter_xz }, + { "zstd", archive_write_add_filter_zstd }, { NULL, NULL } }; diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_gzip.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_gzip.c index bbcc1789a..986123ad8 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_gzip.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_gzip.c @@ -226,7 +226,12 @@ archive_compressor_gzip_open(struct archive_write_filter *f) data->compressed[7] = (uint8_t)(t>>24)&0xff; } else memset(&data->compressed[4], 0, 4); - data->compressed[8] = 0; /* No deflate options */ + if (data->compression_level == 9) + data->compressed[8] = 2; + else if(data->compression_level == 1) + data->compressed[8] = 4; + else + data->compressed[8] = 0; data->compressed[9] = 3; /* OS=Unix */ data->stream.next_out += 10; data->stream.avail_out -= 10; diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_zstd.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_zstd.c new file mode 100644 index 000000000..671fc6aff --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_zstd.c @@ -0,0 +1,335 @@ +/*- + * Copyright (c) 2017 Sean Purcell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_ZSTD_H +#include <zstd.h> +#endif + +#include "archive.h" +#include "archive_private.h" +#include "archive_string.h" +#include "archive_write_private.h" + +/* Don't compile this if we don't have zstd.h */ + +struct private_data { + int compression_level; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + ZSTD_CStream *cstream; + int64_t total_in; + ZSTD_outBuffer out; +#else + struct archive_write_program_data *pdata; +#endif +}; + +static int archive_compressor_zstd_options(struct archive_write_filter *, + const char *, const char *); +static int archive_compressor_zstd_open(struct archive_write_filter *); +static int archive_compressor_zstd_write(struct archive_write_filter *, + const void *, size_t); +static int archive_compressor_zstd_close(struct archive_write_filter *); +static int archive_compressor_zstd_free(struct archive_write_filter *); +#if HAVE_ZSTD_H && HAVE_LIBZSTD +static int drive_compressor(struct archive_write_filter *, + struct private_data *, int, const void *, size_t); +#endif + + +/* + * Add a zstd compression filter to this write handle. + */ +int +archive_write_add_filter_zstd(struct archive *_a) +{ + struct archive_write *a = (struct archive_write *)_a; + struct archive_write_filter *f = __archive_write_allocate_filter(_a); + struct private_data *data; + archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, + ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd"); + + data = calloc(1, sizeof(*data)); + if (data == NULL) { + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + f->data = data; + f->open = &archive_compressor_zstd_open; + f->options = &archive_compressor_zstd_options; + f->close = &archive_compressor_zstd_close; + f->free = &archive_compressor_zstd_free; + f->code = ARCHIVE_FILTER_ZSTD; + f->name = "zstd"; + data->compression_level = 3; /* Default level used by the zstd CLI */ +#if HAVE_ZSTD_H && HAVE_LIBZSTD + data->cstream = ZSTD_createCStream(); + if (data->cstream == NULL) { + free(data); + archive_set_error(&a->archive, ENOMEM, + "Failed to allocate zstd compressor object"); + return (ARCHIVE_FATAL); + } + + return (ARCHIVE_OK); +#else + data->pdata = __archive_write_program_allocate("zstd"); + if (data->pdata == NULL) { + free(data); + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Using external zstd program"); + return (ARCHIVE_WARN); +#endif +} + +static int +archive_compressor_zstd_free(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; +#if HAVE_ZSTD_H && HAVE_LIBZSTD + ZSTD_freeCStream(data->cstream); + free(data->out.dst); +#else + __archive_write_program_free(data->pdata); +#endif + free(data); + f->data = NULL; + return (ARCHIVE_OK); +} + +/* + * Set write options. + */ +static int +archive_compressor_zstd_options(struct archive_write_filter *f, const char *key, + const char *value) +{ + struct private_data *data = (struct private_data *)f->data; + + if (strcmp(key, "compression-level") == 0) { + int level = atoi(value); +#if HAVE_ZSTD_H && HAVE_LIBZSTD + if (level < 1 || level > ZSTD_maxCLevel()) { +#else + /* If we don't have the library, hard-code the max level */ + if (level < 1 || level > 22) { +#endif + return (ARCHIVE_WARN); + } + data->compression_level = level; + return (ARCHIVE_OK); + } + + /* Note: The "warn" return is just to inform the options + * supervisor that we didn't handle it. It will generate + * a suitable error if no one used this option. */ + return (ARCHIVE_WARN); +} + +#if HAVE_ZSTD_H && HAVE_LIBZSTD +/* + * Setup callback. + */ +static int +archive_compressor_zstd_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int ret; + + ret = __archive_write_open_filter(f->next_filter); + if (ret != ARCHIVE_OK) + return (ret); + + if (data->out.dst == NULL) { + size_t bs = ZSTD_CStreamOutSize(), bpb; + if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { + /* Buffer size should be a multiple number of + * the of bytes per block for performance. */ + bpb = archive_write_get_bytes_per_block(f->archive); + if (bpb > bs) + bs = bpb; + else if (bpb != 0) + bs -= bs % bpb; + } + data->out.size = bs; + data->out.pos = 0; + data->out.dst + = (unsigned char *)malloc(data->out.size); + if (data->out.dst == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression buffer"); + return (ARCHIVE_FATAL); + } + } + + f->write = archive_compressor_zstd_write; + + if (ZSTD_isError(ZSTD_initCStream(data->cstream, + data->compression_level))) { + archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, + "Internal error initializing zstd compressor object"); + return (ARCHIVE_FATAL); + } + + return (ARCHIVE_OK); +} + +/* + * Write data to the compressed stream. + */ +static int +archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + int ret; + + /* Update statistics */ + data->total_in += length; + + if ((ret = drive_compressor(f, data, 0, buff, length)) != ARCHIVE_OK) + return (ret); + + return (ARCHIVE_OK); +} + +/* + * Finish the compression... + */ +static int +archive_compressor_zstd_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int r1, r2; + + /* Finish zstd frame */ + r1 = drive_compressor(f, data, 1, NULL, 0); + + r2 = __archive_write_close_filter(f->next_filter); + + return r1 < r2 ? r1 : r2; +} + +/* + * Utility function to push input data through compressor, + * writing full output blocks as necessary. + * + * Note that this handles both the regular write case (finishing == + * false) and the end-of-archive case (finishing == true). + */ +static int +drive_compressor(struct archive_write_filter *f, + struct private_data *data, int finishing, const void *src, size_t length) +{ + ZSTD_inBuffer in = (ZSTD_inBuffer) { src, length, 0 }; + + for (;;) { + if (data->out.pos == data->out.size) { + const int ret = __archive_write_filter(f->next_filter, + data->out.dst, data->out.size); + if (ret != ARCHIVE_OK) + return (ARCHIVE_FATAL); + data->out.pos = 0; + } + + /* If there's nothing to do, we're done. */ + if (!finishing && in.pos == in.size) + return (ARCHIVE_OK); + + { + const size_t zstdret = !finishing ? + ZSTD_compressStream(data->cstream, &data->out, &in) + : ZSTD_endStream(data->cstream, &data->out); + + if (ZSTD_isError(zstdret)) { + archive_set_error(f->archive, + ARCHIVE_ERRNO_MISC, + "Zstd compression failed: %s", + ZSTD_getErrorName(zstdret)); + return (ARCHIVE_FATAL); + } + + /* If we're finishing, 0 means nothing left to flush */ + if (finishing && zstdret == 0) { + const int ret = __archive_write_filter(f->next_filter, + data->out.dst, data->out.pos); + return (ret); + } + } + } +} + +#else /* HAVE_ZSTD_H && HAVE_LIBZSTD */ + +static int +archive_compressor_zstd_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + struct archive_string as; + int r; + + archive_string_init(&as); + archive_string_sprintf(&as, "zstd -%d", data->compression_level); + + f->write = archive_compressor_zstd_write; + r = __archive_write_program_open(f, data->pdata, as.s); + archive_string_free(&as); + return (r); +} + +static int +archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_write(f, data->pdata, buff, length); +} + +static int +archive_compressor_zstd_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_close(f, data->pdata); +} + +#endif /* HAVE_ZSTD_H && HAVE_LIBZSTD */ diff --git a/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c b/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c index 4a42a3beb..affa50381 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c @@ -835,7 +835,7 @@ _archive_write_disk_header(struct archive *_a, struct archive_entry *entry) } int -archive_write_disk_set_skip_file(struct archive *_a, int64_t d, int64_t i) +archive_write_disk_set_skip_file(struct archive *_a, la_int64_t d, la_int64_t i) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1786,7 +1786,7 @@ finish_metadata: int archive_write_disk_set_group_lookup(struct archive *_a, void *private_data, - int64_t (*lookup_gid)(void *private, const char *gname, int64_t gid), + la_int64_t (*lookup_gid)(void *private, const char *gname, la_int64_t gid), void (*cleanup_gid)(void *private)) { struct archive_write_disk *a = (struct archive_write_disk *)_a; @@ -1822,7 +1822,7 @@ archive_write_disk_set_user_lookup(struct archive *_a, } int64_t -archive_write_disk_gid(struct archive *_a, const char *name, int64_t id) +archive_write_disk_gid(struct archive *_a, const char *name, la_int64_t id) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1833,7 +1833,7 @@ archive_write_disk_gid(struct archive *_a, const char *name, int64_t id) } int64_t -archive_write_disk_uid(struct archive *_a, const char *name, int64_t id) +archive_write_disk_uid(struct archive *_a, const char *name, la_int64_t id) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1981,6 +1981,10 @@ restore_entry(struct archive_write_disk *a) if ((en == EISDIR || en == EEXIST) && (a->flags & ARCHIVE_EXTRACT_NO_OVERWRITE)) { /* If we're not overwriting, we're done. */ + if (S_ISDIR(a->mode)) { + /* Don't overwrite any settings on existing directories. */ + a->todo = 0; + } archive_entry_unset_size(a->entry); return (ARCHIVE_OK); } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c b/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c index 94b016edf..78eda4abc 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c @@ -906,7 +906,7 @@ _archive_write_disk_header(struct archive *_a, struct archive_entry *entry) } int -archive_write_disk_set_skip_file(struct archive *_a, int64_t d, int64_t i) +archive_write_disk_set_skip_file(struct archive *_a, la_int64_t d, la_int64_t i) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1156,7 +1156,7 @@ _archive_write_disk_finish_entry(struct archive *_a) int archive_write_disk_set_group_lookup(struct archive *_a, void *private_data, - int64_t (*lookup_gid)(void *private, const char *gname, int64_t gid), + la_int64_t (*lookup_gid)(void *private, const char *gname, la_int64_t gid), void (*cleanup_gid)(void *private)) { struct archive_write_disk *a = (struct archive_write_disk *)_a; @@ -1192,7 +1192,7 @@ archive_write_disk_set_user_lookup(struct archive *_a, } int64_t -archive_write_disk_gid(struct archive *_a, const char *name, int64_t id) +archive_write_disk_gid(struct archive *_a, const char *name, la_int64_t id) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1203,7 +1203,7 @@ archive_write_disk_gid(struct archive *_a, const char *name, int64_t id) } int64_t -archive_write_disk_uid(struct archive *_a, const char *name, int64_t id) +archive_write_disk_uid(struct archive *_a, const char *name, la_int64_t id) { struct archive_write_disk *a = (struct archive_write_disk *)_a; archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, @@ -1322,9 +1322,20 @@ restore_entry(struct archive_write_disk *a) } } + if ((en == ENOENT) && (archive_entry_hardlink(a->entry) != NULL)) { + archive_set_error(&a->archive, en, + "Hard-link target '%s' does not exist.", + archive_entry_hardlink(a->entry)); + return (ARCHIVE_FAILED); + } + if ((en == EISDIR || en == EEXIST) && (a->flags & ARCHIVE_EXTRACT_NO_OVERWRITE)) { /* If we're not overwriting, we're done. */ + if (S_ISDIR(a->mode)) { + /* Don't overwrite any settings on existing directories. */ + a->todo = 0; + } archive_entry_unset_size(a->entry); return (ARCHIVE_OK); } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_filter.3 b/Utilities/cmlibarchive/libarchive/archive_write_filter.3 index e1d189150..d6fa07131 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_filter.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_filter.3 @@ -42,7 +42,8 @@ .Nm archive_write_add_filter_none , .Nm archive_write_add_filter_program , .Nm archive_write_add_filter_uuencode , -.Nm archive_write_add_filter_xz +.Nm archive_write_add_filter_xz , +.Nm archive_write_add_filter_zstd , .Nd functions enabling output filters .Sh LIBRARY Streaming Archive Library (libarchive, -larchive) @@ -76,6 +77,8 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_write_add_filter_uuencode "struct archive *" .Ft int .Fn archive_write_add_filter_xz "struct archive *" +.Ft int +.Fn archive_write_add_filter_zstd "struct archive *" .Sh DESCRIPTION .Bl -tag -width indent .It Xo @@ -89,6 +92,7 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_write_add_filter_lzma , .Fn archive_write_add_filter_lzop , .Fn archive_write_add_filter_xz , +.Fn archive_write_add_filter_zstd , .Xc The resulting archive will be compressed as specified. Note that the compressed output is always properly blocked. diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_7zip.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_7zip.c index 3fc5a07d7..2bd4ec41f 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_7zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_7zip.c @@ -2095,19 +2095,6 @@ compression_init_encoder_lzma2(struct archive *a, /* * _7_PPMD compressor. */ -static void * -ppmd_alloc(void *p, size_t size) -{ - (void)p; - return malloc(size); -} -static void -ppmd_free(void *p, void *address) -{ - (void)p; - free(address); -} -static ISzAlloc g_szalloc = { ppmd_alloc, ppmd_free }; static void ppmd_write(void *p, Byte b) { @@ -2167,7 +2154,7 @@ compression_init_encoder_ppmd(struct archive *a, archive_le32enc(props+1, msize); __archive_ppmd7_functions.Ppmd7_Construct(&strm->ppmd7_context); r = __archive_ppmd7_functions.Ppmd7_Alloc( - &strm->ppmd7_context, msize, &g_szalloc); + &strm->ppmd7_context, msize); if (r == 0) { free(strm->buff); free(strm); @@ -2243,7 +2230,7 @@ compression_end_ppmd(struct archive *a, struct la_zstream *lastrm) (void)a; /* UNUSED */ strm = (struct ppmd_stream *)lastrm->real_stream; - __archive_ppmd7_functions.Ppmd7_Free(&strm->ppmd7_context, &g_szalloc); + __archive_ppmd7_functions.Ppmd7_Free(&strm->ppmd7_context); free(strm->buff); free(strm); lastrm->real_stream = NULL; diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_ar.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_ar.c index c9771d81a..50305ccbe 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_ar.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_ar.c @@ -180,7 +180,7 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) } memset(buff, ' ', 60); - strncpy(&buff[AR_fmag_offset], "`\n", 2); + memcpy(&buff[AR_fmag_offset], "`\n", 2); if (strcmp(pathname, "/") == 0 ) { /* Entry is archive symbol table in GNU format */ @@ -189,7 +189,7 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) } if (strcmp(pathname, "__.SYMDEF") == 0) { /* Entry is archive symbol table in BSD format */ - strncpy(buff + AR_name_offset, "__.SYMDEF", 9); + memcpy(buff + AR_name_offset, "__.SYMDEF", 9); goto stat; } if (strcmp(pathname, "//") == 0) { @@ -225,7 +225,7 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) * actually 15 bytes. */ if (strlen(filename) <= 15) { - strncpy(&buff[AR_name_offset], + memcpy(&buff[AR_name_offset], filename, strlen(filename)); buff[AR_name_offset + strlen(filename)] = '/'; } else { @@ -248,7 +248,7 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) return (ARCHIVE_FATAL); } - strncpy(se, filename, strlen(filename)); + memcpy(se, filename, strlen(filename)); strcpy(se + strlen(filename), "/\n"); ss = strstr(ar->strtab, se); @@ -285,11 +285,11 @@ archive_write_ar_header(struct archive_write *a, struct archive_entry *entry) * archive header. */ if (strlen(filename) <= 16 && strchr(filename, ' ') == NULL) { - strncpy(&buff[AR_name_offset], filename, strlen(filename)); + memcpy(&buff[AR_name_offset], filename, strlen(filename)); buff[AR_name_offset + strlen(filename)] = ' '; } else { - strncpy(buff + AR_name_offset, "#1/", 3); + memcpy(buff + AR_name_offset, "#1/", 3); if (format_decimal(strlen(filename), buff + AR_name_offset + 3, AR_name_size - 3)) { @@ -374,13 +374,14 @@ archive_write_ar_data(struct archive_write *a, const void *buff, size_t s) return (ARCHIVE_WARN); } - ar->strtab = (char *)malloc(s); + ar->strtab = (char *)malloc(s + 1); if (ar->strtab == NULL) { archive_set_error(&a->archive, ENOMEM, "Can't allocate strtab buffer"); return (ARCHIVE_FATAL); } - strncpy(ar->strtab, buff, s); + memcpy(ar->strtab, buff, s); + ar->strtab[s] = '\0'; ar->has_strtab = 1; } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_pax.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_pax.c index 0eaf733cd..3cebeae18 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_pax.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_pax.c @@ -1654,7 +1654,7 @@ build_pax_attribute_name(char *dest, const char *src) * GNU PAX Format 1.0 requires the special name, which pattern is: * <dir>/GNUSparseFile.<pid>/<original file name> * - * Since reproducable archives are more important, use 0 as pid. + * Since reproducible archives are more important, use 0 as pid. * * This function is used for only Sparse file, a file type of which * is regular file. diff --git a/Utilities/cmliblzma/CMakeLists.txt b/Utilities/cmliblzma/CMakeLists.txt index bb3b8a71f..e9f88266d 100644 --- a/Utilities/cmliblzma/CMakeLists.txt +++ b/Utilities/cmliblzma/CMakeLists.txt @@ -6,22 +6,6 @@ include(CheckSymbolExists) include(CheckTypeSize) include(TestBigEndian) -CHECK_C_SOURCE_COMPILES( - "int test (void *restrict x);\nint main (void) {return 0;}" - HAVE_RESTRICT) - -CHECK_C_SOURCE_COMPILES( -"typedef struct abc *d;\nint test (d __restrict x);\nint main (void) {return 0;}" - HAVE___RESTRICT) - -CHECK_C_SOURCE_COMPILES( - "static inline int test (void) {return 0;}\nint main (void) {return test();}" - HAVE_INLINE) - -CHECK_C_SOURCE_COMPILES ( - "static __inline int test (void) {return 0;}\nint main (void) {return test();}" - HAVE___INLINE) - CHECK_INCLUDE_FILE(byteswap.h HAVE_BYTESWAP_H) CHECK_INCLUDE_FILE(inttypes.h HAVE_INTTYPES_H) CHECK_INCLUDE_FILE(limits.h HAVE_LIMITS_H) @@ -95,7 +79,7 @@ CHECK_TYPE_SIZE("unsigned short" SIZE_OF_UNSIGNED_SHORT) CHECK_TYPE_SIZE("unsigned" SIZE_OF_UNSIGNED) CHECK_TYPE_SIZE("unsigned long" SIZE_OF_UNSIGNED_LONG) CHECK_TYPE_SIZE("unsigned long long" SIZE_OF_UNSIGNED_LONG_LONG) -CHECK_TYPE_SIZE("size_t" SIZE_OF_SIZE_T) +CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T) CHECK_TYPE_SIZE("__int64" __INT64) CHECK_TYPE_SIZE("unsigned __int64" UNSIGNED___INT64) diff --git a/Utilities/cmliblzma/COPYING b/Utilities/cmliblzma/COPYING index 43c90d059..20e60d5b2 100644 --- a/Utilities/cmliblzma/COPYING +++ b/Utilities/cmliblzma/COPYING @@ -47,7 +47,7 @@ XZ Utils Licensing naturally it is not legally required. Here is an example of a good notice to put into "about box" or into documentation: - This software includes code from XZ Utils <http://tukaani.org/xz/>. + This software includes code from XZ Utils <https://tukaani.org/xz/>. The following license texts are included in the following files: - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1 diff --git a/Utilities/cmliblzma/common/common_w32res.rc b/Utilities/cmliblzma/common/common_w32res.rc index fdb88d184..a70de3432 100644 --- a/Utilities/cmliblzma/common/common_w32res.rc +++ b/Utilities/cmliblzma/common/common_w32res.rc @@ -17,7 +17,7 @@ #define MY_VERSION LZMA_VERSION_MAJOR,LZMA_VERSION_MINOR,LZMA_VERSION_PATCH,MY_BUILD #define MY_FILENAME MY_NAME MY_SUFFIX -#define MY_COMPANY "The Tukaani Project <http://tukaani.org/>" +#define MY_COMPANY "The Tukaani Project <https://tukaani.org/>" #define MY_PRODUCT PACKAGE_NAME " <" PACKAGE_URL ">" LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US diff --git a/Utilities/cmliblzma/common/sysdefs.h b/Utilities/cmliblzma/common/sysdefs.h index a6edea8b0..22f487b9e 100644 --- a/Utilities/cmliblzma/common/sysdefs.h +++ b/Utilities/cmliblzma/common/sysdefs.h @@ -18,6 +18,7 @@ #if defined(_MSC_VER) # pragma warning(push,1) +# pragma warning(disable: 4028) /* formal parameter different from decl */ # pragma warning(disable: 4142) /* benign redefinition of type */ # pragma warning(disable: 4761) /* integral size mismatch in argument */ #endif @@ -124,9 +125,9 @@ // The code currently assumes that size_t is either 32-bit or 64-bit. #ifndef SIZE_MAX -# if SIZE_OF_SIZE_T == 4 +# if SIZEOF_SIZE_T == 4 # define SIZE_MAX UINT32_MAX -# elif SIZE_OF_SIZE_T == 8 +# elif SIZEOF_SIZE_T == 8 # define SIZE_MAX UINT64_MAX # else # error size_t is not 32-bit or 64-bit @@ -175,6 +176,16 @@ typedef unsigned char _Bool; # include <memory.h> #endif +// As of MSVC 2013, inline and restrict are supported with +// non-standard keywords. +#if defined(_WIN32) && defined(_MSC_VER) +# ifndef inline +# define inline __inline +# endif +# ifndef restrict +# define restrict __restrict +# endif +#endif //////////// // Macros // diff --git a/Utilities/cmliblzma/common/tuklib_integer.h b/Utilities/cmliblzma/common/tuklib_integer.h index 5e8262a11..5d8268580 100644 --- a/Utilities/cmliblzma/common/tuklib_integer.h +++ b/Utilities/cmliblzma/common/tuklib_integer.h @@ -106,6 +106,17 @@ #endif +//////////////////////////////// +// Compiler-specific features // +//////////////////////////////// + +// Newer Intel C compilers require immintrin.h for _bit_scan_reverse() +// and such functions. +#if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1500) +# include <immintrin.h> +#endif + + /////////////////// // Byte swapping // /////////////////// @@ -329,8 +340,8 @@ unaligned_read32le(const uint8_t *buf) static inline void unaligned_write16be(uint8_t *buf, uint16_t num) { - buf[0] = num >> 8; - buf[1] = num; + buf[0] = (uint8_t)(num >> 8); + buf[1] = (uint8_t)num; return; } @@ -338,8 +349,8 @@ unaligned_write16be(uint8_t *buf, uint16_t num) static inline void unaligned_write16le(uint8_t *buf, uint16_t num) { - buf[0] = num; - buf[1] = num >> 8; + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); return; } @@ -347,10 +358,10 @@ unaligned_write16le(uint8_t *buf, uint16_t num) static inline void unaligned_write32be(uint8_t *buf, uint32_t num) { - buf[0] = num >> 24; - buf[1] = num >> 16; - buf[2] = num >> 8; - buf[3] = num; + buf[0] = (uint8_t)(num >> 24); + buf[1] = (uint8_t)(num >> 16); + buf[2] = (uint8_t)(num >> 8); + buf[3] = (uint8_t)num; return; } @@ -358,10 +369,10 @@ unaligned_write32be(uint8_t *buf, uint32_t num) static inline void unaligned_write32le(uint8_t *buf, uint32_t num) { - buf[0] = num; - buf[1] = num >> 8; - buf[2] = num >> 16; - buf[3] = num >> 24; + buf[0] = (uint8_t)num; + buf[1] = (uint8_t)(num >> 8); + buf[2] = (uint8_t)(num >> 16); + buf[3] = (uint8_t)(num >> 24); return; } diff --git a/Utilities/cmliblzma/config.h.in b/Utilities/cmliblzma/config.h.in index 9c53150bf..06f7fcb37 100644 --- a/Utilities/cmliblzma/config.h.in +++ b/Utilities/cmliblzma/config.h.in @@ -29,7 +29,7 @@ @SIZE_OF_UNSIGNED_CODE@ @SIZE_OF_UNSIGNED_LONG_CODE@ @SIZE_OF_UNSIGNED_LONG_LONG_CODE@ -@SIZE_OF_SIZE_T_CODE@ +@SIZEOF_SIZE_T_CODE@ /* * If we lack int64_t, define it to the first of __int64, int, long, and long long @@ -180,32 +180,6 @@ typedef uint64_t uintmax_t; #cmakedefine uintptr_t @uintptr_t@ - -#cmakedefine HAVE_RESTRICT -#cmakedefine HAVE___RESTRICT - -#cmakedefine HAVE_INLINE -#cmakedefine HAVE___INLINE - -#ifndef HAVE_RESTRICT -# ifdef HAVE___RESTRICT -# define LZMA_RESTRICT __restrict -# else -# define LZMA_RESTRICT -# endif -#else -# define LZMA_RESTRICT restrict -#endif /* HAVE_RESTRICT */ - -#ifndef HAVE_INLINE -# ifdef HAVE___INLINE -# define inline __inline -# else -# define inline -# endif -#endif /* HAVE_INLINE */ - - #cmakedefine WORDS_BIGENDIAN 1 #cmakedefine HAVE_BYTESWAP_H 1 diff --git a/Utilities/cmliblzma/liblzma/api/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma.h index fb874c3e1..aa88e4243 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma.h @@ -82,12 +82,20 @@ # if !defined(UINT32_C) || !defined(UINT64_C) \ || !defined(UINT32_MAX) || !defined(UINT64_MAX) /* - * MSVC has no C99 support, and thus it cannot be used to - * compile liblzma. The liblzma API has to still be usable - * from MSVC, so we need to define the required standard - * integer types here. + * MSVC versions older than 2013 have no C99 support, and + * thus they cannot be used to compile liblzma. Using an + * existing liblzma.dll with old MSVC can work though(*), + * but we need to define the required standard integer + * types here in a MSVC-specific way. + * + * (*) If you do this, the existing liblzma.dll probably uses + * a different runtime library than your MSVC-built + * application. Mixing runtimes is generally bad, but + * in this case it should work as long as you avoid + * the few rarely-needed liblzma functions that allocate + * memory and expect the caller to free it using free(). */ -# if defined(_WIN32) && defined(_MSC_VER) +# if defined(_WIN32) && defined(_MSC_VER) && _MSC_VER < 1800 typedef unsigned __int8 uint8_t; typedef unsigned __int32 uint32_t; typedef unsigned __int64 uint64_t; @@ -211,7 +219,11 @@ */ #ifndef lzma_nothrow # if defined(__cplusplus) -# define lzma_nothrow throw() +# if __cplusplus >= 201103L +# define lzma_nothrow noexcept +# else +# define lzma_nothrow throw() +# endif # elif __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3) # define lzma_nothrow __attribute__((__nothrow__)) # else @@ -286,7 +298,7 @@ extern "C" { #include "lzma/filter.h" #include "lzma/bcj.h" #include "lzma/delta.h" -#include "lzma/lzma.h" +#include "lzma/lzma12.h" /* Container formats */ #include "lzma/container.h" diff --git a/Utilities/cmliblzma/liblzma/api/lzma/base.h b/Utilities/cmliblzma/liblzma/api/lzma/base.h index 43dde8d60..a6005accc 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/base.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/base.h @@ -240,12 +240,12 @@ typedef enum { /** * \brief The `action' argument for lzma_code() * - * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, or LZMA_FINISH, - * the same `action' must is used until lzma_code() returns LZMA_STREAM_END. - * Also, the amount of input (that is, strm->avail_in) must not be modified - * by the application until lzma_code() returns LZMA_STREAM_END. Changing the - * `action' or modifying the amount of input will make lzma_code() return - * LZMA_PROG_ERROR. + * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER, + * or LZMA_FINISH, the same `action' must is used until lzma_code() returns + * LZMA_STREAM_END. Also, the amount of input (that is, strm->avail_in) must + * not be modified by the application until lzma_code() returns + * LZMA_STREAM_END. Changing the `action' or modifying the amount of input + * will make lzma_code() return LZMA_PROG_ERROR. */ typedef enum { LZMA_RUN = 0, @@ -293,7 +293,7 @@ typedef enum { * * All the input data going to the current Block must have * been given to the encoder (the last bytes can still be - * pending in* next_in). Call lzma_code() with LZMA_FULL_FLUSH + * pending in *next_in). Call lzma_code() with LZMA_FULL_FLUSH * until it returns LZMA_STREAM_END. Then continue normally * with LZMA_RUN or finish the Stream with LZMA_FINISH. * @@ -302,6 +302,29 @@ typedef enum { * no unfinished Block, no empty Block is created. */ + LZMA_FULL_BARRIER = 4, + /**< + * \brief Finish encoding of the current Block + * + * This is like LZMA_FULL_FLUSH except that this doesn't + * necessarily wait until all the input has been made + * available via the output buffer. That is, lzma_code() + * might return LZMA_STREAM_END as soon as all the input + * has been consumed (avail_in == 0). + * + * LZMA_FULL_BARRIER is useful with a threaded encoder if + * one wants to split the .xz Stream into Blocks at specific + * offsets but doesn't care if the output isn't flushed + * immediately. Using LZMA_FULL_BARRIER allows keeping + * the threads busy while LZMA_FULL_FLUSH would make + * lzma_code() wait until all the threads have finished + * until more data could be passed to the encoder. + * + * With a lzma_stream initialized with the single-threaded + * lzma_stream_encoder() or lzma_easy_encoder(), + * LZMA_FULL_BARRIER is an alias for LZMA_FULL_FLUSH. + */ + LZMA_FINISH = 3 /**< * \brief Finish the coding operation @@ -332,11 +355,19 @@ typedef enum { * malloc() and free(). C++ users should note that the custom memory * handling functions must not throw exceptions. * - * liblzma doesn't make an internal copy of lzma_allocator. Thus, it is - * OK to change these function pointers in the middle of the coding - * process, but obviously it must be done carefully to make sure that the - * replacement `free' can deallocate memory allocated by the earlier - * `alloc' function(s). + * Single-threaded mode only: liblzma doesn't make an internal copy of + * lzma_allocator. Thus, it is OK to change these function pointers in + * the middle of the coding process, but obviously it must be done + * carefully to make sure that the replacement `free' can deallocate + * memory allocated by the earlier `alloc' function(s). + * + * Multithreaded mode: liblzma might internally store pointers to the + * lzma_allocator given via the lzma_stream structure. The application + * must not change the allocator pointer in lzma_stream or the contents + * of the pointed lzma_allocator structure until lzma_end() has been used + * to free the memory associated with that lzma_stream. The allocation + * functions might be called simultaneously from multiple threads, and + * thus they must be thread safe. */ typedef struct { /** @@ -448,7 +479,8 @@ typedef struct lzma_internal_s lzma_internal; * * Application may modify the values of total_in and total_out as it wants. * They are updated by liblzma to match the amount of data read and - * written, but aren't used for anything else. + * written but aren't used for anything else except as a possible return + * values from lzma_get_progress(). */ typedef struct { const uint8_t *next_in; /**< Pointer to the next input byte. */ @@ -464,8 +496,10 @@ typedef struct { * * In most cases this is NULL which makes liblzma use * the standard malloc() and free(). + * + * \note In 5.0.x this is not a const pointer. */ - lzma_allocator *allocator; + const lzma_allocator *allocator; /** Internal state is not visible to applications. */ lzma_internal *internal; @@ -547,6 +581,25 @@ extern LZMA_API(void) lzma_end(lzma_stream *strm) lzma_nothrow; /** + * \brief Get progress information + * + * In single-threaded mode, applications can get progress information from + * strm->total_in and strm->total_out. In multi-threaded mode this is less + * useful because a significant amount of both input and output data gets + * buffered internally by liblzma. This makes total_in and total_out give + * misleading information and also makes the progress indicator updates + * non-smooth. + * + * This function gives realistic progress information also in multi-threaded + * mode by taking into account the progress made by each thread. In + * single-threaded mode *progress_in and *progress_out are set to + * strm->total_in and strm->total_out, respectively. + */ +extern LZMA_API(void) lzma_get_progress(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) lzma_nothrow; + + +/** * \brief Get the memory usage of decoder filter chain * * This function is currently supported only when *strm has been initialized @@ -591,11 +644,16 @@ extern LZMA_API(uint64_t) lzma_memlimit_get(const lzma_stream *strm) * This function is supported only when *strm has been initialized with * a function that takes a memlimit argument. * + * liblzma 5.2.3 and earlier has a bug where memlimit value of 0 causes + * this function to do nothing (leaving the limit unchanged) and still + * return LZMA_OK. Later versions treat 0 as if 1 had been specified (so + * lzma_memlimit_get() will return 1 even if you specify 0 here). + * * \return - LZMA_OK: New memory usage limit successfully set. * - LZMA_MEMLIMIT_ERROR: The new limit is too small. * The limit was not changed. * - LZMA_PROG_ERROR: Invalid arguments, e.g. *strm doesn't - * support memory usage limit or memlimit was zero. + * support memory usage limit. */ extern LZMA_API(lzma_ret) lzma_memlimit_set( lzma_stream *strm, uint64_t memlimit) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/block.h b/Utilities/cmliblzma/liblzma/api/lzma/block.h index e6710a7bc..7bdcfd7cb 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/block.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/block.h @@ -31,11 +31,16 @@ typedef struct { /** * \brief Block format version * - * To prevent API and ABI breakages if new features are needed in - * the Block field, a version number is used to indicate which - * fields in this structure are in use. For now, version must always - * be zero. With non-zero version, most Block related functions will - * return LZMA_OPTIONS_ERROR. + * To prevent API and ABI breakages when new features are needed, + * a version number is used to indicate which fields in this + * structure are in use: + * - liblzma >= 5.0.0: version = 0 is supported. + * - liblzma >= 5.1.4beta: Support for version = 1 was added, + * which adds the ignore_check field. + * + * If version is greater than one, most Block related functions + * will return LZMA_OPTIONS_ERROR (lzma_block_header_decode() works + * with any version value). * * Read by: * - All functions that take pointer to lzma_block as argument, @@ -233,7 +238,28 @@ typedef struct { lzma_reserved_enum reserved_enum2; lzma_reserved_enum reserved_enum3; lzma_reserved_enum reserved_enum4; - lzma_bool reserved_bool1; + + /** + * \brief A flag to Block decoder to not verify the Check field + * + * This field is supported by liblzma >= 5.1.4beta if .version >= 1. + * + * If this is set to true, the integrity check won't be calculated + * and verified. Unless you know what you are doing, you should + * leave this to false. (A reason to set this to true is when the + * file integrity is verified externally anyway and you want to + * speed up the decompression, which matters mostly when using + * SHA-256 as the integrity check.) + * + * If .version >= 1, read by: + * - lzma_block_decoder() + * - lzma_block_buffer_decode() + * + * Written by (.version is ignored): + * - lzma_block_header_decode() always sets this to false + */ + lzma_bool ignore_check; + lzma_bool reserved_bool2; lzma_bool reserved_bool3; lzma_bool reserved_bool4; @@ -310,10 +336,14 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( /** * \brief Decode Block Header * - * block->version should be set to the highest value supported by the - * application; currently the only possible version is zero. This function - * will set version to the lowest value that still supports all the features - * required by the Block Header. + * block->version should (usually) be set to the highest value supported + * by the application. If the application sets block->version to a value + * higher than supported by the current liblzma version, this function will + * downgrade block->version to the highest value supported by it. Thus one + * should check the value of block->version after calling this function if + * block->version was set to a non-zero value and the application doesn't + * otherwise know that the liblzma version being used is new enough to + * support the specified block->version. * * The size of the Block Header must have already been decoded with * lzma_block_header_size_decode() macro and stored to block->header_size. @@ -344,7 +374,7 @@ extern LZMA_API(lzma_ret) lzma_block_header_encode( * block->header_size is invalid or block->filters is NULL. */ extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, - lzma_allocator *allocator, const uint8_t *in) + const lzma_allocator *allocator, const uint8_t *in) lzma_nothrow lzma_attr_warn_unused_result; @@ -493,7 +523,25 @@ extern LZMA_API(size_t) lzma_block_buffer_bound(size_t uncompressed_size) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_buffer_encode( - lzma_block *block, lzma_allocator *allocator, + lzma_block *block, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** + * \brief Single-call uncompressed .xz Block encoder + * + * This is like lzma_block_buffer_encode() except this doesn't try to + * compress the data and instead encodes the data using LZMA2 uncompressed + * chunks. The required output buffer size can be determined with + * lzma_block_buffer_bound(). + * + * Since the data won't be compressed, this function ignores block->filters. + * This function doesn't take lzma_allocator because this function doesn't + * allocate any memory from the heap. + */ +extern LZMA_API(lzma_ret) lzma_block_uncomp_encode(lzma_block *block, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -527,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_block_buffer_encode( * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_block_buffer_decode( - lzma_block *block, lzma_allocator *allocator, + lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/container.h b/Utilities/cmliblzma/liblzma/api/lzma/container.h index 7a9ffc645..9fbf4df06 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/container.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/container.h @@ -61,6 +61,131 @@ /** + * \brief Multithreading options + */ +typedef struct { + /** + * \brief Flags + * + * Set this to zero if no flags are wanted. + * + * No flags are currently supported. + */ + uint32_t flags; + + /** + * \brief Number of worker threads to use + */ + uint32_t threads; + + /** + * \brief Maximum uncompressed size of a Block + * + * The encoder will start a new .xz Block every block_size bytes. + * Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code() + * the caller may tell liblzma to start a new Block earlier. + * + * With LZMA2, a recommended block size is 2-4 times the LZMA2 + * dictionary size. With very small dictionaries, it is recommended + * to use at least 1 MiB block size for good compression ratio, even + * if this is more than four times the dictionary size. Note that + * these are only recommendations for typical use cases; feel free + * to use other values. Just keep in mind that using a block size + * less than the LZMA2 dictionary size is waste of RAM. + * + * Set this to 0 to let liblzma choose the block size depending + * on the compression options. For LZMA2 it will be 3*dict_size + * or 1 MiB, whichever is more. + * + * For each thread, about 3 * block_size bytes of memory will be + * allocated. This may change in later liblzma versions. If so, + * the memory usage will probably be reduced, not increased. + */ + uint64_t block_size; + + /** + * \brief Timeout to allow lzma_code() to return early + * + * Multithreading can make liblzma to consume input and produce + * output in a very bursty way: it may first read a lot of input + * to fill internal buffers, then no input or output occurs for + * a while. + * + * In single-threaded mode, lzma_code() won't return until it has + * either consumed all the input or filled the output buffer. If + * this is done in multithreaded mode, it may cause a call + * lzma_code() to take even tens of seconds, which isn't acceptable + * in all applications. + * + * To avoid very long blocking times in lzma_code(), a timeout + * (in milliseconds) may be set here. If lzma_code() would block + * longer than this number of milliseconds, it will return with + * LZMA_OK. Reasonable values are 100 ms or more. The xz command + * line tool uses 300 ms. + * + * If long blocking times are fine for you, set timeout to a special + * value of 0, which will disable the timeout mechanism and will make + * lzma_code() block until all the input is consumed or the output + * buffer has been filled. + * + * \note Even with a timeout, lzma_code() might sometimes take + * somewhat long time to return. No timing guarantees + * are made. + */ + uint32_t timeout; + + /** + * \brief Compression preset (level and possible flags) + * + * The preset is set just like with lzma_easy_encoder(). + * The preset is ignored if filters below is non-NULL. + */ + uint32_t preset; + + /** + * \brief Filter chain (alternative to a preset) + * + * If this is NULL, the preset above is used. Otherwise the preset + * is ignored and the filter chain specified here is used. + */ + const lzma_filter *filters; + + /** + * \brief Integrity check type + * + * See check.h for available checks. The xz command line tool + * defaults to LZMA_CHECK_CRC64, which is a good choice if you + * are unsure. + */ + lzma_check check; + + /* + * Reserved space to allow possible future extensions without + * breaking the ABI. You should not touch these, because the names + * of these variables may change. These are and will never be used + * with the currently supported options, so it is safe to leave these + * uninitialized. + */ + lzma_reserved_enum reserved_enum1; + lzma_reserved_enum reserved_enum2; + lzma_reserved_enum reserved_enum3; + uint32_t reserved_int1; + uint32_t reserved_int2; + uint32_t reserved_int3; + uint32_t reserved_int4; + uint64_t reserved_int5; + uint64_t reserved_int6; + uint64_t reserved_int7; + uint64_t reserved_int8; + void *reserved_ptr1; + void *reserved_ptr2; + void *reserved_ptr3; + void *reserved_ptr4; + +} lzma_mt; + + +/** * \brief Calculate approximate memory usage of easy encoder * * This function is a wrapper for lzma_raw_encoder_memusage(). @@ -165,7 +290,8 @@ extern LZMA_API(lzma_ret) lzma_easy_encoder( */ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode( uint32_t preset, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -191,6 +317,49 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, /** + * \brief Calculate approximate memory usage of multithreaded .xz encoder + * + * Since doing the encoding in threaded mode doesn't affect the memory + * requirements of single-threaded decompressor, you can use + * lzma_easy_decoder_memusage(options->preset) or + * lzma_raw_decoder_memusage(options->filters) to calculate + * the decompressor memory requirements. + * + * \param options Compression options + * + * \return Number of bytes of memory required for encoding with the + * given options. If an error occurs, for example due to + * unsupported preset or filter chain, UINT64_MAX is returned. + */ +extern LZMA_API(uint64_t) lzma_stream_encoder_mt_memusage( + const lzma_mt *options) lzma_nothrow lzma_attr_pure; + + +/** + * \brief Initialize multithreaded .xz Stream encoder + * + * This provides the functionality of lzma_easy_encoder() and + * lzma_stream_encoder() as a single function for multithreaded use. + * + * The supported actions for lzma_code() are LZMA_RUN, LZMA_FULL_FLUSH, + * LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be + * added in the future. + * + * \param strm Pointer to properly prepared lzma_stream + * \param options Pointer to multithreaded compression options + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_UNSUPPORTED_CHECK + * - LZMA_OPTIONS_ERROR + * - LZMA_PROG_ERROR + */ +extern LZMA_API(lzma_ret) lzma_stream_encoder_mt( + lzma_stream *strm, const lzma_mt *options) + lzma_nothrow lzma_attr_warn_unused_result; + + +/** * \brief Initialize .lzma encoder (legacy file format) * * The .lzma format is sometimes called the LZMA_Alone format, which is the @@ -269,7 +438,8 @@ extern LZMA_API(size_t) lzma_stream_buffer_bound(size_t uncompressed_size) */ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( lzma_filter *filters, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -305,6 +475,30 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( /** + * This flag makes lzma_code() not calculate and verify the integrity check + * of the compressed data in .xz files. This means that invalid integrity + * check values won't be detected and LZMA_DATA_ERROR won't be returned in + * such cases. + * + * This flag only affects the checks of the compressed data itself; the CRC32 + * values in the .xz headers will still be verified normally. + * + * Don't use this flag unless you know what you are doing. Possible reasons + * to use this flag: + * + * - Trying to recover data from a corrupt .xz file. + * + * - Speeding up decompression, which matters mostly with SHA-256 + * or with files that have compressed extremely well. It's recommended + * to not use this flag for this purpose unless the file integrity is + * verified externally in some other way. + * + * Support for this flag was added in liblzma 5.1.4beta. + */ +#define LZMA_IGNORE_CHECK UINT32_C(0x10) + + +/** * This flag enables decoding of concatenated files with file formats that * allow concatenating compressed files as is. From the formats currently * supported by liblzma, only the .xz format allows concatenated files. @@ -326,7 +520,10 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_encode( * * \param strm Pointer to properly prepared lzma_stream * \param memlimit Memory usage limit as bytes. Use UINT64_MAX - * to effectively disable the limiter. + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. * \param flags Bitwise-or of zero or more of the decoder flags: * LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, * LZMA_TELL_ANY_CHECK, LZMA_CONCATENATED @@ -350,7 +547,10 @@ extern LZMA_API(lzma_ret) lzma_stream_decoder( * * \param strm Pointer to properly prepared lzma_stream * \param memlimit Memory usage limit as bytes. Use UINT64_MAX - * to effectively disable the limiter. + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. * \param flags Bitwise-or of flags, or zero for no flags. * * \return - LZMA_OK: Initialization was successful. @@ -366,9 +566,16 @@ extern LZMA_API(lzma_ret) lzma_auto_decoder( /** * \brief Initialize .lzma decoder (legacy file format) * + * \param strm Pointer to properly prepared lzma_stream + * \param memlimit Memory usage limit as bytes. Use UINT64_MAX + * to effectively disable the limiter. liblzma + * 5.2.3 and earlier don't allow 0 here and return + * LZMA_PROG_ERROR; later versions treat 0 as if 1 + * had been specified. + * * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. - * There is no need to use LZMA_FINISH, but allowing it may simplify - * certain types of applications. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. * * \return - LZMA_OK * - LZMA_MEM_ERROR @@ -418,7 +625,8 @@ extern LZMA_API(lzma_ret) lzma_alone_decoder( * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode( - uint64_t *memlimit, uint32_t flags, lzma_allocator *allocator, + uint64_t *memlimit, uint32_t flags, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/filter.h b/Utilities/cmliblzma/liblzma/api/lzma/filter.h index e0bc163ad..4e78752b8 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/filter.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/filter.h @@ -116,8 +116,9 @@ extern LZMA_API(lzma_bool) lzma_filter_decoder_is_supported(lzma_vli id) * is not NULL. * - LZMA_PROG_ERROR: src or dest is NULL. */ -extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, - lzma_filter *dest, lzma_allocator *allocator) lzma_nothrow; +extern LZMA_API(lzma_ret) lzma_filters_copy( + const lzma_filter *src, lzma_filter *dest, + const lzma_allocator *allocator) lzma_nothrow; /** @@ -256,7 +257,7 @@ extern LZMA_API(lzma_ret) lzma_filters_update( * won't necessarily meet that bound.) */ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( - const lzma_filter *filters, lzma_allocator *allocator, + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -280,7 +281,7 @@ extern LZMA_API(lzma_ret) lzma_raw_buffer_encode( * which no data is written to is out[out_size]. */ extern LZMA_API(lzma_ret) lzma_raw_buffer_decode( - const lzma_filter *filters, lzma_allocator *allocator, + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) lzma_nothrow; @@ -356,7 +357,7 @@ extern LZMA_API(lzma_ret) lzma_properties_encode( * - LZMA_MEM_ERROR */ extern LZMA_API(lzma_ret) lzma_properties_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) lzma_nothrow; @@ -419,6 +420,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow lzma_attr_warn_unused_result; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h index e7dd03c3e..5321d9af8 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/hardware.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/hardware.h @@ -48,3 +48,17 @@ * of RAM on the specific operating system. */ extern LZMA_API(uint64_t) lzma_physmem(void) lzma_nothrow; + + +/** + * \brief Get the number of processor cores or threads + * + * This function may be useful when determining how many threads to use. + * If the hardware supports more than one thread per CPU core, the number + * of hardware threads is returned if that information is available. + * + * \brief On success, the number of available CPU threads or cores is + * returned. If this information isn't available or an error + * occurs, zero is returned. + */ +extern LZMA_API(uint32_t) lzma_cputhreads(void) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index.h b/Utilities/cmliblzma/liblzma/api/lzma/index.h index 16bacc287..3dac6fb85 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index.h @@ -303,7 +303,7 @@ extern LZMA_API(uint64_t) lzma_index_memused(const lzma_index *i) * \return On success, a pointer to an empty initialized lzma_index is * returned. If allocation fails, NULL is returned. */ -extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator) +extern LZMA_API(lzma_index *) lzma_index_init(const lzma_allocator *allocator) lzma_nothrow; @@ -312,8 +312,8 @@ extern LZMA_API(lzma_index *) lzma_index_init(lzma_allocator *allocator) * * If i is NULL, this does nothing. */ -extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator) - lzma_nothrow; +extern LZMA_API(void) lzma_index_end( + lzma_index *i, const lzma_allocator *allocator) lzma_nothrow; /** @@ -341,7 +341,7 @@ extern LZMA_API(void) lzma_index_end(lzma_index *i, lzma_allocator *allocator) * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_index_append( - lzma_index *i, lzma_allocator *allocator, + lzma_index *i, const lzma_allocator *allocator, lzma_vli unpadded_size, lzma_vli uncompressed_size) lzma_nothrow lzma_attr_warn_unused_result; @@ -564,8 +564,8 @@ extern LZMA_API(lzma_bool) lzma_index_iter_locate( * - LZMA_MEM_ERROR * - LZMA_PROG_ERROR */ -extern LZMA_API(lzma_ret) lzma_index_cat( - lzma_index *dest, lzma_index *src, lzma_allocator *allocator) +extern LZMA_API(lzma_ret) lzma_index_cat(lzma_index *dest, lzma_index *src, + const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -575,7 +575,7 @@ extern LZMA_API(lzma_ret) lzma_index_cat( * \return A copy of the lzma_index, or NULL if memory allocation failed. */ extern LZMA_API(lzma_index *) lzma_index_dup( - const lzma_index *i, lzma_allocator *allocator) + const lzma_index *i, const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -586,8 +586,7 @@ extern LZMA_API(lzma_index *) lzma_index_dup( * \param i Pointer to lzma_index which should be encoded. * * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. - * It is enough to use only one of them (you can choose freely; use LZMA_RUN - * to support liblzma versions older than 5.0.0). + * It is enough to use only one of them (you can choose freely). * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR @@ -610,16 +609,21 @@ extern LZMA_API(lzma_ret) lzma_index_encoder( * to a new lzma_index, which the application * has to later free with lzma_index_end(). * \param memlimit How much memory the resulting lzma_index is - * allowed to require. + * allowed to require. liblzma 5.2.3 and earlier + * don't allow 0 here and return LZMA_PROG_ERROR; + * later versions treat 0 as if 1 had been specified. * - * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. - * It is enough to use only one of them (you can choose freely; use LZMA_RUN - * to support liblzma versions older than 5.0.0). + * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * There is no need to use LZMA_FINISH, but it's allowed because it may + * simplify certain types of applications. * * \return - LZMA_OK: Initialization succeeded, continue with lzma_code(). * - LZMA_MEM_ERROR - * - LZMA_MEMLIMIT_ERROR * - LZMA_PROG_ERROR + * + * liblzma 5.2.3 and older list also LZMA_MEMLIMIT_ERROR here + * but that error code has never been possible from this + * initialization function. */ extern LZMA_API(lzma_ret) lzma_index_decoder( lzma_stream *strm, lzma_index **i, uint64_t memlimit) @@ -677,6 +681,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, * - LZMA_PROG_ERROR */ extern LZMA_API(lzma_ret) lzma_index_buffer_decode(lzma_index **i, - uint64_t *memlimit, lzma_allocator *allocator, + uint64_t *memlimit, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h index fa2e048d5..9287f1dfd 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/index_hash.h @@ -37,7 +37,7 @@ typedef struct lzma_index_hash_s lzma_index_hash; * pointer than the index_hash that was given as an argument. */ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( - lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_index_hash *index_hash, const lzma_allocator *allocator) lzma_nothrow lzma_attr_warn_unused_result; @@ -45,7 +45,7 @@ extern LZMA_API(lzma_index_hash *) lzma_index_hash_init( * \brief Deallocate lzma_index_hash structure */ extern LZMA_API(void) lzma_index_hash_end( - lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_index_hash *index_hash, const lzma_allocator *allocator) lzma_nothrow; diff --git a/Utilities/cmliblzma/liblzma/api/lzma/lzma.h b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h index 3f8e095f7..4e32fa3a2 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/lzma.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/lzma12.h @@ -1,5 +1,5 @@ /** - * \file lzma/lzma.h + * \file lzma/lzma12.h * \brief LZMA1 and LZMA2 filters */ diff --git a/Utilities/cmliblzma/liblzma/api/lzma/version.h b/Utilities/cmliblzma/liblzma/api/lzma/version.h index 09866b982..143c7dea6 100644 --- a/Utilities/cmliblzma/liblzma/api/lzma/version.h +++ b/Utilities/cmliblzma/liblzma/api/lzma/version.h @@ -21,8 +21,8 @@ * Version number split into components */ #define LZMA_VERSION_MAJOR 5 -#define LZMA_VERSION_MINOR 0 -#define LZMA_VERSION_PATCH 8 +#define LZMA_VERSION_MINOR 2 +#define LZMA_VERSION_PATCH 4 #define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE #ifndef LZMA_VERSION_COMMIT diff --git a/Utilities/cmliblzma/liblzma/check/check.c b/Utilities/cmliblzma/liblzma/check/check.c index 979b0a818..428ddaeb7 100644 --- a/Utilities/cmliblzma/liblzma/check/check.c +++ b/Utilities/cmliblzma/liblzma/check/check.c @@ -16,6 +16,9 @@ extern LZMA_API(lzma_bool) lzma_check_is_supported(lzma_check type) { + if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) + return false; + static const lzma_bool available_checks[LZMA_CHECK_ID_MAX + 1] = { true, // LZMA_CHECK_NONE @@ -53,9 +56,6 @@ lzma_check_is_supported(lzma_check type) false, // Reserved }; - if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) - return false; - return available_checks[(unsigned int)(type)]; } @@ -63,6 +63,9 @@ lzma_check_is_supported(lzma_check type) extern LZMA_API(uint32_t) lzma_check_size(lzma_check type) { + if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) + return UINT32_MAX; + // See file-format.txt section 2.1.1.2. static const uint8_t check_sizes[LZMA_CHECK_ID_MAX + 1] = { 0, @@ -73,9 +76,6 @@ lzma_check_size(lzma_check type) 64, 64, 64 }; - if ((unsigned int)(type) > LZMA_CHECK_ID_MAX) - return UINT32_MAX; - return check_sizes[(unsigned int)(type)]; } diff --git a/Utilities/cmliblzma/liblzma/check/check.h b/Utilities/cmliblzma/liblzma/check/check.h index e100d2b85..3007d889b 100644 --- a/Utilities/cmliblzma/liblzma/check/check.h +++ b/Utilities/cmliblzma/liblzma/check/check.h @@ -15,6 +15,53 @@ #include "common.h" +// If the function for external SHA-256 is missing, use the internal SHA-256 +// code. Due to how configure works, these defines can only get defined when +// both a usable header and a type have already been found. +#if !(defined(HAVE_CC_SHA256_INIT) \ + || defined(HAVE_SHA256_INIT) \ + || defined(HAVE_SHA256INIT)) +# define HAVE_INTERNAL_SHA256 1 +#endif + +#if defined(HAVE_INTERNAL_SHA256) +// Nothing +#elif defined(HAVE_COMMONCRYPTO_COMMONDIGEST_H) +# include <CommonCrypto/CommonDigest.h> +#elif defined(HAVE_SHA256_H) +# include <sys/types.h> +# include <sha256.h> +#elif defined(HAVE_SHA2_H) +# include <sys/types.h> +# include <sha2.h> +#endif + +#if defined(HAVE_INTERNAL_SHA256) +/// State for the internal SHA-256 implementation +typedef struct { + /// Internal state + uint32_t state[8]; + + /// Size of the message excluding padding + uint64_t size; +} lzma_sha256_state; +#elif defined(HAVE_CC_SHA256_CTX) +typedef CC_SHA256_CTX lzma_sha256_state; +#elif defined(HAVE_SHA256_CTX) +typedef SHA256_CTX lzma_sha256_state; +#elif defined(HAVE_SHA2_CTX) +typedef SHA2_CTX lzma_sha256_state; +#endif + +#if defined(HAVE_INTERNAL_SHA256) +// Nothing +#elif defined(HAVE_CC_SHA256_INIT) +# define LZMA_SHA256FUNC(x) CC_SHA256_ ## x +#elif defined(HAVE_SHA256_INIT) +# define LZMA_SHA256FUNC(x) SHA256_ ## x +#elif defined(HAVE_SHA256INIT) +# define LZMA_SHA256FUNC(x) SHA256 ## x +#endif // Index hashing needs the best possible hash function (preferably // a cryptographic hash) for maximum reliability. @@ -43,14 +90,7 @@ typedef struct { union { uint32_t crc32; uint64_t crc64; - - struct { - /// Internal state - uint32_t state[8]; - - /// Size of the message excluding padding - uint64_t size; - } sha256; + lzma_sha256_state sha256; } state; } lzma_check_state; @@ -82,6 +122,8 @@ extern void lzma_check_update(lzma_check_state *check, lzma_check type, extern void lzma_check_finish(lzma_check_state *check, lzma_check type); +#ifndef LZMA_SHA256FUNC + /// Prepare SHA-256 state for new input. extern void lzma_sha256_init(lzma_check_state *check); @@ -92,4 +134,39 @@ extern void lzma_sha256_update( /// Finish the SHA-256 calculation and store the result to check->buffer.u8. extern void lzma_sha256_finish(lzma_check_state *check); + +#else + +static inline void +lzma_sha256_init(lzma_check_state *check) +{ + LZMA_SHA256FUNC(Init)(&check->state.sha256); +} + + +static inline void +lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) +{ +#if defined(HAVE_CC_SHA256_INIT) && SIZE_MAX > UINT32_MAX + // Darwin's CC_SHA256_Update takes uint32_t as the buffer size, + // so use a loop to support size_t. + while (size > UINT32_MAX) { + LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, UINT32_MAX); + buf += UINT32_MAX; + size -= UINT32_MAX; + } +#endif + + LZMA_SHA256FUNC(Update)(&check->state.sha256, buf, size); +} + + +static inline void +lzma_sha256_finish(lzma_check_state *check) +{ + LZMA_SHA256FUNC(Final)(check->buffer.u8, &check->state.sha256); +} + +#endif + #endif diff --git a/Utilities/cmliblzma/liblzma/check/crc32_fast.c b/Utilities/cmliblzma/liblzma/check/crc32_fast.c index c2c3cb79b..3de02638d 100644 --- a/Utilities/cmliblzma/liblzma/check/crc32_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc32_fast.c @@ -33,8 +33,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) #endif if (size > 8) { - const uint8_t * limit; - // Fix the alignment, if needed. The if statement above // ensures that this won't read past the end of buf[]. while ((uintptr_t)(buf) & 7) { @@ -43,7 +41,7 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) } // Calculate the position where to stop. - limit = buf + (size & ~(size_t)(7)); + const uint8_t *const limit = buf + (size & ~(size_t)(7)); // Calculate how many bytes must be calculated separately // before returning the result. @@ -51,8 +49,6 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) // Calculate the CRC32 using the slice-by-eight algorithm. while (buf < limit) { - uint32_t tmp; - crc ^= *(const uint32_t *)(buf); buf += 4; @@ -61,7 +57,7 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) ^ lzma_crc32_table[5][C(crc)] ^ lzma_crc32_table[4][D(crc)]; - tmp = *(const uint32_t *)(buf); + const uint32_t tmp = *(const uint32_t *)(buf); buf += 4; // At least with some compilers, it is critical for diff --git a/Utilities/cmliblzma/liblzma/check/crc64_fast.c b/Utilities/cmliblzma/liblzma/check/crc64_fast.c index 1436557af..52af29ed4 100644 --- a/Utilities/cmliblzma/liblzma/check/crc64_fast.c +++ b/Utilities/cmliblzma/liblzma/check/crc64_fast.c @@ -36,14 +36,12 @@ lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) #endif if (size > 4) { - const uint8_t *limit; - while ((uintptr_t)(buf) & 3) { crc = lzma_crc64_table[0][*buf++ ^ A1(crc)] ^ S8(crc); --size; } - limit = buf + (size & ~(size_t)(3)); + const uint8_t *const limit = buf + (size & ~(size_t)(3)); size &= (size_t)(3); while (buf < limit) { diff --git a/Utilities/cmliblzma/liblzma/check/sha256.c b/Utilities/cmliblzma/liblzma/check/sha256.c index 3af6aa67d..5eede5ce0 100644 --- a/Utilities/cmliblzma/liblzma/check/sha256.c +++ b/Utilities/cmliblzma/liblzma/check/sha256.c @@ -21,22 +21,22 @@ // /////////////////////////////////////////////////////////////////////////////// -// Avoid bogus warnings in transform(). -#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 2) || __GNUC__ > 4 -# pragma GCC diagnostic ignored "-Wuninitialized" -#endif - #include "check.h" -// At least on x86, GCC is able to optimize this to a rotate instruction. -#define rotr_32(num, amount) ((num) >> (amount) | (num) << (32 - (amount))) +// Rotate a uint32_t. GCC can optimize this to a rotate instruction +// at least on x86. +static inline uint32_t +rotr_32(uint32_t num, unsigned amount) +{ + return (num >> amount) | (num << (32 - amount)); +} -#define blk0(i) (W[i] = data[i]) +#define blk0(i) (W[i] = conv32be(data[i])) #define blk2(i) (W[i & 15] += s1(W[(i - 2) & 15]) + W[(i - 7) & 15] \ + s0(W[(i - 15) & 15])) #define Ch(x, y, z) (z ^ (x & (y ^ z))) -#define Maj(x, y, z) ((x & y) | (z & (x | y))) +#define Maj(x, y, z) ((x & (y ^ z)) + (y & z)) #define a(i) T[(0 - i) & 7] #define b(i) T[(1 - i) & 7] @@ -47,16 +47,17 @@ #define g(i) T[(6 - i) & 7] #define h(i) T[(7 - i) & 7] -#define R(i) \ - h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] \ - + (j ? blk2(i) : blk0(i)); \ +#define R(i, j, blk) \ + h(i) += S1(e(i)) + Ch(e(i), f(i), g(i)) + SHA256_K[i + j] + blk; \ d(i) += h(i); \ h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) +#define R0(i) R(i, 0, blk0(i)) +#define R2(i) R(i, j, blk2(i)) -#define S0(x) (rotr_32(x, 2) ^ rotr_32(x, 13) ^ rotr_32(x, 22)) -#define S1(x) (rotr_32(x, 6) ^ rotr_32(x, 11) ^ rotr_32(x, 25)) -#define s0(x) (rotr_32(x, 7) ^ rotr_32(x, 18) ^ (x >> 3)) -#define s1(x) (rotr_32(x, 17) ^ rotr_32(x, 19) ^ (x >> 10)) +#define S0(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 9), 11), 2) +#define S1(x) rotr_32(x ^ rotr_32(x ^ rotr_32(x, 14), 5), 6) +#define s0(x) (rotr_32(x ^ rotr_32(x, 11), 7) ^ (x >> 3)) +#define s1(x) (rotr_32(x ^ rotr_32(x, 2), 17) ^ (x >> 10)) static const uint32_t SHA256_K[64] = { @@ -84,17 +85,22 @@ transform(uint32_t state[8], const uint32_t data[16]) { uint32_t W[16]; uint32_t T[8]; - unsigned int j; // Copy state[] to working vars. memcpy(T, state, sizeof(T)); - // 64 operations, partially loop unrolled - for (j = 0; j < 64; j += 16) { - R( 0); R( 1); R( 2); R( 3); - R( 4); R( 5); R( 6); R( 7); - R( 8); R( 9); R(10); R(11); - R(12); R(13); R(14); R(15); + // The first 16 operations unrolled + R0( 0); R0( 1); R0( 2); R0( 3); + R0( 4); R0( 5); R0( 6); R0( 7); + R0( 8); R0( 9); R0(10); R0(11); + R0(12); R0(13); R0(14); R0(15); + + // The remaining 48 operations partially unrolled + for (unsigned int j = 16; j < 64; j += 16) { + R2( 0); R2( 1); R2( 2); R2( 3); + R2( 4); R2( 5); R2( 6); R2( 7); + R2( 8); R2( 9); R2(10); R2(11); + R2(12); R2(13); R2(14); R2(15); } // Add the working vars back into state[]. @@ -112,19 +118,7 @@ transform(uint32_t state[8], const uint32_t data[16]) static void process(lzma_check_state *check) { -#ifdef WORDS_BIGENDIAN transform(check->state.sha256.state, check->buffer.u32); - -#else - uint32_t data[16]; - size_t i; - - for (i = 0; i < 16; ++i) - data[i] = bswap32(check->buffer.u32[i]); - - transform(check->state.sha256.state, data); -#endif - return; } @@ -174,8 +168,6 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) extern void lzma_sha256_finish(lzma_check_state *check) { - size_t i; - // Add padding as described in RFC 3174 (it describes SHA-1 but // the same padding style is used for SHA-256 too). size_t pos = check->state.sha256.size & 0x3F; @@ -197,7 +189,7 @@ lzma_sha256_finish(lzma_check_state *check) process(check); - for (i = 0; i < 8; ++i) + for (size_t i = 0; i < 8; ++i) check->buffer.u32[i] = conv32be(check->state.sha256.state[i]); return; diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.c b/Utilities/cmliblzma/liblzma/common/alone_decoder.c index 5f5e564ab..77d0a9b10 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.c @@ -15,7 +15,7 @@ #include "lz_decoder.h" -struct lzma_coder_s { +typedef struct { lzma_next_coder next; enum { @@ -46,17 +46,19 @@ struct lzma_coder_s { /// Options decoded from the header needed to initialize /// the LZMA decoder lzma_options_lzma options; -}; +} lzma_alone_coder; static lzma_ret -alone_decode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +alone_decode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_alone_coder *coder = coder_ptr; + while (*out_pos < out_size && (coder->sequence == SEQ_CODE || *in_pos < in_size)) switch (coder->sequence) { @@ -126,17 +128,19 @@ alone_decode(lzma_coder *coder, // Fall through case SEQ_CODER_INIT: { - lzma_ret ret; + if (coder->memusage > coder->memlimit) + return LZMA_MEMLIMIT_ERROR; lzma_filter_info filters[2] = { - { 0, &lzma_lzma_decoder_init, &coder->options }, - { 0, NULL, NULL } + { + .init = &lzma_lzma_decoder_init, + .options = &coder->options, + }, { + .init = NULL, + } }; - if (coder->memusage > coder->memlimit) - return LZMA_MEMLIMIT_ERROR; - - ret = lzma_next_filter_init(&coder->next, + const lzma_ret ret = lzma_next_filter_init(&coder->next, allocator, filters); if (ret != LZMA_OK) return ret; @@ -164,8 +168,9 @@ alone_decode(lzma_coder *coder, static void -alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +alone_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_alone_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -173,9 +178,11 @@ alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +alone_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_alone_coder *coder = coder_ptr; + *memusage = coder->memusage; *old_memlimit = coder->memlimit; @@ -191,34 +198,34 @@ alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, extern lzma_ret -lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_alone_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, bool picky) { lzma_next_coder_init(&lzma_alone_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; + lzma_alone_coder *coder = next->coder; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_alone_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &alone_decode; next->end = &alone_decoder_end; next->memconfig = &alone_decoder_memconfig; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } - next->coder->sequence = SEQ_PROPERTIES; - next->coder->picky = picky; - next->coder->pos = 0; - next->coder->options.dict_size = 0; - next->coder->options.preset_dict = NULL; - next->coder->options.preset_dict_size = 0; - next->coder->uncompressed_size = 0; - next->coder->memlimit = memlimit; - next->coder->memusage = LZMA_MEMUSAGE_BASE; + coder->sequence = SEQ_PROPERTIES; + coder->picky = picky; + coder->pos = 0; + coder->options.dict_size = 0; + coder->options.preset_dict = NULL; + coder->options.preset_dict_size = 0; + coder->uncompressed_size = 0; + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; return LZMA_OK; } @@ -227,7 +234,7 @@ lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) { - lzma_next_strm_init2(lzma_alone_decoder_init, strm, memlimit, false); + lzma_next_strm_init(lzma_alone_decoder_init, strm, memlimit, false); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/alone_decoder.h b/Utilities/cmliblzma/liblzma/common/alone_decoder.h index f666fc382..dfa031aa7 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/alone_decoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_alone_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, bool picky); #endif diff --git a/Utilities/cmliblzma/liblzma/common/alone_encoder.c b/Utilities/cmliblzma/liblzma/common/alone_encoder.c index 4207b4a52..4853cfd1d 100644 --- a/Utilities/cmliblzma/liblzma/common/alone_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/alone_encoder.c @@ -17,7 +17,7 @@ #define ALONE_HEADER_SIZE (1 + 4 + 8) -struct lzma_coder_s { +typedef struct { lzma_next_coder next; enum { @@ -27,17 +27,19 @@ struct lzma_coder_s { size_t header_pos; uint8_t header[ALONE_HEADER_SIZE]; -}; +} lzma_alone_coder; static lzma_ret -alone_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +alone_encode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_alone_coder *coder = coder_ptr; + while (*out_pos < out_size) switch (coder->sequence) { case SEQ_HEADER: @@ -65,8 +67,9 @@ alone_encode(lzma_coder *coder, static void -alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +alone_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_alone_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -75,36 +78,31 @@ alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) // At least for now, this is not used by any internal function. static lzma_ret -alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_lzma *options) { - uint32_t d; - - // Initialize the LZMA encoder. - const lzma_filter_info filters[2] = { - { 0, &lzma_lzma_encoder_init, (void *)(options) }, - { 0, NULL, NULL } - }; - lzma_next_coder_init(&alone_encoder_init, next, allocator); - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_alone_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_alone_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &alone_encode; next->end = &alone_encoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_HEADER; - next->coder->header_pos = 0; + coder->sequence = SEQ_HEADER; + coder->header_pos = 0; // Encode the header: // - Properties (1 byte) - if (lzma_lzma_lclppb_encode(options, next->coder->header)) + if (lzma_lzma_lclppb_encode(options, coder->header)) return LZMA_OPTIONS_ERROR; // - Dictionary size (4 bytes) @@ -115,7 +113,7 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // one is the next unless it is UINT32_MAX. While the header would // allow any 32-bit integer, we do this to keep the decoder of liblzma // accepting the resulting files. - d = options->dict_size - 1; + uint32_t d = options->dict_size - 1; d |= d >> 2; d |= d >> 3; d |= d >> 4; @@ -124,18 +122,28 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, if (d != UINT32_MAX) ++d; - unaligned_write32le(next->coder->header + 1, d); + unaligned_write32le(coder->header + 1, d); // - Uncompressed size (always unknown and using EOPM) - memset(next->coder->header + 1 + 4, 0xFF, 8); + memset(coder->header + 1 + 4, 0xFF, 8); + + // Initialize the LZMA encoder. + const lzma_filter_info filters[2] = { + { + .init = &lzma_lzma_encoder_init, + .options = (void *)(options), + }, { + .init = NULL, + } + }; - return lzma_next_filter_init(&next->coder->next, allocator, filters); + return lzma_next_filter_init(&coder->next, allocator, filters); } /* extern lzma_ret -lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_alone_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_options_alone *options) { lzma_next_coder_init(&alone_encoder_init, next, allocator, options); @@ -146,7 +154,7 @@ lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options) { - lzma_next_strm_init1(alone_encoder_init, strm, options); + lzma_next_strm_init(alone_encoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/auto_decoder.c b/Utilities/cmliblzma/liblzma/common/auto_decoder.c index 24cf48905..6895c7ccf 100644 --- a/Utilities/cmliblzma/liblzma/common/auto_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/auto_decoder.c @@ -14,7 +14,7 @@ #include "alone_decoder.h" -struct lzma_coder_s { +typedef struct { /// Stream decoder or LZMA_Alone decoder lzma_next_coder next; @@ -26,15 +26,17 @@ struct lzma_coder_s { SEQ_CODE, SEQ_FINISH, } sequence; -}; +} lzma_auto_coder; static lzma_ret -auto_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +auto_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_auto_coder *coder = coder_ptr; + switch (coder->sequence) { case SEQ_INIT: if (*in_pos >= in_size) @@ -100,8 +102,9 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator, static void -auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +auto_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_auto_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -109,8 +112,10 @@ auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_check -auto_decoder_get_check(const lzma_coder *coder) +auto_decoder_get_check(const void *coder_ptr) { + const lzma_auto_coder *coder = coder_ptr; + // It is LZMA_Alone if get_check is NULL. return coder->next.get_check == NULL ? LZMA_CHECK_NONE : coder->next.get_check(coder->next.coder); @@ -118,9 +123,11 @@ auto_decoder_get_check(const lzma_coder *coder) static lzma_ret -auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +auto_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_auto_coder *coder = coder_ptr; + lzma_ret ret; if (coder->next.memconfig != NULL) { @@ -132,7 +139,10 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, // the current memory usage. *memusage = LZMA_MEMUSAGE_BASE; *old_memlimit = coder->memlimit; + ret = LZMA_OK; + if (new_memlimit != 0 && new_memlimit < *memusage) + ret = LZMA_MEMLIMIT_ERROR; } if (ret == LZMA_OK && new_memlimit != 0) @@ -143,32 +153,31 @@ auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, static lzma_ret -auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +auto_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(&auto_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; - if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_auto_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_auto_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &auto_decode; next->end = &auto_decoder_end; next->get_check = &auto_decoder_get_check; next->memconfig = &auto_decoder_memconfig; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } - next->coder->memlimit = memlimit; - next->coder->flags = flags; - next->coder->sequence = SEQ_INIT; + coder->memlimit = my_max(1, memlimit); + coder->flags = flags; + coder->sequence = SEQ_INIT; return LZMA_OK; } @@ -177,7 +186,7 @@ auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init2(auto_decoder_init, strm, memlimit, flags); + lzma_next_strm_init(auto_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c index b4bd388eb..b0ded90dd 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_decoder.c @@ -14,13 +14,10 @@ extern LZMA_API(lzma_ret) -lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, +lzma_block_buffer_decode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder block_decoder; - lzma_ret ret; - if (in_pos == NULL || (in == NULL && *in_pos != in_size) || *in_pos > in_size || out_pos == NULL || (out == NULL && *out_pos != out_size) @@ -28,8 +25,9 @@ lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, return LZMA_PROG_ERROR; // Initialize the Block decoder. - block_decoder = LZMA_NEXT_CODER_INIT; - ret = lzma_block_decoder_init(&block_decoder, allocator, block); + lzma_next_coder block_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_block_decoder_init( + &block_decoder, allocator, block); if (ret == LZMA_OK) { // Save the positions so that we can restore them in case diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c index 136f7f573..39e263aa4 100644 --- a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.c @@ -10,6 +10,7 @@ // /////////////////////////////////////////////////////////////////////////////// +#include "block_buffer_encoder.h" #include "block_encoder.h" #include "filter_encoder.h" #include "lzma2_encoder.h" @@ -28,11 +29,9 @@ + LZMA_CHECK_SIZE_MAX + 3) & ~3) -static lzma_vli -lzma2_bound(lzma_vli uncompressed_size) +static uint64_t +lzma2_bound(uint64_t uncompressed_size) { - lzma_vli overhead; - // Prevent integer overflow in overhead calculation. if (uncompressed_size > COMPRESSED_SIZE_MAX) return 0; @@ -41,7 +40,7 @@ lzma2_bound(lzma_vli uncompressed_size) // uncompressed_size up to the next multiple of LZMA2_CHUNK_MAX, // multiply by the size of per-chunk header, and add one byte for // the end marker. - overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) + const uint64_t overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) / LZMA2_CHUNK_MAX) * LZMA2_HEADER_UNCOMPRESSED + 1; @@ -53,30 +52,36 @@ lzma2_bound(lzma_vli uncompressed_size) } -extern LZMA_API(size_t) -lzma_block_buffer_bound(size_t uncompressed_size) +extern uint64_t +lzma_block_buffer_bound64(uint64_t uncompressed_size) { - // For now, if the data doesn't compress, we always use uncompressed - // chunks of LZMA2. In future we may use Subblock filter too, but - // but for simplicity we probably will still use the same bound - // calculation even though Subblock filter would have slightly less - // overhead. - lzma_vli lzma2_size = lzma2_bound(uncompressed_size); + // If the data doesn't compress, we always use uncompressed + // LZMA2 chunks. + uint64_t lzma2_size = lzma2_bound(uncompressed_size); if (lzma2_size == 0) return 0; // Take Block Padding into account. - lzma2_size = (lzma2_size + 3) & ~LZMA_VLI_C(3); + lzma2_size = (lzma2_size + 3) & ~UINT64_C(3); -#if SIZE_MAX < LZMA_VLI_MAX - // Catch the possible integer overflow on 32-bit systems. There's no - // overflow on 64-bit systems, because lzma2_bound() already takes + // No risk of integer overflow because lzma2_bound() already takes // into account the size of the headers in the Block. - if (SIZE_MAX - HEADERS_BOUND < lzma2_size) + return HEADERS_BOUND + lzma2_size; +} + + +extern LZMA_API(size_t) +lzma_block_buffer_bound(size_t uncompressed_size) +{ + uint64_t ret = lzma_block_buffer_bound64(uncompressed_size); + +#if SIZE_MAX < UINT64_MAX + // Catch the possible integer overflow on 32-bit systems. + if (ret > SIZE_MAX) return 0; #endif - return HEADERS_BOUND + lzma2_size; + return ret; } @@ -84,17 +89,12 @@ static lzma_ret block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - size_t in_pos = 0; - uint8_t control = 0x01; // Dictionary reset - lzma_filter *filters_orig; - - // TODO: Figure out if the last filter is LZMA2 or Subblock and use - // that filter to encode the uncompressed chunks. - // Use LZMA2 uncompressed chunks. We wouldn't need a dictionary at // all, but LZMA2 always requires a dictionary, so use the minimum // value to minimize memory usage of the decoder. - lzma_options_lzma lzma2 = { LZMA_DICT_SIZE_MIN }; + lzma_options_lzma lzma2 = { + .dict_size = LZMA_DICT_SIZE_MIN, + }; lzma_filter filters[2]; filters[0].id = LZMA_FILTER_LZMA2; @@ -103,7 +103,7 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, // Set the above filter options to *block temporarily so that we can // encode the Block Header. - filters_orig = block->filters; + lzma_filter *filters_orig = block->filters; block->filters = filters; if (lzma_block_header_size(block) != LZMA_OK) { @@ -132,17 +132,18 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, *out_pos += block->header_size; // Encode the data using LZMA2 uncompressed chunks. + size_t in_pos = 0; + uint8_t control = 0x01; // Dictionary reset while (in_pos < in_size) { - size_t copy_size; - // Control byte: Indicate uncompressed chunk, of which // the first resets the dictionary. out[(*out_pos)++] = control; control = 0x02; // No dictionary reset // Size of the uncompressed chunk - copy_size = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); + const size_t copy_size + = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); out[(*out_pos)++] = (copy_size - 1) >> 8; out[(*out_pos)++] = (copy_size - 1) & 0xFF; @@ -163,27 +164,18 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, static lzma_ret -block_encode_normal(lzma_block *block, lzma_allocator *allocator, +block_encode_normal(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - size_t out_start; - lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; - lzma_ret ret; - // Find out the size of the Block Header. - block->compressed_size = lzma2_bound(in_size); - if (block->compressed_size == 0) - return LZMA_DATA_ERROR; - - block->uncompressed_size = in_size; return_if_error(lzma_block_header_size(block)); // Reserve space for the Block Header and skip it for now. if (out_size - *out_pos <= block->header_size) return LZMA_BUF_ERROR; - out_start = *out_pos; + const size_t out_start = *out_pos; *out_pos += block->header_size; // Limit out_size so that we stop encoding if the output would grow @@ -193,7 +185,8 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator, // TODO: In many common cases this could be optimized to use // significantly less memory. - ret = lzma_raw_encoder_init( + lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_raw_encoder_init( &raw_encoder, allocator, block->filters); if (ret == LZMA_OK) { @@ -227,15 +220,12 @@ block_encode_normal(lzma_block *block, lzma_allocator *allocator, } -extern LZMA_API(lzma_ret) -lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, +static lzma_ret +block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, const uint8_t *in, size_t in_size, - uint8_t *out, size_t *out_pos, size_t out_size) + uint8_t *out, size_t *out_pos, size_t out_size, + bool try_to_compress) { - size_t check_size; - lzma_ret ret; - size_t i; - // Validate the arguments. if (block == NULL || (in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) @@ -243,11 +233,11 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, // The contents of the structure may depend on the version so // check the version before validating the contents of *block. - if (block->version != 0) + if (block->version > 1) return LZMA_OPTIONS_ERROR; if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX - || block->filters == NULL) + || (try_to_compress && block->filters == NULL)) return LZMA_PROG_ERROR; if (!lzma_check_is_supported(block->check)) @@ -259,7 +249,7 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, out_size -= (out_size - *out_pos) & 3; // Get the size of the Check field. - check_size = lzma_check_size(block->check); + const size_t check_size = lzma_check_size(block->check); assert(check_size != UINT32_MAX); // Reserve space for the Check field. @@ -268,9 +258,19 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, out_size -= check_size; + // Initialize block->uncompressed_size and calculate the worst-case + // value for block->compressed_size. + block->uncompressed_size = in_size; + block->compressed_size = lzma2_bound(in_size); + if (block->compressed_size == 0) + return LZMA_DATA_ERROR; + // Do the actual compression. - ret = block_encode_normal(block, allocator, - in, in_size, out, out_pos, out_size); + lzma_ret ret = LZMA_BUF_ERROR; + if (try_to_compress) + ret = block_encode_normal(block, allocator, + in, in_size, out, out_pos, out_size); + if (ret != LZMA_OK) { // If the error was something else than output buffer // becoming full, return the error now. @@ -291,7 +291,7 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, // Block Padding. No buffer overflow here, because we already adjusted // out_size so that (out_size - out_start) is a multiple of four. // Thus, if the buffer is full, the loop body can never run. - for (i = (size_t)(block->compressed_size); i & 3; ++i) { + for (size_t i = (size_t)(block->compressed_size); i & 3; ++i) { assert(*out_pos < out_size); out[(*out_pos)++] = 0x00; } @@ -313,3 +313,25 @@ lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, return LZMA_OK; } + + +extern LZMA_API(lzma_ret) +lzma_block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + return block_buffer_encode(block, allocator, + in, in_size, out, out_pos, out_size, true); +} + + +extern LZMA_API(lzma_ret) +lzma_block_uncomp_encode(lzma_block *block, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // It won't allocate any memory from heap so no need + // for lzma_allocator. + return block_buffer_encode(block, NULL, + in, in_size, out, out_pos, out_size, false); +} diff --git a/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h new file mode 100644 index 000000000..653207f73 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/block_buffer_encoder.h @@ -0,0 +1,24 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_buffer_encoder.h +/// \brief Single-call .xz Block encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_BUFFER_ENCODER_H +#define LZMA_BLOCK_BUFFER_ENCODER_H + +#include "common.h" + + +/// uint64_t version of lzma_block_buffer_bound(). It is used by +/// stream_encoder_mt.c. Probably the original lzma_block_buffer_bound() +/// should have been 64-bit, but fixing it would break the ABI. +extern uint64_t lzma_block_buffer_bound64(uint64_t uncompressed_size); + +#endif diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.c b/Utilities/cmliblzma/liblzma/common/block_decoder.c index 35996e7cf..075bd279f 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_CODE, SEQ_PADDING, @@ -45,7 +45,10 @@ struct lzma_coder_s { /// Check of the uncompressed data lzma_check_state check; -}; + + /// True if the integrity check won't be calculated and verified. + bool ignore_check; +} lzma_block_coder; static inline bool @@ -71,11 +74,13 @@ is_size_valid(lzma_vli size, lzma_vli reference) static lzma_ret -block_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +block_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_block_coder *coder = coder_ptr; + switch (coder->sequence) { case SEQ_CODE: { const size_t in_start = *in_pos; @@ -97,8 +102,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, coder->block->uncompressed_size)) return LZMA_DATA_ERROR; - lzma_check_update(&coder->check, coder->block->check, - out + out_start, out_used); + if (!coder->ignore_check) + lzma_check_update(&coder->check, coder->block->check, + out + out_start, out_used); if (ret != LZMA_STREAM_END) return ret; @@ -140,7 +146,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, if (coder->block->check == LZMA_CHECK_NONE) return LZMA_STREAM_END; - lzma_check_finish(&coder->check, coder->block->check); + if (!coder->ignore_check) + lzma_check_finish(&coder->check, coder->block->check); + coder->sequence = SEQ_CHECK; // Fall through @@ -155,7 +163,8 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, // Validate the Check only if we support it. // coder->check.buffer may be uninitialized // when the Check ID is not supported. - if (lzma_check_is_supported(coder->block->check) + if (!coder->ignore_check + && lzma_check_is_supported(coder->block->check) && memcmp(coder->block->raw_check, coder->check.buffer.u8, check_size) != 0) @@ -170,8 +179,9 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, static void -block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +block_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_block_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -179,7 +189,7 @@ block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_block_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_block *block) { lzma_next_coder_init(&lzma_block_decoder_init, next, allocator); @@ -191,27 +201,29 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, || !lzma_vli_is_valid(block->uncompressed_size)) return LZMA_PROG_ERROR; - // Allocate and initialize *next->coder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + // Allocate *next->coder if needed. + lzma_block_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_block_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &block_decode; next->end = &block_decoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_CODE; - next->coder->block = block; - next->coder->compressed_size = 0; - next->coder->uncompressed_size = 0; + coder->sequence = SEQ_CODE; + coder->block = block; + coder->compressed_size = 0; + coder->uncompressed_size = 0; // If Compressed Size is not known, we calculate the maximum allowed // value so that encoded size of the Block (including Block Padding) // is still a valid VLI and a multiple of four. - next->coder->compressed_limit + coder->compressed_limit = block->compressed_size == LZMA_VLI_UNKNOWN ? (LZMA_VLI_MAX & ~LZMA_VLI_C(3)) - block->header_size @@ -221,11 +233,14 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Initialize the check. It's caller's problem if the Check ID is not // supported, and the Block decoder cannot verify the Check field. // Caller can test lzma_check_is_supported(block->check). - next->coder->check_pos = 0; - lzma_check_init(&next->coder->check, block->check); + coder->check_pos = 0; + lzma_check_init(&coder->check, block->check); + + coder->ignore_check = block->version >= 1 + ? block->ignore_check : false; // Initialize the filter chain. - return lzma_raw_decoder_init(&next->coder->next, allocator, + return lzma_raw_decoder_init(&coder->next, allocator, block->filters); } @@ -233,7 +248,7 @@ lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_block_decoder(lzma_stream *strm, lzma_block *block) { - lzma_next_strm_init1(lzma_block_decoder_init, strm, block); + lzma_next_strm_init(lzma_block_decoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_decoder.h b/Utilities/cmliblzma/liblzma/common/block_decoder.h index 7da9df63f..718c5ced8 100644 --- a/Utilities/cmliblzma/liblzma/common/block_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_decoder.h @@ -17,6 +17,6 @@ extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_block *block); + const lzma_allocator *allocator, lzma_block *block); #endif diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.c b/Utilities/cmliblzma/liblzma/common/block_encoder.c index ed748273e..168846ad6 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { /// The filters in the chain; initialized with lzma_raw_decoder_init(). lzma_next_coder next; @@ -41,15 +41,17 @@ struct lzma_coder_s { /// Check of the uncompressed data lzma_check_state check; -}; +} lzma_block_coder; static lzma_ret -block_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +block_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_block_coder *coder = coder_ptr; + // Check that our amount of input stays in proper limits. if (LZMA_VLI_MAX - coder->uncompressed_size < in_size - *in_pos) return LZMA_DATA_ERROR; @@ -134,8 +136,9 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator, static void -block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +block_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_block_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -143,10 +146,12 @@ block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +block_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_block_coder *coder = coder_ptr; + if (coder->sequence != SEQ_CODE) return LZMA_PROG_ERROR; @@ -156,7 +161,7 @@ block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_block_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_block *block) { lzma_next_coder_init(&lzma_block_encoder_init, next, allocator); @@ -166,7 +171,7 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // The contents of the structure may depend on the version so // check the version first. - if (block->version != 0) + if (block->version > 1) return LZMA_OPTIONS_ERROR; // If the Check ID is not supported, we cannot calculate the check and @@ -178,37 +183,38 @@ lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_UNSUPPORTED_CHECK; // Allocate and initialize *next->coder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_block_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_block_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &block_encode; next->end = &block_encoder_end; next->update = &block_encoder_update; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Basic initializations - next->coder->sequence = SEQ_CODE; - next->coder->block = block; - next->coder->compressed_size = 0; - next->coder->uncompressed_size = 0; - next->coder->pos = 0; + coder->sequence = SEQ_CODE; + coder->block = block; + coder->compressed_size = 0; + coder->uncompressed_size = 0; + coder->pos = 0; // Initialize the check - lzma_check_init(&next->coder->check, block->check); + lzma_check_init(&coder->check, block->check); // Initialize the requested filters. - return lzma_raw_encoder_init(&next->coder->next, allocator, - block->filters); + return lzma_raw_encoder_init(&coder->next, allocator, block->filters); } extern LZMA_API(lzma_ret) lzma_block_encoder(lzma_stream *strm, lzma_block *block) { - lzma_next_strm_init1(lzma_block_encoder_init, strm, block); + lzma_next_strm_init(lzma_block_encoder_init, strm, block); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/block_encoder.h b/Utilities/cmliblzma/liblzma/common/block_encoder.h index b9eff0be2..bd97c186e 100644 --- a/Utilities/cmliblzma/liblzma/common/block_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/block_encoder.h @@ -42,6 +42,6 @@ extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_block *block); + const lzma_allocator *allocator, lzma_block *block); #endif diff --git a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c index f6e470e69..1dd982f6b 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_decoder.c @@ -15,14 +15,12 @@ static void -free_properties(lzma_block *block, lzma_allocator *allocator) +free_properties(lzma_block *block, const lzma_allocator *allocator) { - size_t i; - // Free allocated filter options. The last array member is not // touched after the initialization in the beginning of // lzma_block_header_decode(), so we don't need to touch that here. - for (i = 0; i < LZMA_FILTERS_MAX; ++i) { + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) { lzma_free(block->filters[i].options, allocator); block->filters[i].id = LZMA_VLI_UNKNOWN; block->filters[i].options = NULL; @@ -34,15 +32,8 @@ free_properties(lzma_block *block, lzma_allocator *allocator) extern LZMA_API(lzma_ret) lzma_block_header_decode(lzma_block *block, - lzma_allocator *allocator, const uint8_t *in) + const lzma_allocator *allocator, const uint8_t *in) { - const size_t filter_count = (in[1] & 3) + 1; - size_t in_size; - size_t i; - - // Start after the Block Header Size and Block Flags fields. - size_t in_pos = 2; - // NOTE: We consider the header to be corrupt not only when the // CRC32 doesn't match, but also when variable-length integers // are invalid or over 63 bits, or if the header is too small @@ -50,13 +41,21 @@ lzma_block_header_decode(lzma_block *block, // Initialize the filter options array. This way the caller can // safely free() the options even if an error occurs in this function. - for (i = 0; i <= LZMA_FILTERS_MAX; ++i) { + for (size_t i = 0; i <= LZMA_FILTERS_MAX; ++i) { block->filters[i].id = LZMA_VLI_UNKNOWN; block->filters[i].options = NULL; } - // Always zero for now. - block->version = 0; + // Versions 0 and 1 are supported. If a newer version was specified, + // we need to downgrade it. + if (block->version > 1) + block->version = 1; + + // This isn't a Block Header option, but since the decompressor will + // read it if version >= 1, it's better to initialize it here than + // to expect the caller to do it since in almost all cases this + // should be false. + block->ignore_check = false; // Validate Block Header Size and Check type. The caller must have // already set these, so it is a programming error if this test fails. @@ -65,7 +64,7 @@ lzma_block_header_decode(lzma_block *block, return LZMA_PROG_ERROR; // Exclude the CRC32 field. - in_size = block->header_size - 4; + const size_t in_size = block->header_size - 4; // Verify CRC32 if (lzma_crc32(in, in_size, 0) != unaligned_read32le(in + in_size)) @@ -75,6 +74,9 @@ lzma_block_header_decode(lzma_block *block, if (in[1] & 0x3C) return LZMA_OPTIONS_ERROR; + // Start after the Block Header Size and Block Flags fields. + size_t in_pos = 2; + // Compressed Size if (in[1] & 0x40) { return_if_error(lzma_vli_decode(&block->compressed_size, @@ -96,7 +98,8 @@ lzma_block_header_decode(lzma_block *block, block->uncompressed_size = LZMA_VLI_UNKNOWN; // Filter Flags - for (i = 0; i < filter_count; ++i) { + const size_t filter_count = (in[1] & 3) + 1; + for (size_t i = 0; i < filter_count; ++i) { const lzma_ret ret = lzma_filter_flags_decode( &block->filters[i], allocator, in, &in_pos, in_size); diff --git a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c index 650295c00..5c5f5424a 100644 --- a/Utilities/cmliblzma/liblzma/common/block_header_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/block_header_encoder.c @@ -17,14 +17,12 @@ extern LZMA_API(lzma_ret) lzma_block_header_size(lzma_block *block) { - size_t i; + if (block->version > 1) + return LZMA_OPTIONS_ERROR; // Block Header Size + Block Flags + CRC32. uint32_t size = 1 + 1 + 4; - if (block->version != 0) - return LZMA_OPTIONS_ERROR; - // Compressed Size if (block->compressed_size != LZMA_VLI_UNKNOWN) { const uint32_t add = lzma_vli_size(block->compressed_size); @@ -47,13 +45,12 @@ lzma_block_header_size(lzma_block *block) if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) return LZMA_PROG_ERROR; - for (i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { - uint32_t add; - + for (size_t i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { // Don't allow too many filters. if (i == LZMA_FILTERS_MAX) return LZMA_PROG_ERROR; + uint32_t add; return_if_error(lzma_filter_flags_size(&add, block->filters + i)); @@ -76,23 +73,20 @@ lzma_block_header_size(lzma_block *block) extern LZMA_API(lzma_ret) lzma_block_header_encode(const lzma_block *block, uint8_t *out) { - size_t out_size; - size_t out_pos = 2; - size_t filter_count = 0; - // Validate everything but filters. if (lzma_block_unpadded_size(block) == 0 || !lzma_vli_is_valid(block->uncompressed_size)) return LZMA_PROG_ERROR; // Indicate the size of the buffer _excluding_ the CRC32 field. - out_size = block->header_size - 4; + const size_t out_size = block->header_size - 4; // Store the Block Header Size. out[0] = out_size / 4; // We write Block Flags in pieces. out[1] = 0x00; + size_t out_pos = 2; // Compressed Size if (block->compressed_size != LZMA_VLI_UNKNOWN) { @@ -114,6 +108,7 @@ lzma_block_header_encode(const lzma_block *block, uint8_t *out) if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) return LZMA_PROG_ERROR; + size_t filter_count = 0; do { // There can be a maximum of four filters. if (filter_count == LZMA_FILTERS_MAX) diff --git a/Utilities/cmliblzma/liblzma/common/block_util.c b/Utilities/cmliblzma/liblzma/common/block_util.c index 4cd34d100..00c7fe8d5 100644 --- a/Utilities/cmliblzma/liblzma/common/block_util.c +++ b/Utilities/cmliblzma/liblzma/common/block_util.c @@ -17,14 +17,11 @@ extern LZMA_API(lzma_ret) lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) { - uint32_t container_size; - lzma_vli compressed_size; - // Validate everything but Uncompressed Size and filters. if (lzma_block_unpadded_size(block) == 0) return LZMA_PROG_ERROR; - container_size = block->header_size + const uint32_t container_size = block->header_size + lzma_check_size(block->check); // Validate that Compressed Size will be greater than zero. @@ -34,7 +31,7 @@ lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) // Calculate what Compressed Size is supposed to be. // If Compressed Size was present in Block Header, // compare that the new value matches it. - compressed_size = unpadded_size - container_size; + const lzma_vli compressed_size = unpadded_size - container_size; if (block->compressed_size != LZMA_VLI_UNKNOWN && block->compressed_size != compressed_size) return LZMA_DATA_ERROR; @@ -48,15 +45,13 @@ lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size) extern LZMA_API(lzma_vli) lzma_block_unpadded_size(const lzma_block *block) { - lzma_vli unpadded_size; - // Validate the values that we are interested in i.e. all but // Uncompressed Size and the filters. // // NOTE: This function is used for validation too, so it is // essential that these checks are always done even if // Compressed Size is unknown. - if (block == NULL || block->version != 0 + if (block == NULL || block->version > 1 || block->header_size < LZMA_BLOCK_HEADER_SIZE_MIN || block->header_size > LZMA_BLOCK_HEADER_SIZE_MAX || (block->header_size & 3) @@ -71,7 +66,7 @@ lzma_block_unpadded_size(const lzma_block *block) return LZMA_VLI_UNKNOWN; // Calculate Unpadded Size and validate it. - unpadded_size = block->compressed_size + const lzma_vli unpadded_size = block->compressed_size + block->header_size + lzma_check_size(block->check); diff --git a/Utilities/cmliblzma/liblzma/common/common.c b/Utilities/cmliblzma/liblzma/common/common.c index 2e723c8fe..57e3f8ebd 100644 --- a/Utilities/cmliblzma/liblzma/common/common.c +++ b/Utilities/cmliblzma/liblzma/common/common.c @@ -36,14 +36,14 @@ lzma_version_string(void) /////////////////////// extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) -lzma_alloc(size_t size, lzma_allocator *allocator) +lzma_alloc(size_t size, const lzma_allocator *allocator) { - void *ptr; - // Some malloc() variants return NULL if called with size == 0. if (size == 0) size = 1; + void *ptr; + if (allocator != NULL && allocator->alloc != NULL) ptr = allocator->alloc(allocator->opaque, 1, size); else @@ -53,8 +53,29 @@ lzma_alloc(size_t size, lzma_allocator *allocator) } +extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) +lzma_alloc_zero(size_t size, const lzma_allocator *allocator) +{ + // Some calloc() variants return NULL if called with size == 0. + if (size == 0) + size = 1; + + void *ptr; + + if (allocator != NULL && allocator->alloc != NULL) { + ptr = allocator->alloc(allocator->opaque, 1, size); + if (ptr != NULL) + memzero(ptr, size); + } else { + ptr = calloc(1, size); + } + + return ptr; +} + + extern void -lzma_free(void *ptr, lzma_allocator *allocator) +lzma_free(void *ptr, const lzma_allocator *allocator) { if (allocator != NULL && allocator->free != NULL) allocator->free(allocator->opaque, ptr); @@ -70,9 +91,9 @@ lzma_free(void *ptr, lzma_allocator *allocator) ////////// extern size_t -lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size) +lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) { const size_t in_avail = in_size - *in_pos; const size_t out_avail = out_size - *out_pos; @@ -88,7 +109,7 @@ lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, extern lzma_ret -lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_next_filter_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { lzma_next_coder_init(filters[0].init, next, allocator); @@ -99,7 +120,7 @@ lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, +lzma_next_filter_update(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *reversed_filters) { // Check that the application isn't trying to change the Filter ID. @@ -117,7 +138,7 @@ lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, extern void -lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator) +lzma_next_end(lzma_next_coder *next, const lzma_allocator *allocator) { if (next->init != (uintptr_t)(NULL)) { // To avoid tiny end functions that simply call @@ -156,10 +177,8 @@ lzma_strm_init(lzma_stream *strm) strm->internal->next = LZMA_NEXT_CODER_INIT; } - strm->internal->supported_actions[LZMA_RUN] = false; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = false; - strm->internal->supported_actions[LZMA_FULL_FLUSH] = false; - strm->internal->supported_actions[LZMA_FINISH] = false; + memzero(strm->internal->supported_actions, + sizeof(strm->internal->supported_actions)); strm->internal->sequence = ISEQ_RUN; strm->internal->allow_buf_error = false; @@ -173,16 +192,12 @@ lzma_strm_init(lzma_stream *strm) extern LZMA_API(lzma_ret) lzma_code(lzma_stream *strm, lzma_action action) { - size_t in_pos = 0; - size_t out_pos = 0; - lzma_ret ret; - // Sanity checks if ((strm->next_in == NULL && strm->avail_in != 0) || (strm->next_out == NULL && strm->avail_out != 0) || strm->internal == NULL || strm->internal->next.code == NULL - || (unsigned int)(action) > LZMA_FINISH + || (unsigned int)(action) > LZMA_ACTION_MAX || !strm->internal->supported_actions[action]) return LZMA_PROG_ERROR; @@ -217,6 +232,10 @@ lzma_code(lzma_stream *strm, lzma_action action) case LZMA_FINISH: strm->internal->sequence = ISEQ_FINISH; break; + + case LZMA_FULL_BARRIER: + strm->internal->sequence = ISEQ_FULL_BARRIER; + break; } break; @@ -244,6 +263,13 @@ lzma_code(lzma_stream *strm, lzma_action action) break; + case ISEQ_FULL_BARRIER: + if (action != LZMA_FULL_BARRIER + || strm->internal->avail_in != strm->avail_in) + return LZMA_PROG_ERROR; + + break; + case ISEQ_END: return LZMA_STREAM_END; @@ -252,7 +278,9 @@ lzma_code(lzma_stream *strm, lzma_action action) return LZMA_PROG_ERROR; } - ret = strm->internal->next.code( + size_t in_pos = 0; + size_t out_pos = 0; + lzma_ret ret = strm->internal->next.code( strm->internal->next.coder, strm->allocator, strm->next_in, &in_pos, strm->avail_in, strm->next_out, &out_pos, strm->avail_out, action); @@ -267,7 +295,9 @@ lzma_code(lzma_stream *strm, lzma_action action) strm->internal->avail_in = strm->avail_in; - switch (ret) { + // Cast is needed to silence a warning about LZMA_TIMED_OUT, which + // isn't part of lzma_ret enumeration. + switch ((unsigned int)(ret)) { case LZMA_OK: // Don't return LZMA_BUF_ERROR when it happens the first time. // This is to avoid returning LZMA_BUF_ERROR when avail_out @@ -283,9 +313,16 @@ lzma_code(lzma_stream *strm, lzma_action action) } break; + case LZMA_TIMED_OUT: + strm->internal->allow_buf_error = false; + ret = LZMA_OK; + break; + case LZMA_STREAM_END: if (strm->internal->sequence == ISEQ_SYNC_FLUSH - || strm->internal->sequence == ISEQ_FULL_FLUSH) + || strm->internal->sequence == ISEQ_FULL_FLUSH + || strm->internal->sequence + == ISEQ_FULL_BARRIER) strm->internal->sequence = ISEQ_RUN; else strm->internal->sequence = ISEQ_END; @@ -325,6 +362,22 @@ lzma_end(lzma_stream *strm) } +extern LZMA_API(void) +lzma_get_progress(lzma_stream *strm, + uint64_t *progress_in, uint64_t *progress_out) +{ + if (strm->internal->next.get_progress != NULL) { + strm->internal->next.get_progress(strm->internal->next.coder, + progress_in, progress_out); + } else { + *progress_in = strm->total_in; + *progress_out = strm->total_out; + } + + return; +} + + extern LZMA_API(lzma_check) lzma_get_check(const lzma_stream *strm) { @@ -382,8 +435,10 @@ lzma_memlimit_set(lzma_stream *strm, uint64_t new_memlimit) || strm->internal->next.memconfig == NULL) return LZMA_PROG_ERROR; - if (new_memlimit != 0 && new_memlimit < LZMA_MEMUSAGE_BASE) - return LZMA_MEMLIMIT_ERROR; + // Zero is a special value that cannot be used as an actual limit. + // If 0 was specified, use 1 instead. + if (new_memlimit == 0) + new_memlimit = 1; return strm->internal->next.memconfig(strm->internal->next.coder, &memusage, &old_memlimit, new_memlimit); diff --git a/Utilities/cmliblzma/liblzma/common/common.h b/Utilities/cmliblzma/liblzma/common/common.h index a6a28189e..dde3ae0ec 100644 --- a/Utilities/cmliblzma/liblzma/common/common.h +++ b/Utilities/cmliblzma/liblzma/common/common.h @@ -48,6 +48,13 @@ #define LZMA_BUFFER_SIZE 4096 +/// Maximum number of worker threads within one multithreaded component. +/// The limit exists solely to make it simpler to prevent integer overflows +/// when allocating structures etc. This should be big enough for now... +/// the code won't scale anywhere close to this number anyway. +#define LZMA_THREADS_MAX 16384 + + /// Starting value for memory usage estimates. Instead of calculating size /// of _every_ structure and taking into account malloc() overhead etc., we /// add a base size to all memory usage estimates. It's not very accurate @@ -65,12 +72,20 @@ ( LZMA_TELL_NO_CHECK \ | LZMA_TELL_UNSUPPORTED_CHECK \ | LZMA_TELL_ANY_CHECK \ + | LZMA_IGNORE_CHECK \ | LZMA_CONCATENATED ) -/// Type of encoder/decoder specific data; the actual structure is defined -/// differently in different coders. -typedef struct lzma_coder_s lzma_coder; +/// Largest valid lzma_action value as unsigned integer. +#define LZMA_ACTION_MAX ((unsigned int)(LZMA_FULL_BARRIER)) + + +/// Special return value (lzma_ret) to indicate that a timeout was reached +/// and lzma_code() must not return LZMA_BUF_ERROR. This is converted to +/// LZMA_OK in lzma_code(). This is not in the lzma_ret enumeration because +/// there's no need to have it in the public API. +#define LZMA_TIMED_OUT 32 + typedef struct lzma_next_coder_s lzma_next_coder; @@ -79,7 +94,7 @@ typedef struct lzma_filter_info_s lzma_filter_info; /// Type of a function used to initialize a filter encoder or decoder typedef lzma_ret (*lzma_init_function)( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); /// Type of a function to do some kind of coding work (filters, Stream, @@ -87,15 +102,15 @@ typedef lzma_ret (*lzma_init_function)( /// input and output buffers, but for simplicity they still use this same /// function prototype. typedef lzma_ret (*lzma_code_function)( - lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, + void *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action); /// Type of a function to free the memory allocated for the coder typedef void (*lzma_end_function)( - lzma_coder *coder, lzma_allocator *allocator); + void *coder, const lzma_allocator *allocator); /// Raw coder validates and converts an array of lzma_filter structures to @@ -118,7 +133,7 @@ struct lzma_filter_info_s { /// Hold data and function pointers of the next filter in the chain. struct lzma_next_coder_s { /// Pointer to coder-specific data - lzma_coder *coder; + void *coder; /// Filter ID. This is LZMA_VLI_UNKNOWN when this structure doesn't /// point to a filter coder. @@ -138,35 +153,41 @@ struct lzma_next_coder_s { /// lzma_next_coder.coder. lzma_end_function end; + /// Pointer to a function to get progress information. If this is NULL, + /// lzma_stream.total_in and .total_out are used instead. + void (*get_progress)(void *coder, + uint64_t *progress_in, uint64_t *progress_out); + /// Pointer to function to return the type of the integrity check. /// Most coders won't support this. - lzma_check (*get_check)(const lzma_coder *coder); + lzma_check (*get_check)(const void *coder); /// Pointer to function to get and/or change the memory usage limit. /// If new_memlimit == 0, the limit is not changed. - lzma_ret (*memconfig)(lzma_coder *coder, uint64_t *memusage, + lzma_ret (*memconfig)(void *coder, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit); /// Update the filter-specific options or the whole filter chain /// in the encoder. - lzma_ret (*update)(lzma_coder *coder, lzma_allocator *allocator, + lzma_ret (*update)(void *coder, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters); }; -/// Constant to initialize lzma_next_coder structure -static const lzma_next_coder LZMA_NEXT_CODER_INIT = - { - NULL, - LZMA_VLI_UNKNOWN, - (uintptr_t)(NULL), - NULL, - NULL, - NULL, - NULL, - NULL, - }; +/// Macro to initialize lzma_next_coder structure +#define LZMA_NEXT_CODER_INIT \ + (lzma_next_coder){ \ + .coder = NULL, \ + .init = (uintptr_t)(NULL), \ + .id = LZMA_VLI_UNKNOWN, \ + .code = NULL, \ + .end = NULL, \ + .get_progress = NULL, \ + .get_check = NULL, \ + .memconfig = NULL, \ + .update = NULL, \ + } /// Internal data for lzma_strm_init, lzma_code, and lzma_end. A pointer to @@ -184,6 +205,7 @@ struct lzma_internal_s { ISEQ_SYNC_FLUSH, ISEQ_FULL_FLUSH, ISEQ_FINISH, + ISEQ_FULL_BARRIER, ISEQ_END, ISEQ_ERROR, } sequence; @@ -194,7 +216,7 @@ struct lzma_internal_s { size_t avail_in; /// Indicates which lzma_action values are allowed by next.code. - bool supported_actions[4]; + bool supported_actions[LZMA_ACTION_MAX + 1]; /// If true, lzma_code will return LZMA_BUF_ERROR if no progress was /// made (no input consumed and no output produced by next.code). @@ -203,15 +225,21 @@ struct lzma_internal_s { /// Allocates memory -extern void *lzma_alloc(size_t size, lzma_allocator *allocator) +extern void *lzma_alloc(size_t size, const lzma_allocator *allocator) lzma_attribute((__malloc__)) lzma_attr_alloc_size(1); +/// Allocates memory and zeroes it (like calloc()). This can be faster +/// than lzma_alloc() + memzero() while being backward compatible with +/// custom allocators. +extern void * lzma_attribute((__malloc__)) lzma_attr_alloc_size(1) + lzma_alloc_zero(size_t size, const lzma_allocator *allocator); + /// Frees memory -extern void lzma_free(void *ptr, lzma_allocator *allocator); +extern void lzma_free(void *ptr, const lzma_allocator *allocator); /// Allocates strm->internal if it is NULL, and initializes *strm and -/// strm->internal. This function is only called via lzma_next_strm_init2 macro. +/// strm->internal. This function is only called via lzma_next_strm_init macro. extern lzma_ret lzma_strm_init(lzma_stream *strm); /// Initializes the next filter in the chain, if any. This takes care of @@ -219,24 +247,26 @@ extern lzma_ret lzma_strm_init(lzma_stream *strm); /// than the filter being initialized now. This way the actual filter /// initialization functions don't need to use lzma_next_coder_init macro. extern lzma_ret lzma_next_filter_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); /// Update the next filter in the chain, if any. This checks that /// the application is not trying to change the Filter IDs. extern lzma_ret lzma_next_filter_update( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *reversed_filters); /// Frees the memory allocated for next->coder either using next->end or, /// if next->end is NULL, using lzma_free. -extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator); +extern void lzma_next_end(lzma_next_coder *next, + const lzma_allocator *allocator); /// Copy as much data as possible from in[] to out[] and update *in_pos /// and *out_pos accordingly. Returns the number of bytes copied. -extern size_t lzma_bufcpy(const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size); +extern size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// \brief Return if expression doesn't evaluate to LZMA_OK @@ -269,37 +299,15 @@ do { \ /// (The function being called will use lzma_next_coder_init()). If /// initialization fails, memory that wasn't freed by func() is freed /// along strm->internal. -#define lzma_next_strm_init1(func, strm, arg1) \ +#define lzma_next_strm_init(func, strm, ...) \ do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ -} while (0) - -#define lzma_next_strm_init2(func, strm, arg1, arg2) \ -do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1, arg2); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ -} while (0) - -#define lzma_next_strm_init3(func, strm, arg1, arg2, arg3) \ -do { \ - lzma_ret ret_; \ - return_if_error(lzma_strm_init(strm)); \ - ret_ = func(&(strm)->internal->next, (strm)->allocator, arg1, arg2, arg3); \ - if (ret_ != LZMA_OK) { \ - lzma_end(strm); \ - return ret_; \ - } \ + return_if_error(lzma_strm_init(strm)); \ + const lzma_ret ret_ = func(&(strm)->internal->next, \ + (strm)->allocator, __VA_ARGS__); \ + if (ret_ != LZMA_OK) { \ + lzma_end(strm); \ + return ret_; \ + } \ } while (0) #endif diff --git a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c index c4be34ccf..48eb56f5c 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_buffer_encoder.c @@ -15,8 +15,8 @@ extern LZMA_API(lzma_ret) lzma_easy_buffer_encode(uint32_t preset, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, - uint8_t *out, size_t *out_pos, size_t out_size) + const lzma_allocator *allocator, const uint8_t *in, + size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { lzma_options_easy opt_easy; if (lzma_easy_preset(&opt_easy, preset)) diff --git a/Utilities/cmliblzma/liblzma/common/easy_encoder.c b/Utilities/cmliblzma/liblzma/common/easy_encoder.c index d13ccd735..5cb492dd0 100644 --- a/Utilities/cmliblzma/liblzma/common/easy_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/easy_encoder.c @@ -11,7 +11,6 @@ /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" -#include "stream_encoder.h" extern LZMA_API(lzma_ret) diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c index 65665c17d..6620986ee 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_decoder.c @@ -14,30 +14,27 @@ extern LZMA_API(lzma_ret) -lzma_raw_buffer_decode(const lzma_filter *filters, lzma_allocator *allocator, +lzma_raw_buffer_decode( + const lzma_filter *filters, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder next = LZMA_NEXT_CODER_INIT; - size_t in_start; - size_t out_start; - lzma_ret ret; - // Validate what isn't validated later in filter_common.c. if (in == NULL || in_pos == NULL || *in_pos > in_size || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // Initialize the decoer. + lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_decoder_init(&next, allocator, filters)); // Store the positions so that we can restore them if something // goes wrong. - in_start = *in_pos; - out_start = *out_pos; + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; // Do the actual decoding and free decoder's memory. - ret = next.code(next.coder, allocator, in, in_pos, in_size, + lzma_ret ret = next.code(next.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, LZMA_FINISH); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c index b23329f3e..dda18e3d8 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_buffer_encoder.c @@ -14,29 +14,27 @@ extern LZMA_API(lzma_ret) -lzma_raw_buffer_encode(const lzma_filter *filters, lzma_allocator *allocator, - const uint8_t *in, size_t in_size, uint8_t *out, - size_t *out_pos, size_t out_size) +lzma_raw_buffer_encode( + const lzma_filter *filters, const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder next = LZMA_NEXT_CODER_INIT; - size_t out_start; - size_t in_pos = 0; - lzma_ret ret; - // Validate what isn't validated later in filter_common.c. if ((in == NULL && in_size != 0) || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; // Initialize the encoder + lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_encoder_init(&next, allocator, filters)); // Store the output position so that we can restore it if // something goes wrong. - out_start = *out_pos; + const size_t out_start = *out_pos; // Do the actual encoding and free coder's memory. - ret = next.code(next.coder, allocator, in, &in_pos, in_size, + size_t in_pos = 0; + lzma_ret ret = next.code(next.coder, allocator, in, &in_pos, in_size, out, out_pos, out_size, LZMA_FINISH); lzma_next_end(&next, allocator); diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.c b/Utilities/cmliblzma/liblzma/common/filter_common.c index d2b9e086e..9ad5d5d8e 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.c +++ b/Utilities/cmliblzma/liblzma/common/filter_common.c @@ -36,101 +36,100 @@ static const struct { } features[] = { #if defined (HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) { - LZMA_FILTER_LZMA1, - sizeof(lzma_options_lzma), - false, - true, - true, + .id = LZMA_FILTER_LZMA1, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, }, #endif #if defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) { - LZMA_FILTER_LZMA2, - sizeof(lzma_options_lzma), - false, - true, - true, + .id = LZMA_FILTER_LZMA2, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, }, #endif #if defined(HAVE_ENCODER_X86) || defined(HAVE_DECODER_X86) { - LZMA_FILTER_X86, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_X86, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) { - LZMA_FILTER_POWERPC, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_POWERPC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_IA64) || defined(HAVE_DECODER_IA64) { - LZMA_FILTER_IA64, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_IA64, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM) { - LZMA_FILTER_ARM, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_ARM, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB) { - LZMA_FILTER_ARMTHUMB, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_ARMTHUMB, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) { - LZMA_FILTER_SPARC, - sizeof(lzma_options_bcj), - true, - false, - false, + .id = LZMA_FILTER_SPARC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif #if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) { - LZMA_FILTER_DELTA, - sizeof(lzma_options_delta), - true, - false, - false, + .id = LZMA_FILTER_DELTA, + .options_size = sizeof(lzma_options_delta), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, }, #endif { - LZMA_VLI_UNKNOWN + .id = LZMA_VLI_UNKNOWN } }; extern LZMA_API(lzma_ret) lzma_filters_copy(const lzma_filter *src, lzma_filter *dest, - lzma_allocator *allocator) + const lzma_allocator *allocator) { - size_t i; - lzma_ret ret; - if (src == NULL || dest == NULL) return LZMA_PROG_ERROR; + lzma_ret ret; + size_t i; for (i = 0; src[i].id != LZMA_VLI_UNKNOWN; ++i) { // There must be a maximum of four filters plus // the array terminator. @@ -194,6 +193,10 @@ error: static lzma_ret validate_chain(const lzma_filter *filters, size_t *count) { + // There must be at least one filter. + if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_PROG_ERROR; + // Number of non-last filters that may change the size of the data // significantly (that is, more than 1-2 % or so). size_t changes_size_count = 0; @@ -207,11 +210,6 @@ validate_chain(const lzma_filter *filters, size_t *count) bool last_ok = false; size_t i = 0; - - // There must be at least one filter. - if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) - return LZMA_PROG_ERROR; - do { size_t j; for (j = 0; filters[i].id != features[j].id; ++j) @@ -241,21 +239,18 @@ validate_chain(const lzma_filter *filters, size_t *count) extern lzma_ret -lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options, lzma_filter_find coder_find, bool is_encoder) { - lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; - size_t count; - size_t i; - lzma_ret ret; - // Do some basic validation and get the number of filters. + size_t count; return_if_error(validate_chain(options, &count)); // Set the filter functions and copy the options pointer. + lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; if (is_encoder) { - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { // The order of the filters is reversed in the // encoder. It allows more efficient handling // of the uncompressed data. @@ -271,7 +266,7 @@ lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, filters[j].options = options[i].options; } } else { - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { const lzma_filter_coder *const fc = coder_find(options[i].id); if (fc == NULL || fc->init == NULL) @@ -288,7 +283,7 @@ lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, filters[count].init = NULL; // Initialize the filters. - ret = lzma_next_filter_init(next, allocator, filters); + const lzma_ret ret = lzma_next_filter_init(next, allocator, filters); if (ret != LZMA_OK) lzma_next_end(next, allocator); @@ -300,9 +295,6 @@ extern uint64_t lzma_raw_coder_memusage(lzma_filter_find coder_find, const lzma_filter *filters) { - uint64_t total = 0; - size_t i = 0; - // The chain has to have at least one filter. { size_t tmp; @@ -310,6 +302,9 @@ lzma_raw_coder_memusage(lzma_filter_find coder_find, return UINT64_MAX; } + uint64_t total = 0; + size_t i = 0; + do { const lzma_filter_coder *const fc = coder_find(filters[i].id); diff --git a/Utilities/cmliblzma/liblzma/common/filter_common.h b/Utilities/cmliblzma/liblzma/common/filter_common.h index cd61fc072..42a26a24a 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_common.h +++ b/Utilities/cmliblzma/liblzma/common/filter_common.h @@ -36,7 +36,7 @@ typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id); extern lzma_ret lzma_raw_coder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters, lzma_filter_find coder_find, bool is_encoder); diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_decoder.c index cce2b30ea..c75b0a89c 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.c @@ -35,7 +35,8 @@ typedef struct { /// \return - LZMA_OK: Properties decoded successfully. /// - LZMA_OPTIONS_ERROR: Unsupported properties /// - LZMA_MEM_ERROR: Memory allocation failed. - lzma_ret (*props_decode)(void **options, lzma_allocator *allocator, + lzma_ret (*props_decode)( + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); } lzma_filter_decoder; @@ -44,74 +45,74 @@ typedef struct { static const lzma_filter_decoder decoders[] = { #ifdef HAVE_DECODER_LZMA1 { - LZMA_FILTER_LZMA1, - &lzma_lzma_decoder_init, - &lzma_lzma_decoder_memusage, - &lzma_lzma_props_decode, + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_decoder_init, + .memusage = &lzma_lzma_decoder_memusage, + .props_decode = &lzma_lzma_props_decode, }, #endif #ifdef HAVE_DECODER_LZMA2 { - LZMA_FILTER_LZMA2, - &lzma_lzma2_decoder_init, - &lzma_lzma2_decoder_memusage, - &lzma_lzma2_props_decode, + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_decoder_init, + .memusage = &lzma_lzma2_decoder_memusage, + .props_decode = &lzma_lzma2_props_decode, }, #endif #ifdef HAVE_DECODER_X86 { - LZMA_FILTER_X86, - &lzma_simple_x86_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_POWERPC { - LZMA_FILTER_POWERPC, - &lzma_simple_powerpc_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_IA64 { - LZMA_FILTER_IA64, - &lzma_simple_ia64_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_ARM { - LZMA_FILTER_ARM, - &lzma_simple_arm_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_ARMTHUMB { - LZMA_FILTER_ARMTHUMB, - &lzma_simple_armthumb_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_SPARC { - LZMA_FILTER_SPARC, - &lzma_simple_sparc_decoder_init, - NULL, - &lzma_simple_props_decode, + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, }, #endif #ifdef HAVE_DECODER_DELTA { - LZMA_FILTER_DELTA, - &lzma_delta_decoder_init, - &lzma_delta_coder_memusage, - &lzma_delta_props_decode, + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_decoder_init, + .memusage = &lzma_delta_coder_memusage, + .props_decode = &lzma_delta_props_decode, }, #endif }; @@ -120,8 +121,7 @@ static const lzma_filter_decoder decoders[] = { static const lzma_filter_decoder * decoder_find(lzma_vli id) { - size_t i; - for (i = 0; i < ARRAY_SIZE(decoders); ++i) + for (size_t i = 0; i < ARRAY_SIZE(decoders); ++i) if (decoders[i].id == id) return decoders + i; @@ -137,7 +137,7 @@ lzma_filter_decoder_is_supported(lzma_vli id) extern lzma_ret -lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, @@ -148,7 +148,7 @@ lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options) { - lzma_next_strm_init1(lzma_raw_decoder_init, strm, options); + lzma_next_strm_init(lzma_raw_decoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -166,14 +166,13 @@ lzma_raw_decoder_memusage(const lzma_filter *filters) extern LZMA_API(lzma_ret) -lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator, +lzma_properties_decode(lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - const lzma_filter_decoder *const fd = decoder_find(filter->id); - // Make it always NULL so that the caller can always safely free() it. filter->options = NULL; + const lzma_filter_decoder *const fd = decoder_find(filter->id); if (fd == NULL) return LZMA_OPTIONS_ERROR; diff --git a/Utilities/cmliblzma/liblzma/common/filter_decoder.h b/Utilities/cmliblzma/liblzma/common/filter_decoder.h index d5c68bdd4..a2e255fe5 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_decoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_raw_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options); #endif diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_encoder.c index 9fdb10008..c5d8f3972 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.c @@ -30,11 +30,11 @@ typedef struct { /// invalid, UINT64_MAX is returned. uint64_t (*memusage)(const void *options); - /// Calculates the minimum sane size for Blocks (or other types of - /// chunks) to which the input data can be split to make - /// multithreaded encoding possible. If this is NULL, it is assumed - /// that the encoder is fast enough with single thread. - lzma_vli (*chunk_size)(const void *options); + /// Calculates the recommended Uncompressed Size for .xz Blocks to + /// which the input data can be split to make multithreaded + /// encoding possible. If this is NULL, it is assumed that + /// the encoder is fast enough with single thread. + uint64_t (*block_size)(const void *options); /// Tells the size of the Filter Properties field. If options are /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed @@ -56,101 +56,95 @@ typedef struct { static const lzma_filter_encoder encoders[] = { #ifdef HAVE_ENCODER_LZMA1 { - LZMA_FILTER_LZMA1, - &lzma_lzma_encoder_init, - &lzma_lzma_encoder_memusage, - NULL, // FIXME - NULL, - 5, - &lzma_lzma_props_encode, + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_encoder_init, + .memusage = &lzma_lzma_encoder_memusage, + .block_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 5, + .props_encode = &lzma_lzma_props_encode, }, #endif #ifdef HAVE_ENCODER_LZMA2 { - LZMA_FILTER_LZMA2, - &lzma_lzma2_encoder_init, - &lzma_lzma2_encoder_memusage, - NULL, // FIXME - NULL, - 1, - &lzma_lzma2_props_encode, + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_encoder_init, + .memusage = &lzma_lzma2_encoder_memusage, + .block_size = &lzma_lzma2_block_size, // FIXME + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_lzma2_props_encode, }, #endif #ifdef HAVE_ENCODER_X86 { - LZMA_FILTER_X86, - &lzma_simple_x86_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_POWERPC { - LZMA_FILTER_POWERPC, - &lzma_simple_powerpc_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_IA64 { - LZMA_FILTER_IA64, - &lzma_simple_ia64_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_ARM { - LZMA_FILTER_ARM, - &lzma_simple_arm_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_ARMTHUMB { - LZMA_FILTER_ARMTHUMB, - &lzma_simple_armthumb_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_SPARC { - LZMA_FILTER_SPARC, - &lzma_simple_sparc_encoder_init, - NULL, - NULL, - &lzma_simple_props_size, - 0, - &lzma_simple_props_encode, + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, }, #endif #ifdef HAVE_ENCODER_DELTA { - LZMA_FILTER_DELTA, - &lzma_delta_encoder_init, - &lzma_delta_coder_memusage, - NULL, - NULL, - 1, - &lzma_delta_props_encode, + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_encoder_init, + .memusage = &lzma_delta_coder_memusage, + .block_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_delta_props_encode, }, #endif }; @@ -159,8 +153,7 @@ static const lzma_filter_encoder encoders[] = { static const lzma_filter_encoder * encoder_find(lzma_vli id) { - size_t i; - for (i = 0; i < ARRAY_SIZE(encoders); ++i) + for (size_t i = 0; i < ARRAY_SIZE(encoders); ++i) if (encoders[i].id == id) return encoders + i; @@ -178,10 +171,6 @@ lzma_filter_encoder_is_supported(lzma_vli id) extern LZMA_API(lzma_ret) lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) { - size_t i; - size_t count = 1; - lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; - if (strm->internal->next.update == NULL) return LZMA_PROG_ERROR; @@ -191,10 +180,12 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) // The actual filter chain in the encoder is reversed. Some things // still want the normal order chain, so we provide both. + size_t count = 1; while (filters[count].id != LZMA_VLI_UNKNOWN) ++count; - for (i = 0; i < count; ++i) + lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; + for (size_t i = 0; i < count; ++i) reversed_filters[count - i - 1] = filters[i]; reversed_filters[count].id = LZMA_VLI_UNKNOWN; @@ -205,7 +196,7 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) extern lzma_ret -lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_raw_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, @@ -216,7 +207,7 @@ lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) { - lzma_next_strm_init3(lzma_raw_coder_init, strm, options, + lzma_next_strm_init(lzma_raw_coder_init, strm, options, (lzma_filter_find)(&encoder_find), true); strm->internal->supported_actions[LZMA_RUN] = true; @@ -235,20 +226,19 @@ lzma_raw_encoder_memusage(const lzma_filter *filters) } -/* -extern LZMA_API(lzma_vli) -lzma_chunk_size(const lzma_filter *filters) +extern uint64_t +lzma_mt_block_size(const lzma_filter *filters) { - lzma_vli max = 0; + uint64_t max = 0; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { const lzma_filter_encoder *const fe = encoder_find(filters[i].id); - if (fe->chunk_size != NULL) { - const lzma_vli size - = fe->chunk_size(filters[i].options); - if (size == LZMA_VLI_UNKNOWN) - return LZMA_VLI_UNKNOWN; + if (fe->block_size != NULL) { + const uint64_t size + = fe->block_size(filters[i].options); + if (size == 0) + return 0; if (size > max) max = size; @@ -257,7 +247,6 @@ lzma_chunk_size(const lzma_filter *filters) return max; } -*/ extern LZMA_API(lzma_ret) diff --git a/Utilities/cmliblzma/liblzma/common/filter_encoder.h b/Utilities/cmliblzma/liblzma/common/filter_encoder.h index 5bc137f64..f1d5683fe 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/filter_encoder.h @@ -16,12 +16,12 @@ #include "common.h" -// FIXME: Might become a part of the public API once finished. -// extern lzma_vli lzma_chunk_size(const lzma_filter *filters); +// FIXME: Might become a part of the public API. +extern uint64_t lzma_mt_block_size(const lzma_filter *filters); extern lzma_ret lzma_raw_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c index aa2dbd560..ddfb08594 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_decoder.c @@ -15,12 +15,9 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_decode( - lzma_filter *filter, lzma_allocator *allocator, + lzma_filter *filter, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { - lzma_vli props_size; - lzma_ret ret; - // Set the pointer to NULL so the caller can always safely free it. filter->options = NULL; @@ -32,6 +29,7 @@ lzma_filter_flags_decode( return LZMA_DATA_ERROR; // Size of Properties + lzma_vli props_size; return_if_error(lzma_vli_decode(&props_size, NULL, in, in_pos, in_size)); @@ -39,7 +37,7 @@ lzma_filter_flags_decode( if (in_size - *in_pos < props_size) return LZMA_DATA_ERROR; - ret = lzma_properties_decode( + const lzma_ret ret = lzma_properties_decode( filter, allocator, in + *in_pos, props_size); *in_pos += props_size; diff --git a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c index 755c407f6..d110566de 100644 --- a/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/filter_flags_encoder.c @@ -31,8 +31,6 @@ extern LZMA_API(lzma_ret) lzma_filter_flags_encode(const lzma_filter *filter, uint8_t *out, size_t *out_pos, size_t out_size) { - uint32_t props_size; - // Filter ID if (filter->id >= LZMA_FILTER_RESERVED_START) return LZMA_PROG_ERROR; @@ -41,6 +39,7 @@ lzma_filter_flags_encode(const lzma_filter *filter, out, out_pos, out_size)); // Size of Properties + uint32_t props_size; return_if_error(lzma_properties_size(&props_size, filter)); return_if_error(lzma_vli_encode(props_size, NULL, out, out_pos, out_size)); diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.h b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c index 46a7aed72..f468366a6 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/hardware_cputhreads.c @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file stream_encoder.h -/// \brief Encodes .xz Streams +/// \file hardware_cputhreads.c +/// \brief Get the number of CPU threads or cores // // Author: Lasse Collin // @@ -10,14 +10,13 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_STREAM_ENCODER_H -#define LZMA_STREAM_ENCODER_H - #include "common.h" +#include "tuklib_cpucores.h" -extern lzma_ret lzma_stream_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter *filters, lzma_check check); -#endif +extern LZMA_API(uint32_t) +lzma_cputhreads(void) +{ + return tuklib_cpucores(); +} diff --git a/Utilities/cmliblzma/liblzma/common/index.c b/Utilities/cmliblzma/liblzma/common/index.c index 26135d22c..26e4e519b 100644 --- a/Utilities/cmliblzma/liblzma/common/index.c +++ b/Utilities/cmliblzma/liblzma/common/index.c @@ -191,8 +191,8 @@ index_tree_init(index_tree *tree) /// Helper for index_tree_end() static void -index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, - void (*free_func)(void *node, lzma_allocator *allocator)) +index_tree_node_end(index_tree_node *node, const lzma_allocator *allocator, + void (*free_func)(void *node, const lzma_allocator *allocator)) { // The tree won't ever be very huge, so recursion should be fine. // 20 levels in the tree is likely quite a lot already in practice. @@ -202,22 +202,21 @@ index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, if (node->right != NULL) index_tree_node_end(node->right, allocator, free_func); - if (free_func != NULL) - free_func(node, allocator); - - lzma_free(node, allocator); + free_func(node, allocator); return; } -/// Free the meory allocated for a tree. If free_func is not NULL, -/// it is called on each node before freeing the node. This is used -/// to free the Record groups from each index_stream before freeing -/// the index_stream itself. +/// Free the memory allocated for a tree. Each node is freed using the +/// given free_func which is either &lzma_free or &index_stream_end. +/// The latter is used to free the Record groups from each index_stream +/// before freeing the index_stream itself. static void -index_tree_end(index_tree *tree, lzma_allocator *allocator, - void (*free_func)(void *node, lzma_allocator *allocator)) +index_tree_end(index_tree *tree, const lzma_allocator *allocator, + void (*free_func)(void *node, const lzma_allocator *allocator)) { + assert(free_func != NULL); + if (tree->root != NULL) index_tree_node_end(tree->root, allocator, free_func); @@ -230,7 +229,6 @@ index_tree_end(index_tree *tree, lzma_allocator *allocator, static void index_tree_append(index_tree *tree, index_tree_node *node) { - uint32_t up; node->parent = tree->rightmost; node->left = NULL; node->right = NULL; @@ -259,10 +257,8 @@ index_tree_append(index_tree *tree, index_tree_node *node) // and thus know the state of the tree just by looking at the node // count. From the node count we can calculate how many steps to go // up in the tree to find the rotation root. - up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); + uint32_t up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); if (up != 0) { - index_tree_node *pivot; - // Locate the root node for the rotation. up = ctz32(tree->count) + 2; do { @@ -270,7 +266,7 @@ index_tree_append(index_tree *tree, index_tree_node *node) } while (--up > 0); // Rotate left using node as the rotation root. - pivot = node->right; + index_tree_node *pivot = node->right; if (node->parent == NULL) { tree->root = pivot; @@ -342,8 +338,8 @@ index_tree_locate(const index_tree *tree, lzma_vli target) /// Allocate and initialize a new Stream using the given base offsets. static index_stream * index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, - lzma_vli stream_number, lzma_vli block_number_base, - lzma_allocator *allocator) + uint32_t stream_number, lzma_vli block_number_base, + const lzma_allocator *allocator) { index_stream *s = lzma_alloc(sizeof(index_stream), allocator); if (s == NULL) @@ -371,16 +367,17 @@ index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, /// Free the memory allocated for a Stream and its Record groups. static void -index_stream_end(void *node, lzma_allocator *allocator) +index_stream_end(void *node, const lzma_allocator *allocator) { index_stream *s = node; - index_tree_end(&s->groups, allocator, NULL); + index_tree_end(&s->groups, allocator, &lzma_free); + lzma_free(s, allocator); return; } static lzma_index * -index_init_plain(lzma_allocator *allocator) +index_init_plain(const lzma_allocator *allocator) { lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); if (i != NULL) { @@ -398,15 +395,13 @@ index_init_plain(lzma_allocator *allocator) extern LZMA_API(lzma_index *) -lzma_index_init(lzma_allocator *allocator) +lzma_index_init(const lzma_allocator *allocator) { - index_stream *s; - lzma_index *i = index_init_plain(allocator); if (i == NULL) return NULL; - s = index_stream_init(0, 0, 1, 0, allocator); + index_stream *s = index_stream_init(0, 0, 1, 0, allocator); if (s == NULL) { lzma_free(i, allocator); return NULL; @@ -419,7 +414,7 @@ lzma_index_init(lzma_allocator *allocator) extern LZMA_API(void) -lzma_index_end(lzma_index *i, lzma_allocator *allocator) +lzma_index_end(lzma_index *i, const lzma_allocator *allocator) { // NOTE: If you modify this function, check also the bottom // of lzma_index_cat(). @@ -605,8 +600,6 @@ lzma_index_padding_size(const lzma_index *i) extern LZMA_API(lzma_ret) lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) { - index_stream *s; - if (i == NULL || stream_flags == NULL) return LZMA_PROG_ERROR; @@ -614,7 +607,7 @@ lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) return_if_error(lzma_stream_flags_compare( stream_flags, stream_flags)); - s = (index_stream *)(i->streams.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); s->stream_flags = *stream_flags; return LZMA_OK; @@ -624,17 +617,14 @@ lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) extern LZMA_API(lzma_ret) lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) { - index_stream *s; - lzma_vli old_stream_padding; - if (i == NULL || stream_padding > LZMA_VLI_MAX || (stream_padding & 3) != 0) return LZMA_PROG_ERROR; - s = (index_stream *)(i->streams.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); // Check that the new value won't make the file grow too big. - old_stream_padding = s->stream_padding; + const lzma_vli old_stream_padding = s->stream_padding; s->stream_padding = 0; if (lzma_index_file_size(i) + stream_padding > LZMA_VLI_MAX) { s->stream_padding = old_stream_padding; @@ -647,29 +637,23 @@ lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) extern LZMA_API(lzma_ret) -lzma_index_append(lzma_index *i, lzma_allocator *allocator, +lzma_index_append(lzma_index *i, const lzma_allocator *allocator, lzma_vli unpadded_size, lzma_vli uncompressed_size) { - index_stream *s; - index_group *g; - lzma_vli compressed_base; - lzma_vli uncompressed_base; - uint32_t index_list_size_add; - // Validate. if (i == NULL || unpadded_size < UNPADDED_SIZE_MIN || unpadded_size > UNPADDED_SIZE_MAX || uncompressed_size > LZMA_VLI_MAX) return LZMA_PROG_ERROR; - s = (index_stream *)(i->streams.rightmost); - g = (index_group *)(s->groups.rightmost); + index_stream *s = (index_stream *)(i->streams.rightmost); + index_group *g = (index_group *)(s->groups.rightmost); - compressed_base = g == NULL ? 0 + const lzma_vli compressed_base = g == NULL ? 0 : vli_ceil4(g->records[g->last].unpadded_sum); - uncompressed_base = g == NULL ? 0 + const lzma_vli uncompressed_base = g == NULL ? 0 : g->records[g->last].uncompressed_sum; - index_list_size_add = lzma_vli_size(unpadded_size) + const uint32_t index_list_size_add = lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); // Check that the file size will stay within limits. @@ -780,10 +764,9 @@ index_cat_helper(const index_cat_info *info, index_stream *this) extern LZMA_API(lzma_ret) -lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, - lzma_allocator *allocator) +lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src, + const lzma_allocator *allocator) { - index_cat_info info; const lzma_vli dest_file_size = lzma_index_file_size(dest); // Check that we don't exceed the file size limits. @@ -813,12 +796,10 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, index_stream *s = (index_stream *)(dest->streams.rightmost); index_group *g = (index_group *)(s->groups.rightmost); if (g != NULL && g->last + 1 < g->allocated) { - index_group *newg; - assert(g->node.left == NULL); assert(g->node.right == NULL); - newg = lzma_alloc(sizeof(index_group) + index_group *newg = lzma_alloc(sizeof(index_group) + (g->last + 1) * sizeof(index_record), allocator); @@ -848,17 +829,21 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, s->groups.rightmost = &newg->node; lzma_free(g, allocator); + + // NOTE: newg isn't leaked here because + // newg == (void *)&newg->node. } } // Add all the Streams from src to dest. Update the base offsets // of each Stream from src. - info.uncompressed_size = dest->uncompressed_size; - info.file_size = dest_file_size; - info.stream_number_add = dest->streams.count; - info.block_number_add = dest->record_count; - info.streams = &dest->streams; - + const index_cat_info info = { + .uncompressed_size = dest->uncompressed_size, + .file_size = dest_file_size, + .stream_number_add = dest->streams.count, + .block_number_add = dest->record_count, + .streams = &dest->streams, + }; index_cat_helper(&info, (index_stream *)(src->streams.root)); // Update info about all the combined Streams. @@ -877,26 +862,18 @@ lzma_index_cat(lzma_index *LZMA_RESTRICT dest, lzma_index *LZMA_RESTRICT src, /// Duplicate an index_stream. static index_stream * -index_dup_stream(const index_stream *src, lzma_allocator *allocator) +index_dup_stream(const index_stream *src, const lzma_allocator *allocator) { - index_stream *dest; - index_group *destg; - index_group *srcg; - size_t i = 0; - // Catch a somewhat theoretical integer overflow. if (src->record_count > PREALLOC_MAX) return NULL; // Allocate and initialize a new Stream. - dest = index_stream_init(src->node.compressed_base, + index_stream *dest = index_stream_init(src->node.compressed_base, src->node.uncompressed_base, src->number, src->block_number_base, allocator); - - // Return immediately if allocation failed or if there are - // no groups to duplicate. - if (dest == NULL || src->groups.leftmost == NULL) - return dest; + if (dest == NULL) + return NULL; // Copy the overall information. dest->record_count = src->record_count; @@ -904,10 +881,14 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) dest->stream_flags = src->stream_flags; dest->stream_padding = src->stream_padding; + // Return if there are no groups to duplicate. + if (src->groups.leftmost == NULL) + return dest; + // Allocate memory for the Records. We put all the Records into // a single group. It's simplest and also tends to make // lzma_index_locate() a little bit faster with very big Indexes. - destg = lzma_alloc(sizeof(index_group) + index_group *destg = lzma_alloc(sizeof(index_group) + src->record_count * sizeof(index_record), allocator); if (destg == NULL) { @@ -923,7 +904,8 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) destg->last = src->record_count - 1; // Go through all the groups in src and copy the Records into destg. - srcg = (index_group *)(src->groups.leftmost); + const index_group *srcg = (const index_group *)(src->groups.leftmost); + size_t i = 0; do { memcpy(destg->records + i, srcg->records, (srcg->last + 1) * sizeof(index_record)); @@ -941,11 +923,8 @@ index_dup_stream(const index_stream *src, lzma_allocator *allocator) extern LZMA_API(lzma_index *) -lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) +lzma_index_dup(const lzma_index *src, const lzma_allocator *allocator) { - index_stream *srcstream; - index_stream *deststream; - // Allocate the base structure (no initial Stream). lzma_index *dest = index_init_plain(allocator); if (dest == NULL) @@ -958,9 +937,11 @@ lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) dest->index_list_size = src->index_list_size; // Copy the Streams and the groups in them. - srcstream = (index_stream *)(src->streams.leftmost); + const index_stream *srcstream + = (const index_stream *)(src->streams.leftmost); do { - deststream = index_dup_stream(srcstream, allocator); + index_stream *deststream = index_dup_stream( + srcstream, allocator); if (deststream == NULL) { lzma_index_end(dest, allocator); return NULL; @@ -1031,6 +1012,8 @@ iter_set_info(lzma_index_iter *iter) iter->internal[ITER_GROUP].p = NULL; } + // NOTE: lzma_index_iter.stream.number is lzma_vli but we use uint32_t + // internally. iter->stream.number = stream->number; iter->stream.block_count = stream->record_count; iter->stream.compressed_offset = stream->node.compressed_base; @@ -1119,19 +1102,14 @@ lzma_index_iter_rewind(lzma_index_iter *iter) extern LZMA_API(lzma_bool) lzma_index_iter_next(lzma_index_iter *iter, lzma_index_iter_mode mode) { - const lzma_index *i; - const index_stream *stream; - const index_group *group; - size_t record; - // Catch unsupported mode values. if ((unsigned int)(mode) > LZMA_INDEX_ITER_NONEMPTY_BLOCK) return true; - i = iter->internal[ITER_INDEX].p; - stream = iter->internal[ITER_STREAM].p; - group = NULL; - record = iter->internal[ITER_RECORD].s; + const lzma_index *i = iter->internal[ITER_INDEX].p; + const index_stream *stream = iter->internal[ITER_STREAM].p; + const index_group *group = NULL; + size_t record = iter->internal[ITER_RECORD].s; // If we are being asked for the next Stream, leave group to NULL // so that the rest of the this function thinks that this Stream @@ -1231,10 +1209,6 @@ again: extern LZMA_API(lzma_bool) lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) { - const index_stream *stream; - const index_group *group; - size_t left, right; - const lzma_index *i = iter->internal[ITER_INDEX].p; // If the target is past the end of the file, return immediately. @@ -1242,12 +1216,12 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) return true; // Locate the Stream containing the target offset. - stream = index_tree_locate(&i->streams, target); + const index_stream *stream = index_tree_locate(&i->streams, target); assert(stream != NULL); target -= stream->node.uncompressed_base; // Locate the group containing the target offset. - group = index_tree_locate(&stream->groups, target); + const index_group *group = index_tree_locate(&stream->groups, target); assert(group != NULL); // Use binary search to locate the exact Record. It is the first @@ -1255,8 +1229,8 @@ lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) // This is because we want the rightmost Record that fullfills the // search criterion. It is possible that there are empty Blocks; // we don't want to return them. - left = 0; - right = group->last; + size_t left = 0; + size_t right = group->last; while (left < right) { const size_t pos = left + (right - left) / 2; diff --git a/Utilities/cmliblzma/liblzma/common/index_decoder.c b/Utilities/cmliblzma/liblzma/common/index_decoder.c index 943cfd58a..cc07a1b8c 100644 --- a/Utilities/cmliblzma/liblzma/common/index_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_decoder.c @@ -14,7 +14,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INDICATOR, SEQ_COUNT, @@ -50,18 +50,20 @@ struct lzma_coder_s { /// CRC32 of the List of Records field uint32_t crc32; -}; +} lzma_index_coder; static lzma_ret -index_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +index_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *LZMA_RESTRICT out lzma_attribute((__unused__)), - size_t *LZMA_RESTRICT out_pos lzma_attribute((__unused__)), + uint8_t *restrict out lzma_attribute((__unused__)), + size_t *restrict out_pos lzma_attribute((__unused__)), size_t out_size lzma_attribute((__unused__)), lzma_action action lzma_attribute((__unused__))) { + lzma_index_coder *coder = coder_ptr; + // Similar optimization as in index_encoder.c const size_t in_start = *in_pos; lzma_ret ret = LZMA_OK; @@ -207,8 +209,9 @@ out: static void -index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +index_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_index_coder *coder = coder_ptr; lzma_index_end(coder->index, allocator); lzma_free(coder, allocator); return; @@ -216,9 +219,11 @@ index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +index_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_index_coder *coder = coder_ptr; + *memusage = lzma_index_memusage(1, coder->count); *old_memlimit = coder->memlimit; @@ -234,7 +239,7 @@ index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, static lzma_ret -index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, +index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { // Remember the pointer given by the application. We will set it @@ -251,7 +256,7 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, // Initialize the rest. coder->sequence = SEQ_INDICATOR; - coder->memlimit = memlimit; + coder->memlimit = my_max(1, memlimit); coder->count = 0; // Needs to be initialized due to _memconfig(). coder->pos = 0; coder->crc32 = 0; @@ -261,35 +266,37 @@ index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, static lzma_ret -index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_index **i, uint64_t memlimit) { lzma_next_coder_init(&index_decoder_init, next, allocator); - if (i == NULL || memlimit == 0) + if (i == NULL) return LZMA_PROG_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_index_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_index_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &index_decode; next->end = &index_decoder_end; next->memconfig = &index_decoder_memconfig; - next->coder->index = NULL; + coder->index = NULL; } else { - lzma_index_end(next->coder->index, allocator); + lzma_index_end(coder->index, allocator); } - return index_decoder_reset(next->coder, allocator, i, memlimit); + return index_decoder_reset(coder, allocator, i, memlimit); } extern LZMA_API(lzma_ret) lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) { - lzma_next_strm_init2(index_decoder_init, strm, i, memlimit); + lzma_next_strm_init(index_decoder_init, strm, i, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -299,27 +306,25 @@ lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) extern LZMA_API(lzma_ret) -lzma_index_buffer_decode( - lzma_index **i, uint64_t *memlimit, lzma_allocator *allocator, +lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { - lzma_coder coder; - lzma_ret ret; - - // Store the input start position so that we can restore it in case - // of an error. - const size_t in_start = *in_pos; - // Sanity checks if (i == NULL || memlimit == NULL || in == NULL || in_pos == NULL || *in_pos > in_size) return LZMA_PROG_ERROR; // Initialize the decoder. + lzma_index_coder coder; return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit)); + // Store the input start position so that we can restore it in case + // of an error. + const size_t in_start = *in_pos; + // Do the actual decoding. - ret = index_decode(&coder, allocator, in, in_pos, in_size, + lzma_ret ret = index_decode(&coder, allocator, in, in_pos, in_size, NULL, NULL, 0, LZMA_RUN); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.c b/Utilities/cmliblzma/liblzma/common/index_encoder.c index 194bf2113..ac97d0ceb 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.c @@ -15,7 +15,7 @@ #include "check.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INDICATOR, SEQ_COUNT, @@ -37,19 +37,21 @@ struct lzma_coder_s { /// CRC32 of the List of Records field uint32_t crc32; -}; +} lzma_index_coder; static lzma_ret -index_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in lzma_attribute((__unused__)), - size_t *LZMA_RESTRICT in_pos lzma_attribute((__unused__)), +index_encode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in lzma_attribute((__unused__)), + size_t *restrict in_pos lzma_attribute((__unused__)), size_t in_size lzma_attribute((__unused__)), - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action lzma_attribute((__unused__))) { + lzma_index_coder *coder = coder_ptr; + // Position where to start calculating CRC32. The idea is that we // need to call lzma_crc32() only once per call to index_encode(). const size_t out_start = *out_pos; @@ -159,7 +161,7 @@ out: static void -index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +index_encoder_end(void *coder, const lzma_allocator *allocator) { lzma_free(coder, allocator); return; @@ -167,7 +169,7 @@ index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static void -index_encoder_reset(lzma_coder *coder, const lzma_index *i) +index_encoder_reset(lzma_index_coder *coder, const lzma_index *i) { lzma_index_iter_init(&coder->iter, i); @@ -181,7 +183,7 @@ index_encoder_reset(lzma_coder *coder, const lzma_index *i) extern lzma_ret -lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_index_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_index *i) { lzma_next_coder_init(&lzma_index_encoder_init, next, allocator); @@ -190,7 +192,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_PROG_ERROR; if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + next->coder = lzma_alloc(sizeof(lzma_index_coder), allocator); if (next->coder == NULL) return LZMA_MEM_ERROR; @@ -207,7 +209,7 @@ lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_index_encoder(lzma_stream *strm, const lzma_index *i) { - lzma_next_strm_init1(lzma_index_encoder_init, strm, i); + lzma_next_strm_init(lzma_index_encoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; @@ -220,10 +222,6 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_coder coder; - size_t out_start; - lzma_ret ret; - // Validate the arguments. if (i == NULL || out == NULL || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; @@ -234,12 +232,13 @@ lzma_index_buffer_encode(const lzma_index *i, // The Index encoder needs just one small data structure so we can // allocate it on stack. + lzma_index_coder coder; index_encoder_reset(&coder, i); // Do the actual encoding. This should never fail, but store // the original *out_pos just in case. - out_start = *out_pos; - ret = index_encode(&coder, NULL, NULL, NULL, 0, + const size_t out_start = *out_pos; + lzma_ret ret = index_encode(&coder, NULL, NULL, NULL, 0, out, out_pos, out_size, LZMA_RUN); if (ret == LZMA_STREAM_END) { diff --git a/Utilities/cmliblzma/liblzma/common/index_encoder.h b/Utilities/cmliblzma/liblzma/common/index_encoder.h index a13c94dcd..4d55cd104 100644 --- a/Utilities/cmliblzma/liblzma/common/index_encoder.h +++ b/Utilities/cmliblzma/liblzma/common/index_encoder.h @@ -17,7 +17,7 @@ extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_index *i); + const lzma_allocator *allocator, const lzma_index *i); #endif diff --git a/Utilities/cmliblzma/liblzma/common/index_hash.c b/Utilities/cmliblzma/liblzma/common/index_hash.c index 0cf86b307..d7a0344b7 100644 --- a/Utilities/cmliblzma/liblzma/common/index_hash.c +++ b/Utilities/cmliblzma/liblzma/common/index_hash.c @@ -70,7 +70,8 @@ struct lzma_index_hash_s { extern LZMA_API(lzma_index_hash *) -lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) +lzma_index_hash_init(lzma_index_hash *index_hash, + const lzma_allocator *allocator) { if (index_hash == NULL) { index_hash = lzma_alloc(sizeof(lzma_index_hash), allocator); @@ -101,7 +102,8 @@ lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) extern LZMA_API(void) -lzma_index_hash_end(lzma_index_hash *index_hash, lzma_allocator *allocator) +lzma_index_hash_end(lzma_index_hash *index_hash, + const lzma_allocator *allocator) { lzma_free(index_hash, allocator); return; @@ -124,14 +126,13 @@ static lzma_ret hash_append(lzma_index_hash_info *info, lzma_vli unpadded_size, lzma_vli uncompressed_size) { - const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; - info->blocks_size += vli_ceil4(unpadded_size); info->uncompressed_size += uncompressed_size; info->index_list_size += lzma_vli_size(unpadded_size) + lzma_vli_size(uncompressed_size); ++info->count; + const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; lzma_check_update(&info->check, LZMA_CHECK_BEST, (const uint8_t *)(sizes), sizeof(sizes)); @@ -174,9 +175,6 @@ extern LZMA_API(lzma_ret) lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, size_t *in_pos, size_t in_size) { - size_t in_start; - lzma_ret ret; - // Catch zero input buffer here, because in contrast to Index encoder // and decoder functions, applications call this function directly // instead of via lzma_code(), which does the buffer checking. @@ -186,8 +184,8 @@ lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, // NOTE: This function has many similarities to index_encode() and // index_decode() functions found from index_encoder.c and // index_decoder.c. See the comments especially in index_encoder.c. - in_start = *in_pos; - ret = LZMA_OK; + const size_t in_start = *in_pos; + lzma_ret ret = LZMA_OK; while (*in_pos < in_size) switch (index_hash->sequence) { diff --git a/Utilities/cmliblzma/liblzma/common/memcmplen.h b/Utilities/cmliblzma/liblzma/common/memcmplen.h new file mode 100644 index 000000000..c1efc9e28 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/memcmplen.h @@ -0,0 +1,175 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file memcmplen.h +/// \brief Optimized comparison of two buffers +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_MEMCMPLEN_H +#define LZMA_MEMCMPLEN_H + +#include "common.h" + +#ifdef HAVE_IMMINTRIN_H +# include <immintrin.h> +#endif + + +/// Find out how many equal bytes the two buffers have. +/// +/// \param buf1 First buffer +/// \param buf2 Second buffer +/// \param len How many bytes have already been compared and will +/// be assumed to match +/// \param limit How many bytes to compare at most, including the +/// already-compared bytes. This must be significantly +/// smaller than UINT32_MAX to avoid integer overflows. +/// Up to LZMA_MEMCMPLEN_EXTRA bytes may be read past +/// the specified limit from both buf1 and buf2. +/// +/// \return Number of equal bytes in the buffers is returned. +/// This is always at least len and at most limit. +/// +/// \note LZMA_MEMCMPLEN_EXTRA defines how many extra bytes may be read. +/// It's rounded up to 2^n. This extra amount needs to be +/// allocated in the buffers being used. It needs to be +/// initialized too to keep Valgrind quiet. +static inline uint32_t lzma_attribute((__always_inline__)) +lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2, + uint32_t len, uint32_t limit) +{ + assert(len <= limit); + assert(limit <= UINT32_MAX / 2); + +#if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && ((TUKLIB_GNUC_REQ(3, 4) && defined(__x86_64__)) \ + || (defined(__INTEL_COMPILER) && defined(__x86_64__)) \ + || (defined(__INTEL_COMPILER) && defined(_M_X64)) \ + || (defined(_MSC_VER) && defined(_M_X64))) + // NOTE: This will use 64-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit, but + // it's convenient here at least as long as it's x86-64 only. + // + // I keep this x86-64 only for now since that's where I know this + // to be a good method. This may be fine on other 64-bit CPUs too. + // On big endian one should use xor instead of subtraction and switch + // to __builtin_clzll(). +#define LZMA_MEMCMPLEN_EXTRA 8 + while (len < limit) { + const uint64_t x = *(const uint64_t *)(buf1 + len) + - *(const uint64_t *)(buf2 + len); + if (x != 0) { +# if defined(_M_X64) // MSVC or Intel C compiler on Windows + unsigned long tmp; + _BitScanForward64(&tmp, x); + len += (uint32_t)tmp >> 3; +# else // GCC, clang, or Intel C compiler + len += (uint32_t)__builtin_ctzll(x) >> 3; +# endif + return my_min(len, limit); + } + + len += 8; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ + && defined(HAVE__MM_MOVEMASK_EPI8) \ + && ((defined(__GNUC__) && defined(__SSE2_MATH__)) \ + || (defined(__INTEL_COMPILER) && defined(__SSE2__)) \ + || (defined(_MSC_VER) && defined(_M_IX86_FP) \ + && _M_IX86_FP >= 2)) + // NOTE: Like above, this will use 128-bit unaligned access which + // TUKLIB_FAST_UNALIGNED_ACCESS wasn't meant to permit. + // + // SSE2 version for 32-bit and 64-bit x86. On x86-64 the above + // version is sometimes significantly faster and sometimes + // slightly slower than this SSE2 version, so this SSE2 + // version isn't used on x86-64. +# define LZMA_MEMCMPLEN_EXTRA 16 + while (len < limit) { + const uint32_t x = 0xFFFF ^ _mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128((const __m128i *)(buf1 + len)), + _mm_loadu_si128((const __m128i *)(buf2 + len)))); + + if (x != 0) { +# if defined(__INTEL_COMPILER) + len += _bit_scan_forward(x); +# elif defined(_MSC_VER) + unsigned long tmp; + _BitScanForward(&tmp, x); + len += tmp; +# else + len += __builtin_ctz(x); +# endif + return my_min(len, limit); + } + + len += 16; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && !defined(WORDS_BIGENDIAN) + // Generic 32-bit little endian method +# define LZMA_MEMCMPLEN_EXTRA 4 + while (len < limit) { + uint32_t x = *(const uint32_t *)(buf1 + len) + - *(const uint32_t *)(buf2 + len); + if (x != 0) { + if ((x & 0xFFFF) == 0) { + len += 2; + x >>= 16; + } + + if ((x & 0xFF) == 0) + ++len; + + return my_min(len, limit); + } + + len += 4; + } + + return limit; + +#elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && defined(WORDS_BIGENDIAN) + // Generic 32-bit big endian method +# define LZMA_MEMCMPLEN_EXTRA 4 + while (len < limit) { + uint32_t x = *(const uint32_t *)(buf1 + len) + ^ *(const uint32_t *)(buf2 + len); + if (x != 0) { + if ((x & 0xFFFF0000) == 0) { + len += 2; + x <<= 16; + } + + if ((x & 0xFF000000) == 0) + ++len; + + return my_min(len, limit); + } + + len += 4; + } + + return limit; + +#else + // Simple portable version that doesn't use unaligned access. +# define LZMA_MEMCMPLEN_EXTRA 0 + while (len < limit && buf1[len] == buf2[len]) + ++len; + + return len; +#endif +} + +#endif diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.c b/Utilities/cmliblzma/liblzma/common/outqueue.c new file mode 100644 index 000000000..2dc8a38d1 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/outqueue.c @@ -0,0 +1,184 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file outqueue.c +/// \brief Output queue handling in multithreaded coding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "outqueue.h" + + +/// This is to ease integer overflow checking: We may allocate up to +/// 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other +/// data structures (that's the second /2). +#define BUF_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2) + + +static lzma_ret +get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count, + uint64_t buf_size_max, uint32_t threads) +{ + if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX) + return LZMA_OPTIONS_ERROR; + + // The number of buffers is twice the number of threads. + // This wastes RAM but keeps the threads busy when buffers + // finish out of order. + // + // NOTE: If this is changed, update BUF_SIZE_MAX too. + *bufs_count = threads * 2; + *bufs_alloc_size = *bufs_count * buf_size_max; + + return LZMA_OK; +} + + +extern uint64_t +lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads) +{ + uint64_t bufs_alloc_size; + uint32_t bufs_count; + + if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads) + != LZMA_OK) + return UINT64_MAX; + + return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf) + + bufs_alloc_size; +} + + +extern lzma_ret +lzma_outq_init(lzma_outq *outq, const lzma_allocator *allocator, + uint64_t buf_size_max, uint32_t threads) +{ + uint64_t bufs_alloc_size; + uint32_t bufs_count; + + // Set bufs_count and bufs_alloc_size. + return_if_error(get_options(&bufs_alloc_size, &bufs_count, + buf_size_max, threads)); + + // Allocate memory if needed. + if (outq->buf_size_max != buf_size_max + || outq->bufs_allocated != bufs_count) { + lzma_outq_end(outq, allocator); + +#if SIZE_MAX < UINT64_MAX + if (bufs_alloc_size > SIZE_MAX) + return LZMA_MEM_ERROR; +#endif + + outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf), + allocator); + outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size), + allocator); + + if (outq->bufs == NULL || outq->bufs_mem == NULL) { + lzma_outq_end(outq, allocator); + return LZMA_MEM_ERROR; + } + } + + // Initialize the rest of the main structure. Initialization of + // outq->bufs[] is done when they are actually needed. + outq->buf_size_max = (size_t)(buf_size_max); + outq->bufs_allocated = bufs_count; + outq->bufs_pos = 0; + outq->bufs_used = 0; + outq->read_pos = 0; + + return LZMA_OK; +} + + +extern void +lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator) +{ + lzma_free(outq->bufs, allocator); + outq->bufs = NULL; + + lzma_free(outq->bufs_mem, allocator); + outq->bufs_mem = NULL; + + return; +} + + +extern lzma_outbuf * +lzma_outq_get_buf(lzma_outq *outq) +{ + // Caller must have checked it with lzma_outq_has_buf(). + assert(outq->bufs_used < outq->bufs_allocated); + + // Initialize the new buffer. + lzma_outbuf *buf = &outq->bufs[outq->bufs_pos]; + buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max; + buf->size = 0; + buf->finished = false; + + // Update the queue state. + if (++outq->bufs_pos == outq->bufs_allocated) + outq->bufs_pos = 0; + + ++outq->bufs_used; + + return buf; +} + + +extern bool +lzma_outq_is_readable(const lzma_outq *outq) +{ + uint32_t i = outq->bufs_pos - outq->bufs_used; + if (outq->bufs_pos < outq->bufs_used) + i += outq->bufs_allocated; + + return outq->bufs[i].finished; +} + + +extern lzma_ret +lzma_outq_read(lzma_outq *restrict outq, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_vli *restrict unpadded_size, + lzma_vli *restrict uncompressed_size) +{ + // There must be at least one buffer from which to read. + if (outq->bufs_used == 0) + return LZMA_OK; + + // Get the buffer. + uint32_t i = outq->bufs_pos - outq->bufs_used; + if (outq->bufs_pos < outq->bufs_used) + i += outq->bufs_allocated; + + lzma_outbuf *buf = &outq->bufs[i]; + + // If it isn't finished yet, we cannot read from it. + if (!buf->finished) + return LZMA_OK; + + // Copy from the buffer to output. + lzma_bufcpy(buf->buf, &outq->read_pos, buf->size, + out, out_pos, out_size); + + // Return if we didn't get all the data from the buffer. + if (outq->read_pos < buf->size) + return LZMA_OK; + + // The buffer was finished. Tell the caller its size information. + *unpadded_size = buf->unpadded_size; + *uncompressed_size = buf->uncompressed_size; + + // Free this buffer for further use. + --outq->bufs_used; + outq->read_pos = 0; + + return LZMA_STREAM_END; +} diff --git a/Utilities/cmliblzma/liblzma/common/outqueue.h b/Utilities/cmliblzma/liblzma/common/outqueue.h new file mode 100644 index 000000000..079634de4 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/outqueue.h @@ -0,0 +1,156 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file outqueue.h +/// \brief Output queue handling in multithreaded coding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +/// Output buffer for a single thread +typedef struct { + /// Pointer to the output buffer of lzma_outq.buf_size_max bytes + uint8_t *buf; + + /// Amount of data written to buf + size_t size; + + /// Additional size information + lzma_vli unpadded_size; + lzma_vli uncompressed_size; + + /// True when no more data will be written into this buffer. + /// + /// \note This is read by another thread and thus access + /// to this variable needs a mutex. + bool finished; + +} lzma_outbuf; + + +typedef struct { + /// Array of buffers that are used cyclically. + lzma_outbuf *bufs; + + /// Memory allocated for all the buffers + uint8_t *bufs_mem; + + /// Amount of buffer space available in each buffer + size_t buf_size_max; + + /// Number of buffers allocated + uint32_t bufs_allocated; + + /// Position in the bufs array. The next buffer to be taken + /// into use is bufs[bufs_pos]. + uint32_t bufs_pos; + + /// Number of buffers in use + uint32_t bufs_used; + + /// Position in the buffer in lzma_outq_read() + size_t read_pos; + +} lzma_outq; + + +/** + * \brief Calculate the memory usage of an output queue + * + * \return Approximate memory usage in bytes or UINT64_MAX on error. + */ +extern uint64_t lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads); + + +/// \brief Initialize an output queue +/// +/// \param outq Pointer to an output queue. Before calling +/// this function the first time, *outq should +/// have been zeroed with memzero() so that this +/// function knows that there are no previous +/// allocations to free. +/// \param allocator Pointer to allocator or NULL +/// \param buf_size_max Maximum amount of data that a single buffer +/// in the queue may need to store. +/// \param threads Number of buffers that may be in use +/// concurrently. Note that more than this number +/// of buffers will actually get allocated to +/// improve performance when buffers finish +/// out of order. +/// +/// \return - LZMA_OK +/// - LZMA_MEM_ERROR +/// +extern lzma_ret lzma_outq_init( + lzma_outq *outq, const lzma_allocator *allocator, + uint64_t buf_size_max, uint32_t threads); + + +/// \brief Free the memory associated with the output queue +extern void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator); + + +/// \brief Get a new buffer +/// +/// lzma_outq_has_buf() must be used to check that there is a buffer +/// available before calling lzma_outq_get_buf(). +/// +extern lzma_outbuf *lzma_outq_get_buf(lzma_outq *outq); + + +/// \brief Test if there is data ready to be read +/// +/// Call to this function must be protected with the same mutex that +/// is used to protect lzma_outbuf.finished. +/// +extern bool lzma_outq_is_readable(const lzma_outq *outq); + + +/// \brief Read finished data +/// +/// \param outq Pointer to an output queue +/// \param out Beginning of the output buffer +/// \param out_pos The next byte will be written to +/// out[*out_pos]. +/// \param out_size Size of the out buffer; the first byte into +/// which no data is written to is out[out_size]. +/// \param unpadded_size Unpadded Size from the Block encoder +/// \param uncompressed_size Uncompressed Size from the Block encoder +/// +/// \return - LZMA: All OK. Either no data was available or the buffer +/// being read didn't become empty yet. +/// - LZMA_STREAM_END: The buffer being read was finished. +/// *unpadded_size and *uncompressed_size were set. +/// +/// \note This reads lzma_outbuf.finished variables and thus call +/// to this function needs to be protected with a mutex. +/// +extern lzma_ret lzma_outq_read(lzma_outq *restrict outq, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_vli *restrict unpadded_size, + lzma_vli *restrict uncompressed_size); + + +/// \brief Test if there is at least one buffer free +/// +/// This must be used before getting a new buffer with lzma_outq_get_buf(). +/// +static inline bool +lzma_outq_has_buf(const lzma_outq *outq) +{ + return outq->bufs_used < outq->bufs_allocated; +} + + +/// \brief Test if the queue is completely empty +static inline bool +lzma_outq_is_empty(const lzma_outq *outq) +{ + return outq->bufs_used == 0; +} diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c index 9e2e1da83..b9745b5db 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_decoder.c @@ -15,13 +15,10 @@ extern LZMA_API(lzma_ret) lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, - lzma_allocator *allocator, + const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, uint8_t *out, size_t *out_pos, size_t out_size) { - lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; - lzma_ret ret; - // Sanity checks if (in_pos == NULL || (in == NULL && *in_pos != in_size) || *in_pos > in_size || out_pos == NULL @@ -36,7 +33,8 @@ lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, // Initialize the Stream decoder. // TODO: We need something to tell the decoder that it can use the // output buffer as workspace, and thus save significant amount of RAM. - ret = lzma_stream_decoder_init( + lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_stream_decoder_init( &stream_decoder, allocator, *memlimit, flags); if (ret == LZMA_OK) { diff --git a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c index 8bca87f47..af49554a6 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_buffer_encoder.c @@ -42,13 +42,10 @@ lzma_stream_buffer_bound(size_t uncompressed_size) extern LZMA_API(lzma_ret) lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, - lzma_allocator *allocator, const uint8_t *in, size_t in_size, + const lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, size_t *out_pos_ptr, size_t out_size) { - lzma_stream_flags stream_flags = { 0 }; - lzma_block block = { 0 }; - size_t out_pos; - // Sanity checks if (filters == NULL || (unsigned int)(check) > LZMA_CHECK_ID_MAX || (in == NULL && in_size != 0) || out == NULL @@ -65,7 +62,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, // Use a local copy. We update *out_pos_ptr only if everything // succeeds. - out_pos = *out_pos_ptr; + size_t out_pos = *out_pos_ptr; // Check that there's enough space for both Stream Header and // Stream Footer. @@ -77,7 +74,10 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, out_size -= LZMA_STREAM_HEADER_SIZE; // Encode the Stream Header. - stream_flags.check = check; + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; if (lzma_stream_header_encode(&stream_flags, out + out_pos) != LZMA_OK) @@ -86,8 +86,11 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, out_pos += LZMA_STREAM_HEADER_SIZE; // Encode a Block but only if there is at least one byte of input. - block.check = check; - block.filters = filters; + lzma_block block = { + .version = 0, + .check = check, + .filters = filters, + }; if (in_size > 0) return_if_error(lzma_block_buffer_encode(&block, allocator, @@ -95,8 +98,6 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, // Index { - lzma_ret ret; - // Create an Index. It will have one Record if there was // at least one byte of input to encode. Otherwise the // Index will be empty. @@ -104,7 +105,7 @@ lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, if (i == NULL) return LZMA_MEM_ERROR; - ret = LZMA_OK; + lzma_ret ret = LZMA_OK; if (in_size > 0) ret = lzma_index_append(i, allocator, diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_decoder.c index 5e9a22069..fdd8ff2f9 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.c @@ -14,7 +14,7 @@ #include "block_decoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_HEADER, @@ -57,6 +57,10 @@ struct lzma_coder_s { /// If true, LZMA_GET_CHECK is returned after decoding Stream Header. bool tell_any_check; + /// If true, we will tell the Block decoder to skip calculating + /// and verifying the integrity check. + bool ignore_check; + /// If true, we will decode concatenated Streams that possibly have /// Stream Padding between or after them. LZMA_STREAM_END is returned /// once the application isn't giving us any new input, and we aren't @@ -76,11 +80,11 @@ struct lzma_coder_s { /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; -}; +} lzma_stream_coder; static lzma_ret -stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) +stream_decoder_reset(lzma_stream_coder *coder, const lzma_allocator *allocator) { // Initialize the Index hash used to verify the Index. coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator); @@ -96,18 +100,18 @@ stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +stream_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_stream_coder *coder = coder_ptr; + // When decoding the actual Block, it may be able to produce more // output even if we don't give it any new input. while (true) switch (coder->sequence) { case SEQ_STREAM_HEADER: { - lzma_ret ret; - // Copy the Stream Header to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); @@ -119,7 +123,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->pos = 0; // Decode the Stream Header. - ret = lzma_stream_header_decode( + const lzma_ret ret = lzma_stream_header_decode( &coder->stream_flags, coder->buffer); if (ret != LZMA_OK) return ret == LZMA_FORMAT_ERROR && !coder->first_stream @@ -156,11 +160,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_BLOCK_HEADER: { - lzma_filter filters[LZMA_FILTERS_MAX + 1]; - uint64_t memusage; - lzma_ret ret; - size_t i; - if (*in_pos >= in_size) return LZMA_OK; @@ -189,20 +188,28 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->pos = 0; - // Version 0 is currently the only possible version. - coder->block_options.version = 0; + // Version 1 is needed to support the .ignore_check option. + coder->block_options.version = 1; // Set up a buffer to hold the filter chain. Block Header // decoder will initialize all members of this array so // we don't need to do it here. + lzma_filter filters[LZMA_FILTERS_MAX + 1]; coder->block_options.filters = filters; // Decode the Block Header. return_if_error(lzma_block_header_decode(&coder->block_options, allocator, coder->buffer)); + // If LZMA_IGNORE_CHECK was used, this flag needs to be set. + // It has to be set after lzma_block_header_decode() because + // it always resets this to false. + coder->block_options.ignore_check = coder->ignore_check; + // Check the memory usage limit. - memusage = lzma_raw_decoder_memusage(filters); + const uint64_t memusage = lzma_raw_decoder_memusage(filters); + lzma_ret ret; + if (memusage == UINT64_MAX) { // One or more unknown Filter IDs. ret = LZMA_OPTIONS_ERROR; @@ -228,7 +235,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Free the allocated filter options since they are needed // only to initialize the Block decoder. - for (i = 0; i < LZMA_FILTERS_MAX; ++i) + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) lzma_free(filters[i].options, allocator); coder->block_options.filters = NULL; @@ -264,8 +271,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX: { - lzma_ret ret; - // If we don't have any input, don't call // lzma_index_hash_decode() since it would return // LZMA_BUF_ERROR, which we must not do here. @@ -274,7 +279,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Decode the Index and compare it to the hash calculated // from the sizes of the Blocks (if any). - ret = lzma_index_hash_decode(coder->index_hash, + const lzma_ret ret = lzma_index_hash_decode(coder->index_hash, in, in_pos, in_size); if (ret != LZMA_STREAM_END) return ret; @@ -285,9 +290,6 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_STREAM_FOOTER: { - lzma_stream_flags footer_flags; - lzma_ret ret; - // Copy the Stream Footer to the internal buffer. lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); @@ -301,7 +303,8 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, // Decode the Stream Footer. The decoder gives // LZMA_FORMAT_ERROR if the magic bytes don't match, // so convert that return code to LZMA_DATA_ERROR. - ret = lzma_stream_footer_decode( + lzma_stream_flags footer_flags; + const lzma_ret ret = lzma_stream_footer_decode( &footer_flags, coder->buffer); if (ret != LZMA_OK) return ret == LZMA_FORMAT_ERROR @@ -374,8 +377,9 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, static void -stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +stream_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_stream_coder *coder = coder_ptr; lzma_next_end(&coder->block_decoder, allocator); lzma_index_hash_end(coder->index_hash, allocator); lzma_free(coder, allocator); @@ -384,16 +388,19 @@ stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_check -stream_decoder_get_check(const lzma_coder *coder) +stream_decoder_get_check(const void *coder_ptr) { + const lzma_stream_coder *coder = coder_ptr; return coder->stream_flags.check; } static lzma_ret -stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, +stream_decoder_memconfig(void *coder_ptr, uint64_t *memusage, uint64_t *old_memlimit, uint64_t new_memlimit) { + lzma_stream_coder *coder = coder_ptr; + *memusage = coder->memusage; *old_memlimit = coder->memlimit; @@ -409,48 +416,49 @@ stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, extern lzma_ret -lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_stream_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, uint64_t memlimit, uint32_t flags) { lzma_next_coder_init(&lzma_stream_decoder_init, next, allocator); - if (memlimit == 0) - return LZMA_PROG_ERROR; - if (flags & ~LZMA_SUPPORTED_FLAGS) return LZMA_OPTIONS_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_stream_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &stream_decode; next->end = &stream_decoder_end; next->get_check = &stream_decoder_get_check; next->memconfig = &stream_decoder_memconfig; - next->coder->block_decoder = LZMA_NEXT_CODER_INIT; - next->coder->index_hash = NULL; + coder->block_decoder = LZMA_NEXT_CODER_INIT; + coder->index_hash = NULL; } - next->coder->memlimit = memlimit; - next->coder->memusage = LZMA_MEMUSAGE_BASE; - next->coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; - next->coder->tell_unsupported_check + coder->memlimit = my_max(1, memlimit); + coder->memusage = LZMA_MEMUSAGE_BASE; + coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; + coder->tell_unsupported_check = (flags & LZMA_TELL_UNSUPPORTED_CHECK) != 0; - next->coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; - next->coder->concatenated = (flags & LZMA_CONCATENATED) != 0; - next->coder->first_stream = true; + coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; + coder->ignore_check = (flags & LZMA_IGNORE_CHECK) != 0; + coder->concatenated = (flags & LZMA_CONCATENATED) != 0; + coder->first_stream = true; - return stream_decoder_reset(next->coder, allocator); + return stream_decoder_reset(coder, allocator); } extern LZMA_API(lzma_ret) lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init2(lzma_stream_decoder_init, strm, memlimit, flags); + lzma_next_strm_init(lzma_stream_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/Utilities/cmliblzma/liblzma/common/stream_decoder.h b/Utilities/cmliblzma/liblzma/common/stream_decoder.h index e54ac28f4..c13c6ba12 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_decoder.h +++ b/Utilities/cmliblzma/liblzma/common/stream_decoder.h @@ -15,7 +15,8 @@ #include "common.h" -extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, uint64_t memlimit, uint32_t flags); +extern lzma_ret lzma_stream_decoder_init( + lzma_next_coder *next, const lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags); #endif diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_encoder.c index 1ba45acec..858cba473 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder.c @@ -10,12 +10,11 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_encoder.h" #include "block_encoder.h" #include "index_encoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_STREAM_HEADER, SEQ_BLOCK_INIT, @@ -26,7 +25,7 @@ struct lzma_coder_s { } sequence; /// True if Block encoder has been initialized by - /// lzma_stream_encoder_init() or stream_encoder_update() + /// stream_encoder_init() or stream_encoder_update() /// and thus doesn't need to be initialized in stream_encode(). bool block_encoder_is_initialized; @@ -56,11 +55,11 @@ struct lzma_coder_s { /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; -}; +} lzma_stream_coder; static lzma_ret -block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) +block_encoder_init(lzma_stream_coder *coder, const lzma_allocator *allocator) { // Prepare the Block options. Even though Block encoder doesn't need // compressed_size, uncompressed_size, and header_size to be @@ -79,11 +78,13 @@ block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +stream_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_stream_coder *coder = coder_ptr; + // Main loop while (*out_pos < out_size) switch (coder->sequence) { @@ -126,7 +127,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } // Initialize the Block encoder unless it was already - // initialized by lzma_stream_encoder_init() or + // initialized by stream_encoder_init() or // stream_encoder_update(). if (!coder->block_encoder_is_initialized) return_if_error(block_encoder_init(coder, allocator)); @@ -147,13 +148,12 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_BLOCK_ENCODE: { - lzma_vli unpadded_size; - - static const lzma_action convert[4] = { + static const lzma_action convert[LZMA_ACTION_MAX + 1] = { LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FINISH, LZMA_FINISH, + LZMA_FINISH, }; const lzma_ret ret = coder->block_encoder.code( @@ -164,7 +164,7 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, return ret; // Add a new Index Record. - unpadded_size = lzma_block_unpadded_size( + const lzma_vli unpadded_size = lzma_block_unpadded_size( &coder->block_options); assert(unpadded_size != 0); return_if_error(lzma_index_append(coder->index, allocator, @@ -176,12 +176,6 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX_ENCODE: { - const lzma_stream_flags stream_flags = { - 0, - lzma_index_size(coder->index), - coder->block_options.check, - }; - // Call the Index encoder. It doesn't take any input, so // those pointers can be NULL. const lzma_ret ret = coder->index_encoder.code( @@ -192,6 +186,11 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, return ret; // Encode the Stream Footer into coder->buffer. + const lzma_stream_flags stream_flags = { + .version = 0, + .backward_size = lzma_index_size(coder->index), + .check = coder->block_options.check, + }; if (lzma_stream_footer_encode(&stream_flags, coder->buffer) != LZMA_OK) @@ -212,15 +211,15 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, static void -stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +stream_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { - size_t i; + lzma_stream_coder *coder = coder_ptr; lzma_next_end(&coder->block_encoder, allocator); lzma_next_end(&coder->index_encoder, allocator); lzma_index_end(coder->index, allocator); - for (i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) lzma_free(coder->filters[i].options, allocator); lzma_free(coder, allocator); @@ -229,22 +228,20 @@ stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +stream_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters, const lzma_filter *reversed_filters) { - size_t i; + lzma_stream_coder *coder = coder_ptr; if (coder->sequence <= SEQ_BLOCK_INIT) { - lzma_ret ret; - // There is no incomplete Block waiting to be finished, // thus we can change the whole filter chain. Start by // trying to initialize the Block encoder with the new // chain. This way we detect if the chain is valid. coder->block_encoder_is_initialized = false; coder->block_options.filters = (lzma_filter *)(filters); - ret = block_encoder_init(coder, allocator); + const lzma_ret ret = block_encoder_init(coder, allocator); coder->block_options.filters = coder->filters; if (ret != LZMA_OK) return ret; @@ -264,62 +261,66 @@ stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, } // Free the copy of the old chain and make a copy of the new chain. - for (i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) lzma_free(coder->filters[i].options, allocator); return lzma_filters_copy(filters, coder->filters, allocator); } -extern lzma_ret -lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +static lzma_ret +stream_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters, lzma_check check) { - lzma_stream_flags stream_flags = { 0, 0, check }; - - lzma_next_coder_init(&lzma_stream_encoder_init, next, allocator); + lzma_next_coder_init(&stream_encoder_init, next, allocator); if (filters == NULL) return LZMA_PROG_ERROR; - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_stream_coder *coder = next->coder; + + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &stream_encode; next->end = &stream_encoder_end; next->update = &stream_encoder_update; - next->coder->filters[0].id = LZMA_VLI_UNKNOWN; - next->coder->block_encoder = LZMA_NEXT_CODER_INIT; - next->coder->index_encoder = LZMA_NEXT_CODER_INIT; - next->coder->index = NULL; + coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->block_encoder = LZMA_NEXT_CODER_INIT; + coder->index_encoder = LZMA_NEXT_CODER_INIT; + coder->index = NULL; } // Basic initializations - next->coder->sequence = SEQ_STREAM_HEADER; - next->coder->block_options.version = 0; - next->coder->block_options.check = check; + coder->sequence = SEQ_STREAM_HEADER; + coder->block_options.version = 0; + coder->block_options.check = check; // Initialize the Index - lzma_index_end(next->coder->index, allocator); - next->coder->index = lzma_index_init(allocator); - if (next->coder->index == NULL) + lzma_index_end(coder->index, allocator); + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) return LZMA_MEM_ERROR; // Encode the Stream Header + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; return_if_error(lzma_stream_header_encode( - &stream_flags, next->coder->buffer)); + &stream_flags, coder->buffer)); - next->coder->buffer_pos = 0; - next->coder->buffer_size = LZMA_STREAM_HEADER_SIZE; + coder->buffer_pos = 0; + coder->buffer_size = LZMA_STREAM_HEADER_SIZE; // Initialize the Block encoder. This way we detect unsupported // filter chains when initializing the Stream encoder instead of // giving an error after Stream Header has already written out. - return stream_encoder_update( - next->coder, allocator, filters, NULL); + return stream_encoder_update(coder, allocator, filters, NULL); } @@ -327,11 +328,12 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder(lzma_stream *strm, const lzma_filter *filters, lzma_check check) { - lzma_next_strm_init2(lzma_stream_encoder_init, strm, filters, check); + lzma_next_strm_init(stream_encoder_init, strm, filters, check); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_BARRIER] = true; strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; diff --git a/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c new file mode 100644 index 000000000..2efe44c25 --- /dev/null +++ b/Utilities/cmliblzma/liblzma/common/stream_encoder_mt.c @@ -0,0 +1,1143 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_encoder_mt.c +/// \brief Multithreaded .xz Stream encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" +#include "easy_preset.h" +#include "block_encoder.h" +#include "block_buffer_encoder.h" +#include "index_encoder.h" +#include "outqueue.h" + + +/// Maximum supported block size. This makes it simpler to prevent integer +/// overflows if we are given unusually large block size. +#define BLOCK_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX) + + +typedef enum { + /// Waiting for work. + THR_IDLE, + + /// Encoding is in progress. + THR_RUN, + + /// Encoding is in progress but no more input data will + /// be read. + THR_FINISH, + + /// The main thread wants the thread to stop whatever it was doing + /// but not exit. + THR_STOP, + + /// The main thread wants the thread to exit. We could use + /// cancellation but since there's stopped anyway, this is lazier. + THR_EXIT, + +} worker_state; + +typedef struct lzma_stream_coder_s lzma_stream_coder; + +typedef struct worker_thread_s worker_thread; +struct worker_thread_s { + worker_state state; + + /// Input buffer of coder->block_size bytes. The main thread will + /// put new input into this and update in_size accordingly. Once + /// no more input is coming, state will be set to THR_FINISH. + uint8_t *in; + + /// Amount of data available in the input buffer. This is modified + /// only by the main thread. + size_t in_size; + + /// Output buffer for this thread. This is set by the main + /// thread every time a new Block is started with this thread + /// structure. + lzma_outbuf *outbuf; + + /// Pointer to the main structure is needed when putting this + /// thread back to the stack of free threads. + lzma_stream_coder *coder; + + /// The allocator is set by the main thread. Since a copy of the + /// pointer is kept here, the application must not change the + /// allocator before calling lzma_end(). + const lzma_allocator *allocator; + + /// Amount of uncompressed data that has already been compressed. + uint64_t progress_in; + + /// Amount of compressed data that is ready. + uint64_t progress_out; + + /// Block encoder + lzma_next_coder block_encoder; + + /// Compression options for this Block + lzma_block block_options; + + /// Next structure in the stack of free worker threads. + worker_thread *next; + + mythread_mutex mutex; + mythread_cond cond; + + /// The ID of this thread is used to join the thread + /// when it's not needed anymore. + mythread thread_id; +}; + + +struct lzma_stream_coder_s { + enum { + SEQ_STREAM_HEADER, + SEQ_BLOCK, + SEQ_INDEX, + SEQ_STREAM_FOOTER, + } sequence; + + /// Start a new Block every block_size bytes of input unless + /// LZMA_FULL_FLUSH or LZMA_FULL_BARRIER is used earlier. + size_t block_size; + + /// The filter chain currently in use + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + + + /// Index to hold sizes of the Blocks + lzma_index *index; + + /// Index encoder + lzma_next_coder index_encoder; + + + /// Stream Flags for encoding the Stream Header and Stream Footer. + lzma_stream_flags stream_flags; + + /// Buffer to hold Stream Header and Stream Footer. + uint8_t header[LZMA_STREAM_HEADER_SIZE]; + + /// Read position in header[] + size_t header_pos; + + + /// Output buffer queue for compressed data + lzma_outq outq; + + + /// Maximum wait time if cannot use all the input and cannot + /// fill the output buffer. This is in milliseconds. + uint32_t timeout; + + + /// Error code from a worker thread + lzma_ret thread_error; + + /// Array of allocated thread-specific structures + worker_thread *threads; + + /// Number of structures in "threads" above. This is also the + /// number of threads that will be created at maximum. + uint32_t threads_max; + + /// Number of thread structures that have been initialized, and + /// thus the number of worker threads actually created so far. + uint32_t threads_initialized; + + /// Stack of free threads. When a thread finishes, it puts itself + /// back into this stack. This starts as empty because threads + /// are created only when actually needed. + worker_thread *threads_free; + + /// The most recent worker thread to which the main thread writes + /// the new input from the application. + worker_thread *thr; + + + /// Amount of uncompressed data in Blocks that have already + /// been finished. + uint64_t progress_in; + + /// Amount of compressed data in Stream Header + Blocks that + /// have already been finished. + uint64_t progress_out; + + + mythread_mutex mutex; + mythread_cond cond; +}; + + +/// Tell the main thread that something has gone wrong. +static void +worker_error(worker_thread *thr, lzma_ret ret) +{ + assert(ret != LZMA_OK); + assert(ret != LZMA_STREAM_END); + + mythread_sync(thr->coder->mutex) { + if (thr->coder->thread_error == LZMA_OK) + thr->coder->thread_error = ret; + + mythread_cond_signal(&thr->coder->cond); + } + + return; +} + + +static worker_state +worker_encode(worker_thread *thr, worker_state state) +{ + assert(thr->progress_in == 0); + assert(thr->progress_out == 0); + + // Set the Block options. + thr->block_options = (lzma_block){ + .version = 0, + .check = thr->coder->stream_flags.check, + .compressed_size = thr->coder->outq.buf_size_max, + .uncompressed_size = thr->coder->block_size, + + // TODO: To allow changing the filter chain, the filters + // array must be copied to each worker_thread. + .filters = thr->coder->filters, + }; + + // Calculate maximum size of the Block Header. This amount is + // reserved in the beginning of the buffer so that Block Header + // along with Compressed Size and Uncompressed Size can be + // written there. + lzma_ret ret = lzma_block_header_size(&thr->block_options); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + // Initialize the Block encoder. + ret = lzma_block_encoder_init(&thr->block_encoder, + thr->allocator, &thr->block_options); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + size_t in_pos = 0; + size_t in_size = 0; + + thr->outbuf->size = thr->block_options.header_size; + const size_t out_size = thr->coder->outq.buf_size_max; + + do { + mythread_sync(thr->mutex) { + // Store in_pos and out_pos into *thr so that + // an application may read them via + // lzma_get_progress() to get progress information. + // + // NOTE: These aren't updated when the encoding + // finishes. Instead, the final values are taken + // later from thr->outbuf. + thr->progress_in = in_pos; + thr->progress_out = thr->outbuf->size; + + while (in_size == thr->in_size + && thr->state == THR_RUN) + mythread_cond_wait(&thr->cond, &thr->mutex); + + state = thr->state; + in_size = thr->in_size; + } + + // Return if we were asked to stop or exit. + if (state >= THR_STOP) + return state; + + lzma_action action = state == THR_FINISH + ? LZMA_FINISH : LZMA_RUN; + + // Limit the amount of input given to the Block encoder + // at once. This way this thread can react fairly quickly + // if the main thread wants us to stop or exit. + static const size_t in_chunk_max = 16384; + size_t in_limit = in_size; + if (in_size - in_pos > in_chunk_max) { + in_limit = in_pos + in_chunk_max; + action = LZMA_RUN; + } + + ret = thr->block_encoder.code( + thr->block_encoder.coder, thr->allocator, + thr->in, &in_pos, in_limit, thr->outbuf->buf, + &thr->outbuf->size, out_size, action); + } while (ret == LZMA_OK && thr->outbuf->size < out_size); + + switch (ret) { + case LZMA_STREAM_END: + assert(state == THR_FINISH); + + // Encode the Block Header. By doing it after + // the compression, we can store the Compressed Size + // and Uncompressed Size fields. + ret = lzma_block_header_encode(&thr->block_options, + thr->outbuf->buf); + if (ret != LZMA_OK) { + worker_error(thr, ret); + return THR_STOP; + } + + break; + + case LZMA_OK: + // The data was incompressible. Encode it using uncompressed + // LZMA2 chunks. + // + // First wait that we have gotten all the input. + mythread_sync(thr->mutex) { + while (thr->state == THR_RUN) + mythread_cond_wait(&thr->cond, &thr->mutex); + + state = thr->state; + in_size = thr->in_size; + } + + if (state >= THR_STOP) + return state; + + // Do the encoding. This takes care of the Block Header too. + thr->outbuf->size = 0; + ret = lzma_block_uncomp_encode(&thr->block_options, + thr->in, in_size, thr->outbuf->buf, + &thr->outbuf->size, out_size); + + // It shouldn't fail. + if (ret != LZMA_OK) { + worker_error(thr, LZMA_PROG_ERROR); + return THR_STOP; + } + + break; + + default: + worker_error(thr, ret); + return THR_STOP; + } + + // Set the size information that will be read by the main thread + // to write the Index field. + thr->outbuf->unpadded_size + = lzma_block_unpadded_size(&thr->block_options); + assert(thr->outbuf->unpadded_size != 0); + thr->outbuf->uncompressed_size = thr->block_options.uncompressed_size; + + return THR_FINISH; +} + + +static MYTHREAD_RET_TYPE +worker_start(void *thr_ptr) +{ + worker_thread *thr = thr_ptr; + worker_state state = THR_IDLE; // Init to silence a warning + + while (true) { + // Wait for work. + mythread_sync(thr->mutex) { + while (true) { + // The thread is already idle so if we are + // requested to stop, just set the state. + if (thr->state == THR_STOP) { + thr->state = THR_IDLE; + mythread_cond_signal(&thr->cond); + } + + state = thr->state; + if (state != THR_IDLE) + break; + + mythread_cond_wait(&thr->cond, &thr->mutex); + } + } + + assert(state != THR_IDLE); + assert(state != THR_STOP); + + if (state <= THR_FINISH) + state = worker_encode(thr, state); + + if (state == THR_EXIT) + break; + + // Mark the thread as idle unless the main thread has + // told us to exit. Signal is needed for the case + // where the main thread is waiting for the threads to stop. + mythread_sync(thr->mutex) { + if (thr->state != THR_EXIT) { + thr->state = THR_IDLE; + mythread_cond_signal(&thr->cond); + } + } + + mythread_sync(thr->coder->mutex) { + // Mark the output buffer as finished if + // no errors occurred. + thr->outbuf->finished = state == THR_FINISH; + + // Update the main progress info. + thr->coder->progress_in + += thr->outbuf->uncompressed_size; + thr->coder->progress_out += thr->outbuf->size; + thr->progress_in = 0; + thr->progress_out = 0; + + // Return this thread to the stack of free threads. + thr->next = thr->coder->threads_free; + thr->coder->threads_free = thr; + + mythread_cond_signal(&thr->coder->cond); + } + } + + // Exiting, free the resources. + mythread_mutex_destroy(&thr->mutex); + mythread_cond_destroy(&thr->cond); + + lzma_next_end(&thr->block_encoder, thr->allocator); + lzma_free(thr->in, thr->allocator); + return MYTHREAD_RET_VALUE; +} + + +/// Make the threads stop but not exit. Optionally wait for them to stop. +static void +threads_stop(lzma_stream_coder *coder, bool wait_for_threads) +{ + // Tell the threads to stop. + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_STOP; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + if (!wait_for_threads) + return; + + // Wait for the threads to settle in the idle state. + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + while (coder->threads[i].state != THR_IDLE) + mythread_cond_wait(&coder->threads[i].cond, + &coder->threads[i].mutex); + } + } + + return; +} + + +/// Stop the threads and free the resources associated with them. +/// Wait until the threads have exited. +static void +threads_end(lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + coder->threads[i].state = THR_EXIT; + mythread_cond_signal(&coder->threads[i].cond); + } + } + + for (uint32_t i = 0; i < coder->threads_initialized; ++i) { + int ret = mythread_join(coder->threads[i].thread_id); + assert(ret == 0); + (void)ret; + } + + lzma_free(coder->threads, allocator); + return; +} + + +/// Initialize a new worker_thread structure and create a new thread. +static lzma_ret +initialize_new_thread(lzma_stream_coder *coder, + const lzma_allocator *allocator) +{ + worker_thread *thr = &coder->threads[coder->threads_initialized]; + + thr->in = lzma_alloc(coder->block_size, allocator); + if (thr->in == NULL) + return LZMA_MEM_ERROR; + + if (mythread_mutex_init(&thr->mutex)) + goto error_mutex; + + if (mythread_cond_init(&thr->cond)) + goto error_cond; + + thr->state = THR_IDLE; + thr->allocator = allocator; + thr->coder = coder; + thr->progress_in = 0; + thr->progress_out = 0; + thr->block_encoder = LZMA_NEXT_CODER_INIT; + + if (mythread_create(&thr->thread_id, &worker_start, thr)) + goto error_thread; + + ++coder->threads_initialized; + coder->thr = thr; + + return LZMA_OK; + +error_thread: + mythread_cond_destroy(&thr->cond); + +error_cond: + mythread_mutex_destroy(&thr->mutex); + +error_mutex: + lzma_free(thr->in, allocator); + return LZMA_MEM_ERROR; +} + + +static lzma_ret +get_thread(lzma_stream_coder *coder, const lzma_allocator *allocator) +{ + // If there are no free output subqueues, there is no + // point to try getting a thread. + if (!lzma_outq_has_buf(&coder->outq)) + return LZMA_OK; + + // If there is a free structure on the stack, use it. + mythread_sync(coder->mutex) { + if (coder->threads_free != NULL) { + coder->thr = coder->threads_free; + coder->threads_free = coder->threads_free->next; + } + } + + if (coder->thr == NULL) { + // If there are no uninitialized structures left, return. + if (coder->threads_initialized == coder->threads_max) + return LZMA_OK; + + // Initialize a new thread. + return_if_error(initialize_new_thread(coder, allocator)); + } + + // Reset the parts of the thread state that have to be done + // in the main thread. + mythread_sync(coder->thr->mutex) { + coder->thr->state = THR_RUN; + coder->thr->in_size = 0; + coder->thr->outbuf = lzma_outq_get_buf(&coder->outq); + mythread_cond_signal(&coder->thr->cond); + } + + return LZMA_OK; +} + + +static lzma_ret +stream_encode_in(lzma_stream_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, lzma_action action) +{ + while (*in_pos < in_size + || (coder->thr != NULL && action != LZMA_RUN)) { + if (coder->thr == NULL) { + // Get a new thread. + const lzma_ret ret = get_thread(coder, allocator); + if (coder->thr == NULL) + return ret; + } + + // Copy the input data to thread's buffer. + size_t thr_in_size = coder->thr->in_size; + lzma_bufcpy(in, in_pos, in_size, coder->thr->in, + &thr_in_size, coder->block_size); + + // Tell the Block encoder to finish if + // - it has got block_size bytes of input; or + // - all input was used and LZMA_FINISH, LZMA_FULL_FLUSH, + // or LZMA_FULL_BARRIER was used. + // + // TODO: LZMA_SYNC_FLUSH and LZMA_SYNC_BARRIER. + const bool finish = thr_in_size == coder->block_size + || (*in_pos == in_size && action != LZMA_RUN); + + bool block_error = false; + + mythread_sync(coder->thr->mutex) { + if (coder->thr->state == THR_IDLE) { + // Something has gone wrong with the Block + // encoder. It has set coder->thread_error + // which we will read a few lines later. + block_error = true; + } else { + // Tell the Block encoder its new amount + // of input and update the state if needed. + coder->thr->in_size = thr_in_size; + + if (finish) + coder->thr->state = THR_FINISH; + + mythread_cond_signal(&coder->thr->cond); + } + } + + if (block_error) { + lzma_ret ret; + + mythread_sync(coder->mutex) { + ret = coder->thread_error; + } + + return ret; + } + + if (finish) + coder->thr = NULL; + } + + return LZMA_OK; +} + + +/// Wait until more input can be consumed, more output can be read, or +/// an optional timeout is reached. +static bool +wait_for_work(lzma_stream_coder *coder, mythread_condtime *wait_abs, + bool *has_blocked, bool has_input) +{ + if (coder->timeout != 0 && !*has_blocked) { + // Every time when stream_encode_mt() is called via + // lzma_code(), *has_blocked starts as false. We set it + // to true here and calculate the absolute time when + // we must return if there's nothing to do. + // + // The idea of *has_blocked is to avoid unneeded calls + // to mythread_condtime_set(), which may do a syscall + // depending on the operating system. + *has_blocked = true; + mythread_condtime_set(wait_abs, &coder->cond, coder->timeout); + } + + bool timed_out = false; + + mythread_sync(coder->mutex) { + // There are four things that we wait. If one of them + // becomes possible, we return. + // - If there is input left, we need to get a free + // worker thread and an output buffer for it. + // - Data ready to be read from the output queue. + // - A worker thread indicates an error. + // - Time out occurs. + while ((!has_input || coder->threads_free == NULL + || !lzma_outq_has_buf(&coder->outq)) + && !lzma_outq_is_readable(&coder->outq) + && coder->thread_error == LZMA_OK + && !timed_out) { + if (coder->timeout != 0) + timed_out = mythread_cond_timedwait( + &coder->cond, &coder->mutex, + wait_abs) != 0; + else + mythread_cond_wait(&coder->cond, + &coder->mutex); + } + } + + return timed_out; +} + + +static lzma_ret +stream_encode_mt(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + lzma_stream_coder *coder = coder_ptr; + + switch (coder->sequence) { + case SEQ_STREAM_HEADER: + lzma_bufcpy(coder->header, &coder->header_pos, + sizeof(coder->header), + out, out_pos, out_size); + if (coder->header_pos < sizeof(coder->header)) + return LZMA_OK; + + coder->header_pos = 0; + coder->sequence = SEQ_BLOCK; + + // Fall through + + case SEQ_BLOCK: { + // Initialized to silence warnings. + lzma_vli unpadded_size = 0; + lzma_vli uncompressed_size = 0; + lzma_ret ret = LZMA_OK; + + // These are for wait_for_work(). + bool has_blocked = false; + mythread_condtime wait_abs; + + while (true) { + mythread_sync(coder->mutex) { + // Check for Block encoder errors. + ret = coder->thread_error; + if (ret != LZMA_OK) { + assert(ret != LZMA_STREAM_END); + break; + } + + // Try to read compressed data to out[]. + ret = lzma_outq_read(&coder->outq, + out, out_pos, out_size, + &unpadded_size, + &uncompressed_size); + } + + if (ret == LZMA_STREAM_END) { + // End of Block. Add it to the Index. + ret = lzma_index_append(coder->index, + allocator, unpadded_size, + uncompressed_size); + + // If we didn't fill the output buffer yet, + // try to read more data. Maybe the next + // outbuf has been finished already too. + if (*out_pos < out_size) + continue; + } + + if (ret != LZMA_OK) { + // coder->thread_error was set or + // lzma_index_append() failed. + threads_stop(coder, false); + return ret; + } + + // Try to give uncompressed data to a worker thread. + ret = stream_encode_in(coder, allocator, + in, in_pos, in_size, action); + if (ret != LZMA_OK) { + threads_stop(coder, false); + return ret; + } + + // See if we should wait or return. + // + // TODO: LZMA_SYNC_FLUSH and LZMA_SYNC_BARRIER. + if (*in_pos == in_size) { + // LZMA_RUN: More data is probably coming + // so return to let the caller fill the + // input buffer. + if (action == LZMA_RUN) + return LZMA_OK; + + // LZMA_FULL_BARRIER: The same as with + // LZMA_RUN but tell the caller that the + // barrier was completed. + if (action == LZMA_FULL_BARRIER) + return LZMA_STREAM_END; + + // Finishing or flushing isn't completed until + // all input data has been encoded and copied + // to the output buffer. + if (lzma_outq_is_empty(&coder->outq)) { + // LZMA_FINISH: Continue to encode + // the Index field. + if (action == LZMA_FINISH) + break; + + // LZMA_FULL_FLUSH: Return to tell + // the caller that flushing was + // completed. + if (action == LZMA_FULL_FLUSH) + return LZMA_STREAM_END; + } + } + + // Return if there is no output space left. + // This check must be done after testing the input + // buffer, because we might want to use a different + // return code. + if (*out_pos == out_size) + return LZMA_OK; + + // Neither in nor out has been used completely. + // Wait until there's something we can do. + if (wait_for_work(coder, &wait_abs, &has_blocked, + *in_pos < in_size)) + return LZMA_TIMED_OUT; + } + + // All Blocks have been encoded and the threads have stopped. + // Prepare to encode the Index field. + return_if_error(lzma_index_encoder_init( + &coder->index_encoder, allocator, + coder->index)); + coder->sequence = SEQ_INDEX; + + // Update the progress info to take the Index and + // Stream Footer into account. Those are very fast to encode + // so in terms of progress information they can be thought + // to be ready to be copied out. + coder->progress_out += lzma_index_size(coder->index) + + LZMA_STREAM_HEADER_SIZE; + } + + // Fall through + + case SEQ_INDEX: { + // Call the Index encoder. It doesn't take any input, so + // those pointers can be NULL. + const lzma_ret ret = coder->index_encoder.code( + coder->index_encoder.coder, allocator, + NULL, NULL, 0, + out, out_pos, out_size, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + // Encode the Stream Footer into coder->buffer. + coder->stream_flags.backward_size + = lzma_index_size(coder->index); + if (lzma_stream_footer_encode(&coder->stream_flags, + coder->header) != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->sequence = SEQ_STREAM_FOOTER; + } + + // Fall through + + case SEQ_STREAM_FOOTER: + lzma_bufcpy(coder->header, &coder->header_pos, + sizeof(coder->header), + out, out_pos, out_size); + return coder->header_pos < sizeof(coder->header) + ? LZMA_OK : LZMA_STREAM_END; + } + + assert(0); + return LZMA_PROG_ERROR; +} + + +static void +stream_encoder_mt_end(void *coder_ptr, const lzma_allocator *allocator) +{ + lzma_stream_coder *coder = coder_ptr; + + // Threads must be killed before the output queue can be freed. + threads_end(coder, allocator); + lzma_outq_end(&coder->outq, allocator); + + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + lzma_next_end(&coder->index_encoder, allocator); + lzma_index_end(coder->index, allocator); + + mythread_cond_destroy(&coder->cond); + mythread_mutex_destroy(&coder->mutex); + + lzma_free(coder, allocator); + return; +} + + +/// Options handling for lzma_stream_encoder_mt_init() and +/// lzma_stream_encoder_mt_memusage() +static lzma_ret +get_options(const lzma_mt *options, lzma_options_easy *opt_easy, + const lzma_filter **filters, uint64_t *block_size, + uint64_t *outbuf_size_max) +{ + // Validate some of the options. + if (options == NULL) + return LZMA_PROG_ERROR; + + if (options->flags != 0 || options->threads == 0 + || options->threads > LZMA_THREADS_MAX) + return LZMA_OPTIONS_ERROR; + + if (options->filters != NULL) { + // Filter chain was given, use it as is. + *filters = options->filters; + } else { + // Use a preset. + if (lzma_easy_preset(opt_easy, options->preset)) + return LZMA_OPTIONS_ERROR; + + *filters = opt_easy->filters; + } + + // Block size + if (options->block_size > 0) { + if (options->block_size > BLOCK_SIZE_MAX) + return LZMA_OPTIONS_ERROR; + + *block_size = options->block_size; + } else { + // Determine the Block size from the filter chain. + *block_size = lzma_mt_block_size(*filters); + if (*block_size == 0) + return LZMA_OPTIONS_ERROR; + + assert(*block_size <= BLOCK_SIZE_MAX); + } + + // Calculate the maximum amount output that a single output buffer + // may need to hold. This is the same as the maximum total size of + // a Block. + *outbuf_size_max = lzma_block_buffer_bound64(*block_size); + if (*outbuf_size_max == 0) + return LZMA_MEM_ERROR; + + return LZMA_OK; +} + + +static void +get_progress(void *coder_ptr, uint64_t *progress_in, uint64_t *progress_out) +{ + lzma_stream_coder *coder = coder_ptr; + + // Lock coder->mutex to prevent finishing threads from moving their + // progress info from the worker_thread structure to lzma_stream_coder. + mythread_sync(coder->mutex) { + *progress_in = coder->progress_in; + *progress_out = coder->progress_out; + + for (size_t i = 0; i < coder->threads_initialized; ++i) { + mythread_sync(coder->threads[i].mutex) { + *progress_in += coder->threads[i].progress_in; + *progress_out += coder->threads[i] + .progress_out; + } + } + } + + return; +} + + +static lzma_ret +stream_encoder_mt_init(lzma_next_coder *next, const lzma_allocator *allocator, + const lzma_mt *options) +{ + lzma_next_coder_init(&stream_encoder_mt_init, next, allocator); + + // Get the filter chain. + lzma_options_easy easy; + const lzma_filter *filters; + uint64_t block_size; + uint64_t outbuf_size_max; + return_if_error(get_options(options, &easy, &filters, + &block_size, &outbuf_size_max)); + +#if SIZE_MAX < UINT64_MAX + if (block_size > SIZE_MAX) + return LZMA_MEM_ERROR; +#endif + + // Validate the filter chain so that we can give an error in this + // function instead of delaying it to the first call to lzma_code(). + // The memory usage calculation verifies the filter chain as + // a side effect so we take advatange of that. + if (lzma_raw_encoder_memusage(filters) == UINT64_MAX) + return LZMA_OPTIONS_ERROR; + + // Validate the Check ID. + if ((unsigned int)(options->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + if (!lzma_check_is_supported(options->check)) + return LZMA_UNSUPPORTED_CHECK; + + // Allocate and initialize the base structure if needed. + lzma_stream_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_stream_coder), allocator); + if (coder == NULL) + return LZMA_MEM_ERROR; + + next->coder = coder; + + // For the mutex and condition variable initializations + // the error handling has to be done here because + // stream_encoder_mt_end() doesn't know if they have + // already been initialized or not. + if (mythread_mutex_init(&coder->mutex)) { + lzma_free(coder, allocator); + next->coder = NULL; + return LZMA_MEM_ERROR; + } + + if (mythread_cond_init(&coder->cond)) { + mythread_mutex_destroy(&coder->mutex); + lzma_free(coder, allocator); + next->coder = NULL; + return LZMA_MEM_ERROR; + } + + next->code = &stream_encode_mt; + next->end = &stream_encoder_mt_end; + next->get_progress = &get_progress; +// next->update = &stream_encoder_mt_update; + + coder->filters[0].id = LZMA_VLI_UNKNOWN; + coder->index_encoder = LZMA_NEXT_CODER_INIT; + coder->index = NULL; + memzero(&coder->outq, sizeof(coder->outq)); + coder->threads = NULL; + coder->threads_max = 0; + coder->threads_initialized = 0; + } + + // Basic initializations + coder->sequence = SEQ_STREAM_HEADER; + coder->block_size = (size_t)(block_size); + coder->thread_error = LZMA_OK; + coder->thr = NULL; + + // Allocate the thread-specific base structures. + assert(options->threads > 0); + if (coder->threads_max != options->threads) { + threads_end(coder, allocator); + + coder->threads = NULL; + coder->threads_max = 0; + + coder->threads_initialized = 0; + coder->threads_free = NULL; + + coder->threads = lzma_alloc( + options->threads * sizeof(worker_thread), + allocator); + if (coder->threads == NULL) + return LZMA_MEM_ERROR; + + coder->threads_max = options->threads; + } else { + // Reuse the old structures and threads. Tell the running + // threads to stop and wait until they have stopped. + threads_stop(coder, true); + } + + // Output queue + return_if_error(lzma_outq_init(&coder->outq, allocator, + outbuf_size_max, options->threads)); + + // Timeout + coder->timeout = options->timeout; + + // Free the old filter chain and copy the new one. + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + return_if_error(lzma_filters_copy( + filters, coder->filters, allocator)); + + // Index + lzma_index_end(coder->index, allocator); + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) + return LZMA_MEM_ERROR; + + // Stream Header + coder->stream_flags.version = 0; + coder->stream_flags.check = options->check; + return_if_error(lzma_stream_header_encode( + &coder->stream_flags, coder->header)); + + coder->header_pos = 0; + + // Progress info + coder->progress_in = 0; + coder->progress_out = LZMA_STREAM_HEADER_SIZE; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_encoder_mt(lzma_stream *strm, const lzma_mt *options) +{ + lzma_next_strm_init(stream_encoder_mt_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_BARRIER] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +// This function name is a monster but it's consistent with the older +// monster names. :-( 31 chars is the max that C99 requires so in that +// sense it's not too long. ;-) +extern LZMA_API(uint64_t) +lzma_stream_encoder_mt_memusage(const lzma_mt *options) +{ + lzma_options_easy easy; + const lzma_filter *filters; + uint64_t block_size; + uint64_t outbuf_size_max; + + if (get_options(options, &easy, &filters, &block_size, + &outbuf_size_max) != LZMA_OK) + return UINT64_MAX; + + // Memory usage of the input buffers + const uint64_t inbuf_memusage = options->threads * block_size; + + // Memory usage of the filter encoders + uint64_t filters_memusage = lzma_raw_encoder_memusage(filters); + if (filters_memusage == UINT64_MAX) + return UINT64_MAX; + + filters_memusage *= options->threads; + + // Memory usage of the output queue + const uint64_t outq_memusage = lzma_outq_memusage( + outbuf_size_max, options->threads); + if (outq_memusage == UINT64_MAX) + return UINT64_MAX; + + // Sum them with overflow checking. + uint64_t total_memusage = LZMA_MEMUSAGE_BASE + + sizeof(lzma_stream_coder) + + options->threads * sizeof(worker_thread); + + if (UINT64_MAX - total_memusage < inbuf_memusage) + return UINT64_MAX; + + total_memusage += inbuf_memusage; + + if (UINT64_MAX - total_memusage < filters_memusage) + return UINT64_MAX; + + total_memusage += filters_memusage; + + if (UINT64_MAX - total_memusage < outq_memusage) + return UINT64_MAX; + + return total_memusage + outq_memusage; +} diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c index 8cf48a4ca..1bc2f97c5 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_decoder.c @@ -30,15 +30,13 @@ stream_flags_decode(lzma_stream_flags *options, const uint8_t *in) extern LZMA_API(lzma_ret) lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) { - uint32_t crc; - // Magic if (memcmp(in, lzma_header_magic, sizeof(lzma_header_magic)) != 0) return LZMA_FORMAT_ERROR; // Verify the CRC32 so we can distinguish between corrupt // and unsupported files. - crc = lzma_crc32(in + sizeof(lzma_header_magic), + const uint32_t crc = lzma_crc32(in + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); if (crc != unaligned_read32le(in + sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE)) @@ -61,15 +59,13 @@ lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) extern LZMA_API(lzma_ret) lzma_stream_footer_decode(lzma_stream_flags *options, const uint8_t *in) { - uint32_t crc; - // Magic if (memcmp(in + sizeof(uint32_t) * 2 + LZMA_STREAM_FLAGS_SIZE, lzma_footer_magic, sizeof(lzma_footer_magic)) != 0) return LZMA_FORMAT_ERROR; // CRC32 - crc = lzma_crc32(in + sizeof(uint32_t), + const uint32_t crc = lzma_crc32(in + sizeof(uint32_t), sizeof(uint32_t) + LZMA_STREAM_FLAGS_SIZE, 0); if (crc != unaligned_read32le(in)) return LZMA_DATA_ERROR; diff --git a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c index 290339e08..4e717159f 100644 --- a/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/stream_flags_encoder.c @@ -29,8 +29,6 @@ stream_flags_encode(const lzma_stream_flags *options, uint8_t *out) extern LZMA_API(lzma_ret) lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) { - uint32_t crc; - assert(sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE + 4 == LZMA_STREAM_HEADER_SIZE); @@ -45,7 +43,7 @@ lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) return LZMA_PROG_ERROR; // CRC32 of the Stream Header - crc = lzma_crc32(out + sizeof(lzma_header_magic), + const uint32_t crc = lzma_crc32(out + sizeof(lzma_header_magic), LZMA_STREAM_FLAGS_SIZE, 0); unaligned_write32le(out + sizeof(lzma_header_magic) @@ -58,8 +56,6 @@ lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) extern LZMA_API(lzma_ret) lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) { - uint32_t crc; - assert(2 * 4 + LZMA_STREAM_FLAGS_SIZE + sizeof(lzma_footer_magic) == LZMA_STREAM_HEADER_SIZE); @@ -77,7 +73,7 @@ lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) return LZMA_PROG_ERROR; // CRC32 - crc = lzma_crc32( + const uint32_t crc = lzma_crc32( out + 4, 4 + LZMA_STREAM_FLAGS_SIZE, 0); unaligned_write32le(out, crc); diff --git a/Utilities/cmliblzma/liblzma/common/vli_decoder.c b/Utilities/cmliblzma/liblzma/common/vli_decoder.c index 1c663844f..c181828bf 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_decoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_decoder.c @@ -14,8 +14,8 @@ extern LZMA_API(lzma_ret) -lzma_vli_decode(lzma_vli *LZMA_RESTRICT vli, size_t *vli_pos, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lzma_vli_decode(lzma_vli *restrict vli, size_t *vli_pos, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { // If we haven't been given vli_pos, work in single-call mode. diff --git a/Utilities/cmliblzma/liblzma/common/vli_encoder.c b/Utilities/cmliblzma/liblzma/common/vli_encoder.c index 09e90cbd7..f8642694e 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_encoder.c +++ b/Utilities/cmliblzma/liblzma/common/vli_encoder.c @@ -15,7 +15,7 @@ extern LZMA_API(lzma_ret) lzma_vli_encode(lzma_vli vli, size_t *vli_pos, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { // If we haven't been given vli_pos, work in single-call mode. diff --git a/Utilities/cmliblzma/liblzma/common/vli_size.c b/Utilities/cmliblzma/liblzma/common/vli_size.c index 8b931e40b..ec1b4fa48 100644 --- a/Utilities/cmliblzma/liblzma/common/vli_size.c +++ b/Utilities/cmliblzma/liblzma/common/vli_size.c @@ -16,11 +16,10 @@ extern LZMA_API(uint32_t) lzma_vli_size(lzma_vli vli) { - uint32_t i = 0; - if (vli > LZMA_VLI_MAX) return 0; + uint32_t i = 0; do { vli >>= 7; ++i; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_common.c b/Utilities/cmliblzma/liblzma/delta/delta_common.c index 803e674a1..4768201d1 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_common.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_common.c @@ -15,8 +15,9 @@ static void -delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) +delta_coder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_delta_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; @@ -24,20 +25,21 @@ delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { - const lzma_options_delta *opt; - // Allocate memory for the decoder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_delta_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_delta_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; + // End function is the same for encoder and decoder. next->end = &delta_coder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Validate the options. @@ -45,16 +47,15 @@ lzma_delta_coder_init(lzma_next_coder *next, lzma_allocator *allocator, return LZMA_OPTIONS_ERROR; // Set the delta distance. - opt = filters[0].options; - next->coder->distance = opt->dist; + const lzma_options_delta *opt = filters[0].options; + coder->distance = opt->dist; // Initialize the rest of the variables. - next->coder->pos = 0; - memzero(next->coder->history, LZMA_DELTA_DIST_MAX); + coder->pos = 0; + memzero(coder->history, LZMA_DELTA_DIST_MAX); // Initialize the next decoder in the chain, if any. - return lzma_next_filter_init(&next->coder->next, - allocator, filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } @@ -68,5 +69,5 @@ lzma_delta_coder_memusage(const void *options) || opt->dist > LZMA_DELTA_DIST_MAX) return UINT64_MAX; - return sizeof(lzma_coder); + return sizeof(lzma_delta_coder); } diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c index 28df72735..6859afa5c 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.c @@ -15,12 +15,11 @@ static void -decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) +decode_buffer(lzma_delta_coder *coder, uint8_t *buffer, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { buffer[i] += coder->history[(distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = buffer[i]; } @@ -28,17 +27,18 @@ decode_buffer(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -delta_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +delta_decode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { - const size_t out_start = *out_pos; - lzma_ret ret; + lzma_delta_coder *coder = coder_ptr; assert(coder->next.code != NULL); - ret = coder->next.code(coder->next.coder, allocator, + const size_t out_start = *out_pos; + + const lzma_ret ret = coder->next.code(coder->next.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, action); @@ -49,7 +49,7 @@ delta_decode(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { next->code = &delta_decode; @@ -58,15 +58,14 @@ lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_delta_props_decode(void **options, lzma_allocator *allocator, +lzma_delta_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_delta *opt; - if (props_size != 1) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_delta), allocator); + lzma_options_delta *opt + = lzma_alloc(sizeof(lzma_options_delta), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h index ae89acc59..ad89cc659 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_decoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_decoder.h @@ -16,10 +16,11 @@ #include "delta_common.h" extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_delta_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c index a39c154f1..384165151 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.c +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.c @@ -18,13 +18,12 @@ /// is the first filter in the chain (and thus the last filter in the /// encoder's filter stack). static void -copy_and_encode(lzma_coder *coder, - const uint8_t *LZMA_RESTRICT in, uint8_t *LZMA_RESTRICT out, size_t size) +copy_and_encode(lzma_delta_coder *coder, + const uint8_t *restrict in, uint8_t *restrict out, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { const uint8_t tmp = coder->history[ (distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = in[i]; @@ -36,12 +35,11 @@ copy_and_encode(lzma_coder *coder, /// Encodes the data in place. This is used when we are the last filter /// in the chain (and thus non-last filter in the encoder's filter stack). static void -encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size) +encode_in_place(lzma_delta_coder *coder, uint8_t *buffer, size_t size) { - size_t i; const size_t distance = coder->distance; - for (i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) { const uint8_t tmp = coder->history[ (distance + coder->pos) & 0xFF]; coder->history[coder->pos-- & 0xFF] = buffer[i]; @@ -51,11 +49,13 @@ encode_in_place(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -delta_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +delta_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_delta_coder *coder = coder_ptr; + lzma_ret ret; if (coder->next.code == NULL) { @@ -86,10 +86,12 @@ delta_encode(lzma_coder *coder, lzma_allocator *allocator, static lzma_ret -delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +delta_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_delta_coder *coder = coder_ptr; + // Delta doesn't and will never support changing the options in // the middle of encoding. If the app tries to change them, we // simply ignore them. @@ -99,7 +101,7 @@ delta_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_delta_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { next->code = &delta_encode; @@ -111,13 +113,12 @@ lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out) { - const lzma_options_delta *opt = options; - // The caller must have already validated the options, so it's // LZMA_PROG_ERROR if they are invalid. if (lzma_delta_coder_memusage(options) == UINT64_MAX) return LZMA_PROG_ERROR; + const lzma_options_delta *opt = options; out[0] = opt->dist - LZMA_DELTA_DIST_MIN; return LZMA_OK; diff --git a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h index a447862f2..4ab984785 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_encoder.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_encoder.h @@ -16,7 +16,8 @@ #include "delta_common.h" extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out); diff --git a/Utilities/cmliblzma/liblzma/delta/delta_private.h b/Utilities/cmliblzma/liblzma/delta/delta_private.h index 62b7fed86..0d6cb3866 100644 --- a/Utilities/cmliblzma/liblzma/delta/delta_private.h +++ b/Utilities/cmliblzma/liblzma/delta/delta_private.h @@ -15,7 +15,7 @@ #include "delta_common.h" -struct lzma_coder_s { +typedef struct { /// Next coder in the chain lzma_next_coder next; @@ -27,11 +27,11 @@ struct lzma_coder_s { /// Buffer to hold history of the original data uint8_t history[LZMA_DELTA_DIST_MAX]; -}; +} lzma_delta_coder; extern lzma_ret lzma_delta_coder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/liblzma.pc.in b/Utilities/cmliblzma/liblzma/liblzma.pc.in index 7f11f1a20..9fa489115 100644 --- a/Utilities/cmliblzma/liblzma/liblzma.pc.in +++ b/Utilities/cmliblzma/liblzma/liblzma.pc.in @@ -16,4 +16,4 @@ URL: @PACKAGE_URL@ Version: @PACKAGE_VERSION@ Cflags: -I${includedir} Libs: -L${libdir} -llzma -Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ +Libs.private: @PTHREAD_CFLAGS@ @LIBS@ diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c index 9fa1bdc3a..c7086440b 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.c @@ -20,7 +20,7 @@ #include "lz_decoder.h" -struct lzma_coder_s { +typedef struct { /// Dictionary (history buffer) lzma_dict dict; @@ -48,7 +48,7 @@ struct lzma_coder_s { size_t size; uint8_t buffer[LZMA_BUFFER_SIZE]; } temp; -}; +} lzma_coder; static void @@ -64,22 +64,18 @@ lz_decoder_reset(lzma_coder *coder) static lzma_ret decode_buffer(lzma_coder *coder, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size) + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) { while (true) { - size_t copy_size; - size_t dict_start; - lzma_ret ret; - // Wrap the dictionary if needed. if (coder->dict.pos == coder->dict.size) coder->dict.pos = 0; // Store the current dictionary position. It is needed to know // where to start copying to the out[] buffer. - dict_start = coder->dict.pos; + const size_t dict_start = coder->dict.pos; // Calculate how much we allow coder->lz.code() to decode. // It must not decode past the end of the dictionary @@ -90,13 +86,13 @@ decode_buffer(lzma_coder *coder, coder->dict.size - coder->dict.pos); // Call the coder->lz.code() to do the actual decoding. - ret = coder->lz.code( + const lzma_ret ret = coder->lz.code( coder->lz.coder, &coder->dict, in, in_pos, in_size); // Copy the decoded data from the dictionary to the out[] // buffer. - copy_size = coder->dict.pos - dict_start; + const size_t copy_size = coder->dict.pos - dict_start; assert(copy_size <= out_size - *out_pos); memcpy(out + *out_pos, coder->dict.buf + dict_start, copy_size); @@ -129,13 +125,15 @@ decode_buffer(lzma_coder *coder, static lzma_ret -lz_decode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((__unused__)), - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +lz_decode(void *coder_ptr, + const lzma_allocator *allocator lzma_attribute((__unused__)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_coder *coder = coder_ptr; + if (coder->next.code == NULL) return decode_buffer(coder, in, in_pos, in_size, out, out_pos, out_size); @@ -143,15 +141,13 @@ lz_decode(lzma_coder *coder, // We aren't the last coder in the chain, we need to decode // our input to a temporary buffer. while (*out_pos < out_size) { - lzma_ret ret; - // Fill the temporary buffer if it is empty. if (!coder->next_finished && coder->temp.pos == coder->temp.size) { coder->temp.pos = 0; coder->temp.size = 0; - ret = coder->next.code( + const lzma_ret ret = coder->next.code( coder->next.coder, allocator, in, in_pos, in_size, coder->temp.buffer, &coder->temp.size, @@ -173,7 +169,7 @@ lz_decode(lzma_coder *coder, return LZMA_OK; } - ret = decode_buffer(coder, coder->temp.buffer, + const lzma_ret ret = decode_buffer(coder, coder->temp.buffer, &coder->temp.pos, coder->temp.size, out, out_pos, out_size); @@ -190,8 +186,10 @@ lz_decode(lzma_coder *coder, static void -lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lz_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_coder *coder = coder_ptr; + lzma_next_end(&coder->next, allocator); lzma_free(coder->dict.buf, allocator); @@ -206,32 +204,33 @@ lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) extern lzma_ret -lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)) { - lzma_lz_options lz_options; - // Allocate the base structure if it isn't already allocated. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &lz_decode; next->end = &lz_decoder_end; - next->coder->dict.buf = NULL; - next->coder->dict.size = 0; - next->coder->lz = LZMA_LZ_DECODER_INIT; - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->dict.buf = NULL; + coder->dict.size = 0; + coder->lz = LZMA_LZ_DECODER_INIT; + coder->next = LZMA_NEXT_CODER_INIT; } // Allocate and initialize the LZ-based decoder. It will also give // us the dictionary size. - return_if_error(lz_init(&next->coder->lz, allocator, + lzma_lz_options lz_options; + return_if_error(lz_init(&coder->lz, allocator, filters[0].options, &lz_options)); // If the dictionary size is very small, increase it to 4096 bytes. @@ -255,14 +254,14 @@ lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, lz_options.dict_size = (lz_options.dict_size + 15) & ~((size_t)(15)); // Allocate and initialize the dictionary. - if (next->coder->dict.size != lz_options.dict_size) { - lzma_free(next->coder->dict.buf, allocator); - next->coder->dict.buf + if (coder->dict.size != lz_options.dict_size) { + lzma_free(coder->dict.buf, allocator); + coder->dict.buf = lzma_alloc(lz_options.dict_size, allocator); - if (next->coder->dict.buf == NULL) + if (coder->dict.buf == NULL) return LZMA_MEM_ERROR; - next->coder->dict.size = lz_options.dict_size; + coder->dict.size = lz_options.dict_size; } lz_decoder_reset(next->coder); @@ -275,21 +274,20 @@ lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const size_t copy_size = my_min(lz_options.preset_dict_size, lz_options.dict_size); const size_t offset = lz_options.preset_dict_size - copy_size; - memcpy(next->coder->dict.buf, lz_options.preset_dict + offset, + memcpy(coder->dict.buf, lz_options.preset_dict + offset, copy_size); - next->coder->dict.pos = copy_size; - next->coder->dict.full = copy_size; + coder->dict.pos = copy_size; + coder->dict.full = copy_size; } // Miscellaneous initializations - next->coder->next_finished = false; - next->coder->this_finished = false; - next->coder->temp.pos = 0; - next->coder->temp.size = 0; + coder->next_finished = false; + coder->this_finished = false; + coder->temp.pos = 0; + coder->temp.size = 0; // Initialize the next filter in the chain, if any. - return lzma_next_filter_init(&next->coder->next, allocator, - filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } @@ -301,7 +299,8 @@ lzma_lz_decoder_memusage(size_t dictionary_size) extern void -lzma_lz_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) +lzma_lz_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { + lzma_coder *coder = coder_ptr; coder->lz.set_uncompressed(coder->lz.coder, uncompressed_size); } diff --git a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h index 76011f2ad..754ccf37c 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_decoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_decoder.h @@ -53,45 +53,45 @@ typedef struct { typedef struct { /// Data specific to the LZ-based decoder - lzma_coder *coder; + void *coder; /// Function to decode from in[] to *dict - lzma_ret (*code)(lzma_coder *LZMA_RESTRICT coder, - lzma_dict *LZMA_RESTRICT dict, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size); + lzma_ret (*code)(void *coder, + lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size); - void (*reset)(lzma_coder *coder, const void *options); + void (*reset)(void *coder, const void *options); /// Set the uncompressed size - void (*set_uncompressed)(lzma_coder *coder, - lzma_vli uncompressed_size); + void (*set_uncompressed)(void *coder, lzma_vli uncompressed_size); /// Free allocated resources - void (*end)(lzma_coder *coder, lzma_allocator *allocator); + void (*end)(void *coder, const lzma_allocator *allocator); } lzma_lz_decoder; -static const lzma_lz_decoder LZMA_LZ_DECODER_INIT = - { - NULL, - NULL, - NULL, - NULL, - NULL, - }; +#define LZMA_LZ_DECODER_INIT \ + (lzma_lz_decoder){ \ + .coder = NULL, \ + .code = NULL, \ + .reset = NULL, \ + .set_uncompressed = NULL, \ + .end = NULL, \ + } extern lzma_ret lzma_lz_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters, + const lzma_allocator *allocator, + const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_decoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)); extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size); extern void lzma_lz_decoder_uncompressed( - lzma_coder *coder, lzma_vli uncompressed_size); + void *coder, lzma_vli uncompressed_size); ////////////////////// @@ -151,15 +151,13 @@ dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) dict->pos += left; } else { - uint32_t copy_pos; - uint32_t copy_size; - // The bigger the dictionary, the more rare this // case occurs. We need to "wrap" the dict, thus // we might need two memcpy() to copy all the data. assert(dict->full == dict->size); - copy_pos = dict->pos - distance - 1 + dict->size; - copy_size = dict->size - copy_pos; + const uint32_t copy_pos + = dict->pos - distance - 1 + dict->size; + uint32_t copy_size = dict->size - copy_pos; if (copy_size < left) { memmove(dict->buf + dict->pos, dict->buf + copy_pos, @@ -202,9 +200,9 @@ dict_put(lzma_dict *dict, uint8_t byte) /// Copies arbitrary amount of data into the dictionary. static inline void -dict_write(lzma_dict *LZMA_RESTRICT dict, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size, - size_t *LZMA_RESTRICT left) +dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size, + size_t *restrict left) { // NOTE: If we are being given more data than the size of the // dictionary, it could be possible to optimize the LZ decoder diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c index 1dae924b4..9a74b7c47 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.c @@ -20,8 +20,10 @@ # include "lz_encoder_hash_table.h" #endif +#include "memcmplen.h" -struct lzma_coder_s { + +typedef struct { /// LZ-based encoder e.g. LZMA lzma_lz_encoder lz; @@ -30,7 +32,7 @@ struct lzma_coder_s { /// Next coder in the chain lzma_next_coder next; -}; +} lzma_coder; /// \brief Moves the data in the input window to free space for new data @@ -43,18 +45,16 @@ struct lzma_coder_s { static void move_window(lzma_mf *mf) { - uint32_t move_offset; - size_t move_size; - // Align the move to a multiple of 16 bytes. Some LZ-based encoders // like LZMA use the lowest bits of mf->read_pos to know the // alignment of the uncompressed data. We also get better speed // for memmove() with aligned buffers. assert(mf->read_pos > mf->keep_size_before); - move_offset = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); + const uint32_t move_offset + = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); assert(mf->write_pos > move_offset); - move_size = mf->write_pos - move_offset; + const size_t move_size = mf->write_pos - move_offset; assert(move_offset + move_size <= mf->size); @@ -78,12 +78,10 @@ move_window(lzma_mf *mf) /// This function must not be called once it has returned LZMA_STREAM_END. /// static lzma_ret -fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, - size_t *in_pos, size_t in_size, lzma_action action) +fill_window(lzma_coder *coder, const lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size, + lzma_action action) { - size_t write_pos; - lzma_ret ret; - assert(coder->mf.read_pos <= coder->mf.write_pos); // Move the sliding window if needed. @@ -93,7 +91,8 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, // Maybe this is ugly, but lzma_mf uses uint32_t for most things // (which I find cleanest), but we need size_t here when filling // the history window. - write_pos = coder->mf.write_pos; + size_t write_pos = coder->mf.write_pos; + lzma_ret ret; if (coder->next.code == NULL) { // Not using a filter, simply memcpy() as much as possible. lzma_bufcpy(in, in_pos, in_size, coder->mf.buffer, @@ -111,6 +110,12 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, coder->mf.write_pos = write_pos; + // Silence Valgrind. lzma_memcmplen() can read extra bytes + // and Valgrind will give warnings if those bytes are uninitialized + // because Valgrind cannot see that the values of the uninitialized + // bytes are eventually ignored. + memzero(coder->mf.buffer + write_pos, LZMA_MEMCMPLEN_EXTRA); + // If end of stream has been reached or flushing completed, we allow // the encoder to process all the input (that is, read_pos is allowed // to reach write_pos). Otherwise we keep keep_size_after bytes @@ -134,7 +139,7 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, && coder->mf.read_pos < coder->mf.read_limit) { // Match finder may update coder->pending and expects it to // start from zero, so use a temporary variable. - const size_t pending = coder->mf.pending; + const uint32_t pending = coder->mf.pending; coder->mf.pending = 0; // Rewind read_pos so that the match finder can hash @@ -152,16 +157,16 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, static lzma_ret -lz_encode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lz_encode(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action) { + lzma_coder *coder = coder_ptr; + while (*out_pos < out_size && (*in_pos < in_size || action != LZMA_RUN)) { - lzma_ret ret; - // Read more data to coder->mf.buffer if needed. if (coder->mf.action == LZMA_RUN && coder->mf.read_pos >= coder->mf.read_limit) @@ -169,7 +174,7 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator, in, in_pos, in_size, action)); // Encode - ret = coder->lz.code(coder->lz.coder, + const lzma_ret ret = coder->lz.code(coder->lz.coder, &coder->mf, out, out_pos, out_size); if (ret != LZMA_OK) { // Setting this to LZMA_RUN for cases when we are @@ -185,17 +190,9 @@ lz_encode(lzma_coder *coder, lzma_allocator *allocator, static bool -lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, +lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator, const lzma_lz_options *lz_options) { - bool is_bt; - uint32_t new_count; - uint32_t reserve; - uint32_t old_size; - uint32_t hash_bytes; - uint32_t hs; - uint32_t old_count; - // For now, the dictionary size is limited to 1.5 GiB. This may grow // in the future if needed, but it needs a little more work than just // changing this check. @@ -221,14 +218,14 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // to size_t. // - Memory usage calculation needs something too, e.g. use uint64_t // for mf->size. - reserve = lz_options->dict_size / 2; + uint32_t reserve = lz_options->dict_size / 2; if (reserve > (UINT32_C(1) << 30)) reserve /= 2; reserve += (lz_options->before_size + lz_options->match_len_max + lz_options->after_size) / 2 + (UINT32_C(1) << 19); - old_size = mf->size; + const uint32_t old_size = mf->size; mf->size = mf->keep_size_before + reserve + mf->keep_size_after; // Deallocate the old history buffer if it exists but has different @@ -298,11 +295,12 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, // Calculate the sizes of mf->hash and mf->son and check that // nice_len is big enough for the selected match finder. - hash_bytes = lz_options->match_finder & 0x0F; + const uint32_t hash_bytes = lz_options->match_finder & 0x0F; if (hash_bytes > mf->nice_len) return true; - is_bt = (lz_options->match_finder & 0x10) != 0; + const bool is_bt = (lz_options->match_finder & 0x10) != 0; + uint32_t hs; if (hash_bytes == 2) { hs = 0xFFFF; @@ -338,25 +336,22 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, hs += HASH_4_SIZE; */ - // If the above code calculating hs is modified, make sure that - // this assertion stays valid (UINT32_MAX / 5 is not strictly the - // exact limit). If it doesn't, you need to calculate that - // hash_size_sum + sons_count cannot overflow. - assert(hs < UINT32_MAX / 5); - - old_count = mf->hash_size_sum + mf->sons_count; - mf->hash_size_sum = hs; + const uint32_t old_hash_count = mf->hash_count; + const uint32_t old_sons_count = mf->sons_count; + mf->hash_count = hs; mf->sons_count = mf->cyclic_size; if (is_bt) mf->sons_count *= 2; - new_count = mf->hash_size_sum + mf->sons_count; - // Deallocate the old hash array if it exists and has different size // than what is needed now. - if (old_count != new_count) { + if (old_hash_count != mf->hash_count + || old_sons_count != mf->sons_count) { lzma_free(mf->hash, allocator); mf->hash = NULL; + + lzma_free(mf->son, allocator); + mf->son = NULL; } // Maximum number of match finder cycles @@ -373,16 +368,23 @@ lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, static bool -lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator, +lz_encoder_init(lzma_mf *mf, const lzma_allocator *allocator, const lzma_lz_options *lz_options) { - size_t alloc_count; - // Allocate the history buffer. if (mf->buffer == NULL) { - mf->buffer = lzma_alloc(mf->size, allocator); + // lzma_memcmplen() is used for the dictionary buffer + // so we need to allocate a few extra bytes to prevent + // it from reading past the end of the buffer. + mf->buffer = lzma_alloc(mf->size + LZMA_MEMCMPLEN_EXTRA, + allocator); if (mf->buffer == NULL) return true; + + // Keep Valgrind happy with lzma_memcmplen() and initialize + // the extra bytes whose value may get read but which will + // effectively get ignored. + memzero(mf->buffer + mf->size, LZMA_MEMCMPLEN_EXTRA); } // Use cyclic_size as initial mf->offset. This allows @@ -396,43 +398,48 @@ lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator, mf->write_pos = 0; mf->pending = 0; - // Allocate match finder's hash array. - alloc_count = mf->hash_size_sum + mf->sons_count; - #if UINT32_MAX >= SIZE_MAX / 4 // Check for integer overflow. (Huge dictionaries are not // possible on 32-bit CPU.) - if (alloc_count > SIZE_MAX / sizeof(uint32_t)) + if (mf->hash_count > SIZE_MAX / sizeof(uint32_t) + || mf->sons_count > SIZE_MAX / sizeof(uint32_t)) return true; #endif + // Allocate and initialize the hash table. Since EMPTY_HASH_VALUE + // is zero, we can use lzma_alloc_zero() or memzero() for mf->hash. + // + // We don't need to initialize mf->son, but not doing that may + // make Valgrind complain in normalization (see normalize() in + // lz_encoder_mf.c). Skipping the initialization is *very* good + // when big dictionary is used but only small amount of data gets + // actually compressed: most of the mf->son won't get actually + // allocated by the kernel, so we avoid wasting RAM and improve + // initialization speed a lot. if (mf->hash == NULL) { - mf->hash = lzma_alloc(alloc_count * sizeof(uint32_t), + mf->hash = lzma_alloc_zero(mf->hash_count * sizeof(uint32_t), + allocator); + mf->son = lzma_alloc(mf->sons_count * sizeof(uint32_t), allocator); - if (mf->hash == NULL) - return true; - } - mf->son = mf->hash + mf->hash_size_sum; - mf->cyclic_pos = 0; + if (mf->hash == NULL || mf->son == NULL) { + lzma_free(mf->hash, allocator); + mf->hash = NULL; + + lzma_free(mf->son, allocator); + mf->son = NULL; - // Initialize the hash table. Since EMPTY_HASH_VALUE is zero, we - // can use memset(). + return true; + } + } else { /* - for (uint32_t i = 0; i < hash_size_sum; ++i) - mf->hash[i] = EMPTY_HASH_VALUE; + for (uint32_t i = 0; i < mf->hash_count; ++i) + mf->hash[i] = EMPTY_HASH_VALUE; */ - memzero(mf->hash, (size_t)(mf->hash_size_sum) * sizeof(uint32_t)); + memzero(mf->hash, mf->hash_count * sizeof(uint32_t)); + } - // We don't need to initialize mf->son, but not doing that will - // make Valgrind complain in normalization (see normalize() in - // lz_encoder_mf.c). - // - // Skipping this initialization is *very* good when big dictionary is - // used but only small amount of data gets actually compressed: most - // of the mf->hash won't get actually allocated by the kernel, so - // we avoid wasting RAM and improve initialization speed a lot. - //memzero(mf->son, (size_t)(mf->sons_count) * sizeof(uint32_t)); + mf->cyclic_pos = 0; // Handle preset dictionary. if (lz_options->preset_dict != NULL @@ -457,24 +464,32 @@ extern uint64_t lzma_lz_encoder_memusage(const lzma_lz_options *lz_options) { // Old buffers must not exist when calling lz_encoder_prepare(). - lzma_mf mf = { NULL }; + lzma_mf mf = { + .buffer = NULL, + .hash = NULL, + .son = NULL, + .hash_count = 0, + .sons_count = 0, + }; // Setup the size information into mf. if (lz_encoder_prepare(&mf, NULL, lz_options)) return UINT64_MAX; // Calculate the memory usage. - return (uint64_t)(mf.hash_size_sum + mf.sons_count) - * sizeof(uint32_t) - + (uint64_t)(mf.size) + sizeof(lzma_coder); + return ((uint64_t)(mf.hash_count) + mf.sons_count) * sizeof(uint32_t) + + mf.size + sizeof(lzma_coder); } static void -lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +lz_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_coder *coder = coder_ptr; + lzma_next_end(&coder->next, allocator); + lzma_free(coder->mf.son, allocator); lzma_free(coder->mf.hash, allocator); lzma_free(coder->mf.buffer, allocator); @@ -489,10 +504,12 @@ lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator, +lz_encoder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_coder *coder = coder_ptr; + if (coder->lz.options_update == NULL) return LZMA_PROG_ERROR; @@ -505,58 +522,63 @@ lz_encoder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)) { - lzma_lz_options lz_options; - #ifdef HAVE_SMALL // We need that the CRC32 table has been initialized. lzma_crc32_init(); #endif // Allocate and initialize the base data structure. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) + lzma_coder *coder = next->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &lz_encode; next->end = &lz_encoder_end; next->update = &lz_encoder_update; - next->coder->lz.coder = NULL; - next->coder->lz.code = NULL; - next->coder->lz.end = NULL; - - next->coder->mf.buffer = NULL; - next->coder->mf.hash = NULL; - next->coder->mf.hash_size_sum = 0; - next->coder->mf.sons_count = 0; - - next->coder->next = LZMA_NEXT_CODER_INIT; + coder->lz.coder = NULL; + coder->lz.code = NULL; + coder->lz.end = NULL; + + // mf.size is initialized to silence Valgrind + // when used on optimized binaries (GCC may reorder + // code in a way that Valgrind gets unhappy). + coder->mf.buffer = NULL; + coder->mf.size = 0; + coder->mf.hash = NULL; + coder->mf.son = NULL; + coder->mf.hash_count = 0; + coder->mf.sons_count = 0; + + coder->next = LZMA_NEXT_CODER_INIT; } // Initialize the LZ-based encoder. - return_if_error(lz_init(&next->coder->lz, allocator, + lzma_lz_options lz_options; + return_if_error(lz_init(&coder->lz, allocator, filters[0].options, &lz_options)); - // Setup the size information into next->coder->mf and deallocate + // Setup the size information into coder->mf and deallocate // old buffers if they have wrong size. - if (lz_encoder_prepare(&next->coder->mf, allocator, &lz_options)) + if (lz_encoder_prepare(&coder->mf, allocator, &lz_options)) return LZMA_OPTIONS_ERROR; // Allocate new buffers if needed, and do the rest of // the initialization. - if (lz_encoder_init(&next->coder->mf, allocator, &lz_options)) + if (lz_encoder_init(&coder->mf, allocator, &lz_options)) return LZMA_MEM_ERROR; // Initialize the next filter in the chain, if any. - return lzma_next_filter_init(&next->coder->next, allocator, - filters + 1); + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h index dcb4b2c5b..426dcd8a3 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder.h @@ -119,7 +119,7 @@ struct lzma_mf_s { lzma_action action; /// Number of elements in hash[] - uint32_t hash_size_sum; + uint32_t hash_count; /// Number of elements in son[] uint32_t sons_count; @@ -191,19 +191,18 @@ typedef struct { typedef struct { /// Data specific to the LZ-based encoder - lzma_coder *coder; + void *coder; /// Function to encode from *dict to out[] - lzma_ret (*code)(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size); + lzma_ret (*code)(void *coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// Free allocated resources - void (*end)(lzma_coder *coder, lzma_allocator *allocator); + void (*end)(void *coder, const lzma_allocator *allocator); /// Update the options in the middle of the encoding. - lzma_ret (*options_update)(lzma_coder *coder, - const lzma_filter *filter); + lzma_ret (*options_update)(void *coder, const lzma_filter *filter); } lzma_lz_encoder; @@ -218,7 +217,7 @@ typedef struct { /// Get pointer to the first byte not ran through the match finder -static inline uint8_t * +static inline const uint8_t * mf_ptr(const lzma_mf *mf) { return mf->buffer + mf->read_pos; @@ -296,10 +295,10 @@ mf_read(lzma_mf *mf, uint8_t *out, size_t *out_pos, size_t out_size, extern lzma_ret lzma_lz_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, lzma_ret (*lz_init)(lzma_lz_encoder *lz, - lzma_allocator *allocator, const void *options, + const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options)); diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h index de17c54fc..342a333d1 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_hash.h @@ -39,22 +39,25 @@ // Endianness doesn't matter in hash_2_calc() (no effect on the output). #ifdef TUKLIB_FAST_UNALIGNED_ACCESS # define hash_2_calc() \ - hash_value = *(const uint16_t *)(cur) + const uint32_t hash_value = *(const uint16_t *)(cur) #else # define hash_2_calc() \ - hash_value = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) + const uint32_t hash_value \ + = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) #endif #define hash_3_calc() \ - temp = hash_table[cur[0]] ^ cur[1]; \ - hash_2_value = temp & HASH_2_MASK; \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask + const uint32_t temp = hash_table[cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask #define hash_4_calc() \ - temp = hash_table[cur[0]] ^ cur[1]; \ - hash_2_value = temp & HASH_2_MASK; \ - hash_3_value = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ + const uint32_t temp = hash_table[cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + const uint32_t hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ ^ (hash_table[cur[3]] << 5)) & mf->hash_mask diff --git a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c index 50c3459aa..78520779f 100644 --- a/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c +++ b/Utilities/cmliblzma/liblzma/lz/lz_encoder_mf.c @@ -13,6 +13,7 @@ #include "lz_encoder.h" #include "lz_encoder_hash.h" +#include "memcmplen.h" /// \brief Find matches starting from the current byte @@ -32,9 +33,8 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) if (count > 0) { #ifndef NDEBUG - uint32_t i; // Validate the matches. - for (i = 0; i < count; ++i) { + for (uint32_t i = 0; i < count; ++i) { assert(matches[i].len <= mf->nice_len); assert(matches[i].dist < mf->read_pos); assert(memcmp(mf_ptr(mf) - 1, @@ -50,9 +50,6 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) // If a match of maximum search length was found, try to // extend the match to maximum possible length. if (len_best == mf->nice_len) { - uint8_t *p1; - uint8_t *p2; - // The limit for the match length is either the // maximum match length supported by the LZ-based // encoder or the number of bytes left in the @@ -63,15 +60,13 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) // Pointer to the byte we just ran through // the match finder. - p1 = mf_ptr(mf) - 1; + const uint8_t *p1 = mf_ptr(mf) - 1; // Pointer to the beginning of the match. We need -1 // here because the match distances are zero based. - p2 = p1 - matches[count - 1].dist - 1; + const uint8_t *p2 = p1 - matches[count - 1].dist - 1; - while (len_best < limit - && p1[len_best] == p2[len_best]) - ++len_best; + len_best = lzma_memcmplen(p1, p2, len_best, limit); } } @@ -112,36 +107,35 @@ lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) static void normalize(lzma_mf *mf) { - uint32_t i; - uint32_t subvalue; - uint32_t count; - uint32_t *hash; - assert(mf->read_pos + mf->offset == MUST_NORMALIZE_POS); // In future we may not want to touch the lowest bits, because there // may be match finders that use larger resolution than one byte. - subvalue = (MUST_NORMALIZE_POS - mf->cyclic_size); + const uint32_t subvalue + = (MUST_NORMALIZE_POS - mf->cyclic_size); // & (~(UINT32_C(1) << 10) - 1); - count = mf->hash_size_sum + mf->sons_count; - hash = mf->hash; - - for (i = 0; i < count; ++i) { + for (uint32_t i = 0; i < mf->hash_count; ++i) { // If the distance is greater than the dictionary size, // we can simply mark the hash element as empty. + if (mf->hash[i] <= subvalue) + mf->hash[i] = EMPTY_HASH_VALUE; + else + mf->hash[i] -= subvalue; + } + + for (uint32_t i = 0; i < mf->sons_count; ++i) { + // Do the same for mf->son. // - // NOTE: Only the first mf->hash_size_sum elements are - // initialized for sure. There may be uninitialized elements - // in mf->son. Since we go through both mf->hash and - // mf->son here in normalization, Valgrind may complain - // that the "if" below depends on uninitialized value. In - // this case it is safe to ignore the warning. See also the - // comments in lz_encoder_init() in lz_encoder.c. - if (hash[i] <= subvalue) - hash[i] = EMPTY_HASH_VALUE; + // NOTE: There may be uninitialized elements in mf->son. + // Valgrind may complain that the "if" below depends on + // an uninitialized value. In this case it is safe to ignore + // the warning. See also the comments in lz_encoder_init() + // in lz_encoder.c. + if (mf->son[i] <= subvalue) + mf->son[i] = EMPTY_HASH_VALUE; else - hash[i] -= subvalue; + mf->son[i] -= subvalue; } // Update offset to match the new locations. @@ -204,14 +198,15 @@ move_pending(lzma_mf *mf) move_pending(mf); \ ret_op; \ } \ - cur = mf_ptr(mf); \ - pos = mf->read_pos + mf->offset + const uint8_t *cur = mf_ptr(mf); \ + const uint32_t pos = mf->read_pos + mf->offset /// Header for find functions. "return 0" indicates that zero matches /// were found. #define header_find(is_bt, len_min) \ - header(is_bt, len_min, return 0) + header(is_bt, len_min, return 0); \ + uint32_t matches_count = 0 /// Header for a loop in a skip function. "continue" tells to skip the rest @@ -268,19 +263,15 @@ hc_find_func( while (true) { const uint32_t delta = pos - cur_match; - const uint8_t *pb; if (depth-- == 0 || delta >= cyclic_size) return matches; - pb = cur - delta; + const uint8_t *const pb = cur - delta; cur_match = son[cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)]; if (pb[len_best] == cur[len_best] && pb[0] == cur[0]) { - uint32_t len = 0; - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + uint32_t len = lzma_memcmplen(pb, cur, 1, len_limit); if (len_best < len) { len_best = len; @@ -313,27 +304,21 @@ do { \ extern uint32_t lzma_mf_hc3_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t delta2, cur_match; - uint32_t len_best = 2; - uint32_t matches_count = 0; - header_find(false, 3); hash_3_calc(); - delta2 = pos - mf->hash[hash_2_value]; - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 2; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen(cur - delta2, cur, + len_best, len_limit); matches[0].len = len_best; matches[0].dist = delta2 - 1; @@ -353,22 +338,18 @@ extern void lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t cur_match; - if (mf_avail(mf) < 3) { move_pending(mf); continue; } - cur = mf_ptr(mf); - pos = mf->read_pos + mf->offset; + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; hash_3_calc(); - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; @@ -384,25 +365,21 @@ lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t delta2, delta3, cur_match; - uint32_t len_best = 1; - uint32_t matches_count = 0; - header_find(false, 4); hash_4_calc(); - delta2 = pos - mf->hash[hash_2_value]; - delta3 = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value ] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 1; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { len_best = 2; matches[0].len = 2; @@ -418,9 +395,8 @@ lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) } if (matches_count != 0) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen(cur - delta2, cur, + len_best, len_limit); matches[matches_count - 1].len = len_best; @@ -441,22 +417,18 @@ extern void lzma_mf_hc4_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t cur_match; - if (mf_avail(mf) < 4) { move_pending(mf); continue; } - cur = mf_ptr(mf); - pos = mf->read_pos + mf->offset; + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; hash_4_calc(); - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; @@ -494,10 +466,6 @@ bt_find_func( uint32_t len1 = 0; while (true) { - uint32_t *pair; - const uint8_t *pb; - uint32_t len; - const uint32_t delta = pos - cur_match; if (depth-- == 0 || delta >= cyclic_size) { *ptr0 = EMPTY_HASH_VALUE; @@ -505,17 +473,15 @@ bt_find_func( return matches; } - pair = son + ((cyclic_pos - delta + uint32_t *const pair = son + ((cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)) << 1); - pb = cur - delta; - len = my_min(len0, len1); + const uint8_t *const pb = cur - delta; + uint32_t len = my_min(len0, len1); if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + len = lzma_memcmplen(pb, cur, len + 1, len_limit); if (len_best < len) { len_best = len; @@ -564,10 +530,6 @@ bt_skip_func( uint32_t len1 = 0; while (true) { - uint32_t *pair; - const uint8_t *pb; - uint32_t len; - const uint32_t delta = pos - cur_match; if (depth-- == 0 || delta >= cyclic_size) { *ptr0 = EMPTY_HASH_VALUE; @@ -575,16 +537,14 @@ bt_skip_func( return; } - pair = son + ((cyclic_pos - delta + uint32_t *pair = son + ((cyclic_pos - delta + (delta > cyclic_pos ? cyclic_size : 0)) << 1); - pb = cur - delta; - len = my_min(len0, len1); + const uint8_t *pb = cur - delta; + uint32_t len = my_min(len0, len1); if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; + len = lzma_memcmplen(pb, cur, len + 1, len_limit); if (len == len_limit) { *ptr1 = pair[0]; @@ -626,17 +586,11 @@ do { \ extern uint32_t lzma_mf_bt2_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t hash_value; /* hash_2_calc */ - uint32_t cur_match; - uint32_t matches_count = 0; - header_find(true, 2); hash_2_calc(); - cur_match = mf->hash[hash_value]; + const uint32_t cur_match = mf->hash[hash_value]; mf->hash[hash_value] = pos; bt_find(1); @@ -647,16 +601,11 @@ extern void lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t hash_value; /* hash_2_calc */ - uint32_t cur_match; - header_skip(true, 2); hash_2_calc(); - cur_match = mf->hash[hash_value]; + const uint32_t cur_match = mf->hash[hash_value]; mf->hash[hash_value] = pos; bt_skip(); @@ -670,27 +619,21 @@ lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_bt3_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t delta2, cur_match; - uint32_t len_best = 2; - uint32_t matches_count = 0; - header_find(true, 3); hash_3_calc(); - delta2 = pos - mf->hash[hash_2_value]; - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 2; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen( + cur, cur - delta2, len_best, len_limit); matches[0].len = len_best; matches[0].dist = delta2 - 1; @@ -710,16 +653,12 @@ extern void lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value; /* hash_3_calc */ - uint32_t cur_match; - header_skip(true, 3); hash_3_calc(); - cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; @@ -735,25 +674,21 @@ lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) extern uint32_t lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t delta2, delta3, cur_match; - uint32_t len_best = 1; - uint32_t matches_count = 0; - header_find(true, 4); hash_4_calc(); - delta2 = pos - mf->hash[hash_2_value]; - delta3 = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + uint32_t len_best = 1; + if (delta2 < mf->cyclic_size && *(cur - delta2) == *cur) { len_best = 2; matches[0].len = 2; @@ -769,9 +704,8 @@ lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) } if (matches_count != 0) { - for ( ; len_best != len_limit; ++len_best) - if (*(cur + len_best - delta2) != cur[len_best]) - break; + len_best = lzma_memcmplen( + cur, cur - delta2, len_best, len_limit); matches[matches_count - 1].len = len_best; @@ -792,16 +726,12 @@ extern void lzma_mf_bt4_skip(lzma_mf *mf, uint32_t amount) { do { - const uint8_t *cur; - uint32_t pos; - uint32_t temp, hash_value, hash_2_value, hash_3_value; /* hash_4_calc */ - uint32_t cur_match; - header_skip(true, 4); hash_4_calc(); - cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; mf->hash[hash_2_value] = pos; mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; diff --git a/Utilities/cmliblzma/liblzma/lzma/fastpos.h b/Utilities/cmliblzma/liblzma/lzma/fastpos.h index 5a834d68d..a3feea58d 100644 --- a/Utilities/cmliblzma/liblzma/lzma/fastpos.h +++ b/Utilities/cmliblzma/liblzma/lzma/fastpos.h @@ -14,15 +14,15 @@ #ifndef LZMA_FASTPOS_H #define LZMA_FASTPOS_H -// LZMA encodes match distances (positions) by storing the highest two -// bits using a six-bit value [0, 63], and then the missing lower bits. -// Dictionary size is also stored using this encoding in the new .lzma +// LZMA encodes match distances by storing the highest two bits using +// a six-bit value [0, 63], and then the missing lower bits. +// Dictionary size is also stored using this encoding in the .xz // file format header. // // fastpos.h provides a way to quickly find out the correct six-bit // values. The following table gives some examples of this encoding: // -// pos return +// dist return // 0 0 // 1 1 // 2 2 @@ -48,10 +48,10 @@ // Provided functions or macros // ---------------------------- // -// get_pos_slot(pos) is the basic version. get_pos_slot_2(pos) -// assumes that pos >= FULL_DISTANCES, thus the result is at least -// FULL_DISTANCES_BITS * 2. Using get_pos_slot(pos) instead of -// get_pos_slot_2(pos) would give the same result, but get_pos_slot_2(pos) +// get_dist_slot(dist) is the basic version. get_dist_slot_2(dist) +// assumes that dist >= FULL_DISTANCES, thus the result is at least +// FULL_DISTANCES_BITS * 2. Using get_dist_slot(dist) instead of +// get_dist_slot_2(dist) would give the same result, but get_dist_slot_2(dist) // should be tiny bit faster due to the assumption being made. // // @@ -75,16 +75,15 @@ // on all systems I have tried. The size optimized version is sometimes // slightly faster, but sometimes it is a lot slower. -#include "config.h" - #ifdef HAVE_SMALL -# define get_pos_slot(pos) ((pos) <= 4 ? (pos) : get_pos_slot_2(pos)) +# define get_dist_slot(dist) \ + ((dist) <= 4 ? (dist) : get_dist_slot_2(dist)) static inline uint32_t -get_pos_slot_2(uint32_t pos) +get_dist_slot_2(uint32_t dist) { - const uint32_t i = bsr32(pos); - return (i + i) + ((pos >> (i - 1)) & 1); + const uint32_t i = bsr32(dist); + return (i + i) + ((dist >> (i - 1)) & 1); } @@ -101,39 +100,39 @@ extern const uint8_t lzma_fastpos[1 << FASTPOS_BITS]; #define fastpos_limit(extra, n) \ (UINT32_C(1) << (FASTPOS_BITS + fastpos_shift(extra, n))) -#define fastpos_result(pos, extra, n) \ - lzma_fastpos[(pos) >> fastpos_shift(extra, n)] \ +#define fastpos_result(dist, extra, n) \ + lzma_fastpos[(dist) >> fastpos_shift(extra, n)] \ + 2 * fastpos_shift(extra, n) static inline uint32_t -get_pos_slot(uint32_t pos) +get_dist_slot(uint32_t dist) { // If it is small enough, we can pick the result directly from // the precalculated table. - if (pos < fastpos_limit(0, 0)) - return lzma_fastpos[pos]; + if (dist < fastpos_limit(0, 0)) + return lzma_fastpos[dist]; - if (pos < fastpos_limit(0, 1)) - return fastpos_result(pos, 0, 1); + if (dist < fastpos_limit(0, 1)) + return fastpos_result(dist, 0, 1); - return fastpos_result(pos, 0, 2); + return fastpos_result(dist, 0, 2); } #ifdef FULL_DISTANCES_BITS static inline uint32_t -get_pos_slot_2(uint32_t pos) +get_dist_slot_2(uint32_t dist) { - assert(pos >= FULL_DISTANCES); + assert(dist >= FULL_DISTANCES); - if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 0); + if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 0); - if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 1)) - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 1); + if (dist < fastpos_limit(FULL_DISTANCES_BITS - 1, 1)) + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 1); - return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 2); + return fastpos_result(dist, FULL_DISTANCES_BITS - 1, 2); } #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c index bd2a73782..878c870ae 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.c @@ -16,7 +16,7 @@ #include "lzma_decoder.h" -struct lzma_coder_s { +typedef struct { enum sequence { SEQ_CONTROL, SEQ_UNCOMPRESSED_1, @@ -50,14 +50,16 @@ struct lzma_coder_s { bool need_dictionary_reset; lzma_options_lzma options; -}; +} lzma_lzma2_coder; static lzma_ret -lzma2_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dict, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, +lzma2_decode(void *coder_ptr, lzma_dict *restrict dict, + const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) { + lzma_lzma2_coder *restrict coder = coder_ptr; + // With SEQ_LZMA it is possible that no new input is needed to do // some progress. The rest of the sequences assume that there is // at least one byte of input. @@ -209,8 +211,10 @@ lzma2_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dict, static void -lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma2_decoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_lzma2_coder *coder = coder_ptr; + assert(coder->lzma.end == NULL); lzma_free(coder->lzma.coder, allocator); @@ -221,34 +225,36 @@ lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma2_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options) { - const lzma_options_lzma *options = opt; - - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (lz->coder == NULL) + lzma_lzma2_coder *coder = lz->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzma2_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + lz->coder = coder; lz->code = &lzma2_decode; lz->end = &lzma2_decoder_end; - lz->coder->lzma = LZMA_LZ_DECODER_INIT; + coder->lzma = LZMA_LZ_DECODER_INIT; } - lz->coder->sequence = SEQ_CONTROL; - lz->coder->need_properties = true; - lz->coder->need_dictionary_reset = options->preset_dict == NULL + const lzma_options_lzma *options = opt; + + coder->sequence = SEQ_CONTROL; + coder->need_properties = true; + coder->need_dictionary_reset = options->preset_dict == NULL || options->preset_dict_size == 0; - return lzma_lzma_decoder_create(&lz->coder->lzma, + return lzma_lzma_decoder_create(&coder->lzma, allocator, options, lz_options); } extern lzma_ret -lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma2_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { // LZMA2 can only be the last filter in the chain. This is enforced @@ -263,17 +269,15 @@ lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern uint64_t lzma_lzma2_decoder_memusage(const void *options) { - return sizeof(lzma_coder) + return sizeof(lzma_lzma2_coder) + lzma_lzma_decoder_memusage_nocheck(options); } extern lzma_ret -lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, +lzma_lzma2_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_lzma *opt; - if (props_size != 1) return LZMA_OPTIONS_ERROR; @@ -285,7 +289,8 @@ lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, if (props[0] > 40) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_lzma), allocator); + lzma_options_lzma *opt = lzma_alloc( + sizeof(lzma_options_lzma), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h index fac4ac487..ef2dcbfa7 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_decoder.h @@ -17,12 +17,13 @@ #include "common.h" extern lzma_ret lzma_lzma2_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma2_decoder_memusage(const void *options); extern lzma_ret lzma_lzma2_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c index a3651a7c6..63588ee30 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.c @@ -17,7 +17,7 @@ #include "lzma2_encoder.h" -struct lzma_coder_s { +typedef struct { enum { SEQ_INIT, SEQ_LZMA_ENCODE, @@ -27,7 +27,7 @@ struct lzma_coder_s { } sequence; /// LZMA encoder - lzma_coder *lzma; + void *lzma; /// LZMA options currently in use. lzma_options_lzma opt_cur; @@ -48,20 +48,19 @@ struct lzma_coder_s { /// Buffer to hold the chunk header and LZMA compressed data uint8_t buf[LZMA2_HEADER_MAX + LZMA2_CHUNK_MAX]; -}; +} lzma_lzma2_coder; static void -lzma2_header_lzma(lzma_coder *coder) +lzma2_header_lzma(lzma_lzma2_coder *coder) { - size_t pos; - size_t size; - assert(coder->uncompressed_size > 0); assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX); assert(coder->compressed_size > 0); assert(coder->compressed_size <= LZMA2_CHUNK_MAX); + size_t pos; + if (coder->need_properties) { pos = 0; @@ -82,7 +81,7 @@ lzma2_header_lzma(lzma_coder *coder) coder->buf_pos = pos; // Uncompressed size - size = coder->uncompressed_size - 1; + size_t size = coder->uncompressed_size - 1; coder->buf[pos++] += size >> 16; coder->buf[pos++] = (size >> 8) & 0xFF; coder->buf[pos++] = size & 0xFF; @@ -109,7 +108,7 @@ lzma2_header_lzma(lzma_coder *coder) static void -lzma2_header_uncompressed(lzma_coder *coder) +lzma2_header_uncompressed(lzma_lzma2_coder *coder) { assert(coder->uncompressed_size > 0); assert(coder->uncompressed_size <= LZMA2_CHUNK_MAX); @@ -134,10 +133,12 @@ lzma2_header_uncompressed(lzma_coder *coder) static lzma_ret -lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma2_encode(void *coder_ptr, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { + lzma_lzma2_coder *restrict coder = coder_ptr; + while (*out_pos < out_size) switch (coder->sequence) { case SEQ_INIT: @@ -163,9 +164,6 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Fall through case SEQ_LZMA_ENCODE: { - uint32_t read_start; - lzma_ret ret; - // Calculate how much more uncompressed data this chunk // could accept. const uint32_t left = LZMA2_UNCOMPRESSED_MAX @@ -186,10 +184,10 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Save the start position so that we can update // coder->uncompressed_size. - read_start = mf->read_pos - mf->read_ahead; + const uint32_t read_start = mf->read_pos - mf->read_ahead; // Call the LZMA encoder until the chunk is finished. - ret = lzma_lzma_encode(coder->lzma, mf, + const lzma_ret ret = lzma_lzma_encode(coder->lzma, mf, coder->buf + LZMA2_HEADER_MAX, &coder->compressed_size, LZMA2_CHUNK_MAX, limit); @@ -266,8 +264,9 @@ lzma2_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static void -lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma2_encoder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_lzma2_coder *coder = coder_ptr; lzma_free(coder->lzma, allocator); lzma_free(coder, allocator); return; @@ -275,9 +274,9 @@ lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) +lzma2_encoder_options_update(void *coder_ptr, const lzma_filter *filter) { - lzma_options_lzma *opt; + lzma_lzma2_coder *coder = coder_ptr; // New options can be set only when there is no incomplete chunk. // This is the case at the beginning of the raw stream and right @@ -287,7 +286,7 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) // Look if there are new options. At least for now, // only lc/lp/pb can be changed. - opt = filter->options; + const lzma_options_lzma *opt = filter->options; if (coder->opt_cur.lc != opt->lc || coder->opt_cur.lp != opt->lp || coder->opt_cur.pb != opt->pb) { // Validate the options. @@ -310,36 +309,38 @@ lzma2_encoder_options_update(lzma_coder *coder, const lzma_filter *filter) static lzma_ret -lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, +lzma2_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { if (options == NULL) return LZMA_PROG_ERROR; - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (lz->coder == NULL) + lzma_lzma2_coder *coder = lz->coder; + if (coder == NULL) { + coder = lzma_alloc(sizeof(lzma_lzma2_coder), allocator); + if (coder == NULL) return LZMA_MEM_ERROR; + lz->coder = coder; lz->code = &lzma2_encode; lz->end = &lzma2_encoder_end; lz->options_update = &lzma2_encoder_options_update; - lz->coder->lzma = NULL; + coder->lzma = NULL; } - lz->coder->opt_cur = *(const lzma_options_lzma *)(options); + coder->opt_cur = *(const lzma_options_lzma *)(options); - lz->coder->sequence = SEQ_INIT; - lz->coder->need_properties = true; - lz->coder->need_state_reset = false; - lz->coder->need_dictionary_reset - = lz->coder->opt_cur.preset_dict == NULL - || lz->coder->opt_cur.preset_dict_size == 0; + coder->sequence = SEQ_INIT; + coder->need_properties = true; + coder->need_state_reset = false; + coder->need_dictionary_reset + = coder->opt_cur.preset_dict == NULL + || coder->opt_cur.preset_dict_size == 0; // Initialize LZMA encoder - return_if_error(lzma_lzma_encoder_create(&lz->coder->lzma, allocator, - &lz->coder->opt_cur, lz_options)); + return_if_error(lzma_lzma_encoder_create(&coder->lzma, allocator, + &coder->opt_cur, lz_options)); // Make sure that we will always have enough history available in // case we need to use uncompressed chunks. They are used when the @@ -355,7 +356,7 @@ lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma2_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma2_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return lzma_lz_encoder_init( @@ -370,7 +371,7 @@ lzma_lzma2_encoder_memusage(const void *options) if (lzma_mem == UINT64_MAX) return UINT64_MAX; - return sizeof(lzma_coder) + lzma_mem; + return sizeof(lzma_lzma2_coder) + lzma_mem; } @@ -393,7 +394,17 @@ lzma_lzma2_props_encode(const void *options, uint8_t *out) if (d == UINT32_MAX) out[0] = 40; else - out[0] = get_pos_slot(d + 1) - 24; + out[0] = get_dist_slot(d + 1) - 24; return LZMA_OK; } + + +extern uint64_t +lzma_lzma2_block_size(const void *options) +{ + const lzma_options_lzma *const opt = options; + + // Use at least 1 MiB to keep compression ratio better. + return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20); +} diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h index ca19ef469..515f18393 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma2_encoder.h @@ -31,11 +31,13 @@ extern lzma_ret lzma_lzma2_encoder_init( - lzma_next_coder *next, lzma_allocator *allocator, + lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); extern uint64_t lzma_lzma2_encoder_memusage(const void *options); extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out); +extern uint64_t lzma_lzma2_block_size(const void *options); + #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h index 36267dc88..09efd3872 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_common.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_common.h @@ -129,15 +129,12 @@ static inline void literal_init(probability (*probs)[LITERAL_CODER_SIZE], uint32_t lc, uint32_t lp) { - uint32_t coders; - uint32_t i, j; - assert(lc + lp <= LZMA_LCLP_MAX); - coders = 1U << (lc + lp); + const uint32_t coders = 1U << (lc + lp); - for (i = 0; i < coders; ++i) - for (j = 0; j < LITERAL_CODER_SIZE; ++j) + for (uint32_t i = 0; i < coders; ++i) + for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) bit_reset(probs[i][j]); return; @@ -174,53 +171,54 @@ literal_init(probability (*probs)[LITERAL_CODER_SIZE], // Match distance // //////////////////// -// Different set of probabilities is used for match distances that have very +// Different sets of probabilities are used for match distances that have very // short match length: Lengths of 2, 3, and 4 bytes have a separate set of // probabilities for each length. The matches with longer length use a shared // set of probabilities. -#define LEN_TO_POS_STATES 4 +#define DIST_STATES 4 // Macro to get the index of the appropriate probability array. -#define get_len_to_pos_state(len) \ - ((len) < LEN_TO_POS_STATES + MATCH_LEN_MIN \ +#define get_dist_state(len) \ + ((len) < DIST_STATES + MATCH_LEN_MIN \ ? (len) - MATCH_LEN_MIN \ - : LEN_TO_POS_STATES - 1) + : DIST_STATES - 1) -// The highest two bits of a match distance (pos slot) are encoded using six -// bits. See fastpos.h for more explanation. -#define POS_SLOT_BITS 6 -#define POS_SLOTS (1 << POS_SLOT_BITS) +// The highest two bits of a match distance (distance slot) are encoded +// using six bits. See fastpos.h for more explanation. +#define DIST_SLOT_BITS 6 +#define DIST_SLOTS (1 << DIST_SLOT_BITS) // Match distances up to 127 are fully encoded using probabilities. Since -// the highest two bits (pos slot) are always encoded using six bits, the -// distances 0-3 don't need any additional bits to encode, since the pos -// slot itself is the same as the actual distance. START_POS_MODEL_INDEX -// indicates the first pos slot where at least one additional bit is needed. -#define START_POS_MODEL_INDEX 4 +// the highest two bits (distance slot) are always encoded using six bits, +// the distances 0-3 don't need any additional bits to encode, since the +// distance slot itself is the same as the actual distance. DIST_MODEL_START +// indicates the first distance slot where at least one additional bit is +// needed. +#define DIST_MODEL_START 4 // Match distances greater than 127 are encoded in three pieces: -// - pos slot: the highest two bits +// - distance slot: the highest two bits // - direct bits: 2-26 bits below the highest two bits // - alignment bits: four lowest bits // // Direct bits don't use any probabilities. // -// The pos slot value of 14 is for distances 128-191 (see the table in +// The distance slot value of 14 is for distances 128-191 (see the table in // fastpos.h to understand why). -#define END_POS_MODEL_INDEX 14 +#define DIST_MODEL_END 14 -// Pos slots that indicate a distance <= 127. -#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2) +// Distance slots that indicate a distance <= 127. +#define FULL_DISTANCES_BITS (DIST_MODEL_END / 2) #define FULL_DISTANCES (1 << FULL_DISTANCES_BITS) // For match distances greater than 127, only the highest two bits and the // lowest four bits (alignment) is encoded using probabilities. #define ALIGN_BITS 4 -#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) -#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) +#define ALIGN_SIZE (1 << ALIGN_BITS) +#define ALIGN_MASK (ALIGN_SIZE - 1) // LZMA remembers the four most recent match distances. Reusing these distances // tends to take less space than re-encoding the actual distance value. -#define REP_DISTANCES 4 +#define REPS 4 #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c index 3c0f39331..d0f29b763 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.c @@ -16,6 +16,12 @@ #include "lzma_decoder.h" #include "range_decoder.h" +// The macros unroll loops with switch statements. +// Silence warnings about missing fall-through comments. +#if TUKLIB_GNUC_REQ(7, 0) +# pragma GCC diagnostic ignored "-Wimplicit-fallthrough" +#endif + #ifdef HAVE_SMALL @@ -114,33 +120,33 @@ do { \ case seq ## _CHOICE: \ rc_if_0(ld.choice, seq ## _CHOICE) { \ rc_update_0(ld.choice); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW0); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW1); \ - rc_bit_case(ld.low[pos_state][symbol], 0, 0, seq ## _LOW2); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ } else { \ rc_update_1(ld.choice); \ case seq ## _CHOICE2: \ rc_if_0(ld.choice2, seq ## _CHOICE2) { \ rc_update_0(ld.choice2); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID0); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID1); \ - rc_bit_case(ld.mid[pos_state][symbol], 0, 0, \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ seq ## _MID2); \ target = symbol - LEN_MID_SYMBOLS \ + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ } else { \ rc_update_1(ld.choice2); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH0); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH1); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH2); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH3); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH4); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH5); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH6); \ - rc_bit_case(ld.high[symbol], 0, 0, seq ## _HIGH7); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ target = symbol - LEN_HIGH_SYMBOLS \ + MATCH_LEN_MIN \ + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ @@ -161,7 +167,7 @@ typedef struct { } lzma_length_decoder; -struct lzma_coder_s { +typedef struct { /////////////////// // Probabilities // /////////////////// @@ -193,15 +199,15 @@ struct lzma_coder_s { /// Probability tree for the highest two bits of the match distance. /// There is a separate probability tree for match lengths of /// 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273]. - probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; + probability dist_slot[DIST_STATES][DIST_SLOTS]; /// Probability trees for additional bits for match distance when the /// distance is in the range [4, 127]. - probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_special[FULL_DISTANCES - DIST_MODEL_END]; /// Probability tree for the lowest four bits of a match distance /// that is equal to or greater than 128. - probability pos_align[ALIGN_TABLE_SIZE]; + probability pos_align[ALIGN_SIZE]; /// Length of a normal match lzma_length_decoder match_len_decoder; @@ -245,8 +251,8 @@ struct lzma_coder_s { SEQ_LITERAL_WRITE, SEQ_IS_REP, seq_len(SEQ_MATCH_LEN), - seq_6(SEQ_POS_SLOT), - SEQ_POS_MODEL, + seq_6(SEQ_DIST_SLOT), + SEQ_DIST_MODEL, SEQ_DIRECT, seq_4(SEQ_ALIGN), SEQ_EOPM, @@ -277,14 +283,27 @@ struct lzma_coder_s { /// If decoding a literal: match byte. /// If decoding a match: length of the match. uint32_t len; -}; +} lzma_lzma1_decoder; static lzma_ret -lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, - const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size) +lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, + const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { + lzma_lzma1_decoder *restrict coder = coder_ptr; + + //////////////////// + // Initialization // + //////////////////// + + { + const lzma_ret ret = rc_read_init( + &coder->rc, in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + return ret; + } + /////////////// // Variables // /////////////// @@ -331,16 +350,6 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, if (no_eopm && coder->uncompressed_size < dict.limit - dict.pos) dict.limit = dict.pos + (size_t)(coder->uncompressed_size); - //////////////////// - // Initialization // - //////////////////// - - if (!rc_read_init(&coder->rc, in, in_pos, in_size)) - return LZMA_OK; - - rc = coder->rc; - rc_in_pos = *in_pos; - // The main decoder loop. The "switch" is used to restart the decoder at // correct location. Once restarted, the "switch" is no longer used. switch (coder->sequence) @@ -356,21 +365,6 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, break; rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { - static const lzma_lzma_state next_state[] = { - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_MATCH_LIT_LIT, - STATE_REP_LIT_LIT, - STATE_SHORTREP_LIT_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT, - STATE_SHORTREP_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT - }; - rc_update_0(coder->is_match[state][pos_state]); // It's a literal i.e. a single 8-bit byte. @@ -388,21 +382,16 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, rc_bit(probs[symbol], , , SEQ_LITERAL); } while (symbol < (1 << 8)); #else - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL0); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL1); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL2); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL3); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL4); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL5); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL6); - rc_bit_case(probs[symbol], 0, 0, SEQ_LITERAL7); + rc_bit_case(probs[symbol], , , SEQ_LITERAL0); + rc_bit_case(probs[symbol], , , SEQ_LITERAL1); + rc_bit_case(probs[symbol], , , SEQ_LITERAL2); + rc_bit_case(probs[symbol], , , SEQ_LITERAL3); + rc_bit_case(probs[symbol], , , SEQ_LITERAL4); + rc_bit_case(probs[symbol], , , SEQ_LITERAL5); + rc_bit_case(probs[symbol], , , SEQ_LITERAL6); + rc_bit_case(probs[symbol], , , SEQ_LITERAL7); #endif } else { -#ifndef HAVE_SMALL - uint32_t match_bit; - uint32_t subcoder_index; -#endif - // Decode literal with match byte. // // We store the byte we compare against @@ -441,6 +430,8 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, } while (symbol < (1 << 8)); #else // Unroll the loop. + uint32_t match_bit; + uint32_t subcoder_index; # define d(seq) \ case seq: \ @@ -474,6 +465,20 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // Use a lookup table to update to literal state, // since compared to other state updates, this would // need two branches. + static const lzma_lzma_state next_state[] = { + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_MATCH_LIT_LIT, + STATE_REP_LIT_LIT, + STATE_SHORTREP_LIT_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT, + STATE_SHORTREP_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT + }; state = next_state[state]; case SEQ_LITERAL_WRITE: @@ -509,28 +514,28 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // Prepare to decode the highest two bits of the // match distance. - probs = coder->pos_slot[get_len_to_pos_state(len)]; + probs = coder->dist_slot[get_dist_state(len)]; symbol = 1; #ifdef HAVE_SMALL - case SEQ_POS_SLOT: + case SEQ_DIST_SLOT: do { - rc_bit(probs[symbol], , , SEQ_POS_SLOT); - } while (symbol < POS_SLOTS); + rc_bit(probs[symbol], , , SEQ_DIST_SLOT); + } while (symbol < DIST_SLOTS); #else - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT0); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT1); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT2); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT3); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT4); - rc_bit_case(probs[symbol], 0, 0, SEQ_POS_SLOT5); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT0); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT1); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT2); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT3); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT4); + rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT5); #endif // Get rid of the highest bit that was needed for // indexing of the probability array. - symbol -= POS_SLOTS; + symbol -= DIST_SLOTS; assert(symbol <= 63); - if (symbol < START_POS_MODEL_INDEX) { + if (symbol < DIST_MODEL_START) { // Match distances [0, 3] have only two bits. rep0 = symbol; } else { @@ -540,7 +545,7 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, assert(limit >= 1 && limit <= 30); rep0 = 2 + (symbol & 1); - if (symbol < END_POS_MODEL_INDEX) { + if (symbol < DIST_MODEL_END) { // Prepare to decode the low bits for // a distance of [4, 127]. assert(limit <= 5); @@ -560,38 +565,38 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, - symbol - 1; symbol = 1; offset = 0; - case SEQ_POS_MODEL: + case SEQ_DIST_MODEL: #ifdef HAVE_SMALL do { rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); } while (++offset < limit); #else switch (limit) { case 5: assert(offset == 0); - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 4: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 3: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 2: - rc_bit(probs[symbol], 0, + rc_bit(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); ++offset; --limit; case 1: @@ -601,9 +606,9 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // rc_bit_last() here to omit // the unneeded updating of // "symbol". - rc_bit_last(probs[symbol], 0, + rc_bit_last(probs[symbol], , rep0 += 1 << offset, - SEQ_POS_MODEL); + SEQ_DIST_MODEL); } #endif } else { @@ -635,19 +640,19 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, } while (++offset < ALIGN_BITS); #else case SEQ_ALIGN0: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 1, SEQ_ALIGN0); case SEQ_ALIGN1: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 2, SEQ_ALIGN1); case SEQ_ALIGN2: - rc_bit(coder->pos_align[symbol], 0, + rc_bit(coder->pos_align[symbol], , rep0 += 4, SEQ_ALIGN2); case SEQ_ALIGN3: - // Like in SEQ_POS_MODEL, we don't + // Like in SEQ_DIST_MODEL, we don't // need "symbol" for anything else // than indexing the probability array. - rc_bit_last(coder->pos_align[symbol], 0, + rc_bit_last(coder->pos_align[symbol], , rep0 += 8, SEQ_ALIGN3); #endif @@ -732,11 +737,9 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, // is stored to rep0 and rep1, rep2 and rep3 // are updated accordingly. rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { - uint32_t distance; - rc_update_0(coder->is_rep1[state]); - distance = rep1; + const uint32_t distance = rep1; rep1 = rep0; rep0 = distance; @@ -745,23 +748,19 @@ lzma_decode(lzma_coder *LZMA_RESTRICT coder, lzma_dict *LZMA_RESTRICT dictptr, case SEQ_IS_REP2: rc_if_0(coder->is_rep2[state], SEQ_IS_REP2) { - uint32_t distance; - rc_update_0(coder->is_rep2[ state]); - distance = rep2; + const uint32_t distance = rep2; rep2 = rep1; rep1 = rep0; rep0 = distance; } else { - uint32_t distance; - rc_update_1(coder->is_rep2[ state]); - distance = rep3; + const uint32_t distance = rep3; rep3 = rep2; rep2 = rep1; rep1 = rep0; @@ -849,26 +848,17 @@ out: static void -lzma_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) +lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) { + lzma_lzma1_decoder *coder = coder_ptr; coder->uncompressed_size = uncompressed_size; } -/* -extern void -lzma_lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) -{ - // This is hack. - (*(lzma_coder **)(coder))->uncompressed_size = uncompressed_size; -} -*/ static void -lzma_decoder_reset(lzma_coder *coder, const void *opt) +lzma_decoder_reset(void *coder_ptr, const void *opt) { - uint32_t i, j, pos_state; - uint32_t num_pos_states; - + lzma_lzma1_decoder *coder = coder_ptr; const lzma_options_lzma *options = opt; // NOTE: We assume that lc/lp/pb are valid since they were @@ -895,8 +885,8 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) rc_reset(coder->rc); // Bit and bittree decoders - for (i = 0; i < STATES; ++i) { - for (j = 0; j <= coder->pos_mask; ++j) { + for (uint32_t i = 0; i < STATES; ++i) { + for (uint32_t j = 0; j <= coder->pos_mask; ++j) { bit_reset(coder->is_match[i][j]); bit_reset(coder->is_rep0_long[i][j]); } @@ -907,22 +897,22 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) bit_reset(coder->is_rep2[i]); } - for (i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + for (uint32_t i = 0; i < DIST_STATES; ++i) + bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); - for (i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + for (uint32_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) bit_reset(coder->pos_special[i]); bittree_reset(coder->pos_align, ALIGN_BITS); // Len decoders (also bit/bittree) - num_pos_states = 1U << options->pb; + const uint32_t num_pos_states = 1U << options->pb; bit_reset(coder->match_len_decoder.choice); bit_reset(coder->match_len_decoder.choice2); bit_reset(coder->rep_len_decoder.choice); bit_reset(coder->rep_len_decoder.choice2); - for (pos_state = 0; pos_state < num_pos_states; ++pos_state) { + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { bittree_reset(coder->match_len_decoder.low[pos_state], LEN_LOW_BITS); bittree_reset(coder->match_len_decoder.mid[pos_state], @@ -949,13 +939,11 @@ lzma_decoder_reset(lzma_coder *coder, const void *opt) extern lzma_ret -lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma_lzma_decoder_create(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options) { - const lzma_options_lzma *options = opt; - if (lz->coder == NULL) { - lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + lz->coder = lzma_alloc(sizeof(lzma_lzma1_decoder), allocator); if (lz->coder == NULL) return LZMA_MEM_ERROR; @@ -966,6 +954,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, // All dictionary sizes are OK here. LZ decoder will take care of // the special cases. + const lzma_options_lzma *options = opt; lz_options->dict_size = options->dict_size; lz_options->preset_dict = options->preset_dict; lz_options->preset_dict_size = options->preset_dict_size; @@ -978,7 +967,7 @@ lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, /// initialization (lzma_lzma_decoder_init() passes function pointer to /// the LZ initialization). static lzma_ret -lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, +lzma_decoder_init(lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { if (!is_lclppb_valid(options)) @@ -995,7 +984,7 @@ lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { // LZMA can only be the last filter in the chain. This is enforced @@ -1027,7 +1016,8 @@ extern uint64_t lzma_lzma_decoder_memusage_nocheck(const void *options) { const lzma_options_lzma *const opt = options; - return sizeof(lzma_coder) + lzma_lz_decoder_memusage(opt->dict_size); + return sizeof(lzma_lzma1_decoder) + + lzma_lz_decoder_memusage(opt->dict_size); } @@ -1042,15 +1032,14 @@ lzma_lzma_decoder_memusage(const void *options) extern lzma_ret -lzma_lzma_props_decode(void **options, lzma_allocator *allocator, +lzma_lzma_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_lzma *opt; - if (props_size != 5) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_lzma), allocator); + lzma_options_lzma *opt + = lzma_alloc(sizeof(lzma_options_lzma), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h index a463a76fc..fa8ecb23e 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_decoder.h @@ -19,12 +19,13 @@ /// Allocates and initializes LZMA decoder extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma_decoder_memusage(const void *options); extern lzma_ret lzma_lzma_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); @@ -40,7 +41,7 @@ extern bool lzma_lzma_lclppb_decode( /// Allocate and setup function pointers only. This is used by LZMA1 and /// LZMA2 decoders. extern lzma_ret lzma_lzma_decoder_create( - lzma_lz_decoder *lz, lzma_allocator *allocator, + lzma_lz_decoder *lz, const lzma_allocator *allocator, const void *opt, lzma_lz_options *lz_options); /// Gets memory usage without validating lc/lp/pb. This is used by LZMA2 diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c index e8738f411..ba9ce6989 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.c @@ -28,14 +28,11 @@ literal_matched(lzma_range_encoder *rc, probability *subcoder, symbol += UINT32_C(1) << 8; do { - uint32_t match_bit; - uint32_t subcoder_index; - uint32_t bit; - match_byte <<= 1; - match_bit = match_byte & offset; - subcoder_index = offset + match_bit + (symbol >> 8); - bit = (symbol >> 7) & 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; rc_bit(rc, &subcoder[subcoder_index], bit); symbol <<= 1; @@ -46,7 +43,7 @@ literal_matched(lzma_range_encoder *rc, probability *subcoder, static inline void -literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) +literal(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t position) { // Locate the literal byte to be encoded and the subcoder. const uint8_t cur_byte = mf->buffer[ @@ -80,19 +77,16 @@ literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) static void length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state) { - uint32_t a0, a1, b0, b1; - uint32_t *prices; - uint32_t i; - const uint32_t table_size = lc->table_size; lc->counters[pos_state] = table_size; - a0 = rc_bit_0_price(lc->choice); - a1 = rc_bit_1_price(lc->choice); - b0 = a1 + rc_bit_0_price(lc->choice2); - b1 = a1 + rc_bit_1_price(lc->choice2); - prices = lc->prices[pos_state]; + const uint32_t a0 = rc_bit_0_price(lc->choice); + const uint32_t a1 = rc_bit_1_price(lc->choice); + const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2); + const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2); + uint32_t *const prices = lc->prices[pos_state]; + uint32_t i; for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i) prices[i] = a0 + rc_bittree_price(lc->low[pos_state], LEN_LOW_BITS, i); @@ -146,39 +140,36 @@ length(lzma_range_encoder *rc, lzma_length_encoder *lc, /////////// static inline void -match(lzma_coder *coder, const uint32_t pos_state, +match(lzma_lzma1_encoder *coder, const uint32_t pos_state, const uint32_t distance, const uint32_t len) { - uint32_t pos_slot; - uint32_t len_to_pos_state; - update_match(coder->state); length(&coder->rc, &coder->match_len_encoder, pos_state, len, coder->fast_mode); - pos_slot = get_pos_slot(distance); - len_to_pos_state = get_len_to_pos_state(len); - rc_bittree(&coder->rc, coder->pos_slot[len_to_pos_state], - POS_SLOT_BITS, pos_slot); + const uint32_t dist_slot = get_dist_slot(distance); + const uint32_t dist_state = get_dist_state(len); + rc_bittree(&coder->rc, coder->dist_slot[dist_state], + DIST_SLOT_BITS, dist_slot); - if (pos_slot >= START_POS_MODEL_INDEX) { - const uint32_t footer_bits = (pos_slot >> 1) - 1; - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; - const uint32_t pos_reduced = distance - base; + if (dist_slot >= DIST_MODEL_START) { + const uint32_t footer_bits = (dist_slot >> 1) - 1; + const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; + const uint32_t dist_reduced = distance - base; - if (pos_slot < END_POS_MODEL_INDEX) { - // Careful here: base - pos_slot - 1 can be -1, but + if (dist_slot < DIST_MODEL_END) { + // Careful here: base - dist_slot - 1 can be -1, but // rc_bittree_reverse starts at probs[1], not probs[0]. rc_bittree_reverse(&coder->rc, - coder->pos_special + base - pos_slot - 1, - footer_bits, pos_reduced); + coder->dist_special + base - dist_slot - 1, + footer_bits, dist_reduced); } else { - rc_direct(&coder->rc, pos_reduced >> ALIGN_BITS, + rc_direct(&coder->rc, dist_reduced >> ALIGN_BITS, footer_bits - ALIGN_BITS); rc_bittree_reverse( - &coder->rc, coder->pos_align, - ALIGN_BITS, pos_reduced & ALIGN_MASK); + &coder->rc, coder->dist_align, + ALIGN_BITS, dist_reduced & ALIGN_MASK); ++coder->align_price_count; } } @@ -196,7 +187,7 @@ match(lzma_coder *coder, const uint32_t pos_state, //////////////////// static inline void -rep_match(lzma_coder *coder, const uint32_t pos_state, +rep_match(lzma_lzma1_encoder *coder, const uint32_t pos_state, const uint32_t rep, const uint32_t len) { if (rep == 0) { @@ -240,7 +231,7 @@ rep_match(lzma_coder *coder, const uint32_t pos_state, ////////// static void -encode_symbol(lzma_coder *coder, lzma_mf *mf, +encode_symbol(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t back, uint32_t len, uint32_t position) { const uint32_t pos_state = position & coder->pos_mask; @@ -256,7 +247,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); - if (back < REP_DISTANCES) { + if (back < REPS) { // It's a repeated match i.e. the same distance // has been used earlier. rc_bit(&coder->rc, &coder->is_rep[coder->state], 1); @@ -264,7 +255,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, } else { // Normal match rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); - match(coder, pos_state, back - REP_DISTANCES, len); + match(coder, pos_state, back - REPS, len); } } @@ -274,7 +265,7 @@ encode_symbol(lzma_coder *coder, lzma_mf *mf, static bool -encode_init(lzma_coder *coder, lzma_mf *mf) +encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf) { assert(mf_position(mf) == 0); @@ -302,7 +293,7 @@ encode_init(lzma_coder *coder, lzma_mf *mf) static void -encode_eopm(lzma_coder *coder, uint32_t position) +encode_eopm(lzma_lzma1_encoder *coder, uint32_t position) { const uint32_t pos_state = position & coder->pos_mask; rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); @@ -318,23 +309,18 @@ encode_eopm(lzma_coder *coder, uint32_t position) extern lzma_ret -lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, uint32_t limit) { - uint32_t position; - // Initialize the stream if no data has been encoded yet. if (!coder->is_initialized && !encode_init(coder, mf)) return LZMA_OK; // Get the lowest bits of the uncompressed offset from the LZ layer. - position = mf_position(mf); + uint32_t position = mf_position(mf); while (true) { - uint32_t len; - uint32_t back; - // Encode pending bits, if any. Calling this before encoding // the next symbol is needed only with plain LZMA, since // LZMA2 always provides big enough buffer to flush @@ -367,12 +353,14 @@ lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, // Get optimal match (repeat position and length). // Value ranges for pos: - // - [0, REP_DISTANCES): repeated match - // - [REP_DISTANCES, UINT32_MAX): - // match at (pos - REP_DISTANCES) + // - [0, REPS): repeated match + // - [REPS, UINT32_MAX): + // match at (pos - REPS) // - UINT32_MAX: not a match but a literal // Value ranges for len: // - [MATCH_LEN_MIN, MATCH_LEN_MAX] + uint32_t len; + uint32_t back; if (coder->fast_mode) lzma_lzma_optimum_fast(coder, mf, &back, &len); @@ -414,8 +402,8 @@ lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static lzma_ret -lzma_encode(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint8_t *LZMA_RESTRICT out, size_t *LZMA_RESTRICT out_pos, +lzma_encode(void *coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, size_t out_size) { // Plain LZMA has no support for sync-flushing. @@ -465,12 +453,10 @@ static void length_encoder_reset(lzma_length_encoder *lencoder, const uint32_t num_pos_states, const bool fast_mode) { - size_t pos_state; - bit_reset(lencoder->choice); bit_reset(lencoder->choice2); - for (pos_state = 0; pos_state < num_pos_states; ++pos_state) { + for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); } @@ -478,7 +464,7 @@ length_encoder_reset(lzma_length_encoder *lencoder, bittree_reset(lencoder->high, LEN_HIGH_BITS); if (!fast_mode) - for (pos_state = 0; pos_state < num_pos_states; + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) length_update_prices(lencoder, pos_state); @@ -487,10 +473,9 @@ length_encoder_reset(lzma_length_encoder *lencoder, extern lzma_ret -lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) +lzma_lzma_encoder_reset(lzma_lzma1_encoder *coder, + const lzma_options_lzma *options) { - size_t i, j; - if (!is_options_valid(options)) return LZMA_OPTIONS_ERROR; @@ -503,14 +488,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) // State coder->state = STATE_LIT_LIT; - for (i = 0; i < REP_DISTANCES; ++i) + for (size_t i = 0; i < REPS; ++i) coder->reps[i] = 0; literal_init(coder->literal, options->lc, options->lp); // Bit encoders - for (i = 0; i < STATES; ++i) { - for (j = 0; j <= coder->pos_mask; ++j) { + for (size_t i = 0; i < STATES; ++i) { + for (size_t j = 0; j <= coder->pos_mask; ++j) { bit_reset(coder->is_match[i][j]); bit_reset(coder->is_rep0_long[i][j]); } @@ -521,14 +506,14 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) bit_reset(coder->is_rep2[i]); } - for (i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) - bit_reset(coder->pos_special[i]); + for (size_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) + bit_reset(coder->dist_special[i]); // Bit tree encoders - for (i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + for (size_t i = 0; i < DIST_STATES; ++i) + bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); - bittree_reset(coder->pos_align, ALIGN_BITS); + bittree_reset(coder->dist_align, ALIGN_BITS); // Length encoders length_encoder_reset(&coder->match_len_encoder, @@ -561,20 +546,18 @@ lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) extern lzma_ret -lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, +lzma_lzma_encoder_create(void **coder_ptr, + const lzma_allocator *allocator, const lzma_options_lzma *options, lzma_lz_options *lz_options) { - lzma_coder *coder; - uint32_t log_size = 0; - - // Allocate lzma_coder if it wasn't already allocated. + // Allocate lzma_lzma1_encoder if it wasn't already allocated. if (*coder_ptr == NULL) { - *coder_ptr = lzma_alloc(sizeof(lzma_coder), allocator); + *coder_ptr = lzma_alloc(sizeof(lzma_lzma1_encoder), allocator); if (*coder_ptr == NULL) return LZMA_MEM_ERROR; } - coder = *coder_ptr; + lzma_lzma1_encoder *coder = *coder_ptr; // Set compression mode. We haven't validates the options yet, // but it's OK here, since nothing bad happens with invalid @@ -590,6 +573,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, // Set dist_table_size. // Round the dictionary size up to next 2^n. + uint32_t log_size = 0; while ((UINT32_C(1) << log_size) < options->dict_size) ++log_size; @@ -622,7 +606,7 @@ lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, static lzma_ret -lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, +lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, const void *options, lzma_lz_options *lz_options) { lz->code = &lzma_encode; @@ -632,7 +616,7 @@ lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, extern lzma_ret -lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_lzma_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters) { return lzma_lz_encoder_init( @@ -643,19 +627,17 @@ lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern uint64_t lzma_lzma_encoder_memusage(const void *options) { - lzma_lz_options lz_options; - uint64_t lz_memusage; - if (!is_options_valid(options)) return UINT64_MAX; + lzma_lz_options lz_options; set_lz_options(&lz_options, options); - lz_memusage = lzma_lz_encoder_memusage(&lz_options); + const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options); if (lz_memusage == UINT64_MAX) return UINT64_MAX; - return (uint64_t)(sizeof(lzma_coder)) + lz_memusage; + return (uint64_t)(sizeof(lzma_lzma1_encoder)) + lz_memusage; } diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h index abb8d8b46..6cfdf228b 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder.h @@ -17,8 +17,12 @@ #include "common.h" +typedef struct lzma_lzma1_encoder_s lzma_lzma1_encoder; + + extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern uint64_t lzma_lzma_encoder_memusage(const void *options); @@ -35,18 +39,18 @@ extern bool lzma_lzma_lclppb_encode( /// Initializes raw LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_create( - lzma_coder **coder_ptr, lzma_allocator *allocator, + void **coder_ptr, const lzma_allocator *allocator, const lzma_options_lzma *options, lzma_lz_options *lz_options); /// Resets an already initialized LZMA encoder; this is used by LZMA2. extern lzma_ret lzma_lzma_encoder_reset( - lzma_coder *coder, const lzma_options_lzma *options); + lzma_lzma1_encoder *coder, const lzma_options_lzma *options); -extern lzma_ret lzma_lzma_encode(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, +extern lzma_ret lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, uint32_t read_limit); #endif diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c index f98312643..6c53d2bd0 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_fast.c @@ -10,6 +10,7 @@ /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" +#include "memcmplen.h" #define change_pair(small_dist, big_dist) \ @@ -17,17 +18,10 @@ extern void -lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res) +lzma_lzma_optimum_fast(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res) { - const uint8_t *buf; - uint32_t buf_avail; - uint32_t i; - uint32_t rep_len = 0; - uint32_t rep_index = 0; - uint32_t back_main = 0; - uint32_t limit; - const uint32_t nice_len = mf->nice_len; uint32_t len_main; @@ -40,8 +34,8 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m matches_count = coder->matches_count; } - buf = mf_ptr(mf) - 1; - buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); + const uint8_t *buf = mf_ptr(mf) - 1; + const uint32_t buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); if (buf_avail < 2) { // There's not enough input left to encode a match. @@ -51,9 +45,10 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m } // Look for repeated matches; scan the previous four match distances - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len; + uint32_t rep_len = 0; + uint32_t rep_index = 0; + for (uint32_t i = 0; i < REPS; ++i) { // Pointer to the beginning of the match candidate const uint8_t *const buf_back = buf - coder->reps[i] - 1; @@ -64,8 +59,8 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // The first two bytes matched. // Calculate the length of the match. - for (len = 2; len < buf_avail - && buf[len] == buf_back[len]; ++len) ; + const uint32_t len = lzma_memcmplen( + buf, buf_back, 2, buf_avail); // If we have found a repeated match that is at least // nice_len long, return it immediately. @@ -85,13 +80,13 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // We didn't find a long enough repeated match. Encode it as a normal // match if the match length is at least nice_len. if (len_main >= nice_len) { - *back_res = coder->matches[matches_count - 1].dist - + REP_DISTANCES; + *back_res = coder->matches[matches_count - 1].dist + REPS; *len_res = len_main; mf_skip(mf, len_main - 1); return; } + uint32_t back_main = 0; if (len_main >= 2) { back_main = coder->matches[matches_count - 1].dist; @@ -158,27 +153,17 @@ lzma_lzma_optimum_fast(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT m // the old buf pointer instead of recalculating it with mf_ptr(). ++buf; - limit = len_main - 1; - - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len; - - const uint8_t *const buf_back = buf - coder->reps[i] - 1; - - if (not_equal_16(buf, buf_back)) - continue; - - for (len = 2; len < limit - && buf[len] == buf_back[len]; ++len) ; + const uint32_t limit = my_max(2, len_main - 1); - if (len >= limit) { + for (uint32_t i = 0; i < REPS; ++i) { + if (memcmp(buf, buf - coder->reps[i] - 1, limit) == 0) { *back_res = UINT32_MAX; *len_res = 1; return; } } - *back_res = back_main + REP_DISTANCES; + *back_res = back_main + REPS; *len_res = len_main; mf_skip(mf, len_main - 2); return; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c index d3a63485c..59f77343e 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_optimum_normal.c @@ -11,6 +11,7 @@ #include "lzma_encoder_private.h" #include "fastpos.h" +#include "memcmplen.h" //////////// @@ -18,7 +19,7 @@ //////////// static uint32_t -get_literal_price(const lzma_coder *const coder, const uint32_t pos, +get_literal_price(const lzma_lzma1_encoder *const coder, const uint32_t pos, const uint32_t prev_byte, const bool match_mode, uint32_t match_byte, uint32_t symbol) { @@ -35,15 +36,12 @@ get_literal_price(const lzma_coder *const coder, const uint32_t pos, symbol += UINT32_C(1) << 8; do { - uint32_t match_bit; - uint32_t subcoder_index; - uint32_t bit; - match_byte <<= 1; - match_bit = match_byte & offset; - subcoder_index = offset + match_bit + (symbol >> 8); - bit = (symbol >> 7) & 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; price += rc_bit_price(subcoder[subcoder_index], bit); symbol <<= 1; @@ -67,7 +65,7 @@ get_len_price(const lzma_length_encoder *const lencoder, static inline uint32_t -get_short_rep_price(const lzma_coder *const coder, +get_short_rep_price(const lzma_lzma1_encoder *const coder, const lzma_lzma_state state, const uint32_t pos_state) { return rc_bit_0_price(coder->is_rep0[state]) @@ -76,7 +74,7 @@ get_short_rep_price(const lzma_coder *const coder, static inline uint32_t -get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, +get_pure_rep_price(const lzma_lzma1_encoder *const coder, const uint32_t rep_index, const lzma_lzma_state state, uint32_t pos_state) { uint32_t price; @@ -101,7 +99,7 @@ get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, static inline uint32_t -get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, +get_rep_price(const lzma_lzma1_encoder *const coder, const uint32_t rep_index, const uint32_t len, const lzma_lzma_state state, const uint32_t pos_state) { @@ -111,18 +109,18 @@ get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, static inline uint32_t -get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, +get_dist_len_price(const lzma_lzma1_encoder *const coder, const uint32_t dist, const uint32_t len, const uint32_t pos_state) { - const uint32_t len_to_pos_state = get_len_to_pos_state(len); + const uint32_t dist_state = get_dist_state(len); uint32_t price; - if (pos < FULL_DISTANCES) { - price = coder->distances_prices[len_to_pos_state][pos]; + if (dist < FULL_DISTANCES) { + price = coder->dist_prices[dist_state][dist]; } else { - const uint32_t pos_slot = get_pos_slot_2(pos); - price = coder->pos_slot_prices[len_to_pos_state][pos_slot] - + coder->align_prices[pos & ALIGN_MASK]; + const uint32_t dist_slot = get_dist_slot_2(dist); + price = coder->dist_slot_prices[dist_state][dist_slot] + + coder->align_prices[dist & ALIGN_MASK]; } price += get_len_price(&coder->match_len_encoder, len, pos_state); @@ -132,59 +130,53 @@ get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, static void -fill_distances_prices(lzma_coder *coder) +fill_dist_prices(lzma_lzma1_encoder *coder) { - uint32_t len_to_pos_state; - uint32_t pos_slot; - uint32_t i; - - for (len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) { + for (uint32_t dist_state = 0; dist_state < DIST_STATES; ++dist_state) { - uint32_t *const pos_slot_prices - = coder->pos_slot_prices[len_to_pos_state]; + uint32_t *const dist_slot_prices + = coder->dist_slot_prices[dist_state]; - // Price to encode the pos_slot. - for (pos_slot = 0; - pos_slot < coder->dist_table_size; ++pos_slot) - pos_slot_prices[pos_slot] = rc_bittree_price( - coder->pos_slot[len_to_pos_state], - POS_SLOT_BITS, pos_slot); + // Price to encode the dist_slot. + for (uint32_t dist_slot = 0; + dist_slot < coder->dist_table_size; ++dist_slot) + dist_slot_prices[dist_slot] = rc_bittree_price( + coder->dist_slot[dist_state], + DIST_SLOT_BITS, dist_slot); // For matches with distance >= FULL_DISTANCES, add the price // of the direct bits part of the match distance. (Align bits // are handled by fill_align_prices()). - for (pos_slot = END_POS_MODEL_INDEX; - pos_slot < coder->dist_table_size; ++pos_slot) - pos_slot_prices[pos_slot] += rc_direct_price( - ((pos_slot >> 1) - 1) - ALIGN_BITS); + for (uint32_t dist_slot = DIST_MODEL_END; + dist_slot < coder->dist_table_size; + ++dist_slot) + dist_slot_prices[dist_slot] += rc_direct_price( + ((dist_slot >> 1) - 1) - ALIGN_BITS); // Distances in the range [0, 3] are fully encoded with - // pos_slot, so they are used for coder->distances_prices + // dist_slot, so they are used for coder->dist_prices // as is. - for (i = 0; i < START_POS_MODEL_INDEX; ++i) - coder->distances_prices[len_to_pos_state][i] - = pos_slot_prices[i]; + for (uint32_t i = 0; i < DIST_MODEL_START; ++i) + coder->dist_prices[dist_state][i] + = dist_slot_prices[i]; } - // Distances in the range [4, 127] depend on pos_slot and pos_special. - // We do this in a loop separate from the above loop to avoid - // redundant calls to get_pos_slot(). - for (i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { - const uint32_t pos_slot = get_pos_slot(i); - const uint32_t footer_bits = ((pos_slot >> 1) - 1); - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; + // Distances in the range [4, 127] depend on dist_slot and + // dist_special. We do this in a loop separate from the above + // loop to avoid redundant calls to get_dist_slot(). + for (uint32_t i = DIST_MODEL_START; i < FULL_DISTANCES; ++i) { + const uint32_t dist_slot = get_dist_slot(i); + const uint32_t footer_bits = ((dist_slot >> 1) - 1); + const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; const uint32_t price = rc_bittree_reverse_price( - coder->pos_special + base - pos_slot - 1, + coder->dist_special + base - dist_slot - 1, footer_bits, i - base); - for (len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) - coder->distances_prices[len_to_pos_state][i] - = price + coder->pos_slot_prices[ - len_to_pos_state][pos_slot]; + for (uint32_t dist_state = 0; dist_state < DIST_STATES; + ++dist_state) + coder->dist_prices[dist_state][i] + = price + coder->dist_slot_prices[ + dist_state][dist_slot]; } coder->match_price_count = 0; @@ -193,12 +185,11 @@ fill_distances_prices(lzma_coder *coder) static void -fill_align_prices(lzma_coder *coder) +fill_align_prices(lzma_lzma1_encoder *coder) { - uint32_t i; - for (i = 0; i < ALIGN_TABLE_SIZE; ++i) + for (uint32_t i = 0; i < ALIGN_SIZE; ++i) coder->align_prices[i] = rc_bittree_reverse_price( - coder->pos_align, ALIGN_BITS, i); + coder->dist_align, ALIGN_BITS, i); coder->align_price_count = 0; return; @@ -230,18 +221,15 @@ make_short_rep(lzma_optimal *optimal) static void -backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, - uint32_t *LZMA_RESTRICT back_res, uint32_t cur) +backward(lzma_lzma1_encoder *restrict coder, uint32_t *restrict len_res, + uint32_t *restrict back_res, uint32_t cur) { + coder->opts_end_index = cur; + uint32_t pos_mem = coder->opts[cur].pos_prev; uint32_t back_mem = coder->opts[cur].back_prev; - coder->opts_end_index = cur; - do { - const uint32_t pos_prev = pos_mem; - const uint32_t back_cur = back_mem; - if (coder->opts[cur].prev_1_is_literal) { make_literal(&coder->opts[pos_mem]); coder->opts[pos_mem].pos_prev = pos_mem - 1; @@ -256,6 +244,9 @@ backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, } } + const uint32_t pos_prev = pos_mem; + const uint32_t back_cur = back_mem; + back_mem = coder->opts[pos_prev].back_prev; pos_mem = coder->opts[pos_prev].pos_prev; @@ -278,27 +269,10 @@ backward(lzma_coder *LZMA_RESTRICT coder, uint32_t *LZMA_RESTRICT len_res, ////////// static inline uint32_t -helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res, +helper1(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, uint32_t position) { - uint32_t buf_avail; - const uint8_t *buf; - uint32_t rep_lens[REP_DISTANCES]; - uint32_t rep_max_index = 0; - uint32_t i; - - uint8_t current_byte; - uint8_t match_byte; - - uint32_t pos_state; - uint32_t match_price; - uint32_t rep_match_price; - uint32_t len_end; - uint32_t len; - - uint32_t normal_match_price; - const uint32_t nice_len = mf->nice_len; uint32_t len_main; @@ -312,18 +286,19 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, matches_count = coder->matches_count; } - buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); + const uint32_t buf_avail = my_min(mf_avail(mf) + 1, MATCH_LEN_MAX); if (buf_avail < 2) { *back_res = UINT32_MAX; *len_res = 1; return UINT32_MAX; } - buf = mf_ptr(mf) - 1; + const uint8_t *const buf = mf_ptr(mf) - 1; - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t len_test; + uint32_t rep_lens[REPS]; + uint32_t rep_max_index = 0; + for (uint32_t i = 0; i < REPS; ++i) { const uint8_t *const buf_back = buf - coder->reps[i] - 1; if (not_equal_16(buf, buf_back)) { @@ -331,12 +306,9 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, continue; } - for (len_test = 2; len_test < buf_avail - && buf[len_test] == buf_back[len_test]; - ++len_test) ; + rep_lens[i] = lzma_memcmplen(buf, buf_back, 2, buf_avail); - rep_lens[i] = len_test; - if (len_test > rep_lens[rep_max_index]) + if (rep_lens[i] > rep_lens[rep_max_index]) rep_max_index = i; } @@ -349,15 +321,14 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, if (len_main >= nice_len) { - *back_res = coder->matches[matches_count - 1].dist - + REP_DISTANCES; + *back_res = coder->matches[matches_count - 1].dist + REPS; *len_res = len_main; mf_skip(mf, len_main - 1); return UINT32_MAX; } - current_byte = *buf; - match_byte = *(buf - coder->reps[0] - 1); + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - coder->reps[0] - 1); if (len_main < 2 && current_byte != match_byte && rep_lens[rep_max_index] < 2) { @@ -368,7 +339,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, coder->opts[0].state = coder->state; - pos_state = position & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; coder->opts[1].price = rc_bit_0_price( coder->is_match[coder->state][pos_state]) @@ -378,9 +349,9 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, make_literal(&coder->opts[1]); - match_price = rc_bit_1_price( + const uint32_t match_price = rc_bit_1_price( coder->is_match[coder->state][pos_state]); - rep_match_price = match_price + const uint32_t rep_match_price = match_price + rc_bit_1_price(coder->is_rep[coder->state]); if (match_byte == current_byte) { @@ -394,7 +365,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, } } - len_end = my_max(len_main, rep_lens[rep_max_index]); + const uint32_t len_end = my_max(len_main, rep_lens[rep_max_index]); if (len_end < 2) { *back_res = coder->opts[1].back_prev; @@ -404,23 +375,21 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, coder->opts[1].pos_prev = 0; - for (i = 0; i < REP_DISTANCES; ++i) + for (uint32_t i = 0; i < REPS; ++i) coder->opts[0].backs[i] = coder->reps[i]; - len = len_end; + uint32_t len = len_end; do { coder->opts[len].price = RC_INFINITY_PRICE; } while (--len >= 2); - for (i = 0; i < REP_DISTANCES; ++i) { - uint32_t price; - + for (uint32_t i = 0; i < REPS; ++i) { uint32_t rep_len = rep_lens[i]; if (rep_len < 2) continue; - price = rep_match_price + get_pure_rep_price( + const uint32_t price = rep_match_price + get_pure_rep_price( coder, i, coder->state, pos_state); do { @@ -439,7 +408,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, } - normal_match_price = match_price + const uint32_t normal_match_price = match_price + rc_bit_0_price(coder->is_rep[coder->state]); len = rep_lens[0] >= 2 ? rep_lens[0] + 1 : 2; @@ -451,14 +420,13 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, for(; ; ++len) { const uint32_t dist = coder->matches[i].dist; const uint32_t cur_and_len_price = normal_match_price - + get_pos_len_price(coder, + + get_dist_len_price(coder, dist, len, pos_state); if (cur_and_len_price < coder->opts[len].price) { coder->opts[len].price = cur_and_len_price; coder->opts[len].pos_prev = 0; - coder->opts[len].back_prev - = dist + REP_DISTANCES; + coder->opts[len].back_prev = dist + REPS; coder->opts[len].prev_1_is_literal = false; } @@ -473,7 +441,7 @@ helper1(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, static inline uint32_t -helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, +helper2(lzma_lzma1_encoder *coder, uint32_t *reps, const uint8_t *buf, uint32_t len_end, uint32_t position, const uint32_t cur, const uint32_t nice_len, const uint32_t buf_avail_full) { @@ -481,19 +449,6 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, uint32_t new_len = coder->longest_match_length; uint32_t pos_prev = coder->opts[cur].pos_prev; lzma_lzma_state state; - uint32_t buf_avail; - uint32_t rep_index; - uint32_t i; - - uint32_t cur_price; - uint8_t current_byte; - uint8_t match_byte; - uint32_t pos_state; - uint32_t cur_and_1_price; - bool next_is_literal = false; - uint32_t match_price; - uint32_t rep_match_price; - uint32_t start_len = 2; if (coder->opts[cur].prev_1_is_literal) { --pos_prev; @@ -501,7 +456,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (coder->opts[cur].prev_2) { state = coder->opts[coder->opts[cur].pos_prev_2].state; - if (coder->opts[cur].back_prev_2 < REP_DISTANCES) + if (coder->opts[cur].back_prev_2 < REPS) update_long_rep(state); else update_match(state); @@ -530,48 +485,49 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, update_long_rep(state); } else { pos = coder->opts[cur].back_prev; - if (pos < REP_DISTANCES) + if (pos < REPS) update_long_rep(state); else update_match(state); } - if (pos < REP_DISTANCES) { - uint32_t i; - + if (pos < REPS) { reps[0] = coder->opts[pos_prev].backs[pos]; + uint32_t i; for (i = 1; i <= pos; ++i) reps[i] = coder->opts[pos_prev].backs[i - 1]; - for (; i < REP_DISTANCES; ++i) + for (; i < REPS; ++i) reps[i] = coder->opts[pos_prev].backs[i]; } else { - reps[0] = pos - REP_DISTANCES; + reps[0] = pos - REPS; - for (i = 1; i < REP_DISTANCES; ++i) + for (uint32_t i = 1; i < REPS; ++i) reps[i] = coder->opts[pos_prev].backs[i - 1]; } } coder->opts[cur].state = state; - for (i = 0; i < REP_DISTANCES; ++i) + for (uint32_t i = 0; i < REPS; ++i) coder->opts[cur].backs[i] = reps[i]; - cur_price = coder->opts[cur].price; + const uint32_t cur_price = coder->opts[cur].price; - current_byte = *buf; - match_byte = *(buf - reps[0] - 1); + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - reps[0] - 1); - pos_state = position & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; - cur_and_1_price = cur_price + const uint32_t cur_and_1_price = cur_price + rc_bit_0_price(coder->is_match[state][pos_state]) + get_literal_price(coder, position, buf[-1], !is_literal_state(state), match_byte, current_byte); + bool next_is_literal = false; + if (cur_and_1_price < coder->opts[cur + 1].price) { coder->opts[cur + 1].price = cur_and_1_price; coder->opts[cur + 1].pos_prev = cur; @@ -579,9 +535,9 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, next_is_literal = true; } - match_price = cur_price + const uint32_t match_price = cur_price + rc_bit_1_price(coder->is_match[state][pos_state]); - rep_match_price = match_price + const uint32_t rep_match_price = match_price + rc_bit_1_price(coder->is_rep[state]); if (match_byte == current_byte @@ -602,40 +558,31 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (buf_avail_full < 2) return len_end; - buf_avail = my_min(buf_avail_full, nice_len); + const uint32_t buf_avail = my_min(buf_avail_full, nice_len); if (!next_is_literal && match_byte != current_byte) { // speed optimization // try literal + rep0 const uint8_t *const buf_back = buf - reps[0] - 1; const uint32_t limit = my_min(buf_avail_full, nice_len + 1); - uint32_t len_test = 1; - while (len_test < limit && buf[len_test] == buf_back[len_test]) - ++len_test; - - --len_test; + const uint32_t len_test = lzma_memcmplen(buf, buf_back, 1, limit) - 1; if (len_test >= 2) { - uint32_t pos_state_next; - uint32_t next_rep_match_price; - uint32_t offset; - uint32_t cur_and_len_price; - lzma_lzma_state state_2 = state; update_literal(state_2); - pos_state_next = (position + 1) & coder->pos_mask; - next_rep_match_price = cur_and_1_price + const uint32_t pos_state_next = (position + 1) & coder->pos_mask; + const uint32_t next_rep_match_price = cur_and_1_price + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); //for (; len_test >= 2; --len_test) { - offset = cur + 1 + len_test; + const uint32_t offset = cur + 1 + len_test; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; - cur_and_len_price = next_rep_match_price + const uint32_t cur_and_len_price = next_rep_match_price + get_rep_price(coder, 0, len_test, state_2, pos_state_next); @@ -651,23 +598,20 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, } - for (rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { - uint32_t len_test, len_test_2, len_test_temp; - uint32_t price, limit; + uint32_t start_len = 2; // speed optimization + for (uint32_t rep_index = 0; rep_index < REPS; ++rep_index) { const uint8_t *const buf_back = buf - reps[rep_index] - 1; if (not_equal_16(buf, buf_back)) continue; - for (len_test = 2; len_test < buf_avail - && buf[len_test] == buf_back[len_test]; - ++len_test) ; + uint32_t len_test = lzma_memcmplen(buf, buf_back, 2, buf_avail); while (len_end < cur + len_test) coder->opts[++len_end].price = RC_INFINITY_PRICE; - len_test_temp = len_test; - price = rep_match_price + get_pure_rep_price( + const uint32_t len_test_temp = len_test; + const uint32_t price = rep_match_price + get_pure_rep_price( coder, rep_index, state, pos_state); do { @@ -689,8 +633,8 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, start_len = len_test + 1; - len_test_2 = len_test + 1; - limit = my_min(buf_avail_full, + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = my_min(buf_avail_full, len_test_2 + nice_len); for (; len_test_2 < limit && buf[len_test_2] == buf_back[len_test_2]; @@ -699,18 +643,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, len_test_2 -= len_test + 1; if (len_test_2 >= 2) { - uint32_t pos_state_next; - uint32_t cur_and_len_literal_price; - uint32_t next_rep_match_price; - uint32_t offset; - uint32_t cur_and_len_price; - lzma_lzma_state state_2 = state; update_long_rep(state_2); - pos_state_next = (position + len_test) & coder->pos_mask; + uint32_t pos_state_next = (position + len_test) & coder->pos_mask; - cur_and_len_literal_price = price + const uint32_t cur_and_len_literal_price = price + get_len_price(&coder->rep_len_encoder, len_test, pos_state) + rc_bit_0_price(coder->is_match[state_2][pos_state_next]) @@ -722,17 +660,17 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, pos_state_next = (position + len_test + 1) & coder->pos_mask; - next_rep_match_price = cur_and_len_literal_price + const uint32_t next_rep_match_price = cur_and_len_literal_price + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); //for(; len_test_2 >= 2; len_test_2--) { - offset = cur + len_test + 1 + len_test_2; + const uint32_t offset = cur + len_test + 1 + len_test_2; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; - cur_and_len_price = next_rep_match_price + const uint32_t cur_and_len_price = next_rep_match_price + get_rep_price(coder, 0, len_test_2, state_2, pos_state_next); @@ -763,29 +701,27 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, if (new_len >= start_len) { - uint32_t len_test; - uint32_t i = 0; - const uint32_t normal_match_price = match_price + rc_bit_0_price(coder->is_rep[state]); while (len_end < cur + new_len) coder->opts[++len_end].price = RC_INFINITY_PRICE; + uint32_t i = 0; while (start_len > coder->matches[i].len) ++i; - for (len_test = start_len; ; ++len_test) { + for (uint32_t len_test = start_len; ; ++len_test) { const uint32_t cur_back = coder->matches[i].dist; uint32_t cur_and_len_price = normal_match_price - + get_pos_len_price(coder, + + get_dist_len_price(coder, cur_back, len_test, pos_state); if (cur_and_len_price < coder->opts[cur + len_test].price) { coder->opts[cur + len_test].price = cur_and_len_price; coder->opts[cur + len_test].pos_prev = cur; coder->opts[cur + len_test].back_prev - = cur_back + REP_DISTANCES; + = cur_back + REPS; coder->opts[cur + len_test].prev_1_is_literal = false; } @@ -803,16 +739,12 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, len_test_2 -= len_test + 1; if (len_test_2 >= 2) { - uint32_t pos_state_next; - uint32_t cur_and_len_literal_price; - uint32_t next_rep_match_price; - uint32_t offset; - lzma_lzma_state state_2 = state; update_match(state_2); - pos_state_next = (position + len_test) & coder->pos_mask; + uint32_t pos_state_next + = (position + len_test) & coder->pos_mask; - cur_and_len_literal_price = cur_and_len_price + const uint32_t cur_and_len_literal_price = cur_and_len_price + rc_bit_0_price( coder->is_match[state_2][pos_state_next]) + get_literal_price(coder, @@ -825,14 +757,14 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, update_literal(state_2); pos_state_next = (pos_state_next + 1) & coder->pos_mask; - next_rep_match_price + const uint32_t next_rep_match_price = cur_and_len_literal_price + rc_bit_1_price( coder->is_match[state_2][pos_state_next]) + rc_bit_1_price(coder->is_rep[state_2]); // for(; len_test_2 >= 2; --len_test_2) { - offset = cur + len_test + 1 + len_test_2; + const uint32_t offset = cur + len_test + 1 + len_test_2; while (len_end < offset) coder->opts[++len_end].price = RC_INFINITY_PRICE; @@ -849,7 +781,7 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, coder->opts[offset].prev_2 = true; coder->opts[offset].pos_prev_2 = cur; coder->opts[offset].back_prev_2 - = cur_back + REP_DISTANCES; + = cur_back + REPS; } //} } @@ -865,14 +797,11 @@ helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, extern void -lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res, +lzma_lzma_optimum_normal(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, uint32_t position) { - uint32_t reps[REP_DISTANCES]; - uint32_t len_end; - uint32_t cur; - // If we have symbols pending, return the next pending symbol. if (coder->opts_end_index != coder->opts_current_index) { assert(mf->read_ahead > 0); @@ -889,9 +818,9 @@ lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT // In liblzma they were moved into this single place. if (mf->read_ahead == 0) { if (coder->match_price_count >= (1 << 7)) - fill_distances_prices(coder); + fill_dist_prices(coder); - if (coder->align_price_count >= ALIGN_TABLE_SIZE) + if (coder->align_price_count >= ALIGN_SIZE) fill_align_prices(coder); } @@ -899,13 +828,14 @@ lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT // the original function into two pieces makes it at least a little // more readable, since those two parts don't share many variables. - len_end = helper1(coder, mf, back_res, len_res, position); + uint32_t len_end = helper1(coder, mf, back_res, len_res, position); if (len_end == UINT32_MAX) return; - + uint32_t reps[REPS]; memcpy(reps, coder->reps, sizeof(reps)); + uint32_t cur; for (cur = 1; cur < len_end; ++cur) { assert(cur < OPTS); diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c index 9332abfa5..711df0255 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_presets.c @@ -2,6 +2,7 @@ // /// \file lzma_encoder_presets.c /// \brief Encoder presets +/// \note xz needs this even when only decoding is enabled. // // Author: Lasse Collin // @@ -16,10 +17,6 @@ extern LZMA_API(lzma_bool) lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) { - static const uint8_t dict_pow2[] - = { 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }; - static const uint8_t depths[] = { 4, 8, 24, 48 }; - const uint32_t level = preset & LZMA_PRESET_LEVEL_MASK; const uint32_t flags = preset & ~LZMA_PRESET_LEVEL_MASK; const uint32_t supported_flags = LZMA_PRESET_EXTREME; @@ -34,12 +31,15 @@ lzma_lzma_preset(lzma_options_lzma *options, uint32_t preset) options->lp = LZMA_LP_DEFAULT; options->pb = LZMA_PB_DEFAULT; + static const uint8_t dict_pow2[] + = { 18, 20, 21, 22, 22, 23, 23, 24, 25, 26 }; options->dict_size = UINT32_C(1) << dict_pow2[level]; if (level <= 3) { options->mode = LZMA_MODE_FAST; options->mf = level == 0 ? LZMA_MF_HC3 : LZMA_MF_HC4; options->nice_len = level <= 1 ? 128 : 273; + static const uint8_t depths[] = { 4, 8, 24, 48 }; options->depth = depths[level]; } else { options->mode = LZMA_MODE_NORMAL; diff --git a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h index 04fb29e90..a2da969f4 100644 --- a/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h +++ b/Utilities/cmliblzma/liblzma/lzma/lzma_encoder_private.h @@ -64,12 +64,12 @@ typedef struct { uint32_t pos_prev; // pos_next; uint32_t back_prev; - uint32_t backs[REP_DISTANCES]; + uint32_t backs[REPS]; } lzma_optimal; -struct lzma_coder_s { +struct lzma_lzma1_encoder_s { /// Range encoder lzma_range_encoder rc; @@ -77,7 +77,7 @@ struct lzma_coder_s { lzma_lzma_state state; /// The four most recent match distances - uint32_t reps[REP_DISTANCES]; + uint32_t reps[REPS]; /// Array of match candidates lzma_match matches[MATCH_LEN_MAX + 1]; @@ -112,9 +112,9 @@ struct lzma_coder_s { probability is_rep1[STATES]; probability is_rep2[STATES]; probability is_rep0_long[STATES][POS_STATES_MAX]; - probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; - probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; - probability pos_align[ALIGN_TABLE_SIZE]; + probability dist_slot[DIST_STATES][DIST_SLOTS]; + probability dist_special[FULL_DISTANCES - DIST_MODEL_END]; + probability dist_align[ALIGN_SIZE]; // These are the same as in lzma_decoder.c except that the encoders // include also price tables. @@ -122,12 +122,12 @@ struct lzma_coder_s { lzma_length_encoder rep_len_encoder; // Price tables - uint32_t pos_slot_prices[LEN_TO_POS_STATES][POS_SLOTS]; - uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES]; + uint32_t dist_slot_prices[DIST_STATES][DIST_SLOTS]; + uint32_t dist_prices[DIST_STATES][FULL_DISTANCES]; uint32_t dist_table_size; uint32_t match_price_count; - uint32_t align_prices[ALIGN_TABLE_SIZE]; + uint32_t align_prices[ALIGN_SIZE]; uint32_t align_price_count; // Optimal @@ -138,11 +138,11 @@ struct lzma_coder_s { extern void lzma_lzma_optimum_fast( - lzma_coder *LZMA_RESTRICT coder, lzma_mf *LZMA_RESTRICT mf, - uint32_t *LZMA_RESTRICT back_res, uint32_t *LZMA_RESTRICT len_res); + lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res); -extern void lzma_lzma_optimum_normal(lzma_coder *LZMA_RESTRICT coder, - lzma_mf *LZMA_RESTRICT mf, uint32_t *LZMA_RESTRICT back_res, - uint32_t *LZMA_RESTRICT len_res, uint32_t position); +extern void lzma_lzma_optimum_normal(lzma_lzma1_encoder *restrict coder, + lzma_mf *restrict mf, uint32_t *restrict back_res, + uint32_t *restrict len_res, uint32_t position); #endif diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h index f15623ea6..2c74dc153 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_common.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_common.h @@ -40,11 +40,8 @@ // This does the same for a complete bit tree. // (A tree represented as an array.) #define bittree_reset(probs, bit_levels) \ - do { \ - uint32_t bt_i; \ - for (bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ - bit_reset((probs)[bt_i]); \ - } while (0) + for (uint32_t bt_i = 0; bt_i < (1 << (bit_levels)); ++bt_i) \ + bit_reset((probs)[bt_i]) ////////////////////// diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h index 199e7b5a6..e0b051fac 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_decoder.h @@ -25,20 +25,26 @@ typedef struct { /// Reads the first five bytes to initialize the range decoder. -static inline bool -rc_read_init(lzma_range_decoder *rc, const uint8_t *LZMA_RESTRICT in, - size_t *LZMA_RESTRICT in_pos, size_t in_size) +static inline lzma_ret +rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { while (rc->init_bytes_left > 0) { if (*in_pos == in_size) - return false; + return LZMA_OK; + + // The first byte is always 0x00. It could have been omitted + // in LZMA2 but it wasn't, so one byte is wasted in every + // LZMA2 chunk. + if (rc->init_bytes_left == 5 && in[*in_pos] != 0x00) + return LZMA_DATA_ERROR; rc->code = (rc->code << 8) | in[*in_pos]; ++*in_pos; --rc->init_bytes_left; } - return true; + return LZMA_STREAM_END; } diff --git a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h index e9614f252..1e1c36995 100644 --- a/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h +++ b/Utilities/cmliblzma/liblzma/rangecoder/range_encoder.h @@ -115,8 +115,7 @@ rc_direct(lzma_range_encoder *rc, static inline void rc_flush(lzma_range_encoder *rc) { - size_t i; - for (i = 0; i < 5; ++i) + for (size_t i = 0; i < 5; ++i) rc->symbols[rc->count++] = RC_FLUSH; } diff --git a/Utilities/cmliblzma/liblzma/simple/arm.c b/Utilities/cmliblzma/liblzma/simple/arm.c index 8dcba39fd..181d0e3b2 100644 --- a/Utilities/cmliblzma/liblzma/simple/arm.c +++ b/Utilities/cmliblzma/liblzma/simple/arm.c @@ -15,19 +15,19 @@ static size_t -arm_code(lzma_simple *simple lzma_attribute((__unused__)), +arm_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { size_t i; for (i = 0; i + 4 <= size; i += 4) { if (buffer[i + 3] == 0xEB) { - uint32_t dest; uint32_t src = (buffer[i + 2] << 16) | (buffer[i + 1] << 8) | (buffer[i + 0]); src <<= 2; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + 8 + src; else @@ -45,7 +45,7 @@ arm_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +arm_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -54,7 +54,8 @@ arm_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_arm_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm_coder_init(next, allocator, filters, true); @@ -62,7 +63,8 @@ lzma_simple_arm_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_arm_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_arm_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return arm_coder_init(next, allocator, filters, false); diff --git a/Utilities/cmliblzma/liblzma/simple/armthumb.c b/Utilities/cmliblzma/liblzma/simple/armthumb.c index 4b890a395..eab4862dd 100644 --- a/Utilities/cmliblzma/liblzma/simple/armthumb.c +++ b/Utilities/cmliblzma/liblzma/simple/armthumb.c @@ -15,7 +15,7 @@ static size_t -armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), +armthumb_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -23,7 +23,6 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), for (i = 0; i + 4 <= size; i += 2) { if ((buffer[i + 1] & 0xF8) == 0xF0 && (buffer[i + 3] & 0xF8) == 0xF8) { - uint32_t dest; uint32_t src = ((buffer[i + 1] & 0x7) << 19) | (buffer[i + 0] << 11) | ((buffer[i + 3] & 0x7) << 8) @@ -31,6 +30,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), src <<= 1; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + 4 + src; else @@ -50,7 +50,7 @@ armthumb_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +armthumb_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -60,7 +60,8 @@ armthumb_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return armthumb_coder_init(next, allocator, filters, true); } @@ -68,7 +69,8 @@ lzma_simple_armthumb_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return armthumb_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/ia64.c b/Utilities/cmliblzma/liblzma/simple/ia64.c index c537caca1..580529e80 100644 --- a/Utilities/cmliblzma/liblzma/simple/ia64.c +++ b/Utilities/cmliblzma/liblzma/simple/ia64.c @@ -15,7 +15,7 @@ static size_t -ia64_code(lzma_simple *simple lzma_attribute((__unused__)), +ia64_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -28,42 +28,36 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), size_t i; for (i = 0; i + 16 <= size; i += 16) { - size_t slot; - const uint32_t instr_template = buffer[i] & 0x1F; const uint32_t mask = BRANCH_TABLE[instr_template]; uint32_t bit_pos = 5; - for (slot = 0; slot < 3; ++slot, bit_pos += 41) { + for (size_t slot = 0; slot < 3; ++slot, bit_pos += 41) { + if (((mask >> slot) & 1) == 0) + continue; + const size_t byte_pos = (bit_pos >> 3); const uint32_t bit_res = bit_pos & 0x7; uint64_t instruction = 0; - uint64_t inst_norm; - size_t j; - - if (((mask >> slot) & 1) == 0) - continue; - for (j = 0; j < 6; ++j) + for (size_t j = 0; j < 6; ++j) instruction += (uint64_t)( buffer[i + j + byte_pos]) << (8 * j); - inst_norm = instruction >> bit_res; + uint64_t inst_norm = instruction >> bit_res; if (((inst_norm >> 37) & 0xF) == 0x5 && ((inst_norm >> 9) & 0x7) == 0 /* && (inst_norm & 0x3F)== 0 */ ) { - uint32_t dest; - size_t j; - uint32_t src = (uint32_t)( (inst_norm >> 13) & 0xFFFFF); src |= ((inst_norm >> 36) & 1) << 20; src <<= 4; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + src; else @@ -79,7 +73,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), instruction &= (1 << bit_res) - 1; instruction |= (inst_norm << bit_res); - for (j = 0; j < 6; j++) + for (size_t j = 0; j < 6; j++) buffer[i + j + byte_pos] = (uint8_t)( instruction >> (8 * j)); @@ -92,7 +86,7 @@ ia64_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +ia64_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -102,7 +96,8 @@ ia64_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return ia64_coder_init(next, allocator, filters, true); } @@ -110,7 +105,8 @@ lzma_simple_ia64_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return ia64_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/powerpc.c b/Utilities/cmliblzma/liblzma/simple/powerpc.c index 6f8351176..54dfbf102 100644 --- a/Utilities/cmliblzma/liblzma/simple/powerpc.c +++ b/Utilities/cmliblzma/liblzma/simple/powerpc.c @@ -15,7 +15,7 @@ static size_t -powerpc_code(lzma_simple *simple lzma_attribute((__unused__)), +powerpc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -49,7 +49,7 @@ powerpc_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +powerpc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -59,7 +59,8 @@ powerpc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return powerpc_coder_init(next, allocator, filters, true); } @@ -67,7 +68,8 @@ lzma_simple_powerpc_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return powerpc_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.c b/Utilities/cmliblzma/liblzma/simple/simple_coder.c index f3bbdd7a7..13ebabc76 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.c @@ -18,10 +18,10 @@ /// Copied or encodes/decodes more data to out[]. static lzma_ret -copy_or_code(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +copy_or_code(lzma_simple_coder *coder, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { assert(!coder->end_was_reached); @@ -55,7 +55,7 @@ copy_or_code(lzma_coder *coder, lzma_allocator *allocator, static size_t -call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) +call_filter(lzma_simple_coder *coder, uint8_t *buffer, size_t size) { const size_t filtered = coder->filter(coder->simple, coder->now_pos, coder->is_encoder, @@ -66,13 +66,12 @@ call_filter(lzma_coder *coder, uint8_t *buffer, size_t size) static lzma_ret -simple_code(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *LZMA_RESTRICT in, size_t *LZMA_RESTRICT in_pos, - size_t in_size, uint8_t *LZMA_RESTRICT out, - size_t *LZMA_RESTRICT out_pos, size_t out_size, lzma_action action) +simple_code(void *coder_ptr, const lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) { - size_t out_avail; - size_t buf_avail; + lzma_simple_coder *coder = coder_ptr; // TODO: Add partial support for LZMA_SYNC_FLUSH. We can support it // in cases when the filter is able to filter everything. With most @@ -108,13 +107,9 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, // more data to out[] hopefully filling it completely. Then filter // the data in out[]. This step is where most of the data gets // filtered if the buffer sizes used by the application are reasonable. - out_avail = out_size - *out_pos; - buf_avail = coder->size - coder->pos; + const size_t out_avail = out_size - *out_pos; + const size_t buf_avail = coder->size - coder->pos; if (out_avail > buf_avail || buf_avail == 0) { - size_t size; - size_t filtered; - size_t unfiltered; - // Store the old position so that we know from which byte // to start filtering. const size_t out_start = *out_pos; @@ -137,10 +132,11 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, } // Filter out[]. - size = *out_pos - out_start; - filtered = call_filter(coder, out + out_start, size); + const size_t size = *out_pos - out_start; + const size_t filtered = call_filter( + coder, out + out_start, size); - unfiltered = size - filtered; + const size_t unfiltered = size - filtered; assert(unfiltered <= coder->allocated / 2); // Now we can update coder->pos and coder->size, because @@ -204,8 +200,9 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, static void -simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) +simple_coder_end(void *coder_ptr, const lzma_allocator *allocator) { + lzma_simple_coder *coder = coder_ptr; lzma_next_end(&coder->next, allocator); lzma_free(coder->simple, allocator); lzma_free(coder, allocator); @@ -214,10 +211,12 @@ simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) static lzma_ret -simple_coder_update(lzma_coder *coder, lzma_allocator *allocator, +simple_coder_update(void *coder_ptr, const lzma_allocator *allocator, const lzma_filter *filters_null lzma_attribute((__unused__)), const lzma_filter *reversed_filters) { + lzma_simple_coder *coder = coder_ptr; + // No update support, just call the next filter in the chain. return lzma_next_filter_update( &coder->next, allocator, reversed_filters + 1); @@ -225,59 +224,59 @@ simple_coder_update(lzma_coder *coder, lzma_allocator *allocator, extern lzma_ret -lzma_simple_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size), size_t simple_size, size_t unfiltered_max, uint32_t alignment, bool is_encoder) { - // Allocate memory for the lzma_coder structure if needed. - if (next->coder == NULL) { + // Allocate memory for the lzma_simple_coder structure if needed. + lzma_simple_coder *coder = next->coder; + if (coder == NULL) { // Here we allocate space also for the temporary buffer. We // need twice the size of unfiltered_max, because then it // is always possible to filter at least unfiltered_max bytes // more data in coder->buffer[] if it can be filled completely. - next->coder = lzma_alloc(sizeof(lzma_coder) + coder = lzma_alloc(sizeof(lzma_simple_coder) + 2 * unfiltered_max, allocator); - if (next->coder == NULL) + if (coder == NULL) return LZMA_MEM_ERROR; + next->coder = coder; next->code = &simple_code; next->end = &simple_coder_end; next->update = &simple_coder_update; - next->coder->next = LZMA_NEXT_CODER_INIT; - next->coder->filter = filter; - next->coder->allocated = 2 * unfiltered_max; + coder->next = LZMA_NEXT_CODER_INIT; + coder->filter = filter; + coder->allocated = 2 * unfiltered_max; // Allocate memory for filter-specific data structure. if (simple_size > 0) { - next->coder->simple = lzma_alloc( - simple_size, allocator); - if (next->coder->simple == NULL) + coder->simple = lzma_alloc(simple_size, allocator); + if (coder->simple == NULL) return LZMA_MEM_ERROR; } else { - next->coder->simple = NULL; + coder->simple = NULL; } } if (filters[0].options != NULL) { const lzma_options_bcj *simple = filters[0].options; - next->coder->now_pos = simple->start_offset; - if (next->coder->now_pos & (alignment - 1)) + coder->now_pos = simple->start_offset; + if (coder->now_pos & (alignment - 1)) return LZMA_OPTIONS_ERROR; } else { - next->coder->now_pos = 0; + coder->now_pos = 0; } // Reset variables. - next->coder->is_encoder = is_encoder; - next->coder->end_was_reached = false; - next->coder->pos = 0; - next->coder->filtered = 0; - next->coder->size = 0; - - return lzma_next_filter_init( - &next->coder->next, allocator, filters + 1); + coder->is_encoder = is_encoder; + coder->end_was_reached = false; + coder->pos = 0; + coder->filtered = 0; + coder->size = 0; + + return lzma_next_filter_init(&coder->next, allocator, filters + 1); } diff --git a/Utilities/cmliblzma/liblzma/simple/simple_coder.h b/Utilities/cmliblzma/liblzma/simple/simple_coder.h index 0952fad33..19c2ee03a 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_coder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_coder.h @@ -17,44 +17,56 @@ extern lzma_ret lzma_simple_x86_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_x86_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_powerpc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_powerpc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_ia64_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_ia64_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_armthumb_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); #endif diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c index 034e158ff..1d864f2bf 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.c +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.c @@ -14,18 +14,17 @@ extern lzma_ret -lzma_simple_props_decode(void **options, lzma_allocator *allocator, +lzma_simple_props_decode(void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size) { - lzma_options_bcj *opt; - if (props_size == 0) return LZMA_OK; if (props_size != 4) return LZMA_OPTIONS_ERROR; - opt = lzma_alloc(sizeof(lzma_options_bcj), allocator); + lzma_options_bcj *opt = lzma_alloc( + sizeof(lzma_options_bcj), allocator); if (opt == NULL) return LZMA_MEM_ERROR; diff --git a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h index b8bf590f7..bed8d37a9 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_decoder.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_decoder.h @@ -16,7 +16,7 @@ #include "simple_coder.h" extern lzma_ret lzma_simple_props_decode( - void **options, lzma_allocator *allocator, + void **options, const lzma_allocator *allocator, const uint8_t *props, size_t props_size); #endif diff --git a/Utilities/cmliblzma/liblzma/simple/simple_private.h b/Utilities/cmliblzma/liblzma/simple/simple_private.h index fcf9f7c19..9d2c0fdd7 100644 --- a/Utilities/cmliblzma/liblzma/simple/simple_private.h +++ b/Utilities/cmliblzma/liblzma/simple/simple_private.h @@ -16,9 +16,7 @@ #include "simple_coder.h" -typedef struct lzma_simple_s lzma_simple; - -struct lzma_coder_s { +typedef struct { /// Next filter in the chain lzma_next_coder next; @@ -33,12 +31,12 @@ struct lzma_coder_s { /// Pointer to filter-specific function, which does /// the actual filtering. - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size); /// Pointer to filter-specific data, or NULL if filter doesn't need /// any extra data. - lzma_simple *simple; + void *simple; /// The lowest 32 bits of the current position in the data. Most /// filters need this to do conversions between absolute and relative @@ -62,12 +60,13 @@ struct lzma_coder_s { /// Temporary buffer uint8_t buffer[]; -}; +} lzma_simple_coder; extern lzma_ret lzma_simple_coder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters, - size_t (*filter)(lzma_simple *simple, uint32_t now_pos, + const lzma_allocator *allocator, + const lzma_filter_info *filters, + size_t (*filter)(void *simple, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size), size_t simple_size, size_t unfiltered_max, uint32_t alignment, bool is_encoder); diff --git a/Utilities/cmliblzma/liblzma/simple/sparc.c b/Utilities/cmliblzma/liblzma/simple/sparc.c index 0ddd2ac9d..74b2655f3 100644 --- a/Utilities/cmliblzma/liblzma/simple/sparc.c +++ b/Utilities/cmliblzma/liblzma/simple/sparc.c @@ -15,7 +15,7 @@ static size_t -sparc_code(lzma_simple *simple lzma_attribute((__unused__)), +sparc_code(void *simple lzma_attribute((__unused__)), uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { @@ -26,8 +26,6 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), || (buffer[i] == 0x7F && (buffer[i + 1] & 0xC0) == 0xC0)) { - uint32_t dest; - uint32_t src = ((uint32_t)buffer[i + 0] << 24) | ((uint32_t)buffer[i + 1] << 16) | ((uint32_t)buffer[i + 2] << 8) @@ -35,6 +33,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), src <<= 2; + uint32_t dest; if (is_encoder) dest = now_pos + (uint32_t)(i) + src; else @@ -58,7 +57,7 @@ sparc_code(lzma_simple *simple lzma_attribute((__unused__)), static lzma_ret -sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +sparc_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { return lzma_simple_coder_init(next, allocator, filters, @@ -68,7 +67,8 @@ sparc_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return sparc_coder_init(next, allocator, filters, true); } @@ -76,7 +76,8 @@ lzma_simple_sparc_encoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_filter_info *filters) + const lzma_allocator *allocator, + const lzma_filter_info *filters) { return sparc_coder_init(next, allocator, filters, false); } diff --git a/Utilities/cmliblzma/liblzma/simple/x86.c b/Utilities/cmliblzma/liblzma/simple/x86.c index 95858e51e..23d0c4293 100644 --- a/Utilities/cmliblzma/liblzma/simple/x86.c +++ b/Utilities/cmliblzma/liblzma/simple/x86.c @@ -17,14 +17,14 @@ #define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) -struct lzma_simple_s { +typedef struct { uint32_t prev_mask; uint32_t prev_pos; -}; +} lzma_simple_x86; static size_t -x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, +x86_code(void *simple_ptr, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { static const bool MASK_TO_ALLOWED_STATUS[8] @@ -33,39 +33,34 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, static const uint32_t MASK_TO_BIT_NUMBER[8] = { 0, 1, 2, 2, 3, 3, 3, 3 }; + lzma_simple_x86 *simple = simple_ptr; uint32_t prev_mask = simple->prev_mask; uint32_t prev_pos = simple->prev_pos; - size_t limit; - size_t buffer_pos; - if (size < 5) return 0; if (now_pos - prev_pos > 5) prev_pos = now_pos - 5; - limit = size - 5; - buffer_pos = 0; + const size_t limit = size - 5; + size_t buffer_pos = 0; while (buffer_pos <= limit) { - uint32_t offset; - uint32_t i; - uint8_t b = buffer[buffer_pos]; if (b != 0xE8 && b != 0xE9) { ++buffer_pos; continue; } - offset = now_pos + (uint32_t)(buffer_pos) + const uint32_t offset = now_pos + (uint32_t)(buffer_pos) - prev_pos; prev_pos = now_pos + (uint32_t)(buffer_pos); if (offset > 5) { prev_mask = 0; } else { - for (i = 0; i < offset; ++i) { + for (uint32_t i = 0; i < offset; ++i) { prev_mask &= 0x77; prev_mask <<= 1; } @@ -84,8 +79,6 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, uint32_t dest; while (true) { - uint32_t i; - if (is_encoder) dest = src + (now_pos + (uint32_t)( buffer_pos) + 5); @@ -96,7 +89,8 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, if (prev_mask == 0) break; - i = MASK_TO_BIT_NUMBER[prev_mask >> 1]; + const uint32_t i = MASK_TO_BIT_NUMBER[ + prev_mask >> 1]; b = (uint8_t)(dest >> (24 - i * 8)); @@ -130,15 +124,17 @@ x86_code(lzma_simple *simple, uint32_t now_pos, bool is_encoder, static lzma_ret -x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, +x86_coder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters, bool is_encoder) { const lzma_ret ret = lzma_simple_coder_init(next, allocator, filters, - &x86_code, sizeof(lzma_simple), 5, 1, is_encoder); + &x86_code, sizeof(lzma_simple_x86), 5, 1, is_encoder); if (ret == LZMA_OK) { - next->coder->simple->prev_mask = 0; - next->coder->simple->prev_pos = (uint32_t)(-5); + lzma_simple_coder *coder = next->coder; + lzma_simple_x86 *simple = coder->simple; + simple->prev_mask = 0; + simple->prev_pos = (uint32_t)(-5); } return ret; @@ -146,7 +142,8 @@ x86_coder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_x86_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return x86_coder_init(next, allocator, filters, true); @@ -154,7 +151,8 @@ lzma_simple_x86_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern lzma_ret -lzma_simple_x86_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, +lzma_simple_x86_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, const lzma_filter_info *filters) { return x86_coder_init(next, allocator, filters, false); diff --git a/Utilities/cmlibrhash/librhash/rhash.c b/Utilities/cmlibrhash/librhash/rhash.c index ad6249b17..34e1eb305 100644 --- a/Utilities/cmlibrhash/librhash/rhash.c +++ b/Utilities/cmlibrhash/librhash/rhash.c @@ -18,6 +18,8 @@ #define _LARGEFILE64_SOURCE #define _FILE_OFFSET_BITS 64 +#include "ustd.h" /* Need this first within CMake. */ + #include <string.h> /* memset() */ #include <stdlib.h> /* free() */ #include <stddef.h> /* ptrdiff_t */ diff --git a/Utilities/cmlibuv/CMakeLists.txt b/Utilities/cmlibuv/CMakeLists.txt index ba1638eb0..a503041be 100644 --- a/Utilities/cmlibuv/CMakeLists.txt +++ b/Utilities/cmlibuv/CMakeLists.txt @@ -208,6 +208,22 @@ if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") ) endif() +if(CMAKE_SYSTEM_NAME STREQUAL "kFreeBSD") + list(APPEND uv_libraries + freebsd-glue + kvm + ) + list(APPEND uv_headers + include/uv-bsd.h + ) + list(APPEND uv_sources + src/unix/bsd-ifaddrs.c + src/unix/freebsd.c + src/unix/kqueue.c + src/unix/posix-hrtime.c + ) +endif() + if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") list(APPEND uv_libraries kvm |
