From 23f3310331474440a8c16d98f808427fad9d0a3c Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 09:21:01 -0600 Subject: [PATCH 01/13] First stab at adding compresslevel to zipfile --- Lib/zipfile.py | 59 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 37ce3281e0928c..64d34eed396376 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -295,6 +295,7 @@ class ZipInfo (object): 'filename', 'date_time', 'compress_type', + 'compress_level', 'comment', 'extra', 'create_system', @@ -334,6 +335,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file + self.compress_level = None # Level or preset for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': @@ -654,14 +656,27 @@ def _check_compression(compression): raise NotImplementedError("That compression method is not supported") -def _get_compressor(compress_type): +def _get_compressor(compress_type, compress_level=None): + compressor_kwargs = {} + # zlib.compressobj defaults to zlib.Z_DEFAULT_COMPRESSION if the level + # isn't given. if compress_type == ZIP_DEFLATED: - return zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, - zlib.DEFLATED, -15) + if compress_level is not None: + compressor_kwargs['level'] = compress_level + compressor_kwargs['method'] = zlib.DEFLATED + compressor_kwargs['wbits'] = -15 + return zlib.compressobj(**compressor_kwargs) + # bz2.BZ2Compressor defaults to compresslevel=9 if the level isn't given. elif compress_type == ZIP_BZIP2: - return bz2.BZ2Compressor() + if compress_level is not None: + compressor_kwargs['compresslevel'] = compress_level + return bz2.BZ2Compressor(**compressor_kwargs) + # lzma.LZMACompressor defaults to lzma.PRESET_DEFAULT if the level isn't + # given. elif compress_type == ZIP_LZMA: - return LZMACompressor() + if compress_level is not None: + compressor_kwargs['preset'] = compress_level + return LZMACompressor(**compressor_kwargs) else: return None @@ -747,6 +762,7 @@ def __init__(self, fileobj, mode, zipinfo, decrypter=None, self._close_fileobj = close_fileobj self._compress_type = zipinfo.compress_type + self._compress_level = zipinfo.compress_level # XXX: Is this needed? self._compress_left = zipinfo.compress_size self._left = zipinfo.file_size @@ -963,7 +979,8 @@ def __init__(self, zf, zinfo, zip64): self._zinfo = zinfo self._zip64 = zip64 self._zipfile = zf - self._compressor = _get_compressor(zinfo.compress_type) + self._compressor = _get_compressor(zinfo.compress_type, + zinfo.compress_level) self._file_size = 0 self._compress_size = 0 self._crc = 0 @@ -1035,7 +1052,8 @@ def close(self): class ZipFile: """ Class with methods to open, read, write, close, list zip files. - z = ZipFile(file, mode="r", compression=ZIP_STORED, allowZip64=True) + z = ZipFile(file, mode="r", compression=ZIP_STORED, allowZip64=True, + compresslevel=None) file: Either the path to the file, or a file-like object. If it is a path, the file will be opened and closed by ZipFile. @@ -1046,13 +1064,17 @@ class ZipFile: allowZip64: if True ZipFile will create files with ZIP64 extensions when needed, otherwise it will raise an exception when this would be necessary. + compresslevel: None (default for the given compression type) or an integer + from 0 to 9 specifying the level or preset to pass to the + compressor. """ fp = None # Set here since __del__ checks it _windows_illegal_name_trans_table = None - def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): + def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, + compresslevel=None): """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', or append 'a'.""" if mode not in ('r', 'w', 'x', 'a'): @@ -1066,6 +1088,7 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): self.NameToInfo = {} # Find file info given name self.filelist = [] # List of ZipInfo instances for archive self.compression = compression # Method of compression + self.compresslevel = compresslevel self.mode = mode self.pwd = None self._comment = b'' @@ -1342,6 +1365,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): elif mode == 'w': zinfo = ZipInfo(name) zinfo.compress_type = self.compression + zinfo.compress_level = self.compresslevel else: # Get info object for name zinfo = self.getinfo(name) @@ -1575,7 +1599,8 @@ def _writecheck(self, zinfo): raise LargeZipFile(requires_zip64 + " would require ZIP64 extensions") - def write(self, filename, arcname=None, compress_type=None): + def write(self, filename, arcname=None, + compress_type=None, compress_level=None): """Put the bytes from filename into the archive under the name arcname.""" if not self.fp: @@ -1597,6 +1622,11 @@ def write(self, filename, arcname=None, compress_type=None): else: zinfo.compress_type = self.compression + if compress_level is not None: + zinfo.compress_level = compress_level + else: + zinfo.compress_level = self.compresslevel + if zinfo.is_dir(): with self._lock: if self._seekable: @@ -1617,7 +1647,8 @@ def write(self, filename, arcname=None, compress_type=None): with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: shutil.copyfileobj(src, dest, 1024*8) - def writestr(self, zinfo_or_arcname, data, compress_type=None): + def writestr(self, zinfo_or_arcname, data, + compress_type=None, compress_level=None): """Write a file into the archive. The contents is 'data', which may be either a 'str' or a 'bytes' instance; if it is a 'str', it is encoded as UTF-8 first. @@ -1629,6 +1660,7 @@ def writestr(self, zinfo_or_arcname, data, compress_type=None): zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression + zinfo.compress_level = self.compresslevel if zinfo.filename[-1] == '/': zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr |= 0x10 # MS-DOS directory flag @@ -1648,6 +1680,9 @@ def writestr(self, zinfo_or_arcname, data, compress_type=None): if compress_type is not None: zinfo.compress_type = compress_type + if compress_level is not None: + zinfo.compress_level = compress_level + zinfo.file_size = len(data) # Uncompressed size with self._lock: with self.open(zinfo, mode='w') as dest: @@ -1791,9 +1826,9 @@ class PyZipFile(ZipFile): """Class to create ZIP archives with Python library files and packages.""" def __init__(self, file, mode="r", compression=ZIP_STORED, - allowZip64=True, optimize=-1): + allowZip64=True, optimize=-1, compresslevel=None): ZipFile.__init__(self, file, mode=mode, compression=compression, - allowZip64=allowZip64) + allowZip64=allowZip64, compresslevel=compresslevel) self._optimize = optimize def writepy(self, pathname, basename="", filterfunc=None): From b02997775bcb5fa200e3951a52bfe7d748079769 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 11:41:09 -0600 Subject: [PATCH 02/13] Smooth out errors for bz2 and lzma --- Lib/zipfile.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 64d34eed396376..a5bc648965de05 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -657,26 +657,22 @@ def _check_compression(compression): def _get_compressor(compress_type, compress_level=None): - compressor_kwargs = {} # zlib.compressobj defaults to zlib.Z_DEFAULT_COMPRESSION if the level # isn't given. if compress_type == ZIP_DEFLATED: if compress_level is not None: - compressor_kwargs['level'] = compress_level - compressor_kwargs['method'] = zlib.DEFLATED - compressor_kwargs['wbits'] = -15 - return zlib.compressobj(**compressor_kwargs) + return zlib.compressobj(compress_level, zlib.DEFLATED, -15) + return zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) # bz2.BZ2Compressor defaults to compresslevel=9 if the level isn't given. + # compresslevel=0 is not valid. elif compress_type == ZIP_BZIP2: if compress_level is not None: - compressor_kwargs['compresslevel'] = compress_level - return bz2.BZ2Compressor(**compressor_kwargs) - # lzma.LZMACompressor defaults to lzma.PRESET_DEFAULT if the level isn't - # given. + return bz2.BZ2Compressor(compress_level) + return bz2.BZ2Compressor() + # LZMACompressor (defined below) doesn't allow setting a preset (i.e., + # compression level) elif compress_type == ZIP_LZMA: - if compress_level is not None: - compressor_kwargs['preset'] = compress_level - return LZMACompressor(**compressor_kwargs) + return LZMACompressor() else: return None @@ -1065,8 +1061,10 @@ class ZipFile: needed, otherwise it will raise an exception when this would be necessary. compresslevel: None (default for the given compression type) or an integer - from 0 to 9 specifying the level or preset to pass to the - compressor. + specifying the level to pass to the compressor. + When using ZIP_STORED or ZIP_LZMA this keyword has no effect. + When using ZIP_DEFLATED integers 0 through 9 are accepted. + When using ZIP_BZIP2 integers 1 through 9 are accepted. """ From 9c5770042ed9e67bea3bc2b162d5cce7aa9f2a4d Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 11:44:30 -0600 Subject: [PATCH 03/13] Remove now-spurious comments --- Lib/zipfile.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index a5bc648965de05..447b1aa720a4ba 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -335,7 +335,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file - self.compress_level = None # Level or preset for the compressor + self.compress_level = None # Level for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': @@ -657,20 +657,15 @@ def _check_compression(compression): def _get_compressor(compress_type, compress_level=None): - # zlib.compressobj defaults to zlib.Z_DEFAULT_COMPRESSION if the level - # isn't given. if compress_type == ZIP_DEFLATED: if compress_level is not None: return zlib.compressobj(compress_level, zlib.DEFLATED, -15) return zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) - # bz2.BZ2Compressor defaults to compresslevel=9 if the level isn't given. - # compresslevel=0 is not valid. elif compress_type == ZIP_BZIP2: if compress_level is not None: return bz2.BZ2Compressor(compress_level) return bz2.BZ2Compressor() - # LZMACompressor (defined below) doesn't allow setting a preset (i.e., - # compression level) + # compress_level is ignored for ZIP_LZMA elif compress_type == ZIP_LZMA: return LZMACompressor() else: From 15080736f30df418d3b9c4cbecd6c039fb1fb361 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 11:55:46 -0600 Subject: [PATCH 04/13] No it was not needed --- Lib/zipfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 447b1aa720a4ba..eea241726022a0 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -753,7 +753,6 @@ def __init__(self, fileobj, mode, zipinfo, decrypter=None, self._close_fileobj = close_fileobj self._compress_type = zipinfo.compress_type - self._compress_level = zipinfo.compress_level # XXX: Is this needed? self._compress_left = zipinfo.compress_size self._left = zipinfo.file_size From 9b5d74ed9a0d2b8fc3c1f4a380f3be618c6e8877 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 12:13:25 -0600 Subject: [PATCH 05/13] Don't open the PyZipFile box yet --- Lib/zipfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index eea241726022a0..9f37b6a40f5a4b 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1818,9 +1818,9 @@ class PyZipFile(ZipFile): """Class to create ZIP archives with Python library files and packages.""" def __init__(self, file, mode="r", compression=ZIP_STORED, - allowZip64=True, optimize=-1, compresslevel=None): + allowZip64=True, optimize=-1): ZipFile.__init__(self, file, mode=mode, compression=compression, - allowZip64=allowZip64, compresslevel=compresslevel) + allowZip64=allowZip64) self._optimize = optimize def writepy(self, pathname, basename="", filterfunc=None): From 65045819e0459b1e40cf79ce39745bd5b531436f Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 13:40:42 -0600 Subject: [PATCH 06/13] Add basic notes about compresslevel to docs --- Doc/library/zipfile.rst | 42 +++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5b8c776ed648cd..7cff9d60adf83e 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -130,10 +130,12 @@ ZipFile Objects --------------- -.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) +.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ + compresslevel=None) Open a ZIP file, where *file* can be a path to a file (a string), a file-like object or a :term:`path-like object`. + The *mode* parameter should be ``'r'`` to read an existing file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an existing file, or ``'x'`` to exclusively create and write a new file. @@ -145,16 +147,25 @@ ZipFile Objects adding a ZIP archive to another file (such as :file:`python.exe`). If *mode* is ``'a'`` and the file does not exist at all, it is created. If *mode* is ``'r'`` or ``'a'``, the file should be seekable. + *compression* is the ZIP compression method to use when writing the archive, and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized - values will cause :exc:`NotImplementedError` to be raised. If :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified but the corresponding module - (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not available, :exc:`RuntimeError` - is raised. The default is :const:`ZIP_STORED`. If *allowZip64* is - ``True`` (the default) zipfile will create ZIP files that use the ZIP64 - extensions when the zipfile is larger than 4 GiB. If it is false :mod:`zipfile` - will raise an exception when the ZIP file would require ZIP64 extensions. + values will cause :exc:`NotImplementedError` to be raised. If + :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified + but the corresponding module (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not + available, :exc:`RuntimeError` is raised. The default is :const:`ZIP_STORED`. + + If *allowZip64* is ``True`` (the default) zipfile will create ZIP files that + use the ZIP64 extensions when the zipfile is larger than 4 GiB. If it is + ``false`` :mod:`zipfile` will raise an exception when the ZIP file would + require ZIP64 extensions. + + The *compresslevel* parameter controls the compression level to use when + writing files to the archive. When using :const:`ZIP_STORED` or + :const:`ZIP_LZMA` it has no effect. When using :const:`ZIP_DEFLATED` + integers ``0`` through ``9`` are accepted. When using :const:`ZIP_BZIP2` + integers ``1`` through ``9`` are accepted. If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then :meth:`closed ` without adding any files to the archive, the appropriate @@ -187,6 +198,9 @@ ZipFile Objects .. versionchanged:: 3.6.2 The *file* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.7 + Add the *compresslevel* parameter. + .. method:: ZipFile.close() @@ -351,13 +365,15 @@ ZipFile Objects :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. -.. method:: ZipFile.write(filename, arcname=None, compress_type=None) +.. method:: ZipFile.write(filename, arcname=None, compress_type=None, \ + compress_level=None) Write the file named *filename* to the archive, giving it the archive name *arcname* (by default, this will be the same as *filename*, but without a drive letter and with leading path separators removed). If given, *compress_type* overrides the value given for the *compression* parameter to the constructor for - the new entry. + the new entry. Similarly, if given, *compress_level* overrides the *compresslevel* + parameter. The archive must be open with mode ``'w'``, ``'x'`` or ``'a'``. .. note:: @@ -383,7 +399,8 @@ ZipFile Objects a :exc:`RuntimeError` was raised. -.. method:: ZipFile.writestr(zinfo_or_arcname, data[, compress_type]) +.. method:: ZipFile.writestr(zinfo_or_arcname, data, compress_type=None, \ + compress_level=None) Write the string *data* to the archive; *zinfo_or_arcname* is either the file name it will be given in the archive, or a :class:`ZipInfo` instance. If it's @@ -393,7 +410,8 @@ ZipFile Objects If given, *compress_type* overrides the value given for the *compression* parameter to the constructor for the new entry, or in the *zinfo_or_arcname* - (if that is a :class:`ZipInfo` instance). + (if that is a :class:`ZipInfo` instance). Similarly, *compress_level* will + override the *compresslevel* parameter. .. note:: From 46eb50360a98d8459bf0fda4d164454c20c39d1e Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 16:43:39 -0600 Subject: [PATCH 07/13] Add basic tests for compresslevel --- Lib/test/test_zipfile.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 3bc867ea51c946..dc10f4b3e64f7d 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -53,9 +53,10 @@ def setUp(self): with open(TESTFN, "wb") as fp: fp.write(self.data) - def make_test_archive(self, f, compression): + def make_test_archive(self, f, compression, compresslevel=None): + kwargs = {'compression': compression, 'compresslevel': compresslevel} # Create the ZIP archive - with zipfile.ZipFile(f, "w", compression) as zipfp: + with zipfile.ZipFile(f, "w", **kwargs) as zipfp: zipfp.write(TESTFN, "another.name") zipfp.write(TESTFN, TESTFN) zipfp.writestr("strfile", self.data) @@ -63,8 +64,8 @@ def make_test_archive(self, f, compression): for line in self.line_gen: f.write(line) - def zip_test(self, f, compression): - self.make_test_archive(f, compression) + def zip_test(self, f, compression, compresslevel=None): + self.make_test_archive(f, compression, compresslevel) # Read the ZIP archive with zipfile.ZipFile(f, "r", compression) as zipfp: @@ -297,6 +298,13 @@ def test_writestr_compression(self): info = zipfp.getinfo('b.txt') self.assertEqual(info.compress_type, self.compression) + def test_writestr_compresslevel(self): + zipfp = zipfile.ZipFile(TESTFN2, "w") + zipfp.writestr("b.txt", "hello world", compress_type=self.compression, + compress_level=1) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, self.compression) + def test_read_return_size(self): # Issue #9837: ZipExtFile.read() shouldn't return more bytes # than requested. @@ -370,6 +378,21 @@ def test_repr(self): self.assertIn('[closed]', repr(zipopen)) self.assertIn('[closed]', repr(zipfp)) + def test_compresslevel_basic(self): + for f in get_files(self): + self.zip_test(f, self.compression, compresslevel=9) + + def test_per_file_compresslevel(self): + """Check that files within a Zip archive can have different + compression options.""" + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.write(TESTFN, 'compress_1', compress_level=1) + zipfp.write(TESTFN, 'compress_9', compress_level=9) + one_info = zipfp.getinfo('compress_1') + nine_info = zipfp.getinfo('compress_9') + self.assertEqual(one_info.compress_level, 1) + self.assertEqual(nine_info.compress_level, 9) + def tearDown(self): unlink(TESTFN) unlink(TESTFN2) From 7ff82720ffb914277300e2dfa0e4b93264b453c1 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 20:06:54 -0600 Subject: [PATCH 08/13] Make ZipInfo compression level private --- Lib/test/test_zipfile.py | 6 +++--- Lib/zipfile.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index dc10f4b3e64f7d..954a89bb303477 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -384,14 +384,14 @@ def test_compresslevel_basic(self): def test_per_file_compresslevel(self): """Check that files within a Zip archive can have different - compression options.""" + compression levels.""" with zipfile.ZipFile(TESTFN2, "w") as zipfp: zipfp.write(TESTFN, 'compress_1', compress_level=1) zipfp.write(TESTFN, 'compress_9', compress_level=9) one_info = zipfp.getinfo('compress_1') nine_info = zipfp.getinfo('compress_9') - self.assertEqual(one_info.compress_level, 1) - self.assertEqual(nine_info.compress_level, 9) + self.assertEqual(one_info._compress_level, 1) + self.assertEqual(nine_info._compress_level, 9) def tearDown(self): unlink(TESTFN) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 9f37b6a40f5a4b..7b9607d133a6b9 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -295,7 +295,7 @@ class ZipInfo (object): 'filename', 'date_time', 'compress_type', - 'compress_level', + '_compress_level', 'comment', 'extra', 'create_system', @@ -335,7 +335,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file - self.compress_level = None # Level for the compressor + self._compress_level = None # Level for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': @@ -970,7 +970,7 @@ def __init__(self, zf, zinfo, zip64): self._zip64 = zip64 self._zipfile = zf self._compressor = _get_compressor(zinfo.compress_type, - zinfo.compress_level) + zinfo._compress_level) self._file_size = 0 self._compress_size = 0 self._crc = 0 @@ -1357,7 +1357,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): elif mode == 'w': zinfo = ZipInfo(name) zinfo.compress_type = self.compression - zinfo.compress_level = self.compresslevel + zinfo._compress_level = self.compresslevel else: # Get info object for name zinfo = self.getinfo(name) @@ -1615,9 +1615,9 @@ def write(self, filename, arcname=None, zinfo.compress_type = self.compression if compress_level is not None: - zinfo.compress_level = compress_level + zinfo._compress_level = compress_level else: - zinfo.compress_level = self.compresslevel + zinfo._compress_level = self.compresslevel if zinfo.is_dir(): with self._lock: @@ -1652,7 +1652,7 @@ def writestr(self, zinfo_or_arcname, data, zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression - zinfo.compress_level = self.compresslevel + zinfo._compress_level = self.compresslevel if zinfo.filename[-1] == '/': zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr |= 0x10 # MS-DOS directory flag @@ -1673,7 +1673,7 @@ def writestr(self, zinfo_or_arcname, data, zinfo.compress_type = compress_type if compress_level is not None: - zinfo.compress_level = compress_level + zinfo._compress_level = compress_level zinfo.file_size = len(data) # Uncompressed size with self._lock: From 8310c7951b8f0f37d6a7b59b4ab570080195d4a2 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sat, 27 Jan 2018 20:25:04 -0600 Subject: [PATCH 09/13] Test implicit and overridden compression levels --- Lib/test/test_zipfile.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 954a89bb303477..256c1698e895bc 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -299,11 +299,20 @@ def test_writestr_compression(self): self.assertEqual(info.compress_type, self.compression) def test_writestr_compresslevel(self): - zipfp = zipfile.ZipFile(TESTFN2, "w") + zipfp = zipfile.ZipFile(TESTFN2, "w", compresslevel=1) + zipfp.writestr("a.txt", "hello world", compress_type=self.compression) zipfp.writestr("b.txt", "hello world", compress_type=self.compression, - compress_level=1) - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, self.compression) + compress_level=2) + + # Compression level follows the constructor. + a_info = zipfp.getinfo('a.txt') + self.assertEqual(a_info.compress_type, self.compression) + self.assertEqual(a_info._compress_level, 1) + + # Compression level is overriden. + b_info = zipfp.getinfo('b.txt') + self.assertEqual(b_info.compress_type, self.compression) + self.assertEqual(b_info._compress_level, 2) def test_read_return_size(self): # Issue #9837: ZipExtFile.read() shouldn't return more bytes @@ -385,8 +394,8 @@ def test_compresslevel_basic(self): def test_per_file_compresslevel(self): """Check that files within a Zip archive can have different compression levels.""" - with zipfile.ZipFile(TESTFN2, "w") as zipfp: - zipfp.write(TESTFN, 'compress_1', compress_level=1) + with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: + zipfp.write(TESTFN, 'compress_1') zipfp.write(TESTFN, 'compress_9', compress_level=9) one_info = zipfp.getinfo('compress_1') nine_info = zipfp.getinfo('compress_9') From 8ab542c3e6198578ebd71a4bd41d135378680607 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 28 Jan 2018 07:55:26 -0600 Subject: [PATCH 10/13] Add Misc/NEWS entry --- .../NEWS.d/next/Library/2018-01-28-07-55-10.bpo-21417.JFnV99.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2018-01-28-07-55-10.bpo-21417.JFnV99.rst diff --git a/Misc/NEWS.d/next/Library/2018-01-28-07-55-10.bpo-21417.JFnV99.rst b/Misc/NEWS.d/next/Library/2018-01-28-07-55-10.bpo-21417.JFnV99.rst new file mode 100644 index 00000000000000..50207a0e4c33ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-01-28-07-55-10.bpo-21417.JFnV99.rst @@ -0,0 +1 @@ +Added support for setting the compression level for zipfile.ZipFile. From 0e3f9ced44bd5121e7f4e8f3c141d44ffab72358 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 28 Jan 2018 21:39:15 -0600 Subject: [PATCH 11/13] Correct typo in test comment. --- Lib/test/test_zipfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 256c1698e895bc..38fafeedc47053 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -309,7 +309,7 @@ def test_writestr_compresslevel(self): self.assertEqual(a_info.compress_type, self.compression) self.assertEqual(a_info._compress_level, 1) - # Compression level is overriden. + # Compression level is overridden. b_info = zipfp.getinfo('b.txt') self.assertEqual(b_info.compress_type, self.compression) self.assertEqual(b_info._compress_level, 2) From 3925fc2f310b4eb4a46f16fa7bb562b11242a8b0 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 28 Jan 2018 21:56:05 -0600 Subject: [PATCH 12/13] compress_level to compresslevel throughout --- Doc/library/zipfile.rst | 22 ++++++++++++---------- Lib/test/test_zipfile.py | 12 ++++++------ Lib/zipfile.py | 36 ++++++++++++++++++------------------ 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 7cff9d60adf83e..b0ae149642bb87 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -162,10 +162,12 @@ ZipFile Objects require ZIP64 extensions. The *compresslevel* parameter controls the compression level to use when - writing files to the archive. When using :const:`ZIP_STORED` or - :const:`ZIP_LZMA` it has no effect. When using :const:`ZIP_DEFLATED` - integers ``0`` through ``9`` are accepted. When using :const:`ZIP_BZIP2` - integers ``1`` through ``9`` are accepted. + writing files to the archive. + When using :const:`ZIP_STORED` or :const:`ZIP_LZMA` it has no effect. + When using :const:`ZIP_DEFLATED` integers ``0`` through ``9`` are accepted + (see :class:`gzip.GzipFile`). + When using :const:`ZIP_BZIP2` integers ``1`` through ``9`` are accepted + (see :class:`bz2.BZ2File`). If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then :meth:`closed ` without adding any files to the archive, the appropriate @@ -366,14 +368,14 @@ ZipFile Objects .. method:: ZipFile.write(filename, arcname=None, compress_type=None, \ - compress_level=None) + compresslevel=None) Write the file named *filename* to the archive, giving it the archive name *arcname* (by default, this will be the same as *filename*, but without a drive letter and with leading path separators removed). If given, *compress_type* overrides the value given for the *compression* parameter to the constructor for - the new entry. Similarly, if given, *compress_level* overrides the *compresslevel* - parameter. + the new entry. Similarly, *compresslevel* will override the constructor if + given. The archive must be open with mode ``'w'``, ``'x'`` or ``'a'``. .. note:: @@ -400,7 +402,7 @@ ZipFile Objects .. method:: ZipFile.writestr(zinfo_or_arcname, data, compress_type=None, \ - compress_level=None) + compresslevel=None) Write the string *data* to the archive; *zinfo_or_arcname* is either the file name it will be given in the archive, or a :class:`ZipInfo` instance. If it's @@ -410,8 +412,8 @@ ZipFile Objects If given, *compress_type* overrides the value given for the *compression* parameter to the constructor for the new entry, or in the *zinfo_or_arcname* - (if that is a :class:`ZipInfo` instance). Similarly, *compress_level* will - override the *compresslevel* parameter. + (if that is a :class:`ZipInfo` instance). Similarly, *compresslevel* will + override the constructor if given. .. note:: diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 38fafeedc47053..94db858a1517c4 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -302,17 +302,17 @@ def test_writestr_compresslevel(self): zipfp = zipfile.ZipFile(TESTFN2, "w", compresslevel=1) zipfp.writestr("a.txt", "hello world", compress_type=self.compression) zipfp.writestr("b.txt", "hello world", compress_type=self.compression, - compress_level=2) + compresslevel=2) # Compression level follows the constructor. a_info = zipfp.getinfo('a.txt') self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info._compress_level, 1) + self.assertEqual(a_info._compresslevel, 1) # Compression level is overridden. b_info = zipfp.getinfo('b.txt') self.assertEqual(b_info.compress_type, self.compression) - self.assertEqual(b_info._compress_level, 2) + self.assertEqual(b_info._compresslevel, 2) def test_read_return_size(self): # Issue #9837: ZipExtFile.read() shouldn't return more bytes @@ -396,11 +396,11 @@ def test_per_file_compresslevel(self): compression levels.""" with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: zipfp.write(TESTFN, 'compress_1') - zipfp.write(TESTFN, 'compress_9', compress_level=9) + zipfp.write(TESTFN, 'compress_9', compresslevel=9) one_info = zipfp.getinfo('compress_1') nine_info = zipfp.getinfo('compress_9') - self.assertEqual(one_info._compress_level, 1) - self.assertEqual(nine_info._compress_level, 9) + self.assertEqual(one_info._compresslevel, 1) + self.assertEqual(nine_info._compresslevel, 9) def tearDown(self): unlink(TESTFN) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 7b9607d133a6b9..f9db45f58a2bde 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -295,7 +295,7 @@ class ZipInfo (object): 'filename', 'date_time', 'compress_type', - '_compress_level', + '_compresslevel', 'comment', 'extra', 'create_system', @@ -335,7 +335,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file - self._compress_level = None # Level for the compressor + self._compresslevel = None # Level for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': @@ -656,16 +656,16 @@ def _check_compression(compression): raise NotImplementedError("That compression method is not supported") -def _get_compressor(compress_type, compress_level=None): +def _get_compressor(compress_type, compresslevel=None): if compress_type == ZIP_DEFLATED: - if compress_level is not None: - return zlib.compressobj(compress_level, zlib.DEFLATED, -15) + if compresslevel is not None: + return zlib.compressobj(compresslevel, zlib.DEFLATED, -15) return zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) elif compress_type == ZIP_BZIP2: - if compress_level is not None: - return bz2.BZ2Compressor(compress_level) + if compresslevel is not None: + return bz2.BZ2Compressor(compresslevel) return bz2.BZ2Compressor() - # compress_level is ignored for ZIP_LZMA + # compresslevel is ignored for ZIP_LZMA elif compress_type == ZIP_LZMA: return LZMACompressor() else: @@ -970,7 +970,7 @@ def __init__(self, zf, zinfo, zip64): self._zip64 = zip64 self._zipfile = zf self._compressor = _get_compressor(zinfo.compress_type, - zinfo._compress_level) + zinfo._compresslevel) self._file_size = 0 self._compress_size = 0 self._crc = 0 @@ -1357,7 +1357,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): elif mode == 'w': zinfo = ZipInfo(name) zinfo.compress_type = self.compression - zinfo._compress_level = self.compresslevel + zinfo._compresslevel = self.compresslevel else: # Get info object for name zinfo = self.getinfo(name) @@ -1592,7 +1592,7 @@ def _writecheck(self, zinfo): " would require ZIP64 extensions") def write(self, filename, arcname=None, - compress_type=None, compress_level=None): + compress_type=None, compresslevel=None): """Put the bytes from filename into the archive under the name arcname.""" if not self.fp: @@ -1614,10 +1614,10 @@ def write(self, filename, arcname=None, else: zinfo.compress_type = self.compression - if compress_level is not None: - zinfo._compress_level = compress_level + if compresslevel is not None: + zinfo._compresslevel = compresslevel else: - zinfo._compress_level = self.compresslevel + zinfo._compresslevel = self.compresslevel if zinfo.is_dir(): with self._lock: @@ -1640,7 +1640,7 @@ def write(self, filename, arcname=None, shutil.copyfileobj(src, dest, 1024*8) def writestr(self, zinfo_or_arcname, data, - compress_type=None, compress_level=None): + compress_type=None, compresslevel=None): """Write a file into the archive. The contents is 'data', which may be either a 'str' or a 'bytes' instance; if it is a 'str', it is encoded as UTF-8 first. @@ -1652,7 +1652,7 @@ def writestr(self, zinfo_or_arcname, data, zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression - zinfo._compress_level = self.compresslevel + zinfo._compresslevel = self.compresslevel if zinfo.filename[-1] == '/': zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr |= 0x10 # MS-DOS directory flag @@ -1672,8 +1672,8 @@ def writestr(self, zinfo_or_arcname, data, if compress_type is not None: zinfo.compress_type = compress_type - if compress_level is not None: - zinfo._compress_level = compress_level + if compresslevel is not None: + zinfo._compresslevel = compresslevel zinfo.file_size = len(data) # Uncompressed size with self._lock: From c390f645d5bccade0d7744164483a87729aa5346 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 29 Jan 2018 11:01:35 -0600 Subject: [PATCH 13/13] Update compresslevel documentation targets --- Doc/library/zipfile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index b0ae149642bb87..d58efe0b417516 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -165,9 +165,9 @@ ZipFile Objects writing files to the archive. When using :const:`ZIP_STORED` or :const:`ZIP_LZMA` it has no effect. When using :const:`ZIP_DEFLATED` integers ``0`` through ``9`` are accepted - (see :class:`gzip.GzipFile`). + (see :class:`zlib ` for more information). When using :const:`ZIP_BZIP2` integers ``1`` through ``9`` are accepted - (see :class:`bz2.BZ2File`). + (see :class:`bz2 ` for more information). If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then :meth:`closed ` without adding any files to the archive, the appropriate