Attachment 1503860 Details for Bug 1648490 – Patch 0002: Add zstd compression

[patch] Patch 0002: Add zstd compression

0002-Add-zstd-compression-support-4.3-18.patch (text/plain), 15.22 KB, created by John Friend on 2018-11-09 22:05:46 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: John Friend

Created: 2018-11-09 22:05:46 UTC

Size: 15.22 KB

patch

obsolete

>From 9270846b9a30a4c0606d63d725c7b97cc846a709 Mon Sep 17 00:00:00 2001
>From: root <root@localhost>
>Date: Sat, 10 Nov 2018 00:27:45 +0300
>Subject: [PATCH 2/2] Add zstd compression support 4.3-18
>
>Patch sourced from:
>
>https://github.com/plougher/squashfs-tools/commit/6113361316d5ce5bfdc118d188e5617a1fcd747c
>Repo contains LICENSING file for GNU General Public License v2.0, like
>the original squashfs-tools. and commit log contains sign-offs by: 
>Sean Purcell <me@seanp.xyz> and Nick Terrell <terrelln@fb.com>
>---
> 0001-squashfs-tools-Add-zstd-support.patch | 413 +++++++++++++++++++++
> mksquashfs.1                               | 150 ++++++++
> sources                                    |   2 +-
> squashfs-tools.spec                        |  12 +-
> 4 files changed, 574 insertions(+), 3 deletions(-)
> create mode 100644 0001-squashfs-tools-Add-zstd-support.patch
> create mode 100644 mksquashfs.1
>
>diff --git a/0001-squashfs-tools-Add-zstd-support.patch b/0001-squashfs-tools-Add-zstd-support.patch
>new file mode 100644
>index 0000000..d0ba23c
>--- /dev/null
>+++ b/0001-squashfs-tools-Add-zstd-support.patch
>@@ -0,0 +1,413 @@
>+From 6113361316d5ce5bfdc118d188e5617a1fcd747c Mon Sep 17 00:00:00 2001
>+From: Sean Purcell <me@seanp.xyz>
>+Date: Mon, 14 Aug 2017 22:46:04 -0700
>+Subject: [PATCH 1/4] squashfs-tools: Add zstd support
>+
>+This patch adds zstd support to squashfs-tools. It works with zstd
>+versions >= 1.0.0. It was originally written by Sean Purcell.
>+
>+Signed-off-by: Sean Purcell <me@seanp.xyz>
>+Signed-off-by: Nick Terrell <terrelln@fb.com>
>+---
>+ squashfs-tools/Makefile       |  20 +++
>+ squashfs-tools/compressor.c   |   8 ++
>+ squashfs-tools/squashfs_fs.h  |   1 +
>+ squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++
>+ squashfs-tools/zstd_wrapper.h |  48 +++++++
>+ 5 files changed, 331 insertions(+)
>+ create mode 100644 squashfs-tools/zstd_wrapper.c
>+ create mode 100644 squashfs-tools/zstd_wrapper.h
>+
>+diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile
>+index 52d2582..22fc559 100644
>+--- a/squashfs-tools/Makefile
>++++ b/squashfs-tools/Makefile
>+@@ -75,6 +75,18 @@ GZIP_SUPPORT = 1
>+ #LZMA_SUPPORT = 1
>+ #LZMA_DIR = ../../../../LZMA/lzma465
>+ 
>++
>++########### Building ZSTD support ############
>++#
>++# The ZSTD library is supported
>++# ZSTD homepage: http://zstd.net
>++# ZSTD source repository: https://github.com/facebook/zstd
>++#
>++# To build using the ZSTD library - install the library and uncomment the
>++# ZSTD_SUPPORT line below.
>++#
>++#ZSTD_SUPPORT = 1
>++
>+ ######## Specifying default compression ########
>+ #
>+ # The next line specifies which compression algorithm is used by default
>+@@ -177,6 +189,14 @@ LIBS += -llz4
>+ COMPRESSORS += lz4
>+ endif
>+ 
>++ifeq ($(ZSTD_SUPPORT),1)
>++CFLAGS += -DZSTD_SUPPORT
>++MKSQUASHFS_OBJS += zstd_wrapper.o
>++UNSQUASHFS_OBJS += zstd_wrapper.o
>++LIBS += -lzstd
>++COMPRESSORS += zstd
>++endif
>++
>+ ifeq ($(XATTR_SUPPORT),1)
>+ ifeq ($(XATTR_DEFAULT),1)
>+ CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT
>+diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c
>+index 525e316..02b5e90 100644
>+--- a/squashfs-tools/compressor.c
>++++ b/squashfs-tools/compressor.c
>+@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = {
>+ extern struct compressor xz_comp_ops;
>+ #endif
>+ 
>++#ifndef ZSTD_SUPPORT
>++static struct compressor zstd_comp_ops = {
>++	ZSTD_COMPRESSION, "zstd"
>++};
>++#else
>++extern struct compressor zstd_comp_ops;
>++#endif
>+ 
>+ static struct compressor unknown_comp_ops = {
>+ 	0, "unknown"
>+@@ -77,6 +84,7 @@ struct compressor *compressor[] = {
>+ 	&lzo_comp_ops,
>+ 	&lz4_comp_ops,
>+ 	&xz_comp_ops,
>++	&zstd_comp_ops,
>+ 	&unknown_comp_ops
>+ };
>+ 
>+diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h
>+index 791fe12..afca918 100644
>+--- a/squashfs-tools/squashfs_fs.h
>++++ b/squashfs-tools/squashfs_fs.h
>+@@ -277,6 +277,7 @@ typedef long long		squashfs_inode;
>+ #define LZO_COMPRESSION		3
>+ #define XZ_COMPRESSION		4
>+ #define LZ4_COMPRESSION		5
>++#define ZSTD_COMPRESSION	6
>+ 
>+ struct squashfs_super_block {
>+ 	unsigned int		s_magic;
>+diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c
>+new file mode 100644
>+index 0000000..dcab75a
>+--- /dev/null
>++++ b/squashfs-tools/zstd_wrapper.c
>+@@ -0,0 +1,254 @@
>++/*
>++ * Copyright (c) 2017
>++ * Phillip Lougher <phillip@squashfs.org.uk>
>++ *
>++ * This program is free software; you can redistribute it and/or
>++ * modify it under the terms of the GNU General Public License
>++ * as published by the Free Software Foundation; either version 2,
>++ * or (at your option) any later version.
>++ *
>++ * This program is distributed in the hope that it will be useful,
>++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
>++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>++ * GNU General Public License for more details.
>++ *
>++ * zstd_wrapper.c
>++ *
>++ * Support for ZSTD compression http://zstd.net
>++ */
>++
>++#include <stdio.h>
>++#include <string.h>
>++#include <stdlib.h>
>++#include <zstd.h>
>++#include <zstd_errors.h>
>++
>++#include "squashfs_fs.h"
>++#include "zstd_wrapper.h"
>++#include "compressor.h"
>++
>++static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
>++
>++/*
>++ * This function is called by the options parsing code in mksquashfs.c
>++ * to parse any -X compressor option.
>++ *
>++ * This function returns:
>++ *	>=0 (number of additional args parsed) on success
>++ *	-1 if the option was unrecognised, or
>++ *	-2 if the option was recognised, but otherwise bad in
>++ *	   some way (e.g. invalid parameter)
>++ *
>++ * Note: this function sets internal compressor state, but does not
>++ * pass back the results of the parsing other than success/failure.
>++ * The zstd_dump_options() function is called later to get the options in
>++ * a format suitable for writing to the filesystem.
>++ */
>++static int zstd_options(char *argv[], int argc)
>++{
>++	if (strcmp(argv[0], "-Xcompression-level") == 0) {
>++		if (argc < 2) {
>++			fprintf(stderr, "zstd: -Xcompression-level missing "
>++				"compression level\n");
>++			fprintf(stderr, "zstd: -Xcompression-level it should "
>++				"be 1 <= n <= %d\n", ZSTD_maxCLevel());
>++			goto failed;
>++		}
>++
>++		compression_level = atoi(argv[1]);
>++		if (compression_level < 1 ||
>++		    compression_level > ZSTD_maxCLevel()) {
>++			fprintf(stderr, "zstd: -Xcompression-level invalid, it "
>++				"should be 1 <= n <= %d\n", ZSTD_maxCLevel());
>++			goto failed;
>++		}
>++
>++		return 1;
>++	}
>++
>++	return -1;
>++failed:
>++	return -2;
>++}
>++
>++/*
>++ * This function is called by mksquashfs to dump the parsed
>++ * compressor options in a format suitable for writing to the
>++ * compressor options field in the filesystem (stored immediately
>++ * after the superblock).
>++ *
>++ * This function returns a pointer to the compression options structure
>++ * to be stored (and the size), or NULL if there are no compression
>++ * options.
>++ */
>++static void *zstd_dump_options(int block_size, int *size)
>++{
>++	static struct zstd_comp_opts comp_opts;
>++
>++	/* don't return anything if the options are all default */
>++	if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL)
>++		return NULL;
>++
>++	comp_opts.compression_level = compression_level;
>++
>++	SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
>++
>++	*size = sizeof(comp_opts);
>++	return &comp_opts;
>++}
>++
>++/*
>++ * This function is a helper specifically for the append mode of
>++ * mksquashfs.  Its purpose is to set the internal compressor state
>++ * to the stored compressor options in the passed compressor options
>++ * structure.
>++ *
>++ * In effect this function sets up the compressor options
>++ * to the same state they were when the filesystem was originally
>++ * generated, this is to ensure on appending, the compressor uses
>++ * the same compression options that were used to generate the
>++ * original filesystem.
>++ *
>++ * Note, even if there are no compressor options, this function is still
>++ * called with an empty compressor structure (size == 0), to explicitly
>++ * set the default options, this is to ensure any user supplied
>++ * -X options on the appending mksquashfs command line are over-ridden.
>++ *
>++ * This function returns 0 on sucessful extraction of options, and -1 on error.
>++ */
>++static int zstd_extract_options(int block_size, void *buffer, int size)
>++{
>++	struct zstd_comp_opts *comp_opts = buffer;
>++
>++	if (size == 0) {
>++		/* Set default values */
>++		compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
>++		return 0;
>++	}
>++
>++	/* we expect a comp_opts structure of sufficient size to be present */
>++	if (size < sizeof(*comp_opts))
>++		goto failed;
>++
>++	SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
>++
>++	if (comp_opts->compression_level < 1 ||
>++	    comp_opts->compression_level > ZSTD_maxCLevel()) {
>++		fprintf(stderr, "zstd: bad compression level in compression "
>++			"options structure\n");
>++		goto failed;
>++	}
>++
>++	compression_level = comp_opts->compression_level;
>++
>++	return 0;
>++
>++failed:
>++	fprintf(stderr, "zstd: error reading stored compressor options from "
>++		"filesystem!\n");
>++
>++	return -1;
>++}
>++
>++static void zstd_display_options(void *buffer, int size)
>++{
>++	struct zstd_comp_opts *comp_opts = buffer;
>++
>++	/* we expect a comp_opts structure of sufficient size to be present */
>++	if (size < sizeof(*comp_opts))
>++		goto failed;
>++
>++	SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
>++
>++	if (comp_opts->compression_level < 1 ||
>++	    comp_opts->compression_level > ZSTD_maxCLevel()) {
>++		fprintf(stderr, "zstd: bad compression level in compression "
>++			"options structure\n");
>++		goto failed;
>++	}
>++
>++	printf("\tcompression-level %d\n", comp_opts->compression_level);
>++
>++	return;
>++
>++failed:
>++	fprintf(stderr, "zstd: error reading stored compressor options from "
>++		"filesystem!\n");
>++}
>++
>++/*
>++ * This function is called by mksquashfs to initialise the
>++ * compressor, before compress() is called.
>++ *
>++ * This function returns 0 on success, and -1 on error.
>++ */
>++static int zstd_init(void **strm, int block_size, int datablock)
>++{
>++	ZSTD_CCtx *cctx = ZSTD_createCCtx();
>++
>++	if (!cctx) {
>++		fprintf(stderr, "zstd: failed to allocate compression "
>++			"context!\n");
>++		return -1;
>++	}
>++
>++	*strm = cctx;
>++	return 0;
>++}
>++
>++static int zstd_compress(void *strm, void *dest, void *src, int size,
>++			 int block_size, int *error)
>++{
>++	const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,
>++					     src, size, compression_level);
>++
>++	if (ZSTD_isError(res)) {
>++		/* FIXME:
>++		 * zstd does not expose stable error codes. The error enum may
>++		 * change between versions. Until upstream zstd stablizes the
>++		 * error codes, we have no way of knowing why the error occurs.
>++		 * zstd shouldn't fail to compress any input unless there isn't
>++		 * enough output space. We assume that is the cause and return
>++		 * the special error code for not enough output space.
>++		 */
>++		return 0;
>++	}
>++
>++	return (int)res;
>++}
>++
>++static int zstd_uncompress(void *dest, void *src, int size, int outsize,
>++			   int *error)
>++{
>++	const size_t res = ZSTD_decompress(dest, outsize, src, size);
>++
>++	if (ZSTD_isError(res)) {
>++		fprintf(stderr, "\t%d %d\n", outsize, size);
>++
>++		*error = (int)ZSTD_getErrorCode(res);
>++		return -1;
>++	}
>++
>++	return (int)res;
>++}
>++
>++static void zstd_usage(void)
>++{
>++	fprintf(stderr, "\t  -Xcompression-level <compression-level>\n");
>++	fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default "
>++		"%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL);
>++}
>++
>++struct compressor zstd_comp_ops = {
>++	.init = zstd_init,
>++	.compress = zstd_compress,
>++	.uncompress = zstd_uncompress,
>++	.options = zstd_options,
>++	.dump_options = zstd_dump_options,
>++	.extract_options = zstd_extract_options,
>++	.display_options = zstd_display_options,
>++	.usage = zstd_usage,
>++	.id = ZSTD_COMPRESSION,
>++	.name = "zstd",
>++	.supported = 1
>++};
>+diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h
>+new file mode 100644
>+index 0000000..4fbef0a
>+--- /dev/null
>++++ b/squashfs-tools/zstd_wrapper.h
>+@@ -0,0 +1,48 @@
>++#ifndef ZSTD_WRAPPER_H
>++#define ZSTD_WRAPPER_H
>++/*
>++ * Squashfs
>++ *
>++ * Copyright (c) 2017
>++ * Phillip Lougher <phillip@squashfs.org.uk>
>++ *
>++ * This program is free software; you can redistribute it and/or
>++ * modify it under the terms of the GNU General Public License
>++ * as published by the Free Software Foundation; either version 2,
>++ * or (at your option) any later version.
>++ *
>++ * This program is distributed in the hope that it will be useful,
>++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
>++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>++ * GNU General Public License for more details.
>++ *
>++ * zstd_wrapper.h
>++ *
>++ */
>++
>++#ifndef linux
>++#define __BYTE_ORDER BYTE_ORDER
>++#define __BIG_ENDIAN BIG_ENDIAN
>++#define __LITTLE_ENDIAN LITTLE_ENDIAN
>++#else
>++#include <endian.h>
>++#endif
>++
>++#if __BYTE_ORDER == __BIG_ENDIAN
>++extern unsigned int inswap_le16(unsigned short);
>++extern unsigned int inswap_le32(unsigned int);
>++
>++#define SQUASHFS_INSWAP_COMP_OPTS(s) { \
>++	(s)->compression_level = inswap_le32((s)->compression_level); \
>++}
>++#else
>++#define SQUASHFS_INSWAP_COMP_OPTS(s)
>++#endif
>++
>++/* Default compression */
>++#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15
>++
>++struct zstd_comp_opts {
>++	int compression_level;
>++};
>++#endif
>+-- 
>+2.19.1
>+
>diff --git a/sources b/sources
>index c1284a9..a287875 100644
>--- a/sources
>+++ b/sources
>@@ -1,3 +1,3 @@
> d92ab59aabf5173f2a59089531e30dbf  squashfs4.3.tar.gz
>-ed8d2bb2cac678a394815d081f1c465c  mksquashfs.1
>+c5c8175e7204005e80a0981914ae017c  mksquashfs.1
> 68bf9bbf3ba00dc6e480a59b24905185  unsquashfs.1
>diff --git a/squashfs-tools.spec b/squashfs-tools.spec
>index df1b6c9..12649b8 100644
>--- a/squashfs-tools.spec
>+++ b/squashfs-tools.spec
>@@ -1,7 +1,7 @@
> Summary: Utility for the creation of squashfs filesystems
> Name: squashfs-tools
> Version: 4.3
>-Release: 17%{?dist}
>+Release: 18%{?dist}
> License: GPLv2+
> Group: System Environment/Base
> URL: http://squashfs.sourceforge.net/
>@@ -27,12 +27,15 @@ Patch3:  cve-2015-4645.patch
> Patch4:  local-cve-fix.patch
> # sys/sysmacros.h defines major()/minor()
> Patch5:  0001-Add-missing-Include.patch
>+# zstd compression support from https://github.com/plougher/squashfs-tools
>+Patch6:  0001-squashfs-tools-Add-zstd-support.patch
> BuildRequires:  gcc
> BuildRequires: zlib-devel
> BuildRequires: xz-devel
> BuildRequires: lzo-devel
> BuildRequires: libattr-devel
> BuildRequires: lz4-devel
>+BuildRequires: libzstd-devel
> 
> %description
> Squashfs is a highly compressed read-only filesystem for Linux.  This package
>@@ -46,10 +49,11 @@ contains the utilities for manipulating squashfs filesystems.
> %patch3 -p1
> %patch4 -p0
> %patch5 -p1
>+%patch6 -p1
> 
> %build
> pushd squashfs-tools
>-CFLAGS="%{optflags}" XZ_SUPPORT=1 LZO_SUPPORT=1 LZMA_XZ_SUPPORT=1 LZ4_SUPPORT=1 make %{?_smp_mflags}
>+CFLAGS="%{optflags}" XZ_SUPPORT=1 LZO_SUPPORT=1 LZMA_XZ_SUPPORT=1 LZ4_SUPPORT=1 ZSTD_SUPPORT=1 make %{?_smp_mflags}
> 
> %install
> mkdir -p %{buildroot}%{_sbindir} %{buildroot}%{_mandir}/man1
>@@ -68,6 +72,10 @@ install -m 644 %{SOURCE2} %{buildroot}%{_mandir}/man1/unsquashfs.1
> %{_sbindir}/unsquashfs
> 
> %changelog
>+* Fri Nov 09 2018 ??? - 4.3-18
>+- Add missing include file
>+- Add zstd compression support (Sean Purcell via github.com/plougher/squashfs-tools)
>+
> * Sat Jul 14 2018 Fedora Release Engineering <releng@fedoraproject.org> - 4.3-17
> - Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild
> 
>-- 
>2.19.1
>

Actions: View | Diff

Attachments on bug 1648490: 1503859 | 1503860 | 1503861